/**
 * Blog Draft Generator API
 *
 * POST /api/blog/generate   — Generate a blog draft via LLM (multi-pass pipeline)
 * GET  /api/blog             — List all drafts
 * GET  /api/blog/:id         — Get a specific draft
 * PUT  /api/blog/:id/status  — Update draft status
 *
 * Pipeline: gather data → LLM master pass → depth improvement → quality control
 * Voice: Senior optical network engineer, not marketing.
 */
import { Router, Request, Response } from "express";
import * as pdfParseModule from "pdf-parse";
const pdfParse: (buffer: Buffer) => Promise<{ text: string; numpages: number; info: Record<string, unknown> }> =
  (pdfParseModule as any).default ?? (pdfParseModule as any);
import { pool } from "../db/client";
import { setLlmProvider, getLlmProvider, refreshLlmAutoDiscovery, pinLlmVersion, unpinLlmVersion } from "../llm/client";

/** In-memory pipeline progress tracker — step updates pushed here, polled via GET /api/blog/:id/progress */
const pipelineProgress = new Map<string, { step: number; total: number; label: string; pct: number }>();

function setProgress(draftId: string, step: number, label: string): void {
  const pct = Math.round((step / 16) * 92) + 2; // 2%..94% during run, 100% on complete
  pipelineProgress.set(draftId, { step, total: 16, label, pct });
}

function clearProgress(draftId: string): void {
  pipelineProgress.delete(draftId);
}
import { semanticSearch } from "../embeddings/client";
import { generate, checkHealth, resetOllamaQueue, getQueueDepth } from "../llm/client";
import {
  SYSTEM_PROMPT,
  DEPTH_PROMPT,
  ANTI_GENERIC_INTRO_PROMPT,
  QUALITY_CONTROL_PROMPT,
  PROCUREMENT_LAYER_PROMPT,
  buildTopicPrompt,
} from "../llm/blog-prompts";

// Anti-patterns list for quality validation
const GENERIC_PHRASES = [
  "plays a key role",
  "increasingly important",
  "it is important to note",
  "in today's rapidly evolving",
  "The optical transceiver market continues",
  "in today's fast-paced world",
  "optimize your network",
  "leverage the power",
  "enhance your",
  "consider implementing",
  "may indicate issues",
  "could potentially",
];

export const blogRouter = Router();

interface BlogTemplate {
  topic: string;
  title: string;
  target_audience: "sales" | "technical" | "customer" | "seo";
  seo_keywords: string[];
}

const BLOG_TEMPLATES: Record<string, BlogTemplate[]> = {
  hype_cycle: [
    {
      topic: "hype_cycle",
      title: "The State of {SPEED} Transceivers in {YEAR}: Technology Adoption Assessment",
      target_audience: "technical",
      seo_keywords: ["transceiver", "technology lifecycle", "optical networking", "adoption curve"],
    },
    {
      topic: "hype_cycle",
      title: "Investment Guide: Which Transceiver Speeds to Bet On in {YEAR}",
      target_audience: "sales",
      seo_keywords: ["transceiver investment", "data center optics", "ROI", "network planning"],
    },
  ],
  comparison: [
    {
      topic: "comparison",
      title: "{FORM_FACTOR} Transceiver Comparison: What Actually Matters for {USE_CASE}",
      target_audience: "technical",
      seo_keywords: ["transceiver comparison", "best transceiver", "compatible vs original"],
    },
    {
      topic: "comparison",
      title: "Original vs. Compatible Transceivers: The Real Cost Difference in {YEAR}",
      target_audience: "seo",
      seo_keywords: ["compatible transceiver", "original vs compatible", "cost savings", "interoperability"],
    },
  ],
  new_product: [
    {
      topic: "new_product",
      title: "{SPEED} Transceivers: What's New and What It Means for Your Network",
      target_audience: "technical",
      seo_keywords: ["new transceiver", "latest optics", "deployment guide"],
    },
  ],
  tutorial: [
    {
      topic: "tutorial",
      title: "Troubleshooting Optical Transceiver Issues: A Field Engineer's Guide",
      target_audience: "technical",
      seo_keywords: ["transceiver troubleshooting", "optical module problems", "low tx power", "BER errors", "400ZR"],
    },
    {
      topic: "tutorial",
      title: "How to Choose the Right Transceiver: A Practical {YEAR} Guide",
      target_audience: "customer",
      seo_keywords: ["transceiver buying guide", "how to choose transceiver", "form factor guide"],
    },
  ],
  technology_deep_dive: [
    {
      topic: "technology_deep_dive",
      title: "Deep Dive: {SPEED} Technology — What the Specs Don't Tell You",
      target_audience: "technical",
      seo_keywords: ["optical transceiver technology", "deep dive", "silicon photonics", "coherent optics"],
    },
    {
      topic: "technology_deep_dive",
      title: "{YEAR} Standards Roundup: What's Actually Production-Ready",
      target_audience: "technical",
      seo_keywords: ["IEEE 802.3", "OIF standards", "MSA", "production optics"],
    },
  ],
  market_alert: [
    {
      topic: "market_alert",
      title: "Market Alert: {SPEED} Transceiver Prices Are Moving — Here's Why",
      target_audience: "sales",
      seo_keywords: ["transceiver price", "market analysis", "optical networking market", "price drop"],
    },
    {
      topic: "market_alert",
      title: "Price War: What {YEAR}'s Transceiver Market Shift Means for Your Budget",
      target_audience: "sales",
      seo_keywords: ["transceiver market", "price trend", "optical module cost", "procurement"],
    },
  ],
  migration_guide: [
    {
      topic: "migration_guide",
      title: "The Complete Migration Guide: Moving to {SPEED} Without Breaking Production",
      target_audience: "technical",
      seo_keywords: ["network migration", "transceiver upgrade", "100G to 400G", "migration guide"],
    },
    {
      topic: "migration_guide",
      title: "{YEAR} Migration Playbook: From Planning to Production in 12 Months",
      target_audience: "technical",
      seo_keywords: ["network upgrade", "migration planning", "optical transceiver migration"],
    },
  ],
  buying_guide: [
    {
      topic: "buying_guide",
      title: "The {YEAR} Transceiver Buying Guide: What to Buy, What to Skip",
      target_audience: "customer",
      seo_keywords: ["transceiver buying guide", "best transceiver", "OEM vs compatible", "procurement"],
    },
    {
      topic: "buying_guide",
      title: "OEM vs Compatible Transceivers in {YEAR}: The Real Numbers",
      target_audience: "customer",
      seo_keywords: ["OEM transceiver", "compatible transceiver", "cost savings", "Flexoptix"],
    },
  ],
  competitor_analysis: [
    {
      topic: "competitor_analysis",
      title: "Competitor Roundup: What's New in {SPEED} Transceivers and What It Means",
      target_audience: "sales",
      seo_keywords: ["transceiver comparison", "competitor analysis", "optical module vendors"],
    },
    {
      topic: "competitor_analysis",
      title: "{YEAR} Vendor Landscape: Who's Winning the {FORM_FACTOR} Market",
      target_audience: "sales",
      seo_keywords: ["transceiver vendor", "market share", "optical networking vendors"],
    },
  ],
};

/** Gather data from vector collections for blog content — with PostgreSQL fallback.
 *  Topic-aware: strategy articles (hype_cycle, comparison) skip troubleshooting data.
 *
 *  IMPORTANT: Always enriches products with REAL verified prices from price_observations.
 *  The LLM may ONLY use prices returned here — never invent pricing. */
async function gatherBlogData(keywords: string[], topic?: string): Promise<{
  products: Array<Record<string, unknown>>;
  news: Array<Record<string, unknown>>;
  faq: Array<Record<string, unknown>>;
  troubleshooting: Array<Record<string, unknown>>;
}> {
  const query = keywords.join(" ");

  // Strategy articles should NOT pull troubleshooting data (topic separation)
  const skipTroubleshooting = topic === "hype_cycle" || topic === "comparison" || topic === "new_product";

  // Extract speed/form_factor hints from keywords for relevance filtering
  const speedHints = keywords.join(" ").match(/\b(10|25|40|100|200|400|800|1600)G\b/gi) || [];
  const speedGbps = speedHints.map(s => parseInt(s)).filter(Boolean);

  // ── Fetch real products with verified prices from DB ──────────────────────
  // Primary: filter by keyword-extracted speed; fallback to top products by speed
  const productQuery = speedGbps.length > 0
    ? `SELECT t.id, t.slug, t.form_factor, t.speed, t.speed_gbps, t.reach_label,
              t.fiber_type, t.standard_name, t.connector, t.power_consumption_w,
              t.tx_power_min_dbm, t.tx_power_max_dbm, t.rx_sensitivity_dbm,
              v.name as vendor, v.type as vendor_type
       FROM transceivers t
       LEFT JOIN vendors v ON t.vendor_id = v.id
       WHERE t.speed_gbps = ANY($1::int[])
       ORDER BY v.type = 'Compatible' DESC, t.speed_gbps DESC
       LIMIT 20`
    : `SELECT t.id, t.slug, t.form_factor, t.speed, t.speed_gbps, t.reach_label,
              t.fiber_type, t.standard_name, t.connector, t.power_consumption_w,
              t.tx_power_min_dbm, t.tx_power_max_dbm, t.rx_sensitivity_dbm,
              v.name as vendor, v.type as vendor_type
       FROM transceivers t
       LEFT JOIN vendors v ON t.vendor_id = v.id
       ORDER BY t.speed_gbps DESC
       LIMIT 20`;

  const [productsDb, newsDb] = await Promise.all([
    pool.query(productQuery, speedGbps.length > 0 ? [speedGbps] : []).catch(() => ({ rows: [] })),
    pool.query(
      `SELECT title, source, category, published_at::text as date
       FROM news_articles
       ORDER BY published_at DESC NULLS LAST
       LIMIT 5`
    ).catch(() => ({ rows: [] })),
  ]);

  // ── Enrich each product with real verified prices ─────────────────────────
  const productIds = productsDb.rows.map((r: Record<string, unknown>) => r.id).filter(Boolean);
  let priceMap: Record<string, Array<{vendor: string; price: number; currency: string; url: string; observed_at: string}>> = {};

  if (productIds.length > 0) {
    const priceResult = await pool.query(
      `SELECT po.transceiver_id,
              v.name as vendor,
              v.type as vendor_type,
              po.price::float as price,
              po.currency,
              po.url,
              po.time::text as observed_at
       FROM price_observations po
       JOIN vendors v ON po.source_vendor_id = v.id
       WHERE po.transceiver_id = ANY($1::int[])
         AND po.time > NOW() - INTERVAL '30 days'
         AND po.price IS NOT NULL
         AND po.price > 0
       ORDER BY po.transceiver_id, po.time DESC`,
      [productIds]
    ).catch(() => ({ rows: [] }));

    // Group by transceiver_id — keep best price per vendor
    for (const row of priceResult.rows) {
      const tid = String(row.transceiver_id);
      if (!priceMap[tid]) priceMap[tid] = [];
      // Deduplicate by vendor — keep most recent
      if (!priceMap[tid].find((p) => p.vendor === row.vendor)) {
        priceMap[tid].push({
          vendor: row.vendor,
          price: row.price,
          currency: row.currency || "EUR",
          url: row.url || "",
          observed_at: row.observed_at,
        });
      }
    }
  }

  // Attach prices to products
  const enrichedProducts = productsDb.rows.map((p: Record<string, unknown>) => ({
    ...p,
    verified_prices: priceMap[String(p.id)] || [],
    has_verified_price: (priceMap[String(p.id)] || []).length > 0,
  }));

  // Try vector search to supplement (but always use DB products as base — they have real prices)
  try {
    const [vectorProducts, news, faq, troubleshooting] = await Promise.all([
      semanticSearch("product_embeddings", query, 10).catch(() => []),
      semanticSearch("news_embeddings", query, 5).catch(() => []),
      skipTroubleshooting ? Promise.resolve([]) : semanticSearch("faq_embeddings", query, 5).catch(() => []),
      skipTroubleshooting ? Promise.resolve([]) : semanticSearch("troubleshooting_embeddings", query, 3).catch(() => []),
    ]);

    return {
      // DB products first (they have real prices) — vector results supplemental only
      products: [
        ...enrichedProducts,
        ...vectorProducts.map((r) => ({ score: r.score, ...r.payload })),
      ].slice(0, 20),
      news: news.length > 0 ? news.map((r) => ({ score: r.score, ...r.payload })) : newsDb.rows,
      faq: faq.map((r) => ({ score: r.score, ...r.payload })),
      troubleshooting: troubleshooting.map((r) => ({ score: r.score, ...r.payload })),
    };
  } catch {
    // Vector search unavailable — use PostgreSQL only
  }

  return {
    products: enrichedProducts,
    news: newsDb.rows,
    faq: [],
    troubleshooting: [],
  };
}

/** Validate article has no placeholder text or empty sections */
function validateArticle(content: string): string[] {
  const issues: string[] = [];

  if (/\b(TODO|NOTE|FIXME|PLACEHOLDER)\b/i.test(content)) {
    issues.push("Contains placeholder text");
  }
  if (/<!--[\s\S]*?-->/.test(content)) {
    issues.push("Contains HTML comments");
  }
  // Check for truly empty sections: heading followed by only whitespace then another heading
  if (/^##\s+.+\n\s*\n(?=##\s)/m.test(content)) {
    issues.push("Empty section detected");
  }
  // Check for generic filler
  for (const phrase of GENERIC_PHRASES) {
    if (content.toLowerCase().includes(phrase.toLowerCase())) {
      issues.push(`Generic phrase: "${phrase}"`);
    }
  }
  // Check minimum depth
  const wordCount = content.split(/\s+/).length;
  if (wordCount < 800) {
    issues.push(`Too short: ${wordCount} words (minimum 800 for template, 1200 for LLM)`);
  }
  // Check for power budget section in troubleshooting articles
  if (content.toLowerCase().includes("troubleshoot") && !content.toLowerCase().includes("power budget")) {
    issues.push("Missing power budget section");
  }

  return issues;
}

/** Generate a template-based draft (fallback when LLM is unavailable) */
function generateTemplateDraft(
  title: string,
  topic: string,
  data: Awaited<ReturnType<typeof gatherBlogData>>,
): string {
  const date = new Date().toISOString().split("T")[0];
  const parts: string[] = [];
  parts.push(`# ${title}\n`);
  parts.push(`*Generated by TIP Blog Engine on ${date}*\n`);
  parts.push(`> **Status**: Template draft — LLM enhancement required for publication.\n`);

  if (topic === "tutorial") {
    parts.push(generateTutorialTemplate(data));
  } else if (topic === "hype_cycle") {
    parts.push(generateHypeCycleTemplate(data));
  } else if (topic === "comparison") {
    parts.push(generateComparisonTemplate(data));
  } else {
    parts.push(generateNewProductTemplate(data));
  }

  return parts.join("\n");
}

function generateTutorialTemplate(data: Awaited<ReturnType<typeof gatherBlogData>>): string {
  const parts: string[] = [];

  parts.push(`
## The Scenario

It's 2 AM. The NOC pager goes off. A core spine link between two pods is flapping — 200G of aggregate capacity is bouncing every 45 seconds. You SSH into the switch and pull the DOM readings:

\`\`\`
switch# show interface transceiver details
Ethernet1/49
  Tx Power:     -14.3 dBm  (alarm: low)
  Rx Power:     -2.1 dBm
  Temperature:  78.4°C     (warning: high)
  Voltage:      3.28 V
  Current:      42.1 mA
\`\`\`

Tx at -14.3 dBm on a module rated for -8.2 to +0.5 dBm. The laser is dying. But is it the only problem? The temperature is also high at 78.4°C — that's above the 75°C warning threshold. Here's how to work through this systematically.


## Quick Diagnosis Framework

Before diving into details, use this decision tree under pressure:

**Link is completely down:**
→ Check Tx power on both ends → If Tx is within spec but Rx is very low or absent → fiber issue (break, dirty connector, wrong fiber type)
→ If Tx is below alarm threshold → transceiver is failing, replace it
→ If both Tx and Rx look normal but link won't establish → check speed/encoding mismatch

**Link is up but errors are climbing:**
→ Check pre-FEC BER → If above 2.4×10⁻⁴ → power budget is marginal, check connectors and fiber path
→ Check CRC counters → If >100/min → dirty fiber end-faces → clean and recheck
→ If >10,000/min → wrong fiber type or failing optic

**Link is flapping (up/down cycling):**
→ Check temperature trends over 24h → rising temperature = airflow or load issue
→ Check DOM Tx power over time → declining Tx = laser degradation
→ Check both ends — flapping is often caused by one side, visible on the other


## Low Transmit Power — Dying Laser

The most common transceiver failure mode. The laser diode degrades over time, especially at elevated temperatures. You'll see Tx power drop gradually over weeks before the module finally dies.

**How to spot it:**
\`\`\`
switch# show interface Ethernet1/1 transceiver details
  Tx Power: -12.8 dBm   ← below -11.0 alarm threshold
  Tx Bias:  58.2 mA      ← laser is drawing more current to compensate
\`\`\`

**What's happening:** The laser is pushing more current (bias) to maintain output. When the bias hits its maximum, the power drops off a cliff. Rising bias current is your early warning — catch it before Tx power actually drops.

**Reference Tx power ranges:**
| Module | Normal Tx Range | Alarm Threshold |
|--------|----------------|-----------------|
| SFP+ SR | -8.2 to +0.5 dBm | -11.0 dBm |
| SFP+ LR | -8.2 to +0.5 dBm | -11.0 dBm |
| QSFP28 SR4 | -7.0 to +2.0 dBm/lane | -10.0 dBm |
| QSFP28 LR4 | -4.3 to +4.5 dBm | -7.0 dBm |
| QSFP-DD DR4 | -2.9 to +3.0 dBm/lane | -5.5 dBm |
| 400ZR | -10.0 to +2.0 dBm | -13.0 dBm |

**What engineers get wrong:** Replacing the module immediately without checking if high temperature caused the degradation. If the replacement runs at 78°C too, it'll die in 6 months. Fix the airflow first.


## High BER and CRC Errors

CRC errors are the most misdiagnosed issue. Engineers often blame the optic when the problem is the fiber path.

**Pre-FEC vs Post-FEC BER — this matters:**
- **Pre-FEC BER < 2.4×10⁻⁴**: FEC can correct this. Normal operation. Don't panic.
- **Pre-FEC BER > 2.4×10⁻⁴**: FEC is at its limit. Errors will leak through. Fix the root cause.
- **Post-FEC BER > 0**: Uncorrectable errors. The link is functionally broken even if it shows "up."

**Diagnosis steps:**
1. Clean fiber end-faces on BOTH ends. Use IBC one-click cleaners, not wipes. Inspect with a 200x fiber scope after cleaning.
2. Check fiber type: SMF for LR/ER/ZR modules, MMF for SR modules. A 10G LR module on OM3 MMF won't throw an error — it just won't work at all.
3. Measure with an optical power meter. Compare Rx power against the spec.
4. If Rx is within spec but BER is high → suspect the optic itself, swap it.

**CRC error rate interpretation:**
- 0-10/min: Normal noise on long runs
- 10-100/min: Marginal — investigate during next window
- 100-1000/min: Dirty connectors or marginal power budget
- >10,000/min: Wrong fiber type, failing optic, or severe fiber damage


## Power Budget — The Most Ignored Cause of Problems

Every link has a power budget. If you don't calculate it, you're guessing. Here's a real example:

**Scenario: 10G LR link over 8km campus fiber**
\`\`\`
Tx Power (SFP+ LR):           -1.0 dBm (typical)
Fiber loss (8km × 0.35 dB/km): -2.8 dB
Connector loss (4 × 0.3 dB):   -1.2 dB  (2 patch panels)
Splice loss (2 × 0.1 dB):      -0.2 dB
─────────────────────────────────────────
Expected Rx Power:              -5.2 dBm
Rx Sensitivity (SFP+ LR):     -14.4 dBm
Margin:                         +9.2 dB  ✓ (need ≥3 dB)
\`\`\`

This link has plenty of margin. But add two dirty connectors (1.5 dB loss each instead of 0.3 dB) and you lose 2.4 dB of margin. Now you're at 6.8 dB — still okay, but you've eaten into your safety margin.

**Common fiber loss values:**
- SMF @ 1310nm: 0.35 dB/km
- SMF @ 1550nm: 0.22 dB/km
- MMF OM3 @ 850nm: 3.5 dB/km (yes, 10x more than SMF)
- Connector (clean): 0.3 dB
- Connector (dirty): 1.0-3.0 dB ← this is where most problems hide
- Fusion splice: 0.1 dB
- Mechanical splice: 0.5 dB


## Temperature and Environmental Issues

Transceivers have a specified operating temperature range. Exceed it and the laser degrades faster, DOM readings become unreliable, and the module may shut down.

- **Commercial (COM)**: 0°C to 70°C — most data center modules
- **Industrial (IND)**: -40°C to +85°C — outdoor/harsh environments
- **Extended**: -5°C to +85°C — some enterprise modules

**Reality check:** Top-of-rack switch positions run 10-15°C hotter than bottom positions. A module rated to 70°C in a switch at the top of a 42U rack might see 68°C ambient. Add the module's own heat dissipation and you're at 75-80°C internally.

**What to check:**
\`\`\`
switch# show interface transceiver details | include Temp
  Temperature: 72.3°C   ← approaching commercial limit
\`\`\`

If temperature is above 65°C consistently, consider:
1. Moving the switch to a cooler rack position
2. Using industrial-temp rated modules ($15-30 more per optic — worth it)
3. Improving rack airflow (blanking panels, hot/cold aisle separation)


## Common Mistakes Engineers Make

1. **Replacing a $2,400 QSFP-DD when the problem is a dirty connector.** Always clean and inspect fiber end-faces first. 40% of RMA'd optics test fine at the vendor.

2. **Using MMF patch cables with LR optics.** The link won't come up at all. No error message — just "down." Check patch cable color: orange = OM3/OM4 (MMF), yellow = SMF.

3. **Ignoring pre-FEC BER trending.** By the time post-FEC errors appear, the link is already failing. Monitor pre-FEC BER and set alerts at 1×10⁻⁵.

4. **Not checking both ends.** A flapping link caused by a dying Tx on the far end shows as unstable Rx on the near end. Always check both sides.

5. **Assuming the optic is bad without measuring.** Use an optical power meter. It takes 30 seconds and eliminates guesswork. A $200 power meter saves thousands in unnecessary RMAs.


## When to Replace the Transceiver vs Fix the Fiber

| Symptom | Check First | Likely Cause |
|---------|-------------|-------------|
| Tx below alarm | DOM Tx + bias current | Optic — replace |
| Rx below alarm, Tx normal on far end | Fiber path, connectors | Fiber — clean/repair |
| High BER, power levels OK | Fiber type, connectors | Fiber — inspect |
| High temperature | Airflow, rack position | Environment — fix cooling |
| Link won't establish | Speed/encoding settings | Config — check both ends |`);

  // Add troubleshooting data if available
  if (data.troubleshooting.length > 0) {
    parts.push(`\n\n## Additional Troubleshooting Reference\n`);
    for (const t of data.troubleshooting) {
      parts.push(`### ${t.symptom}\n`);
      parts.push(`**Root Cause**: ${t.cause}\n`);
      parts.push(`**Resolution**: ${t.solution}\n`);
    }
  }

  parts.push(`

## Key Takeaways

1. **Clean before you swap.** Dirty connectors cause 40%+ of optical issues. Always inspect with a fiber scope first.
2. **Pre-FEC BER is your early warning system.** Set alerts at 1×10⁻⁵ and investigate before post-FEC errors start.
3. **Calculate your power budget.** Don't guess. Measure Tx, subtract expected losses, compare against Rx sensitivity. Need ≥3 dB margin.
4. **Temperature kills optics slowly.** A module running at 72°C will last 2-3 years instead of 7-10. Fix the environment.
5. **Check both ends.** Every link has two sides. One bad Tx looks like a bad Rx on the other end.
`);

  // Only reference products if contextually relevant
  if (data.products.length > 0) {
    parts.push(`\n## Related Transceiver Specifications\n`);
    parts.push(`The following modules were referenced in this guide:\n`);
    for (const p of data.products.slice(0, 5)) {
      parts.push(`- **${p.standard_name || p.slug}**: ${p.form_factor} ${p.speed}, ${p.reach_label || "N/A"} reach, ${p.fiber_type || "N/A"}`);
    }
    parts.push(`\n*Check current availability and pricing at [flexoptix.net](https://www.flexoptix.net/en/)*`);
  }

  return parts.join("\n");
}

function generateHypeCycleTemplate(data: Awaited<ReturnType<typeof gatherBlogData>>): string {
  const parts: string[] = [];
  parts.push(`
## The Thesis

If you're still planning new 100G leaf-spine deployments in 2026, you're designing yesterday's network. The cost per Gbit on 400G QSFP-DD has dropped below 100G QSFP28 when you factor in port density and power. The numbers don't lie — and they've shifted faster than most procurement cycles can keep up.

This is not a technology overview. This is an investment decision framework. You're here because you need to decide what to buy, when to buy it, and what to skip entirely.


## Market Reality

AI and ML workloads have fundamentally changed data center traffic patterns. East-west traffic in GPU clusters is doubling every 12 months. A single 8-GPU training node generates 3.2 Tbps of fabric traffic. The spine layer that was comfortable at 100G two years ago is now the bottleneck.

Hyperscalers moved to 400G spine in 2023-2024 and are deploying 800G in 2025-2026. Enterprise follows with a 2-3 year lag — which means enterprise 400G spine deployments should be happening now, not "next budget cycle."

On the supply side: 400G QSFP-DD compatible pricing has dropped 40% in the last 18 months. A DR4 module that cost $600 in early 2025 is now $250-350. The economics have crossed over — 400G is no longer a premium choice, it's the rational default.


## Speed-by-Speed Investment Analysis

### 100G QSFP28 — LEGACY
**Cost per Gbit**: ~$0.45-1.20 (compatible), ~$3.00-9.00 (OEM)
**Verdict**: Do not design new builds around 100G.

100G QSFP28 is mature, proven, and cheap in absolute terms. But per-Gbit economics now favor 400G for spine interconnects. A 32-port 400G switch gives you 12.8 Tbps in 1RU. You'd need four 32-port 100G switches (4RU, 4x the power, 4x the cabling) for the same bandwidth. The port count math kills 100G for new spine designs.

Where 100G still makes sense: leaf switches connecting to existing 100G server NICs, and brownfield environments where the switch chassis only supports QSFP28. Everywhere else, it's a legacy choice.

### 200G — SKIP
200G never gained meaningful traction. Most vendors skipped it entirely, going from 100G to 400G. Unless you have a specific chassis that only supports 200G line cards, there is no reason to invest here. Skip it.

### 400G QSFP-DD / OSFP — BUY
**Cost per Gbit**: ~$0.50-1.12 (compatible DR4/FR4), ~$2.25-8.00 (OEM)
**Verdict**: The 2026 sweet spot. Deploy broadly.

400G is where the industry has converged. Multi-vendor interop is solid for DR4 and FR4. The compatible supply chain has matured — Flexoptix, FS, Precision OT, and others all ship production-quality modules with full vendor coding.

**QSFP-DD vs OSFP**: QSFP-DD wins on port density (backward-compatible with QSFP28 cages, so 36-port switches are standard). OSFP wins on thermal headroom (wider module = better heat dissipation), which matters for coherent 400ZR and future 800G upgrades. If you're buying switches today: QSFP-DD for pure data center, OSFP if your roadmap includes coherent or 800G.

Power consumption: ~12W per QSFP-DD DR4 port. At $0.12/kWh, that's $12.60/port/year for the optic alone. Scale that to 2,000 ports and you're looking at $25,200/year in optics power cost. Factor this in.

### 800G OSFP / QSFP-DD800 — EARLY ADOPTER
**Cost per Gbit**: ~$1.00-2.50 (compatible, limited availability), ~$4.00-10.00 (OEM)
**Verdict**: Deploy for AI/ML spine only. Not general-purpose yet.

800G is real hardware — Arista 7060X6, Cisco Nexus 9000v with Silicon One G200, and Broadcom Memory-based switches all support it. But the ecosystem is 2 years behind 400G: fewer compatible vendors, more interop edge cases, higher per-port cost.

Deploy 800G if: you're building GPU cluster interconnects where bandwidth density per RU is the constraint. A single 800G port replaces two 400G ports — that's real rack space savings in a 10,000-GPU training cluster.

Do not deploy 800G if: you're running a standard enterprise data center. The 2.5-3x price premium over 400G does not justify the bandwidth for typical east-west traffic patterns. Wait 12-18 months for pricing to normalize.

### 1.6T / CPO / LPO — WATCH
1.6T OSFP-XD exists in lab demos. Co-Packaged Optics (CPO) and Linear Pluggable Optics (LPO) are promising architectures but 3-5 years from enterprise production. Do not design around technology that doesn't ship yet. If a vendor is pushing 1.6T into your RFP, they're selling a roadmap, not a product.


## Investment Decision Matrix

### DO (act now)
- **Deploy 400G QSFP-DD for all new spine interconnects.** The cost/Gbit crossover with 100G has happened. Waiting costs you more than buying.
- **Standardize on 25G SFP28 for server access.** $18-35 per port, universal NIC support, mature supply chain.
- **Budget 800G for AI/ML fabric spine in 2027.** Put it in the 3-year plan, not the current PO.
- **Use compatible optics for everything ≤400G.** OEM pricing is indefensible below 400G. For 400G, evaluate per-platform.

### AVOID (do not invest)
- **New 100G spine designs.** The per-Gbit economics no longer work.
- **200G anything.** Dead-end speed class.
- **1.6T or CPO commitments.** No production ecosystem.
- **OEM optics for SFP+ through QSFP28.** You're paying 5-10x for a logo.

### CONSIDER (evaluate carefully)
- **Fiber infrastructure readiness.** 400G DR4 needs SMF — if your intra-DC cabling is still MMF, factor in the re-cabling cost.
- **Power and cooling capacity.** 400G draws ~12W/port, 800G draws ~18-25W/port. At 2,000 ports, that's 24-50 kW of optics power alone.
- **Switch platform lifecycle.** The optics are 10% of the cost; the switch is 90%. Choose a switch platform that supports your 3-5 year optics roadmap.


## Total Cost of Ownership — A Real Example

Scenario: 200-port leaf-spine fabric upgrade, 3-year lifecycle.

| | 100G QSFP28 | 400G QSFP-DD |
|---|---|---|
| Optics (compatible) | 200 × $80 = $16,000 | 50 × $350 = $17,500 |
| Switch ports needed | 200 | 50 |
| Switch cost (est.) | 4 × $45,000 = $180,000 | 1 × $85,000 = $85,000 |
| Power (optics, 3yr) | 200 × 3.5W × $0.12 × 26,280h = $22,075 | 50 × 12W × $0.12 × 26,280h = $18,922 |
| Rack space | 4 RU | 1 RU |
| Cabling | 200 cables | 50 cables |
| **Total 3yr** | **~$218,000** | **~$121,000** |

The 400G option costs **45% less** while delivering the same aggregate bandwidth in one-quarter of the rack space. This is why 100G is no longer the "safe" choice — it's the expensive one.


## Key Takeaways

1. **400G QSFP-DD is the rational default for 2026.** Not because it's new, but because the economics crossed over. It's cheaper than 100G at scale.
2. **100G is legacy.** Fine for brownfield, wrong for greenfield. Stop designing new 100G spines.
3. **800G is for AI fabrics, not general enterprise.** Deploy when bandwidth density per RU is your actual constraint, not because a vendor told you to "future-proof."
4. **The optic costs 10% of the total.** The switch, fiber, power, cooling, and engineering time are the real investment. Choose optics that fit the platform strategy, not the other way around.
5. **Compatible optics are the industry default.** OEM pricing is a relic of vendor lock-in. The silicon is identical. Test, validate, deploy.

*Check current pricing and availability at [flexoptix.net](https://www.flexoptix.net/en/)*
`);

  return parts.join("\n");
}

function generateComparisonTemplate(data: Awaited<ReturnType<typeof gatherBlogData>>): string {
  const parts: string[] = [];
  parts.push(`
## The Question

You need 200 optics for a new leaf-spine build. The OEM quotes $3,200 per QSFP-DD DR4. A compatible vendor offers the same module — same Broadcom DSP, same laser, same fiber interface — for $350. That's $570,000 in savings on a single PO. Your boss asks: "What's the catch?" Here's the honest answer.


## What You're Actually Paying For

Let's be direct about what's inside a transceiver module, because the pricing gap makes no sense until you understand the supply chain.

**The silicon is identical.** Whether the label says Cisco, Arista, or Flexoptix — the DSP inside a QSFP-DD DR4 is a Broadcom or Marvell chip. The same fab, the same wafer, the same die. The laser diode comes from the same handful of suppliers: II-VI (now Coherent), Lumentum, or Source Photonics. There are maybe 5 companies in the world that actually manufacture the optical subassemblies. Everyone else assembles and brands.

**What OEM vendors add:**
- EEPROM coding with their vendor ID (so the switch recognizes "their" module)
- An additional test cycle in their own facility
- Integration into their TAC support workflow
- A logo and a 10x markup

**What compatible vendors add:**
- The same EEPROM coding, programmed for your specific switch platform
- Their own test cycle (varies by vendor — this is where quality differs)
- Typically faster RMA turnaround (days, not weeks)
- Pricing that reflects the actual manufacturing cost


## The Real Differences That Matter

### 1. Vendor Authentication (The Lock-In Question)
Some switches check the EEPROM for a vendor-specific signature. The behavior varies:

| Platform | Default Behavior | Compatible Fix |
|----------|-----------------|----------------|
| Cisco Nexus 9000 | Logs warning, may limit features | \`service unsupported-transceiver\` |
| Arista 7000 series | Allows all modules by default | None needed |
| Juniper QFX/EX | May reject on some firmware | \`allow-unsupported-sfp\` |
| HPE/Aruba | Strict on some models | Requires correct vendor coding |
| MikroTik | Allows everything | None needed |

The key: vendor locking is a software decision, not a hardware limitation. A properly coded compatible module is electrically identical.

### 2. DOM Accuracy
OEM modules typically report temperature within ±1°C. Some compatible modules are ±2-3°C. In practice, this matters only if your monitoring system triggers alerts on tight thresholds. For most deployments, it's irrelevant. For coherent modules where temperature tracking is critical (400ZR tuning), this gap narrows — the coherent DSP handles its own calibration.

### 3. Quality Variability
This is the real difference. Not all compatible vendors are equal:

**Top tier** (Flexoptix, FS.com, Precision OT): Rigorous testing, platform-specific coding, reliable DOM, fast RMA. Failure rates comparable to OEM (< 0.5% annualized).

**Mid tier** (various regional distributors): Adequate for ≤100G, inconsistent DOM on 400G+, slower RMA. Acceptable for non-critical paths.

**Bottom tier** (generic marketplace modules): Unpredictable quality, generic EEPROM coding, minimal testing. Avoid for production use.

The vendor matters more than the "OEM vs compatible" label.


## Cost Analysis — The Numbers

| Speed | OEM Price | Compatible Price | Savings | Savings at 200 units |
|-------|-----------|-----------------|---------|---------------------|
| 10G SFP+ SR | $80-150 | $12-25 | 5-10x | $13,600-25,000 |
| 25G SFP28 SR | $100-180 | $18-35 | 3-5x | $16,400-29,000 |
| 100G QSFP28 LR4 | $300-900 | $45-120 | 4-8x | $51,000-156,000 |
| 400G QSFP-DD DR4 | $900-3,200 | $250-500 | 2-4x | $130,000-540,000 |
| 400ZR Coherent | $4,000-6,000 | $2,500-3,500 | ~50% | $300,000-500,000 |

Below 100G, the case for compatible is overwhelming. The technology is commoditized, the modules have shipped for 10+ years, and failure rates are negligible.

At 400G, the equation is still strongly in favor of compatible for DR4/FR4 direct-detect modules. The savings fund your next infrastructure project.

At 400ZR coherent, the gap narrows because the DSP dominates the BOM cost. Evaluate case-by-case, and stick with OEM for your first coherent deployment until your team has operational experience.


## When to Buy OEM

1. **Your TAC contract explicitly requires it.** If losing vendor support costs more than the optics savings, the OEM premium is insurance. Do the math: a $50,000/hour outage SLA with a 4-hour response time means one incident = $200,000. If OEM optics eliminate the "we don't support third-party modules" escalation delay, they pay for themselves.

2. **400ZR/ZR+ coherent on your first deployment.** Coherent interop is genuinely more complex. Wavelength assignment, FEC mode negotiation, chromatic dispersion compensation — these have real edge cases. Build experience with OEM, then evaluate compatible for subsequent deployments.

3. **Peering and transit links with revenue SLAs.** Your IX port or paid transit link generates $50,000-200,000/month. The $1,500 OEM premium on those 4 optics is a rounding error.


## When Compatible Is the Only Rational Choice

1. **Leaf-spine fabric at any scale.** 200 QSFP-DD DR4 at $350 vs $3,200 = $570,000 saved. That's an entire network refresh budget.

2. **Lab, staging, and development.** Zero reason for OEM optics outside production. Buy the cheapest compatible that works.

3. **Anything ≤100G.** The compatible market for SFP+ through QSFP28 has been rock-solid for a decade. If you're still buying OEM 10G optics in 2026, you're paying a 5-10x premium for a logo.

4. **Spares inventory.** You need 5-10% spares for every deployment. OEM spares at $3,200 each sit on a shelf depreciating. Compatible spares at $350 each are economically disposable.


## The Procurement Playbook

1. **Select a Tier-1 compatible vendor** with platform-specific coding and documented test procedures.
2. **Order a 10-unit pilot batch** for your specific switch platform and NX-OS/EOS/Junos version.
3. **Run for 2 weeks in production** monitoring DOM, BER, CRC errors, and temperature.
4. **Validate**: Does vendor auth pass cleanly? Are DOM readings within ±2 dB of the OEM reference? Zero CRC errors after 2 weeks?
5. **Order the full batch** only after pilot validation passes.
6. **Document the validated part numbers** — create an internal approved module list per platform.

This process takes 3-4 weeks and saves your organization hundreds of thousands of dollars.
`);

  parts.push(`

## Key Takeaways

1. **The silicon is identical.** OEM and compatible modules use the same DSPs and lasers from the same 5 suppliers. You're paying for a logo and a test cycle.
2. **The vendor matters, not the label.** A top-tier compatible vendor (Flexoptix, FS.com) delivers the same quality as OEM. A bottom-tier marketplace seller does not. Choose your vendor, not your brand.
3. **Below 100G, compatible is the industry default.** OEM pricing at 10G/25G is indefensible in 2026.
4. **At 400G, compatible saves $500K+ per 200-unit deployment.** The math is simple.
5. **Test before you commit.** 10-unit pilot, 2-week soak, then bulk order. This is non-negotiable.
6. **Keep OEM for coherent and SLA-critical links** — until your team has operational experience with compatible coherent modules.

*Compare pricing across vendors at [flexoptix.net](https://www.flexoptix.net/en/)*
`);

  return parts.join("\n");
}

function generateNewProductTemplate(data: Awaited<ReturnType<typeof gatherBlogData>>): string {
  // Group products by speed class for structured analysis
  const bySpeed = new Map<string, typeof data.products>();
  for (const p of data.products) {
    const speed = String(p.speed || "Unknown");
    if (!bySpeed.has(speed)) bySpeed.set(speed, []);
    bySpeed.get(speed)!.push(p);
  }

  // Pick the primary product for the lead
  const primary = data.products[0];
  const primaryName = String(primary?.standard_name || primary?.slug || "New Transceiver Module");
  const primarySpeed = String(primary?.speed || "next-gen");
  const primaryFF = String(primary?.form_factor || "QSFP-DD");

  const parts: string[] = [];

  parts.push(`
## Why This Matters Now

Another transceiver announcement. Before your inbox fills with "revolutionary" press releases and your vendor starts pushing an upgrade cycle you didn't ask for — here's what this actually means for your network, your budget, and your roadmap.

Skip this at your own risk: the difference between early adoption and being stuck with stranded inventory is about 6 months of timing.


## The Short Version

**${primaryName}** (${primaryFF}, ${primarySpeed}) is now available in the compatible transceiver market. Here's the honest assessment:
`);

  // Product-by-product analysis with verdicts
  if (data.products.length > 0) {
    parts.push(`\n## Product Analysis\n`);

    for (const [speed, products] of bySpeed) {
      if (products.length > 1) {
        parts.push(`### ${speed} Class (${products.length} variants)\n`);
      }

      for (const p of products.slice(0, 5)) {
        const name = String(p.standard_name || p.slug || "");
        const reach = String(p.reach_label || "N/A");
        const fiber = String(p.fiber_type || "SMF");

        parts.push(`#### ${name}\n`);
        parts.push(`| Spec | Value |`);
        parts.push(`|------|-------|`);
        parts.push(`| Form Factor | ${p.form_factor} |`);
        parts.push(`| Speed | ${p.speed} |`);
        parts.push(`| Reach | ${reach} |`);
        parts.push(`| Fiber | ${fiber} |`);
        parts.push(``);

        // Generate deployment verdict based on speed + reach + form factor
        const speedNum = parseInt(String(p.speed || "0"));
        const reachStr = String(reach || "").toLowerCase();
        const ffStr = String(p.form_factor || "").toLowerCase();
        const isCoherent = reachStr.includes("dwdm") || reachStr.includes("80km") || reachStr.includes("2000km") || ffStr.includes("cfp") || name.toLowerCase().includes("coherent") || name.toLowerCase().includes("zr");
        const isLongReach = reachStr.includes("40km") || reachStr.includes("80km") || reachStr.includes("er") || reachStr.includes("zr");

        if (speedNum >= 800) {
          parts.push(`**Verdict: EARLY ADOPTER ONLY.** 800G is shipping but the ecosystem is immature. DSP interop between vendors is inconsistent. Unless you're a hyperscaler or running a proof-of-concept, wait for Q3 2026 when second-source silicon drops prices 30-40%.\n`);
        } else if (speedNum >= 400 && isCoherent) {
          parts.push(`**Verdict: BUY for DWDM/DCI.** 400ZR/ZR+ has matured rapidly. If you're running metro or DCI links, this replaces dedicated transponders at a fraction of the cost and power. Verify your ROADM supports the grid spacing.\n`);
        } else if (speedNum >= 400) {
          parts.push(`**Verdict: BUY.** 400G is the production sweet spot in 2026. Multi-vendor interop is proven, prices have dropped 60% from 2024 levels, and every major switch platform supports it natively. If you're still running 100G spine links — this is your upgrade.\n`);
        } else if (speedNum >= 100 && isCoherent) {
          parts.push(`**Verdict: LEGACY COHERENT.** 100G coherent served its purpose for metro DWDM. New deployments should use 400ZR instead — smaller form factor, lower power, better cost per bit. Only buy as spares for existing DWDM lines.\n`);
        } else if (speedNum >= 100 && isLongReach) {
          parts.push(`**Verdict: COMMODITY — niche use.** Long-reach 100G still has a place for campus/metro links where coherent is overkill. Buy on price and lead time. Compatible vendors are fully qualified at this speed.\n`);
        } else if (speedNum >= 100) {
          parts.push(`**Verdict: COMMODITY — buy on price.** 100G is fully mature. The only differentiator is price and lead time. Don't pay more than $45 for a QSFP28 LR4 in 2026.\n`);
        } else if (speedNum >= 25) {
          parts.push(`**Verdict: ACCESS TIER ONLY.** 25G/10G SFP28/SFP+ is server-access and ToR only. Don't invest in new infrastructure at this speed — use it where your existing switch ports demand it.\n`);
        } else {
          parts.push(`**Verdict: LEGACY.** Only buy as replacement stock. Do not expand deployments at this speed class.\n`);
        }
      }
    }
  }


  parts.push(`
## What the Spec Sheet Won't Tell You

### 1. Power Budget Reality
Every new module generation promises "same power, more speed." The actual numbers:

| Generation | Typical Module Power | Per-Port Power (incl. switch ASIC) |
|-----------|---------------------|-----------------------------------|
| 100G QSFP28 | 3.5W | ~12W |
| 400G QSFP-DD | 12-14W | ~28W |
| 800G OSFP | 18-22W | ~45W |

That 800G upgrade isn't just an optics swap — it's a cooling redesign. If your rack can handle 8kW and you're upgrading 48 ports from 400G to 800G, you need an additional ~800W per switch. Most colocation power contracts don't have that headroom without renegotiation.

### 2. Switch Compatibility
New optics don't always work in existing switches, even if the form factor fits:
- **QSFP-DD in QSFP-DD slots**: Usually fine, but verify firmware version supports the specific DR/FR/LR variant
- **400G optics in 800G-capable switches**: Works, but you may need to configure port speed explicitly
- **800G optics in 400G switches**: Will not work — different electrical signaling (PAM4 112G per lane vs 56G)

### 3. Lead Times
New products have unpredictable supply. First-gen compatible modules typically ship in:
- **OEM (Cisco/Arista branded)**: 8-16 weeks
- **Tier-1 compatible (Flexoptix, FS, Prolabs)**: 2-4 weeks
- **Generic compatible**: 1-2 weeks (but verify quality — DOA rates vary wildly)
`);

  if (data.news.length > 0) {
    parts.push(`\n## Industry Context\n`);
    parts.push(`Recent developments that affect this product category:\n`);
    for (const n of data.news.slice(0, 5)) {
      parts.push(`- **${n.title}** *(${n.source || "industry"})* — ${n.summary ? String(n.summary).slice(0, 150) : "See full article for details."}`);
    }
    parts.push(``);
  }

  parts.push(`
## Procurement Playbook

### If You Need This Module NOW:
1. **Order 10% of your target volume** from a compatible vendor with a proven track record
2. **Run interop testing** on your specific switch platform + firmware version
3. **Verify DOM reporting** — check that Tx power, Rx power, and temperature read correctly in your NMS
4. **Order remaining volume** only after testing passes on at least 2 production switches
5. **Keep 5% spares** — new products have higher infant mortality rates than mature ones

### If You Can Wait 3-6 Months:
- Prices will drop 15-25% as more vendors qualify the module
- Second-source silicon may become available (especially for 800G)
- Switch vendors will release firmware updates that improve compatibility
- Field data on failure rates will be available from early adopters

### Red Flags — Do NOT Buy If:
- The vendor can't provide a compatibility matrix for your specific switch model + firmware
- The module price is suspiciously low (below manufacturing cost = quality corners cut)
- The vendor doesn't offer cross-ship RMA
- No DOM data available in the module's EEPROM


## Bottom Line

New transceiver products follow a predictable cycle: hype → early adopter premium → multi-vendor qualification → commodity pricing. The question isn't whether this product is good — it's whether YOU need it NOW at current pricing, or whether waiting 2 quarters saves you 30% and gives you proven interop data.

For most networks: test now, deploy at scale in Q3. Unless your traffic growth forces your hand — in which case, buy compatible, test thoroughly, and keep OEM spares for your most critical links.
`);

  return parts.join("\n");
}

// Simple serial queue to prevent concurrent Ollama requests crashing
const llmQueue: Array<() => Promise<void>> = [];
let llmRunning = false;

async function enqueueLlmPipeline(
  draftId: string,
  title: string,
  selectedTopic: string,
  targetAudience: string,
  data: Awaited<ReturnType<typeof gatherBlogData>>,
  additionalContext?: string,
): Promise<void> {
  return new Promise<void>((resolve) => {
    llmQueue.push(async () => {
      await runLlmPipeline(draftId, title, selectedTopic, targetAudience, data, additionalContext);
      resolve();
    });
    processLlmQueue();
  });
}

async function processLlmQueue(): Promise<void> {
  if (llmRunning || llmQueue.length === 0) return;
  llmRunning = true;
  const task = llmQueue.shift();
  if (task) {
    try {
      await task();
    } catch (err) {
      console.error(`LLM queue task error: ${(err as Error).message}`);
    }
  }
  llmRunning = false;
  // Process next item — small delay between pipelines to avoid nginx rate-limit bursts
  if (llmQueue.length > 0) setTimeout(() => processLlmQueue(), 3000);
}

/**
 * 3-pass pipeline for external content (from-url / from-pdf).
 * Grounds entirely in source text — never expands from parametric knowledge,
 * which prevents the "senior network engineer" persona from drifting to optics.
 */
async function runExternalContentPipeline(
  draftId: string,
  title: string,
  selectedTopic: string,
  targetAudience: string,
  additionalContext: string,
): Promise<void> {
  const {
    FO_BLOG_SYSTEM_PROMPT,
    STEP8_KILL_AI_TONE,
    STEP8c_STYLE_LOCK,
    STEP_HEADLINE_GENERATION,
    withCalibration,
    buildFeedbackContext,
  } = await import("../llm/fo-blog-pipeline");

  const LLM_WRITE = { temperature: 0.7, maxTokens: 2000, timeoutMs: 480000 };
  const LLM_REFINE = { temperature: 0.35, maxTokens: 2000, timeoutMs: 480000 };

  // Extract persona-neutral section of FO system prompt (keep writing style, drop optics persona)
  const mindsetMarker = "YOUR MINDSET:";
  const mindsetStart = FO_BLOG_SYSTEM_PROMPT.indexOf(mindsetMarker);
  const writingStyleRules = mindsetStart > -1
    ? FO_BLOG_SYSTEM_PROMPT.slice(mindsetStart)
    : FO_BLOG_SYSTEM_PROMPT;

  // Load feedback
  let feedbackContext = "";
  try {
    const fbResult = await pool.query(
      `SELECT score_overall, feedback_text, blog_type FROM blog_feedback
       WHERE feedback_text IS NOT NULL AND feedback_text != ''
       ORDER BY score_overall ASC LIMIT 20`
    );
    feedbackContext = buildFeedbackContext(fbResult.rows.map(r => ({
      score: r.score_overall, feedback_text: r.feedback_text, blog_type: r.blog_type || ""
    })));
  } catch { /* no feedback yet */ }

  // Extract just the PDF/URL text from additionalContext
  const pdfStart = additionalContext.indexOf("--- EXTRACTED PDF CONTENT ---");
  const urlStart = additionalContext.indexOf("--- EXTRACTED PAGE CONTENT ---");
  const contentStart = pdfStart > -1 ? pdfStart : urlStart > -1 ? urlStart : -1;
  const sourceText = contentStart > -1
    ? additionalContext.slice(contentStart).slice(0, 5000)
    : additionalContext.slice(0, 5000);

  const externalSysPrompt = withCalibration(
    `You are a senior IT infrastructure engineer and technical writer with 20+ years of experience.\n` +
    `You write practical articles for IT architects, infrastructure managers, and decision-makers.\n\n` +
    `ABSOLUTE RULE: You write ONLY about what the source document says. ` +
    `Do NOT add optical transceivers, fiber optics, 400G, or compatible optics content ` +
    `unless the source document explicitly covers it. ` +
    `Your only source of facts is the document provided.\n\n` +
    `━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n` +
    `WRITING STYLE (apply to everything you write):\n` +
    `━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n` +
    writingStyleRules + feedbackContext
  );

  console.log(`Blog External Pipeline: Starting for ${draftId} — "${title}"`);
  setProgress(draftId, 1, "Step 1/5: Extract source facts");

  // ─── Pass 1: Extract key facts from source document ───────────────────────
  const extractPrompt =
    `Extract the key factual content from the source document below.\n\n` +
    `Return:\n` +
    `- The core problem or challenge the document describes (2-3 sentences)\n` +
    `- 5-7 specific facts, findings, or insights stated in the document\n` +
    `- The main conclusion or practical recommendation\n` +
    `- Who this affects and why it matters to them\n\n` +
    `Do NOT add information not present in the document. Do NOT interpret or expand.\n` +
    `Return only what the document actually says.\n\n` +
    sourceText;

  await generate("You are a precise document analyst.", extractPrompt, { temperature: 0.2, maxTokens: 800, timeoutMs: 120000 }).catch(() => {});
  const extractResult = await generate("You are a precise document analyst.", extractPrompt, { temperature: 0.2, maxTokens: 800, timeoutMs: 120000 });

  setProgress(draftId, 2, "Step 2/5: Write article draft");

  // ─── Pass 2: Write the full article from extracted facts ──────────────────
  const draftPrompt =
    `Write a blog article titled "${title}".\n\n` +
    `Use ONLY the facts below as your source material. Do not add any facts not listed here.\n\n` +
    `SOURCE FACTS:\n${extractResult.text}\n\n` +
    `ARTICLE REQUIREMENTS:\n` +
    `- 600-900 words\n` +
    `- Continuous narrative prose — no section headers, no bullet lists in the body\n` +
    `- First-person engineering voice ("I've seen this happen...", "The problem is...")\n` +
    `- Start with a specific operational scenario, not a general statement\n` +
    `- Explain consequences for real infrastructure decisions\n` +
    `- End with a concrete implication or call to action, not a summary\n` +
    `- Apply all writing style rules from your system prompt\n\n` +
    `Topic: "${title}"\n` +
    `Audience: ${targetAudience} engineers and infrastructure decision-makers`;

  const draftResult = await generate(externalSysPrompt, draftPrompt, LLM_WRITE);
  console.log(`  Draft: ${draftResult.text.split(/\s+/).length} words`);

  setProgress(draftId, 3, "Step 3/5: Kill AI tone");

  // ─── Pass 3: Kill AI tone ─────────────────────────────────────────────────
  const step8 = await generate(externalSysPrompt,
    STEP8_KILL_AI_TONE.replace("{{ARTICLE}}", draftResult.text),
    LLM_REFINE
  );

  setProgress(draftId, 4, "Step 4/5: Style lock");

  // Skip STEP8b_REDUCTION for external content — it targets 1,200-2,000 words
  // with "DO NOT go below 1,000 words" which conflicts with thin source material.
  // Just apply style lock for readability polish.
  const step8c = await generate(externalSysPrompt,
    STEP8c_STYLE_LOCK.replace("{{ARTICLE}}", step8.text),
    LLM_REFINE
  );

  setProgress(draftId, 5, "Step 5/5: LinkedIn + headline");

  // ─── LinkedIn post — topic-neutral version (no optics example) ────────────
  const externalLinkedInPrompt =
    `Write a LinkedIn post for this article.\n\n` +
    `FORMAT:\n` +
    `Line 1-2: HOOK — a reframe or uncomfortable truth from the article. NOT an announcement.\n\n` +
    `3-5 SHORT BEATS — each beat is 1-3 lines. One insight per beat. No bullet markers.\n\n` +
    `Last line before hashtags: "Full breakdown in the blog — link in first comment."\n\n` +
    `HASHTAGS (last line): 3-4 relevant hashtags based on the article topic. Include #Flexoptix.\n` +
    `  Pick hashtags that match what the article is actually about — NOT #OpticalNetworking unless the article covers optics.\n\n` +
    `RULES:\n` +
    `- No emojis\n` +
    `- No "I'm thrilled to share" or "Excited to announce"\n` +
    `- Engineer voice — specific, blunt, useful\n` +
    `- Maximum 2,800 characters\n` +
    `- Return ONLY the post text. No commentary.\n\n` +
    `Article:\n${step8c.text}`;

  const linkedInResult = await generate(externalSysPrompt,
    externalLinkedInPrompt,
    { ...LLM_REFINE, maxTokens: 600 }
  ).catch(() => ({ text: "" }));

  // ─── Headline ──────────────────────────────────────────────────────────────
  const headlineResult = await generate(externalSysPrompt,
    STEP_HEADLINE_GENERATION.replace("{{ARTICLE}}", step8c.text),
    { temperature: 0.5, maxTokens: 80, timeoutMs: 60000 }
  ).catch(() => ({ text: title }));

  const finalTitle = headlineResult.text.trim().replace(/^["']|["']$/g, "").replace(/\n.*$/s, "").trim() || title;
  const wordCount = step8c.text.split(/\s+/).length;

  await pool.query(
    `UPDATE blog_drafts SET
       title = $1,
       draft_content = $2,
       linkedin_post = $3,
       word_count = $4,
       status = 'draft',
       updated_at = NOW()
     WHERE id = $5`,
    [finalTitle, step8c.text, linkedInResult.text || null, wordCount, draftId]
  );

  clearProgress(draftId);
  console.log(`Blog External Pipeline: ${draftId} complete — ${wordCount} words, title: "${finalTitle}"`);
}

/** Run 10-Step Flexoptix Style LLM Pipeline and update draft in-place */
async function runLlmPipeline(
  draftId: string,
  title: string,
  selectedTopic: string,
  targetAudience: string,
  data: Awaited<ReturnType<typeof gatherBlogData>>,
  additionalContext?: string,
): Promise<void> {
  // External content (from-url / from-pdf) uses a grounded 5-pass pipeline
  // that never expands from parametric knowledge — prevents optics topic drift
  if (additionalContext?.startsWith("⚠️ TOPIC LOCK") && additionalContext.length > 200) {
    return runExternalContentPipeline(draftId, title, selectedTopic, targetAudience, additionalContext);
  }

  // Lazy-load the new FO pipeline
  const {
    FO_BLOG_SYSTEM_PROMPT,
    STEP1_TOPIC_EXPANSION,
    STEP2_ANGLE_SELECTION,
    STEP3_OUTLINE,
    STEP4_MASTER_DRAFT,
    STEP4b_NARRATIVE_CONTROL,
    STEP5_REALITY_INJECTION,
    STEP6_TECHNICAL_DEEPENING,
    STEP7_OPINION_LAYER,
    STEP8_KILL_AI_TONE,
    STEP8b_REDUCTION,
    STEP8c_STYLE_LOCK,
    STEP8d_AUTO_KILL,
    AUTO_KILL_SCORING,
    STEP9_QA_CHECK,
    STEP10_QUALITY_SCORE,
    STEP_LINKEDIN_POST,
    STEP_HEADLINE_GENERATION,
    BLOG_TYPES,
    buildFeedbackContext,
    withCalibration,
  } = await import("../llm/fo-blog-pipeline");

  const LLM_OPTS = { temperature: 0.7, maxTokens: 1600, timeoutMs: 480000 };
  const LLM_REFINE = { temperature: 0.4, maxTokens: 1800, timeoutMs: 480000 };
  const TOTAL_STEPS = 16; // 10 original + 4b Narrative Control + 8b Reduction + 8c Style Lock + 8d Auto-Kill + Auto-Kill Score + LinkedIn
  let stepsCompleted = 0;

  try {
    console.log(`Blog FO Pipeline: Starting 10-step generation for ${draftId}`);
    console.log(`  Topic: "${title}" | Type: ${selectedTopic} | Audience: ${targetAudience}`);

    // Load accumulated feedback to inject into system prompt
    let feedbackContext = "";
    try {
      const fbResult = await pool.query(
        `SELECT score_overall, feedback_text, blog_type FROM blog_feedback
         WHERE feedback_text IS NOT NULL AND feedback_text != ''
         ORDER BY score_overall ASC LIMIT 20`
      );
      feedbackContext = buildFeedbackContext(fbResult.rows.map(r => ({
        score: r.score_overall, feedback_text: r.feedback_text, blog_type: r.blog_type || ""
      })));
    } catch { /* no feedback yet, that's fine */ }

    // For external content (from-url / from-pdf), the Flexoptix optical-networking persona
    // must be replaced — otherwise every article drifts back to 400G transceivers regardless
    // of what the TOPIC LOCK says. Strip the Flexoptix mandate and inject a generic persona.
    const isExternalContent = additionalContext?.startsWith("⚠️ TOPIC LOCK");
    const extractedTopicName = isExternalContent
      ? (additionalContext?.match(/TOPIC LOCK[^"]*"([^"]+)"/) || [])[1] || title
      : title;

    // Build the section of FO_BLOG_SYSTEM_PROMPT that's topic-neutral (writing style rules only).
    // The Flexoptix persona block ends at the first ════ separator after line 65 ("YOUR MINDSET").
    const mindsetMarker = "YOUR MINDSET:";
    const mindsetStart = FO_BLOG_SYSTEM_PROMPT.indexOf(mindsetMarker);
    const writingStyleRules = mindsetStart > -1
      ? FO_BLOG_SYSTEM_PROMPT.slice(mindsetStart)
      : FO_BLOG_SYSTEM_PROMPT;

    const externalSystemPrompt = `\
╔══════════════════════════════════════════════════════════════════════╗
║  EXTERNAL CONTENT MODE — these rules override ALL defaults below  ║
╚══════════════════════════════════════════════════════════════════════╝

THIS ARTICLE IS ABOUT: "${extractedTopicName}"

You are a senior IT infrastructure engineer and technical writer.
Your readers are IT architects, infrastructure managers, and decision-makers.

ABSOLUTE RULE: Write ONLY about "${extractedTopicName}".
Do NOT write about optical transceivers, fiber optics, 400G, DR4, compatible optics,
Flexoptix products, or any networking hardware UNLESS the source document below
explicitly covers that topic. The source document is your sole editorial mandate.

The Flexoptix brand rules and compatible-optics framing in this prompt DO NOT APPLY
to this article. This is a general IT/infrastructure piece, not an optics blog post.

━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
WRITING STYLE (applies to all articles — keep these):
━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
${writingStyleRules}`;

    const systemPrompt = isExternalContent
      ? withCalibration(externalSystemPrompt + feedbackContext)
      : withCalibration(FO_BLOG_SYSTEM_PROMPT + feedbackContext);

    // Warmup
    await generate("Test", "OK", { temperature: 0.1, maxTokens: 8, timeoutMs: 60000 }).catch(() => {});

    // Build context data string for injection — REAL DB data only, never fabricated
    type PriceEntry = { vendor: string; price: number; currency: string; url: string; observed_at: string };
    const contextLines: string[] = [];

    for (const p of data.products.slice(0, 20)) {
      const prices = (p.verified_prices as PriceEntry[] | undefined) || [];
      const hasPrice = prices.length > 0;

      // Build product line with real specs
      let line = `[PRODUCT] ${p.standard_name || p.slug || "unknown"}`;
      if (p.form_factor) line += ` | Form factor: ${p.form_factor}`;
      if (p.speed)       line += ` | Speed: ${p.speed}`;
      if (p.reach_label) line += ` | Reach: ${p.reach_label}`;
      if (p.fiber_type)  line += ` | Fiber: ${p.fiber_type}`;
      if (p.connector)   line += ` | Connector: ${p.connector}`;
      if (p.vendor)      line += ` | Vendor: ${p.vendor}`;
      if (p.vendor_type) line += ` (${p.vendor_type})`;

      // Optical specs if available
      if (p.tx_power_min_dbm != null) line += ` | TX min: ${p.tx_power_min_dbm} dBm`;
      if (p.tx_power_max_dbm != null) line += ` TX max: ${p.tx_power_max_dbm} dBm`;
      if (p.rx_sensitivity_dbm != null) line += ` | RX sensitivity: ${p.rx_sensitivity_dbm} dBm`;
      if (p.power_consumption_w != null) line += ` | Power: ${p.power_consumption_w}W`;

      contextLines.push(line);

      // Append verified prices — clearly tagged as real DB observations
      if (hasPrice) {
        for (const pr of prices.slice(0, 3)) {
          const date = pr.observed_at ? pr.observed_at.split("T")[0] : "recent";
          contextLines.push(
            `  [VERIFIED PRICE] ${pr.currency} ${pr.price.toFixed(2)} — ${pr.vendor} (observed ${date}) ${pr.url ? `| ${pr.url}` : ""}`
          );
        }
      } else {
        contextLines.push(`  [NO VERIFIED PRICE IN DB — do NOT invent a price for this product]`);
      }
    }

    if (additionalContext) {
      contextLines.unshift(
        `[HOT TOPIC BRIEFING — editorial direction, do not copy verbatim]\n${additionalContext.slice(0, 4000)}`
      );
    }

    const contextData = contextLines.length > 0
      ? contextLines.join("\n")
      : "[NO PRODUCT DATA AVAILABLE — do NOT invent product names, part numbers, or prices]";

    // Get blog type config
    const blogType = BLOG_TYPES[selectedTopic as keyof typeof BLOG_TYPES] || BLOG_TYPES.tutorial;

    // ═══ STEP 1: Topic Expansion ═══
    console.log("  Step 1/10: Topic Expansion...");
    setProgress(draftId, 1, "Step 1/10: Topic Expansion");

    // For external content: inject a hard topic anchor before the standard prompt so the
    // LLM cannot drift back to optical networking when expanding the topic in Step 1.
    const step1TopicPrefix = isExternalContent
      ? `HARD TOPIC LOCK: This article is about "${extractedTopicName}". ` +
        `It is NOT about optical transceivers, fiber optics, 400G migrations, or compatible optics. ` +
        `Your expansion below must stay strictly within the topic stated above.\n\n`
      : "";

    const step1 = await generate(systemPrompt,
      step1TopicPrefix + STEP1_TOPIC_EXPANSION
        .replace("{{TOPIC}}", isExternalContent ? extractedTopicName : title)
        .replace("{{ADDITIONAL_CONTEXT}}", additionalContext
          ? `\n\n---\nBACKGROUND REFERENCE (use as factual direction ONLY — do not copy verbatim):\n${additionalContext.slice(0, 4000)}\n\n` +
            (isExternalContent
              ? `REMINDER: Write about "${extractedTopicName}" — NOT optical networking. The background above is your source material.`
              : `CRITICAL: Do NOT copy any phrase, sentence, or wording from the above into the article or any step output.`)
          : ""),
      LLM_OPTS
    );
    stepsCompleted = 1;

    // ═══ STEP 2: Angle Selection ═══
    console.log("  Step 2/10: Angle Selection...");
    setProgress(draftId, 2, "Step 2/10: Angle Selection");
    const step2 = await generate(systemPrompt,
      STEP2_ANGLE_SELECTION.replace("{{SCENARIOS}}", step1.text),
      LLM_REFINE
    );
    stepsCompleted = 2;

    // ═══ STEP 3: Outline ═══
    console.log("  Step 3/10: Outline Generation...");
    setProgress(draftId, 3, "Step 3/10: Outline Generation");
    const step3 = await generate(systemPrompt,
      STEP3_OUTLINE
        .replace("{{ANGLE}}", step2.text)
        .replace("{{AUDIENCE}}", targetAudience)
        .replace("{{DECISION}}", title),
      LLM_REFINE
    );
    stepsCompleted = 3;

    // ═══ STEP 4: Master Draft ═══
    console.log("  Step 4/10: Master Draft (this takes a while)...");
    setProgress(draftId, 4, "Step 4/10: Master Draft (longest step…)");
    const step4 = await generate(systemPrompt,
      STEP4_MASTER_DRAFT
        .replace("{{OUTLINE}}", step3.text)
        .replace("{{CONTEXT_DATA}}", contextData),
      { ...LLM_OPTS, maxTokens: 1600 }
    );
    stepsCompleted = 4;
    console.log(`    Draft: ${step4.text.split(/\s+/).length} words`);

    // ═══ STEP 4b: Narrative Control ═══
    console.log("  Step 5/13: Narrative Control (framing check + anti-FUD)...");
    setProgress(draftId, 5, "Step 5/13: Narrative Control");
    const step4b = await generate(systemPrompt,
      STEP4b_NARRATIVE_CONTROL.replace("{{ARTICLE}}", step4.text),
      LLM_REFINE
    );
    stepsCompleted = 5;
    console.log(`    After narrative control: ${step4b.text.split(/\s+/).length} words`);

    // ═══ STEP 5: Reality Injection ═══
    console.log("  Step 6/13: Reality Injection...");
    setProgress(draftId, 6, "Step 6/13: Reality Injection");
    const step5 = await generate(systemPrompt,
      STEP5_REALITY_INJECTION.replace("{{DRAFT}}", step4b.text),
      LLM_REFINE
    );
    stepsCompleted = 6;

    // ═══ STEP 6: Technical Deepening ═══
    console.log("  Step 7/13: Technical Deepening...");
    setProgress(draftId, 7, "Step 7/13: Technical Deepening");
    const step6 = await generate(systemPrompt,
      STEP6_TECHNICAL_DEEPENING.replace("{{ARTICLE}}", step5.text),
      LLM_REFINE
    );
    stepsCompleted = 6;

    // ═══ STEP 7: Opinion Layer ═══
    console.log("  Step 8/13: Opinion Layer...");
    setProgress(draftId, 8, "Step 8/13: Opinion Layer");
    const step7 = await generate(systemPrompt,
      STEP7_OPINION_LAYER.replace("{{ARTICLE}}", step6.text),
      LLM_REFINE
    );
    stepsCompleted = 8;

    // ═══ STEP 8: Kill AI Tone ═══
    console.log("  Step 9/13: Kill AI Tone...");
    setProgress(draftId, 9, "Step 9/13: Kill AI Tone");
    const step8 = await generate(systemPrompt,
      STEP8_KILL_AI_TONE.replace("{{ARTICLE}}", step7.text),
      LLM_REFINE
    );
    stepsCompleted = 9;

    // ═══ STEP 8b: Reduction Engine (5-pass, target: cut 40%) ═══
    console.log("  Step 10/16: Reduction Engine (5-pass, cut 40%, target 600-1000 words)...");
    setProgress(draftId, 10, "Step 10/16: Reduction Engine (cut 40%)");
    const step8b = await generate(systemPrompt,
      STEP8b_REDUCTION.replace("{{ARTICLE}}", step8.text),
      LLM_REFINE
    );
    stepsCompleted = 10;
    const wordsAfter = step8b.text.split(/\s+/).length;
    const wordsBefore = step8.text.split(/\s+/).length;
    const pctChange = Math.round((1 - wordsAfter / wordsBefore) * 100);
    console.log(`    After reduction: ${wordsAfter} words (was ${wordsBefore}, −${pctChange}%) ${wordsAfter > 1200 ? "⚠ WARNING: >1200 words" : wordsAfter < 500 ? "⚠ WARNING: <500 words" : "✓ in target range"}`);

    // ═══ STEP 8c: Style Lock ═══
    console.log("  Step 11/16: Style Lock (tone consistency + scope/SKU fixes)...");
    setProgress(draftId, 11, "Step 11/16: Style Lock");
    const step8c = await generate(systemPrompt,
      STEP8c_STYLE_LOCK.replace("{{ARTICLE}}", step8b.text),
      LLM_REFINE
    );
    stepsCompleted = 11;

    // ═══ STEP 8d: Auto-Kill Layer v1.0 (10 categories A-J) ═══
    console.log("  Step 12/16: Auto-Kill Layer (10 categories A-J)...");
    setProgress(draftId, 12, "Step 12/16: Auto-Kill Layer");
    const step8d = await generate(systemPrompt,
      STEP8d_AUTO_KILL.replace("{{ARTICLE}}", step8c.text),
      LLM_REFINE
    );
    stepsCompleted = 12;
    const wordsAfterKill = step8d.text.split(/\s+/).length;
    console.log(`    After Auto-Kill: ${wordsAfterKill} words (was ${step8c.text.split(/\s+/).length})`);

    // ═══ STEP 9: QA Check ═══
    console.log("  Step 13/16: QA Check...");
    setProgress(draftId, 13, "Step 13/16: QA Check");
    const step9 = await generate(systemPrompt,
      STEP9_QA_CHECK.replace("{{ARTICLE}}", step8d.text),
      LLM_REFINE
    );
    stepsCompleted = 13;

    // ═══ STEP 10: Quality Score ═══
    console.log("  Step 14/16: Quality Score...");
    setProgress(draftId, 14, "Step 14/16: Quality Score");
    let autoQaScore: Record<string, unknown> | null = null;
    try {
      const step10 = await generate(systemPrompt,
        STEP10_QUALITY_SCORE.replace("{{ARTICLE}}", step9.text),
        { temperature: 0.2, maxTokens: 1024, timeoutMs: 120000 }
      );
      // Try to parse JSON score
      const jsonMatch = step10.text.match(/\{[\s\S]*"scores"[\s\S]*\}/);
      if (jsonMatch) {
        autoQaScore = JSON.parse(jsonMatch[0]);
        console.log(`    Auto QA Score: ${(autoQaScore as any)?.overall || "?"}/10`);
      }
    } catch {
      console.log("    Quality scoring skipped (parse error)");
    }
    stepsCompleted = 14;

    // ═══ Auto-Kill Scoring (non-destructive) ═══
    console.log("  Step 15/16: Auto-Kill Scoring...");
    setProgress(draftId, 15, "Step 15/16: Auto-Kill Scoring");
    let autoKillScores: Record<string, unknown> | null = null;
    try {
      const killScoreResult = await generate(systemPrompt,
        AUTO_KILL_SCORING.replace("{{ARTICLE}}", step9.text),
        { temperature: 0.2, maxTokens: 512, timeoutMs: 60000 }
      );
      const killJson = killScoreResult.text.match(/\{[\s\S]*\}/);
      if (killJson) {
        autoKillScores = JSON.parse(killJson[0]);
        console.log(`    Auto-Kill Scores: ${JSON.stringify(autoKillScores)}`);
      }
    } catch {
      console.log("    Auto-Kill scoring skipped");
    }
    stepsCompleted = 15;

    // ═══ LinkedIn Post ═══
    console.log("  Step 16/16: LinkedIn Post (max 2,800 chars)...");
    setProgress(draftId, 16, "Step 16/16: LinkedIn Post");
    let linkedinPost: string | null = null;
    let linkedinCharCount: number | null = null;
    try {
      const stepLinkedIn = await generate(systemPrompt,
        STEP_LINKEDIN_POST.replace("{{ARTICLE}}", step9.text),
        { temperature: 0.6, maxTokens: 1024, timeoutMs: 120000 }
      );
      linkedinPost = stepLinkedIn.text.trim();
      linkedinCharCount = linkedinPost.length;
      // Enforce hard limit — truncate at last sentence before 2800 if too long
      if (linkedinCharCount > 2800) {
        linkedinPost = linkedinPost.slice(0, 2800).replace(/[^.!?]*$/, "").trim();
        linkedinCharCount = linkedinPost.length;
        console.log(`    LinkedIn post truncated to ${linkedinCharCount} chars`);
      } else {
        console.log(`    LinkedIn post: ${linkedinCharCount} chars`);
      }
    } catch {
      console.log("    LinkedIn post generation skipped");
    }
    stepsCompleted = 16;

    // ═══ Headline Generation (post-article — never copies user input) ═══
    console.log("  Step 17: Headline Generation...");
    let generatedHeadline: string | null = null;
    try {
      const headlineResult = await generate(systemPrompt,
        STEP_HEADLINE_GENERATION.replace("{{ARTICLE}}", step9.text),
        { temperature: 0.5, maxTokens: 64, timeoutMs: 60000 }
      );
      const rawHeadline = headlineResult.text.trim().replace(/^["']|["']$/g, "");
      // Sanity check: must be non-empty, ≤120 chars, and not start with "Here is"
      if (rawHeadline.length > 5 && rawHeadline.length <= 120 && !rawHeadline.toLowerCase().startsWith("here is")) {
        generatedHeadline = rawHeadline;
        console.log(`    Generated headline: "${generatedHeadline}"`);
      }
    } catch {
      console.log("    Headline generation skipped");
    }
    // Use generated headline if valid, fall back to original title
    const finalTitle = generatedHeadline || title;

    // Extract only the article from STEP9 output (QA returns review + fixed article)
    // Look for "COMPLETE FIXED ARTICLE" marker and take everything after it
    let finalArticleText = step9.text;
    const articleMarkers = [
      "### COMPLETE FIXED ARTICLE",
      "## COMPLETE FIXED ARTICLE",
      "COMPLETE FIXED ARTICLE",
      "---\n\n**You're",
      "---\n\nYou're",
    ];
    for (const marker of articleMarkers) {
      const idx = step9.text.indexOf(marker);
      if (idx !== -1) {
        // Skip past the marker line itself
        const afterMarker = step9.text.slice(idx + marker.length).trimStart();
        // Strip leading --- separator if present
        finalArticleText = afterMarker.replace(/^---\s*\n/, "").trimStart();
        break;
      }
    }
    // Strip any remaining markdown review headers (### lines) from the article
    finalArticleText = finalArticleText
      .split("\n")
      .filter(line => !line.match(/^#{1,4}\s+(Critical Review|HARD FAIL|QUALITY CHECKS|CALIBRATION FAILS)/))
      .join("\n")
      .trim();

    const draftContent = `# ${finalTitle}\n\n${finalArticleText}`;
    const wordCount = draftContent.split(/\s+/).length;
    const finalIssues = validateArticle(draftContent);

    // Update the draft in DB (title updated to generated headline if available)
    const pipelineModel = getLlmProvider();
    const pipelineGeneratedBy = `fo-blog-engine-${pipelineModel.ollamaModel || pipelineModel.provider || "llm"}`;
    await pool.query(
      `UPDATE blog_drafts
       SET title = $9, draft_content = $1, word_count = $2,
           generated_by = $10,
           pipeline_version = 'v7',
           pipeline_steps_completed = $3,
           auto_qa_score = $4,
           outline = $5,
           linkedin_post = $6,
           linkedin_char_count = $7,
           status = 'draft',
           updated_at = NOW()
       WHERE id = $8::uuid`,
      [
        draftContent,
        wordCount,
        stepsCompleted,
        autoQaScore ? JSON.stringify(autoQaScore) : null,
        JSON.stringify({
          generation_method: "fo-pipeline-v5-autokill",
          auto_kill_scores: autoKillScores,
          steps_completed: stepsCompleted,
          blog_type: selectedTopic,
          quality_issues: finalIssues,
          feedback_entries_used: feedbackContext ? feedbackContext.split("\n").length : 0,
        }),
        linkedinPost,
        linkedinCharCount,
        draftId,
        finalTitle,
        pipelineGeneratedBy,
      ],
    );

    // Auto-submit QA score as self-feedback
    if (autoQaScore && (autoQaScore as any).scores) {
      const s = (autoQaScore as any).scores;
      await pool.query(
        `INSERT INTO blog_feedback (blog_id, score_overall, score_technical_depth, score_real_world,
          score_clarity, score_originality, score_engineer_voice, score_decision_value,
          score_failure_scenarios, score_opinion_strength, reviewer, blog_type, blog_topic, improvements)
         VALUES ($1::uuid, $2, $3, $4, $5, $6, $7, $8, $9, $10, 'llm_self', $11, $12, $13)`,
        [draftId, (autoQaScore as any).overall || 5,
         s.technical_depth, s.real_world_relevance, s.clarity, s.originality,
         s.engineer_voice, s.decision_value, s.failure_scenarios, s.opinion_strength,
         selectedTopic, title,
         (autoQaScore as any).improvements ? JSON.stringify((autoQaScore as any).improvements) : null]
      ).catch(() => {});
    }

    clearProgress(draftId);
    console.log(`Blog FO Pipeline: ${draftId} complete — ${wordCount} words, ${stepsCompleted}/14 steps, QA: ${(autoQaScore as any)?.overall || "N/A"}/10, LinkedIn: ${linkedinCharCount ?? "n/a"} chars`);
  } catch (llmErr) {
    clearProgress(draftId);
    console.warn(`Blog FO Pipeline failed at step ${stepsCompleted + 1}/14 for ${draftId}: ${(llmErr as Error).message}`);
    // Update with partial progress
    await pool.query(
      `UPDATE blog_drafts SET pipeline_steps_completed = $1, pipeline_version = 'v5-narrative-control',
       outline = $2, updated_at = NOW() WHERE id = $3::uuid`,
      [stepsCompleted, JSON.stringify({ error: (llmErr as Error).message, steps_completed: stepsCompleted }), draftId]
    ).catch(() => {});
  }
}

// POST /api/blog/generate — Generate a new blog draft (returns immediately, LLM runs async)
blogRouter.post("/generate", async (req: Request, res: Response) => {
  const { topic, speed, form_factor, use_case, use_llm, custom_title, additional_context } = req.body as {
    topic?: string;
    speed?: string;
    form_factor?: string;
    use_case?: string;
    use_llm?: boolean;
    /** Override the auto-selected template title with a specific topic/title */
    custom_title?: string;
    /** Background context for the LLM — used as factual direction ONLY, never copied verbatim into the article */
    additional_context?: string;
  };

  const selectedTopic = topic || "tutorial";
  const templates = BLOG_TEMPLATES[selectedTopic];

  if (!templates) {
    res.status(400).json({
      success: false,
      error: `Invalid topic. Valid: ${Object.keys(BLOG_TEMPLATES).join(", ")}`,
    });
    return;
  }

  try {
    const year = new Date().getFullYear();
    const template = templates[Math.floor(Math.random() * templates.length)];

    // custom_title overrides the template title — LLM will still generate a better headline at the end
    const title = custom_title || template.title
      .replace("{YEAR}", String(year))
      .replace("{SPEED}", speed || "400G/800G")
      .replace("{FORM_FACTOR}", form_factor || "QSFP-DD/OSFP")
      .replace("{USE_CASE}", use_case || "Data Center Interconnect");

    const keywords = [
      ...template.seo_keywords,
      speed || "400G",
      form_factor || "",
      use_case || "data center",
    ].filter(Boolean);

    const data = await gatherBlogData(keywords, selectedTopic);

    // Always create a template draft first (instant response)
    const draftContent = generateTemplateDraft(title, selectedTopic, data);
    const wordCount = draftContent.split(/\s+/).length;
    const initialIssues = validateArticle(draftContent);

    const result = await pool.query(
      `INSERT INTO blog_drafts (title, topic, target_audience, outline, draft_content, data_sources, status, generated_by, word_count, seo_keywords)
       VALUES ($1, $2, $3, $4, $5, $6, 'draft', $7, $8, $9)
       RETURNING id, created_at`,
      [
        title,
        selectedTopic,
        template.target_audience,
        JSON.stringify({ generation_method: "template", quality_issues: initialIssues }),
        draftContent,
        JSON.stringify({
          products: data.products.length,
          news: data.news.length,
          faq: data.faq.length,
          troubleshooting: data.troubleshooting.length,
        }),
        "tip-blog-engine-template",
        wordCount,
        template.seo_keywords,
      ],
    );

    const draftId = result.rows[0].id;

    // Launch async LLM enhancement if available
    const shouldUseLlm = use_llm !== false;
    let llmStarted = false;
    if (shouldUseLlm) {
      const health = await checkHealth().catch(() => ({ ok: false, model: "", error: "unreachable" }));
      if (health.ok) {
        console.log(`Blog LLM: Using ${health.model} — enhancing draft ${draftId} in background`);
        llmStarted = true;
        // Fire-and-forget: LLM pipeline queued, updates draft when done
        enqueueLlmPipeline(draftId, title, selectedTopic, template.target_audience, data, additional_context).catch((err) => {
          console.error(`Blog LLM background pipeline error: ${(err as Error).message}`);
        });
      }
    }

    res.json({
      success: true,
      draft: {
        id: draftId,
        title,
        topic: selectedTopic,
        target_audience: template.target_audience,
        word_count: wordCount,
        generation_method: "template",
        llm_enhancing: llmStarted,
        quality_issues: initialIssues,
        data_sources: {
          products: data.products.length,
          news: data.news.length,
          faq: data.faq.length,
          troubleshooting: data.troubleshooting.length,
        },
        created_at: result.rows[0].created_at,
      },
    });
  } catch (err) {
    res.status(500).json({
      success: false,
      error: "Blog generation failed",
      detail: (err as Error).message,
    });
  }
});

/** Paywall signal patterns in page HTML */
const PAYWALL_PATTERNS = [
  /class=["'][^"']*paywall[^"']*["']/i,
  /id=["'][^"']*paywall[^"']*["']/i,
  /"paywall"\s*:/i,
  /data-paywall/i,
  /subscribe (to|now) (read|access|continue)/i,
  /sign[- ]?in to (read|continue|access)/i,
  /log[- ]?in to (read|continue|access)/i,
  /create (a free )?account to (read|continue|access)/i,
  /register (now )?to (read|continue|access)/i,
  /this (article|content|paper) is (for|available to) (subscribers?|members?)/i,
  /premium (content|article|paper)/i,
  /access (this|the) (full )?(article|content|paper)/i,
  /metered[- ]?content/i,
  /subscriber[- ]?only/i,
  /intel(ligence)?\.theregister\.com\/paper/i,
];

/** Fetch a URL and extract readable text content for use as LLM context.
 *  Returns spaDetected=true when extracted body text is thin (< 300 chars).
 *  Returns paywallDetected=true when login/subscription wall signals are found.
 */
async function fetchUrlContent(rawUrl: string): Promise<{
  pageTitle: string;
  text: string;
  spaDetected: boolean;
  paywallDetected: boolean;
  metaDesc: string;
}> {
  const response = await fetch(rawUrl, {
    headers: { "User-Agent": "TIPBot/1.0 blog-from-url (+https://tip.flexoptix.net)" },
    signal: AbortSignal.timeout(20000),
    redirect: "follow",
  });

  if (!response.ok) throw new Error(`HTTP ${response.status} ${response.statusText}`);

  const contentType = response.headers.get("content-type") || "";
  if (!contentType.includes("text/html") && !contentType.includes("text/plain") && !contentType.includes("application/xhtml")) {
    throw new Error(`Unsupported content type: ${contentType.split(";")[0]}`);
  }

  const html = await response.text();

  // --- Extract OG / meta tags for SPA fallback ---
  const decodeEntities = (s: string) =>
    s.replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">")
     .replace(/&#\d+;/g, "").replace(/&[a-z]+;/gi, " ").trim();

  const ogTitle =
    html.match(/<meta[^>]+property=["']og:title["'][^>]+content=["']([^"']{1,200})["']/i)?.[1] ||
    html.match(/<meta[^>]+content=["']([^"']{1,200})["'][^>]+property=["']og:title["']/i)?.[1] || "";

  const ogDesc =
    html.match(/<meta[^>]+property=["']og:description["'][^>]+content=["']([^"']{1,500})["']/i)?.[1] ||
    html.match(/<meta[^>]+content=["']([^"']{1,500})["'][^>]+property=["']og:description["']/i)?.[1] ||
    html.match(/<meta[^>]+name=["']description["'][^>]+content=["']([^"']{1,500})["']/i)?.[1] ||
    html.match(/<meta[^>]+content=["']([^"']{1,500})["'][^>]+name=["']description["']/i)?.[1] || "";

  const ogSiteName =
    html.match(/<meta[^>]+property=["']og:site_name["'][^>]+content=["']([^"']{1,100})["']/i)?.[1] ||
    html.match(/<meta[^>]+content=["']([^"']{1,100})["'][^>]+property=["']og:site_name["']/i)?.[1] || "";

  // Extract page title: prefer OG title, then <title>, then <h1>
  const titleMatch = html.match(/<title[^>]*>([^<]{1,200})<\/title>/i);
  const h1Match = html.match(/<h1[^>]*>([^<]{1,150})<\/h1>/i);
  const pageTitle = decodeEntities(
    ogTitle || titleMatch?.[1] || h1Match?.[1] || ""
  );

  const metaDesc = decodeEntities(ogDesc);

  // Strip scripts, styles, SVG, navigation boilerplate
  let text = html
    .replace(/<script[\s\S]*?<\/script>/gi, " ")
    .replace(/<style[\s\S]*?<\/style>/gi, " ")
    .replace(/<svg[\s\S]*?<\/svg>/gi, " ")
    .replace(/<nav[\s\S]*?<\/nav>/gi, " ")
    .replace(/<footer[\s\S]*?<\/footer>/gi, " ")
    .replace(/<header[\s\S]*?<\/header>/gi, " ")
    .replace(/<aside[\s\S]*?<\/aside>/gi, " ")
    .replace(/<form[\s\S]*?<\/form>/gi, " ")
    // Block elements → newlines
    .replace(/<\/?(p|div|section|article|h[1-6]|li|br|hr|tr|td|th|blockquote|pre)[^>]*>/gi, "\n")
    // Strip all remaining tags
    .replace(/<[^>]{0,500}>/g, " ")
    // Decode common entities
    .replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">")
    .replace(/&quot;/g, '"').replace(/&#39;/g, "'").replace(/&nbsp;/g, " ")
    .replace(/&[a-z]+;/gi, " ")
    // Collapse whitespace
    .split("\n").map(l => l.trim()).filter(l => l.length > 30).join("\n")
    .replace(/\n{3,}/g, "\n\n")
    .trim();

  // Limit to ~5000 chars — enough for LLM context, not so much it blows the prompt
  if (text.length > 5000) {
    text = text.slice(0, 5000) + "\n[… content truncated for LLM context …]";
  }

  // Detect SPA: very little body text means JS renders the real content
  const spaDetected = text.length < 300;

  // Detect paywall: check raw HTML for subscription/login wall signals
  const paywallDetected = PAYWALL_PATTERNS.some(p => p.test(html.slice(0, 20000)));

  // When SPA/paywall detected, enrich text with what we could extract from meta tags
  if ((spaDetected || paywallDetected) && (metaDesc || ogSiteName)) {
    const parts: string[] = [];
    if (ogSiteName) parts.push(`Site: ${ogSiteName}`);
    if (pageTitle) parts.push(`Title: ${pageTitle}`);
    if (metaDesc) parts.push(`Description: ${metaDesc}`);
    text = parts.join("\n");
  }

  return { pageTitle, text, spaDetected, paywallDetected, metaDesc };
}

// POST /api/blog/from-url — Fetch URL, extract content, generate a blog from it
blogRouter.post("/from-url", async (req: Request, res: Response) => {
  const { url, topic } = req.body as { url?: string; topic?: string };

  if (!url) {
    res.status(400).json({ success: false, error: "url ist erforderlich" });
    return;
  }

  // Validate URL — must be http/https
  let parsedUrl: URL;
  try {
    parsedUrl = new URL(url);
    if (!["http:", "https:"].includes(parsedUrl.protocol)) throw new Error("bad protocol");
  } catch {
    res.status(400).json({ success: false, error: "Ungültige URL — muss http:// oder https:// beginnen" });
    return;
  }

  const selectedTopic = topic || "technology_deep_dive";
  const templates = BLOG_TEMPLATES[selectedTopic];
  if (!templates) {
    res.status(400).json({ success: false, error: `Ungültiger Blog-Typ. Gültig: ${Object.keys(BLOG_TEMPLATES).join(", ")}` });
    return;
  }

  try {
    // Fetch page content server-side (no CORS issues)
    const { pageTitle, text: extractedText, spaDetected, paywallDetected, metaDesc } = await fetchUrlContent(url);

    console.log(
      `Blog from-url: fetched "${pageTitle}" from ${parsedUrl.hostname} ` +
      `(${extractedText.length} chars${spaDetected ? ", SPA" : ""}${paywallDetected ? ", PAYWALL" : ""})`
    );

    // Paywall or inaccessible content — return signal so client can prompt for PDF upload.
    // Also catches meta-refresh redirects and other "content gatekeeping" patterns where
    // the fetched HTML is tiny and yields no usable text or metadata.
    const contentBlocked =
      paywallDetected ||
      (spaDetected && extractedText.length < 50 && !metaDesc && !pageTitle);

    if (contentBlocked) {
      res.json({
        success: false,
        paywall_detected: true,
        page_title: pageTitle,
        meta_desc: metaDesc,
        source_url: url,
        topic: selectedTopic,
        error: paywallDetected
          ? "Paywall erkannt — bitte PDF hochladen"
          : "Seite nicht zugänglich — bitte PDF hochladen",
      });
      return;
    }

    // Build a rich additional_context from the URL content.
    // When a SPA is detected (JS-rendered), body text is a shell — we rely on meta tags instead.
    const spaWarning = spaDetected
      ? `\nNOTE: This URL is a JavaScript Single Page Application. Only meta/OG data was available ` +
        `server-side — the LLM should infer topic from the site name, title, and description above. ` +
        `Do NOT default to optical networking topics unless the page is actually about that.`
      : "";

    const additionalContext =
      `⚠️ TOPIC LOCK — THIS BLOG IS ABOUT: "${pageTitle || parsedUrl.hostname}"\n` +
      `The article MUST cover this topic. Do NOT write about optical transceivers, 400G, fiber optics, or DOM readings unless the source article explicitly covers them.\n\n` +
      `SOURCE URL: ${url}\n` +
      `PAGE TITLE: ${pageTitle}\n` +
      `HOSTNAME: ${parsedUrl.hostname}\n` +
      (metaDesc ? `META DESCRIPTION: ${metaDesc}\n` : "") +
      `\n--- EXTRACTED PAGE CONTENT ---\n` +
      `${extractedText || "(No body text extractable — JavaScript-rendered SPA)"}\n` +
      `--- END PAGE CONTENT ---\n` +
      spaWarning +
      `\n\nIMPORTANT: Use this content as factual background and editorial direction. ` +
      `The blog MUST be about the topic described above, NOT about optical transceivers or fiber unless explicitly relevant. ` +
      `Do NOT copy sentences verbatim. Write a Flexoptix-voice blog article using these facts and insights.`;

    const title = pageTitle || parsedUrl.hostname;
    const template = templates[Math.floor(Math.random() * templates.length)];

    // For from-url flow: ALWAYS use empty data — no transceiver product injection.
    // The URL content IS the data. Injecting transceiver products would cause the
    // fine-tuned model to ignore the source article and write a generic 400G post.
    const data = { products: [] as any[], news: [] as any[], faq: [] as any[], troubleshooting: [] as any[] };

    // Use a minimal placeholder draft — generateTemplateDraft produces transceiver-specific
    // skeleton content (NOC scenarios, DOM readings) that pollutes the LLM context.
    const date = new Date().toISOString().split("T")[0];
    const draftContent =
      `# ${title}\n\n` +
      `*Generated from URL: ${url} on ${date}*\n\n` +
      `> **Status**: Pending LLM enhancement — source article loaded.\n\n` +
      `**Source**: ${url}\n` +
      (metaDesc ? `**Summary**: ${metaDesc}\n` : "");
    const wordCount = draftContent.split(/\s+/).length;
    const initialIssues: string[] = [];

    const activeModel = getLlmProvider();
    const generatedBy = `tip-blog-from-url-${activeModel.ollamaModel || activeModel.provider || "llm"}`;

    const result = await pool.query(
      `INSERT INTO blog_drafts (title, topic, target_audience, outline, draft_content, data_sources, status, generated_by, word_count, seo_keywords)
       VALUES ($1, $2, $3, $4, $5, $6, 'draft', $7, $8, $9)
       RETURNING id, created_at`,
      [
        title,
        selectedTopic,
        template.target_audience,
        JSON.stringify({ generation_method: "from-url", source_url: url, spa_detected: spaDetected, quality_issues: initialIssues }),
        draftContent,
        JSON.stringify({ source_url: url, extracted_chars: extractedText.length, spa_detected: spaDetected, products: data.products.length, news: data.news.length }),
        generatedBy,
        wordCount,
        template.seo_keywords,
      ],
    );

    const draftId = result.rows[0].id;

    // Launch LLM pipeline with URL content as context
    const health = await checkHealth().catch(() => ({ ok: false, model: "", error: "unreachable" }));
    let llmStarted = false;
    if (health.ok) {
      llmStarted = true;
      enqueueLlmPipeline(draftId, title, selectedTopic, template.target_audience, data, additionalContext).catch((err) => {
        console.error(`Blog from-url LLM pipeline error: ${(err as Error).message}`);
      });
    }

    res.json({
      success: true,
      source_url: url,
      page_title: pageTitle,
      extracted_chars: extractedText.length,
      spa_detected: spaDetected,
      draft: {
        id: draftId,
        title,
        topic: selectedTopic,
        target_audience: template.target_audience,
        word_count: wordCount,
        generation_method: "from-url",
        llm_enhancing: llmStarted,
        created_at: result.rows[0].created_at,
      },
    });
  } catch (err) {
    const msg = (err as Error).message;
    console.error(`Blog from-url error for ${url}: ${msg}`);
    res.status(500).json({
      success: false,
      error: `URL konnte nicht verarbeitet werden: ${msg}`,
    });
  }
});

// POST /api/blog/from-pdf — Upload a PDF, extract text, generate blog from it
// Accepts JSON body: { pdf_base64: string, filename: string, url?: string, topic?: string, page_title?: string }
// Using base64 JSON instead of multipart to avoid Cloudflare WAF blocking binary uploads.
blogRouter.post("/from-pdf", async (req: Request, res: Response) => {
  const { pdf_base64, filename, url, topic, page_title } = req.body as {
    pdf_base64?: string;
    filename?: string;
    url?: string;
    topic?: string;
    page_title?: string;
  };

  if (!pdf_base64) {
    res.status(400).json({ success: false, error: "Keine PDF-Daten empfangen (pdf_base64 fehlt)" });
    return;
  }

  // Validate base64 + decode
  let fileBuffer: Buffer;
  let fileSize: number;
  try {
    fileBuffer = Buffer.from(pdf_base64, "base64");
    fileSize = fileBuffer.length;
    if (fileSize < 100) throw new Error("Datei zu klein");
    if (fileSize > 20 * 1024 * 1024) throw new Error("Datei zu groß (max 20 MB)");
  } catch (err) {
    res.status(400).json({ success: false, error: `Ungültige PDF-Daten: ${(err as Error).message}` });
    return;
  }

  const originalName = filename || "upload.pdf";
  const selectedTopic = topic || "technology_deep_dive";
  const templates = BLOG_TEMPLATES[selectedTopic];
  if (!templates) {
    res.status(400).json({ success: false, error: `Ungültiger Blog-Typ. Gültig: ${Object.keys(BLOG_TEMPLATES).join(", ")}` });
    return;
  }

  try {
    // Extract text from PDF
    const pdfData = await pdfParse(fileBuffer);
    let extractedText = pdfData.text
      .split("\n").map((l: string) => l.trim()).filter((l: string) => l.length > 20).join("\n")
      .replace(/\n{3,}/g, "\n\n")
      .trim();

    if (extractedText.length < 100) {
      res.status(422).json({ success: false, error: "PDF enthält zu wenig lesbaren Text (ggf. gescannt/bildbasiert)" });
      return;
    }

    // Limit to ~6000 chars for LLM context
    if (extractedText.length > 6000) {
      extractedText = extractedText.slice(0, 6000) + "\n[… PDF content truncated for LLM context …]";
    }

    const title: string = page_title || (typeof pdfData.info?.Title === "string" ? pdfData.info.Title : null) || originalName.replace(/\.pdf$/i, "") || "Artikel aus PDF";

    console.log(`Blog from-pdf: "${title}" — ${extractedText.length} chars from ${originalName} (${(fileSize / 1024).toFixed(0)} KB)`);

    const additionalContext =
      `⚠️ TOPIC LOCK — THIS BLOG IS ABOUT: "${title}"\n` +
      `The article MUST cover this topic. Do NOT write about optical transceivers, 400G, fiber optics, or DOM readings unless the source document explicitly covers them.\n\n` +
      (url ? `SOURCE URL: ${url}\n` : "") +
      `SOURCE FILE: ${originalName}\n` +
      `PAGE TITLE: ${title}\n` +
      `\n--- EXTRACTED PDF CONTENT ---\n` +
      `${extractedText}\n` +
      `--- END PDF CONTENT ---\n\n` +
      `IMPORTANT: Use this content as factual background and editorial direction. ` +
      `The blog MUST be about the topic described above. ` +
      `Do NOT copy sentences verbatim. Write a Flexoptix-voice blog article using these facts and insights.`;

    const template = templates[Math.floor(Math.random() * templates.length)];
    const data = { products: [] as any[], news: [] as any[], faq: [] as any[], troubleshooting: [] as any[] };

    const date = new Date().toISOString().split("T")[0];
    const draftContent =
      `# ${title}\n\n` +
      `*Generated from PDF: ${originalName} on ${date}*\n\n` +
      `> **Status**: Pending LLM enhancement — PDF content loaded.\n\n` +
      (url ? `**Source URL**: ${url}\n` : "") +
      `**Source file**: ${originalName} (${(fileSize / 1024).toFixed(0)} KB, ${pdfData.numpages} pages)\n`;

    const wordCount = draftContent.split(/\s+/).length;
    const activeModel = getLlmProvider();
    const generatedBy = `tip-blog-from-pdf-${activeModel.ollamaModel || activeModel.provider || "llm"}`;

    const result = await pool.query(
      `INSERT INTO blog_drafts (title, topic, target_audience, outline, draft_content, data_sources, status, generated_by, word_count, seo_keywords)
       VALUES ($1, $2, $3, $4, $5, $6, 'draft', $7, $8, $9)
       RETURNING id, created_at`,
      [
        title,
        selectedTopic,
        template.target_audience,
        JSON.stringify({ generation_method: "from-pdf", source_url: url || null, source_file: originalName, pdf_pages: pdfData.numpages }),
        draftContent,
        JSON.stringify({ source_url: url || null, source_file: originalName, extracted_chars: extractedText.length, pdf_pages: pdfData.numpages }),
        generatedBy,
        wordCount,
        template.seo_keywords,
      ],
    );

    const draftId = result.rows[0].id;

    const health = await checkHealth().catch(() => ({ ok: false, model: "", error: "unreachable" }));
    let llmStarted = false;
    if (health.ok) {
      llmStarted = true;
      enqueueLlmPipeline(draftId, title, selectedTopic, template.target_audience, data, additionalContext).catch((err) => {
        console.error(`Blog from-pdf LLM pipeline error: ${(err as Error).message}`);
      });
    }

    res.json({
      success: true,
      source_file: originalName,
      source_url: url || null,
      page_title: title,
      extracted_chars: extractedText.length,
      pdf_pages: pdfData.numpages,
      draft: {
        id: draftId,
        title,
        topic: selectedTopic,
        target_audience: template.target_audience,
        word_count: wordCount,
        generation_method: "from-pdf",
        llm_enhancing: llmStarted,
        created_at: result.rows[0].created_at,
      },
    });
  } catch (err) {
    const msg = (err as Error).message;
    console.error(`Blog from-pdf error: ${msg}`);
    res.status(500).json({ success: false, error: `PDF konnte nicht verarbeitet werden: ${msg}` });
  }
});

// GET /api/blog — List all drafts
blogRouter.get("/", async (_req: Request, res: Response) => {
  try {
    const result = await pool.query(
      `SELECT id, title, topic, target_audience, status, word_count, seo_keywords, generated_by,
              pipeline_steps_completed, linkedin_post, linkedin_char_count, review_tag, created_at
       FROM blog_drafts
       ORDER BY created_at DESC
       LIMIT 50`,
    );

    res.json({ success: true, drafts: result.rows, count: result.rows.length });
  } catch (err) {
    res.status(500).json({ success: false, error: (err as Error).message });
  }
});

// GET /api/blog/llm/status — Queue depth + Ollama health
blogRouter.get("/llm/status", async (_req: Request, res: Response) => {
  const health = await checkHealth().catch(() => ({ ok: false, model: "", error: "unreachable" }));
  res.json({ success: true, queue_depth: getQueueDepth(), llm: health });
});

// POST /api/blog/llm/reset-queue — Force-reset stuck Ollama queue
blogRouter.post("/llm/reset-queue", (_req: Request, res: Response) => {
  resetOllamaQueue();
  res.json({ success: true, message: "Ollama queue reset — stuck requests cleared" });
});

// GET /api/blog/llm/model-info — Training metadata for the active fo-blog model
// Returns Ollama model details + training manifest stats (pairs, eval, base model, revision, etc.)
blogRouter.get("/llm/model-info", async (_req: Request, res: Response) => {
  try {
    const settings = getLlmProvider();
    const ollamaUrl = process.env.BLOG_OLLAMA_URL || process.env.OLLAMA_URL || "http://localhost:11434";

    // Only meaningful for fo-blog Ollama models
    const modelName = settings.ollamaModel || "";
    const isFoBlog = /^fo-blog-v\d+/.test(modelName);

    // 1. Ollama /api/show — model metadata
    let ollamaInfo: Record<string, unknown> | null = null;
    if (settings.provider === "ollama" && modelName) {
      try {
        const r = await fetch(`${ollamaUrl}/api/show`, {
          method: "POST",
          headers: { "Content-Type": "application/json" },
          body: JSON.stringify({ name: modelName }),
          signal: AbortSignal.timeout(6000),
        });
        if (r.ok) ollamaInfo = (await r.json()) as Record<string, unknown>;
      } catch { /* non-fatal */ }
    }

    // 2. Ollama /api/tags — count available fo-blog-vX revisions
    let availableVersions: string[] = [];
    try {
      const r = await fetch(`${ollamaUrl}/api/tags`, { signal: AbortSignal.timeout(5000) });
      if (r.ok) {
        const data = (await r.json()) as { models?: { name: string; modified_at?: string }[] };
        availableVersions = (data.models || [])
          .map((m) => m.name)
          .filter((n) => /^fo-blog-v\d+(?:-r\d+)?:/.test(n));
      }
    } catch { /* non-fatal */ }

    // 3. Training manifest on disk (training-data/runpod/blog_llm/manifest.json)
    let manifest: Record<string, unknown> | null = null;
    try {
      const { readFileSync } = await import("fs");
      const { resolve } = await import("path");
      const manifestPath = resolve(process.cwd(), "training-data/runpod/blog_llm/manifest.json");
      manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as Record<string, unknown>;
    } catch { /* manifest may not exist on this deployment */ }

    // Extract structured fields from Ollama response
    const details = (ollamaInfo?.details ?? {}) as Record<string, unknown>;
    const modelInfo = (ollamaInfo?.model_info ?? {}) as Record<string, unknown>;
    const params = (ollamaInfo?.parameters ?? "") as string;

    // Parse revision from parent_model e.g. "fo-blog-v13-r16:latest" → r16
    const parentModel = (details?.parent_model ?? "") as string;
    const revMatch = parentModel.match(/-r(\d+)/);
    const revision = revMatch ? `r${revMatch[1]}` : null;

    // Parse context length from parameters string
    const ctxMatch = params.match(/num_ctx\s+(\d+)/);
    const contextLength = ctxMatch ? Number.parseInt(ctxMatch[1], 10) : null;

    // Count major versions (fo-blog-vX:latest, not revisions like fo-blog-vX-rY:latest)
    const majorVersions = availableVersions.filter((n) => !/v\d+-r\d+:/.test(n));
    const allRevisions = availableVersions.filter((n) => /v\d+-r\d+:/.test(n));

    res.json({
      success: true,
      provider: settings.provider,
      model: modelName,
      is_fo_blog: isFoBlog,
      revision,
      parent_model: parentModel || null,
      trained_at: (ollamaInfo?.modified_at ?? null) as string | null,
      quantization: (details?.quantization_level ?? null) as string | null,
      parameter_size: (details?.parameter_size ?? null) as string | null,
      base_model: (modelInfo?.["general.basename"] ?? null) as string | null,
      finetune_id: (modelInfo?.["general.finetune"] ?? null) as string | null,
      parameter_count: (modelInfo?.["general.parameter_count"] ?? null) as number | null,
      context_length: contextLength ?? ((modelInfo?.["qwen2.context_length"] ?? null) as number | null),
      temperature: Number.parseFloat(params.match(/temperature\s+([\d.]+)/)?.[1] ?? "NaN") || null,
      // Training data stats from manifest
      training_pairs: (manifest?.training_pairs ?? null) as number | null,
      train_pairs: (manifest?.train_pairs ?? null) as number | null,
      eval_pairs: (manifest?.eval_pairs ?? null) as number | null,
      raw_pairs: (manifest?.raw_pairs ?? null) as number | null,
      // Version availability
      major_versions_available: majorVersions.length,
      total_revisions_available: allRevisions.length,
      available_versions: majorVersions,
      // Pin status
      pinned_version: settings.pinnedVersion ?? null,
    });
  } catch (err) {
    res.status(500).json({ success: false, error: (err as Error).message });
  }
});

// POST /api/blog/llm/refresh-discovery — Force auto-discovery to pick up newly-trained fo-blog-v* versions
// Useful right after Magatama adopts a new fo-blog-vN model. Otherwise runs every 10 min by itself.
blogRouter.post("/llm/refresh-discovery", async (_req: Request, res: Response) => {
  try {
    const active = await refreshLlmAutoDiscovery();
    res.json({ success: true, active, message: `Auto-discovery refreshed. Active: ${active.provider}${active.ollamaModel ? ` (${active.ollamaModel})` : ""}` });
  } catch (err) {
    res.status(500).json({ success: false, error: (err as Error).message });
  }
});

// POST /api/blog/llm/switch — Switch active LLM provider at runtime (no restart needed)
// Body: { provider: "claude-code" | "anthropic" | "ollama", model?: "fo-blog-v10" | "qwen2.5:14b" | ... }
blogRouter.post("/llm/switch", (req: Request, res: Response) => {
  const { provider, model } = req.body as { provider?: string; model?: string };
  const validProviders = ["claude-code", "anthropic", "ollama"];

  if (!provider || !validProviders.includes(provider)) {
    res.status(400).json({
      success: false,
      error: `Invalid provider. Valid: ${validProviders.join(", ")}`,
    });
    return;
  }

  const prev = getLlmProvider();
  setLlmProvider(provider, model);
  const next = getLlmProvider();

  console.log(`[blog/llm/switch] ${prev.provider}→${next.provider} model=${next.ollamaModel}`);
  res.json({
    success: true,
    previous: { provider: prev.provider, model: prev.ollamaModel },
    active:   { provider: next.provider, model: next.ollamaModel },
    message: `Switched to ${next.provider}${next.ollamaModel ? ` (${next.ollamaModel})` : ""}`,
  });
});

// POST /api/blog/llm/pin — Pin a specific fo-blog-vN model, disabling auto-upgrade
// Body: { version: "fo-blog-v13" } — omit version to pin the current model
blogRouter.post("/llm/pin", (req: Request, res: Response) => {
  const { version } = req.body as { version?: string };
  const current = getLlmProvider();
  const target = version || current.ollamaModel;

  if (!target.startsWith("fo-blog-v")) {
    res.status(400).json({ success: false, error: "Only fo-blog-v* models can be pinned" });
    return;
  }

  pinLlmVersion(target);
  const next = getLlmProvider();
  console.log(`[blog/llm/pin] pinned to ${target}`);
  res.json({
    success: true,
    pinned: target,
    active: { provider: next.provider, model: next.ollamaModel, pinnedVersion: next.pinnedVersion },
    message: `Pinned to ${target} — auto-upgrade disabled`,
  });
});

// POST /api/blog/llm/unpin — Remove version pin, re-enable auto-upgrade
// Immediately reconciles against Ollama so the highest available version is adopted.
blogRouter.post("/llm/unpin", async (_req: Request, res: Response) => {
  try {
    const active = await unpinLlmVersion();
    console.log(`[blog/llm/unpin] unpinned, active → ${active.ollamaModel}`);
    res.json({
      success: true,
      active: { provider: active.provider, model: active.ollamaModel },
      message: `Unpinned — auto-upgrade re-enabled. Active: ${active.ollamaModel}`,
    });
  } catch (err) {
    res.status(500).json({ success: false, error: (err as Error).message });
  }
});

// GET /api/blog/:id — Get a specific draft with full content
// GET /api/blog/:id/progress — Real-time pipeline step progress (in-memory)
blogRouter.get("/:id/progress", (req: Request, res: Response) => {
  const p = pipelineProgress.get(String(req.params.id));
  if (!p) {
    res.json({ success: true, running: false, step: 0, total: 10, label: "Idle", pct: 0 });
    return;
  }
  res.json({ success: true, running: true, ...p });
});

blogRouter.get("/:id", async (req: Request, res: Response) => {
  try {
    const result = await pool.query(
      `SELECT * FROM blog_drafts WHERE id = $1::uuid`,
      [req.params.id],
    );

    if (result.rows.length === 0) {
      res.status(404).json({ success: false, error: "Draft not found" });
      return;
    }

    res.json({ success: true, draft: result.rows[0] });
  } catch (err) {
    res.status(500).json({ success: false, error: (err as Error).message });
  }
});

// PUT /api/blog/:id/status — Update draft status
blogRouter.put("/:id/status", async (req: Request, res: Response) => {
  const { status } = req.body as { status?: string };
  const validStatuses = ["draft", "review", "approved", "published"];

  if (!status || !validStatuses.includes(status)) {
    res.status(400).json({
      success: false,
      error: `Invalid status. Valid: ${validStatuses.join(", ")}`,
    });
    return;
  }

  try {
    const result = await pool.query(
      `UPDATE blog_drafts SET status = $1, updated_at = NOW() WHERE id = $2::uuid RETURNING id, title, status`,
      [status, req.params.id],
    );

    if (result.rows.length === 0) {
      res.status(404).json({ success: false, error: "Draft not found" });
      return;
    }

    res.json({ success: true, draft: result.rows[0] });
  } catch (err) {
    res.status(500).json({ success: false, error: (err as Error).message });
  }
});

// PUT /api/blog/:id/review — Toggle reviewed/unreviewed tag
blogRouter.put("/:id/review", async (req: Request, res: Response) => {
  try {
    const current = await pool.query(
      `SELECT review_tag FROM blog_drafts WHERE id = $1::uuid`,
      [req.params.id]
    );
    if (current.rows.length === 0) {
      res.status(404).json({ success: false, error: "Draft not found" });
      return;
    }
    const newTag = current.rows[0].review_tag === "reviewed" ? null : "reviewed";
    await pool.query(
      `UPDATE blog_drafts SET review_tag = $1, updated_at = NOW() WHERE id = $2::uuid`,
      [newTag, req.params.id]
    );
    res.json({ success: true, review_tag: newTag });
  } catch (err) {
    res.status(500).json({ success: false, error: (err as Error).message });
  }
});

// ═══════════════════════════════════════════════════════
// FEEDBACK SYSTEM (v0.2.0 — FO_Blog_LLM Training Loop)
// ═══════════════════════════════════════════════════════

/**
 * POST /api/blog/:id/feedback — Submit rating + feedback. Fed back to LLM.
 */
blogRouter.post("/:id/feedback", async (req: Request, res: Response) => {
  const {
    score_overall, score_technical_depth, score_real_world, score_clarity,
    score_originality, score_engineer_voice, score_decision_value,
    score_failure_scenarios, score_opinion_strength,
    feedback_text, reviewer = "human", improvements
  } = req.body;

  if (!score_overall) return res.status(400).json({ error: "score_overall required (1-10)" });

  try {
    const blog = await pool.query("SELECT topic, title FROM blog_drafts WHERE id = $1::uuid", [req.params.id]);
    const bd = blog.rows[0];

    const result = await pool.query(
      `INSERT INTO blog_feedback (blog_id, score_overall, score_technical_depth, score_real_world,
        score_clarity, score_originality, score_engineer_voice, score_decision_value,
        score_failure_scenarios, score_opinion_strength, feedback_text, reviewer,
        blog_type, blog_topic, improvements)
       VALUES ($1::uuid,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) RETURNING id`,
      [req.params.id, score_overall, score_technical_depth ?? null, score_real_world ?? null,
       score_clarity ?? null, score_originality ?? null, score_engineer_voice ?? null,
       score_decision_value ?? null, score_failure_scenarios ?? null, score_opinion_strength ?? null,
       feedback_text ?? null, reviewer, bd?.topic ?? null, bd?.title ?? null,
       improvements ? JSON.stringify(improvements) : null]
    );
    res.json({ success: true, feedback_id: result.rows[0].id });
  } catch (err) {
    console.error("Feedback error:", err);
    res.status(500).json({ error: "Failed to save feedback" });
  }
});

/** GET /api/blog/feedback/stats — Aggregate feedback for LLM improvement tracking */
blogRouter.get("/feedback/stats", async (_req: Request, res: Response) => {
  try {
    const [overall, byType] = await Promise.all([
      pool.query(`SELECT COUNT(*) AS total, AVG(score_overall)::numeric(3,1) AS avg FROM blog_feedback`),
      pool.query(`SELECT blog_type, COUNT(*) AS cnt, AVG(score_overall)::numeric(3,1) AS avg
        FROM blog_feedback WHERE blog_type IS NOT NULL GROUP BY blog_type ORDER BY avg ASC`),
    ]);
    res.json({ total: parseInt(overall.rows[0]?.total||"0"), avg_score: overall.rows[0]?.avg, by_type: byType.rows });
  } catch (err) { res.status(500).json({ error: "Failed" }); }
});

/** GET /api/blog/feedback/training-data — Export for FO_Blog_LLM injection */
blogRouter.get("/feedback/training-data", async (_req: Request, res: Response) => {
  try {
    const result = await pool.query(
      `SELECT score_overall, feedback_text, blog_type, improvements FROM blog_feedback
       WHERE feedback_text IS NOT NULL ORDER BY score_overall ASC LIMIT 30`);
    await pool.query(`UPDATE blog_feedback SET fed_to_llm=true, fed_at=NOW() WHERE fed_to_llm=false AND feedback_text IS NOT NULL`);
    res.json({ entries: result.rows, count: result.rowCount });
  } catch (err) { res.status(500).json({ error: "Failed" }); }
});

// POST /api/blog/:id/regenerate — Re-run full LLM pipeline on existing draft (for review/quality-issue cases)
blogRouter.post("/:id/regenerate", async (req: Request, res: Response) => {
  try {
    const result = await pool.query(
      `SELECT id, title, topic, target_audience, seo_keywords FROM blog_drafts WHERE id = $1::uuid`,
      [req.params.id],
    );

    if (result.rows.length === 0) {
      res.status(404).json({ success: false, error: "Draft not found" });
      return;
    }

    const draft = result.rows[0];
    const keywords: string[] = draft.seo_keywords || [];

    // Re-gather fresh data for this topic
    const data = await gatherBlogData(keywords, draft.topic);

    // Reset status to draft + clear quality issues in outline
    await pool.query(
      `UPDATE blog_drafts SET status = 'draft', updated_at = NOW(),
        outline = outline || '{"quality_issues":[],"regeneration_requested":true}'::jsonb
       WHERE id = $1::uuid`,
      [draft.id],
    );

    // Check LLM availability
    const health = await checkHealth().catch(() => ({ ok: false, model: "", error: "unreachable" }));
    if (!health.ok) {
      res.status(503).json({ success: false, error: "LLM not available — cannot regenerate" });
      return;
    }

    console.log(`Blog Regenerate: Re-queuing LLM pipeline for draft ${draft.id} ("${draft.title}")`);
    enqueueLlmPipeline(draft.id, draft.title, draft.topic, draft.target_audience, data).catch((err) => {
      console.error(`Blog regenerate pipeline error: ${(err as Error).message}`);
    });

    res.json({
      success: true,
      draft_id: draft.id,
      title: draft.title,
      message: "LLM pipeline re-queued — poll /api/blog/:id/progress for status",
    });
  } catch (err) {
    res.status(500).json({ success: false, error: (err as Error).message });
  }
});

// DELETE /api/blog/:id — Delete a blog draft
blogRouter.delete("/:id", async (req: Request, res: Response) => {
  try {
    // Delete feedback first (FK constraint)
    await pool.query("DELETE FROM blog_feedback WHERE blog_id = $1::uuid", [req.params.id]);
    const result = await pool.query(
      "DELETE FROM blog_drafts WHERE id = $1::uuid RETURNING id, title",
      [req.params.id]
    );
    if (result.rows.length === 0) {
      return res.status(404).json({ success: false, error: "Draft not found" });
    }
    res.json({ success: true, deleted: result.rows[0] });
  } catch (err) {
    res.status(500).json({ success: false, error: (err as Error).message });
  }
});

// GET /api/blog/linkedin/history — LinkedIn distribution log
blogRouter.get("/linkedin/history", async (_req: Request, res: Response) => {
  try {
    const result = await pool.query(
      `SELECT id, ghost_post_id, ghost_slug, ghost_url, title, state, teaser,
              linkedin_urn, error_message, attempt_count, created_at, posted_at
       FROM blog_linkedin_distribution
       ORDER BY created_at DESC
       LIMIT 50`
    );
    // Also get current DRY_RUN status from env (read from distributor ecosystem config)
    let dryRun = true;
    try {
      const { execSync } = await import("child_process");
      const out = execSync(
        "cat /opt/linkedin-distributor/ecosystem.config.cjs 2>/dev/null | grep DRY_RUN | head -1",
        { timeout: 3000 }
      ).toString();
      dryRun = !out.includes("'false'") && !out.includes('"false"');
    } catch { /* keep default true */ }

    res.json({
      success: true,
      dry_run: dryRun,
      history: result.rows,
      total: result.rows.length,
      stats: {
        posted: result.rows.filter(r => r.state === "posted").length,
        dry_run: result.rows.filter(r => r.state === "dry_run").length,
        skipped: result.rows.filter(r => r.state === "skipped").length,
        failed: result.rows.filter(r => r.state === "failed").length,
      },
    });
  } catch (err) {
    res.status(500).json({ success: false, error: (err as Error).message });
  }
});
