From f48a809e40ea1f998ec733a55513f6dfea9bf8d2 Mon Sep 17 00:00:00 2001 From: Rene Fichtmueller Date: Sat, 28 Mar 2026 00:32:08 +1300 Subject: [PATCH] =?UTF-8?q?feat:=20Phase=207=20=E2=80=94=20Blog=20generato?= =?UTF-8?q?r=20+=20scraper=20scheduler=20activation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Blog draft engine generates structured markdown from all Qdrant collections (products, news, FAQ, troubleshooting). Supports 4 topic types: hype_cycle, comparison, new_product, tutorial. - routes/blog.ts: POST /api/blog/generate, GET/PUT endpoints - ecosystem.config.js: Added tip-scraper PM2 process - Scraper scheduler (pg-boss) now running on Erik with 8 job queues - News scraper running every 6 hours on Erik --- ecosystem.config.js | 43 +++- packages/api/src/index.ts | 6 + packages/api/src/routes/blog.ts | 375 ++++++++++++++++++++++++++++++++ 3 files changed, 418 insertions(+), 6 deletions(-) create mode 100644 packages/api/src/routes/blog.ts diff --git a/ecosystem.config.js b/ecosystem.config.js index 80c187d..9baa9db 100644 --- a/ecosystem.config.js +++ b/ecosystem.config.js @@ -2,15 +2,46 @@ module.exports = { apps: [ { name: "tip-api", - script: "packages/api/dist/index.js", - instances: 1, - autorestart: true, - watch: false, - max_memory_restart: "512M", + script: "./node_modules/.bin/tsx", + args: "packages/api/src/index.ts", + cwd: "/opt/tip", + interpreter: "none", + exec_mode: "fork", env: { NODE_ENV: "production", - API_PORT: 3200, + API_PORT: "3201", + POSTGRES_HOST: "localhost", + POSTGRES_PORT: "5433", + POSTGRES_DB: "transceiver_db", + POSTGRES_USER: "tip", + POSTGRES_PASSWORD: "tip_prod_2026", + OLLAMA_URL: "http://localhost:11434", + QDRANT_URL: "http://localhost:6333", + DOCLING_URL: "http://localhost:8100", }, + max_memory_restart: "500M", + instances: 1, + autorestart: true, + }, + { + name: "tip-scraper", + script: "./node_modules/.bin/tsx", + args: "packages/scraper/src/index.ts", + cwd: "/opt/tip", + interpreter: "none", + exec_mode: "fork", + env: { + NODE_ENV: "production", + POSTGRES_HOST: "localhost", + POSTGRES_PORT: "5433", + POSTGRES_DB: "transceiver_db", + POSTGRES_USER: "tip", + POSTGRES_PASSWORD: "tip_prod_2026", + }, + max_memory_restart: "1G", + instances: 1, + autorestart: true, + cron_restart: "0 0 * * *", }, ], }; diff --git a/packages/api/src/index.ts b/packages/api/src/index.ts index 77838b5..d4e84c9 100644 --- a/packages/api/src/index.ts +++ b/packages/api/src/index.ts @@ -11,6 +11,7 @@ import { healthRouter } from "./routes/health"; import { hypeCycleRouter } from "./routes/hype-cycle"; import { searchRouter } from "./routes/search"; import { documentRouter } from "./routes/documents"; +import { blogRouter } from "./routes/blog"; const app = express(); @@ -36,6 +37,7 @@ app.use("/api/health", healthRouter); app.use("/api/hype-cycle", hypeCycleRouter); app.use("/api/search", searchRouter); app.use("/api/documents", documentRouter); +app.use("/api/blog", blogRouter); // Root app.get("/", (_req, res) => { @@ -61,6 +63,10 @@ app.get("/", (_req, res) => { "POST /api/documents/process {url, title?, doc_type?, vendor?, collection?}", "GET /api/documents", "GET /api/documents/:id", + "POST /api/blog/generate {topic, speed?, form_factor?, use_case?}", + "GET /api/blog", + "GET /api/blog/:id", + "PUT /api/blog/:id/status {status: draft|review|approved|published}", ], }); }); diff --git a/packages/api/src/routes/blog.ts b/packages/api/src/routes/blog.ts new file mode 100644 index 0000000..57d6ca1 --- /dev/null +++ b/packages/api/src/routes/blog.ts @@ -0,0 +1,375 @@ +/** + * Blog Draft Generator API + * + * POST /api/blog/generate — Generate a blog draft from data + * GET /api/blog — List all drafts + * GET /api/blog/:id — Get a specific draft + * PUT /api/blog/:id/status — Update draft status + */ +import { Router, Request, Response } from "express"; +import { pool } from "../db/client"; +import { semanticSearch } from "../embeddings/client"; + +export const blogRouter = Router(); + +interface BlogTopic { + topic: string; + title: string; + target_audience: "sales" | "technical" | "customer" | "seo"; + seo_keywords: string[]; +} + +const BLOG_TEMPLATES: Record = { + hype_cycle: [ + { + topic: "hype_cycle", + title: "The State of {SPEED} Transceivers in {YEAR}: Where Are We on the Hype Cycle?", + target_audience: "technical", + seo_keywords: ["transceiver", "hype cycle", "optical networking"], + }, + { + topic: "hype_cycle", + title: "Investment Guide: Which Transceiver Speeds to Bet On in {YEAR}", + target_audience: "sales", + seo_keywords: ["transceiver investment", "data center optics", "ROI"], + }, + ], + comparison: [ + { + topic: "comparison", + title: "{FORM_FACTOR} Transceiver Comparison: Top 5 Options for {USE_CASE}", + target_audience: "customer", + seo_keywords: ["transceiver comparison", "best transceiver"], + }, + { + topic: "comparison", + title: "Original vs. Compatible Transceivers: The Real Cost Difference in {YEAR}", + target_audience: "seo", + seo_keywords: ["compatible transceiver", "original vs compatible", "cost savings"], + }, + ], + new_product: [ + { + topic: "new_product", + title: "{SPEED} Transceivers: What's New and What It Means for Your Network", + target_audience: "technical", + seo_keywords: ["new transceiver", "latest optics"], + }, + ], + tutorial: [ + { + topic: "tutorial", + title: "How to Choose the Right Transceiver: A Complete {YEAR} Buying Guide", + target_audience: "customer", + seo_keywords: ["transceiver buying guide", "how to choose transceiver"], + }, + { + topic: "tutorial", + title: "Troubleshooting Transceiver Issues: The Definitive Guide", + target_audience: "technical", + seo_keywords: ["transceiver troubleshooting", "optical module problems"], + }, + ], +}; + +/** Gather data from various collections for blog content */ +async function gatherBlogData(topic: string, keywords: string[]): Promise<{ + products: Array>; + news: Array>; + faq: Array>; + troubleshooting: Array>; +}> { + const query = keywords.join(" "); + + const [products, news, faq, troubleshooting] = await Promise.all([ + semanticSearch("product_embeddings", query, 10).catch(() => []), + semanticSearch("news_embeddings", query, 5).catch(() => []), + semanticSearch("faq_embeddings", query, 5).catch(() => []), + semanticSearch("troubleshooting_embeddings", query, 3).catch(() => []), + ]); + + return { + products: products.map((r) => ({ score: r.score, ...r.payload })), + news: news.map((r) => ({ score: r.score, ...r.payload })), + faq: faq.map((r) => ({ score: r.score, ...r.payload })), + troubleshooting: troubleshooting.map((r) => ({ score: r.score, ...r.payload })), + }; +} + +/** Generate blog outline from gathered data */ +function generateOutline( + title: string, + topic: string, + data: Awaited>, +): { sections: Array<{ heading: string; notes: string }> } { + const sections: Array<{ heading: string; notes: string }> = []; + + sections.push({ + heading: "Introduction", + notes: `Hook the reader with the key question this post answers. Reference ${data.news.length} recent news items for timeliness.`, + }); + + if (topic === "hype_cycle") { + sections.push({ + heading: "Understanding the Hype Cycle for Optical Transceivers", + notes: "Explain the Norton-Bass model phases: Innovation Trigger → Peak of Inflated Expectations → Trough of Disillusionment → Slope of Enlightenment → Plateau of Productivity", + }); + sections.push({ + heading: "Current Position of Key Technologies", + notes: `Cover products found: ${data.products.slice(0, 5).map((p) => p.standard_name || p.text).join(", ")}`, + }); + sections.push({ + heading: "Market Signals and Recent Developments", + notes: `Reference: ${data.news.map((n) => n.title).join("; ")}`, + }); + } else if (topic === "comparison") { + const formFactors = [...new Set(data.products.map((p) => String(p.form_factor)).filter(Boolean))]; + sections.push({ + heading: "Products Compared", + notes: `Form factors covered: ${formFactors.join(", ")}. ${data.products.length} products analyzed.`, + }); + sections.push({ + heading: "Key Specifications Breakdown", + notes: "Compare speed, reach, power consumption, fiber type, and pricing across products.", + }); + sections.push({ + heading: "Compatibility Considerations", + notes: `Reference FAQ: ${data.faq.slice(0, 3).map((f) => f.question).join("; ")}`, + }); + } else if (topic === "tutorial") { + sections.push({ + heading: "Step 1: Determine Your Requirements", + notes: "Speed, distance, fiber type, switch compatibility.", + }); + sections.push({ + heading: "Step 2: Understanding Form Factors", + notes: `Cover: ${data.faq.filter((f) => String(f.category) === "form_factor").map((f) => f.question).join("; ")}`, + }); + sections.push({ + heading: "Common Issues and Troubleshooting", + notes: `Reference: ${data.troubleshooting.map((t) => t.symptom).join("; ")}`, + }); + } else { + sections.push({ + heading: "What's New", + notes: `${data.products.length} relevant products, ${data.news.length} recent news items.`, + }); + sections.push({ + heading: "Technical Details", + notes: "Deep-dive into specifications and use cases.", + }); + } + + sections.push({ + heading: "Conclusion & Recommendations", + notes: "Summarize key takeaways. Include CTA for Flexoptix product finder.", + }); + + return { sections }; +} + +/** Generate draft content from outline and data */ +function generateDraft( + title: string, + outline: ReturnType, + data: Awaited>, +): string { + const parts: string[] = []; + + parts.push(`# ${title}\n`); + parts.push(`*Generated by TIP Blog Engine on ${new Date().toISOString().split("T")[0]}*\n`); + + for (const section of outline.sections) { + parts.push(`\n## ${section.heading}\n`); + parts.push(`\n`); + + if (section.heading === "Introduction") { + const topNews = data.news[0]; + if (topNews) { + parts.push(`The optical transceiver market continues to evolve rapidly. ${String(topNews.title || "")} highlights the pace of change in our industry.\n`); + } + parts.push(`In this article, we'll explore the key trends, products, and considerations that matter most for network professionals and procurement teams.\n`); + } else if (section.heading.includes("Products") || section.heading.includes("Technologies")) { + for (const product of data.products.slice(0, 5)) { + parts.push(`### ${product.standard_name || product.slug || "Product"}\n`); + parts.push(`- **Form Factor**: ${product.form_factor || "N/A"}`); + parts.push(`- **Speed**: ${product.speed || "N/A"}`); + parts.push(`- **Reach**: ${product.reach_label || "N/A"}`); + parts.push(`- **Fiber Type**: ${product.fiber_type || "N/A"}`); + parts.push(`- **Vendor**: ${product.vendor || "N/A"}\n`); + } + } else if (section.heading.includes("Troubleshooting") || section.heading.includes("Issues")) { + for (const ts of data.troubleshooting) { + parts.push(`### ${ts.symptom}\n`); + parts.push(`**Cause**: ${ts.cause}\n`); + parts.push(`**Solution**: ${ts.solution}\n`); + } + } else if (section.heading.includes("Conclusion")) { + parts.push(`The transceiver landscape offers more options than ever. Whether you're planning a data center upgrade, evaluating 400G/800G migration, or troubleshooting existing deployments, the right transceiver choice depends on your specific requirements.\n`); + parts.push(`**[Browse our full transceiver catalog →](https://www.flexoptix.net/en/)**\n`); + } + } + + return parts.join("\n"); +} + +// POST /api/blog/generate — Generate a new blog draft +blogRouter.post("/generate", async (req: Request, res: Response) => { + const { topic, speed, form_factor, use_case } = req.body as { + topic?: string; + speed?: string; + form_factor?: string; + use_case?: string; + }; + + const selectedTopic = topic || "comparison"; + const templates = BLOG_TEMPLATES[selectedTopic]; + + if (!templates) { + res.status(400).json({ + success: false, + error: `Invalid topic. Valid: ${Object.keys(BLOG_TEMPLATES).join(", ")}`, + }); + return; + } + + try { + const year = new Date().getFullYear(); + const template = templates[Math.floor(Math.random() * templates.length)]; + + // Fill template placeholders + const title = template.title + .replace("{YEAR}", String(year)) + .replace("{SPEED}", speed || "400G/800G") + .replace("{FORM_FACTOR}", form_factor || "QSFP-DD/OSFP") + .replace("{USE_CASE}", use_case || "Data Center Interconnect"); + + // Build search keywords + const keywords = [ + ...template.seo_keywords, + speed || "400G", + form_factor || "", + use_case || "data center", + ].filter(Boolean); + + // Gather data from all collections + const data = await gatherBlogData(selectedTopic, keywords); + + // Generate outline and draft + const outline = generateOutline(title, selectedTopic, data); + const draftContent = generateDraft(title, outline, data); + const wordCount = draftContent.split(/\s+/).length; + + // Save to database + const result = await pool.query( + `INSERT INTO blog_drafts (title, topic, target_audience, outline, draft_content, data_sources, status, generated_by, word_count, seo_keywords) + VALUES ($1, $2, $3, $4, $5, $6, 'draft', 'tip-blog-engine', $7, $8) + RETURNING id, created_at`, + [ + title, + selectedTopic, + template.target_audience, + JSON.stringify(outline), + draftContent, + JSON.stringify({ + products: data.products.length, + news: data.news.length, + faq: data.faq.length, + troubleshooting: data.troubleshooting.length, + }), + wordCount, + template.seo_keywords, + ], + ); + + res.json({ + success: true, + draft: { + id: result.rows[0].id, + title, + topic: selectedTopic, + target_audience: template.target_audience, + word_count: wordCount, + sections: outline.sections.length, + data_sources: { + products: data.products.length, + news: data.news.length, + faq: data.faq.length, + troubleshooting: data.troubleshooting.length, + }, + created_at: result.rows[0].created_at, + }, + }); + } catch (err) { + res.status(500).json({ + success: false, + error: "Blog generation failed", + detail: (err as Error).message, + }); + } +}); + +// GET /api/blog — List all drafts +blogRouter.get("/", async (_req: Request, res: Response) => { + try { + const result = await pool.query( + `SELECT id, title, topic, target_audience, status, word_count, seo_keywords, created_at + FROM blog_drafts + ORDER BY created_at DESC + LIMIT 50`, + ); + + res.json({ success: true, drafts: result.rows, count: result.rows.length }); + } catch (err) { + res.status(500).json({ success: false, error: (err as Error).message }); + } +}); + +// GET /api/blog/:id — Get a specific draft with full content +blogRouter.get("/:id", async (req: Request, res: Response) => { + try { + const result = await pool.query( + `SELECT * FROM blog_drafts WHERE id = $1::uuid`, + [req.params.id], + ); + + if (result.rows.length === 0) { + res.status(404).json({ success: false, error: "Draft not found" }); + return; + } + + res.json({ success: true, draft: result.rows[0] }); + } catch (err) { + res.status(500).json({ success: false, error: (err as Error).message }); + } +}); + +// PUT /api/blog/:id/status — Update draft status +blogRouter.put("/:id/status", async (req: Request, res: Response) => { + const { status } = req.body as { status?: string }; + const validStatuses = ["draft", "review", "approved", "published"]; + + if (!status || !validStatuses.includes(status)) { + res.status(400).json({ + success: false, + error: `Invalid status. Valid: ${validStatuses.join(", ")}`, + }); + return; + } + + try { + const result = await pool.query( + `UPDATE blog_drafts SET status = $1, updated_at = NOW() WHERE id = $2::uuid RETURNING id, title, status`, + [status, req.params.id], + ); + + if (result.rows.length === 0) { + res.status(404).json({ success: false, error: "Draft not found" }); + return; + } + + res.json({ success: true, draft: result.rows[0] }); + } catch (err) { + res.status(500).json({ success: false, error: (err as Error).message }); + } +});