Rene Fichtmueller c6308e93c0 feat: massive scraper expansion + hype cycle engine + lifecycle prediction
New scrapers:
- GBICS.com (BigCommerce, GBP prices, 10 categories, 78 products)
- Juniper HCT (Next.js SSR parser, 475 transceivers with specs/EOL)
- SFPcables.com (Magento store, 16 categories, 78 products)
- Fluxlight (BigCommerce, 6 pages, 118 products)
- Champion ONE (compatible vendor scraper)

Scraper fixes:
- 10Gtek: rewritten to parse HTML spec tables (152 products)
- Flexoptix: fix price extraction from Magento Hyva HTML
- Register all scrapers in CLI (--gbics, --juniper, --sfpcables, etc.)

Hype Cycle Engine enhancements:
- Data-driven enrichment from scraped vendor/price data
- Revenue lifecycle prediction (peak year, decline, revenue index)
- Regional adoption model (NA, China, APAC, Europe, RoW with lag coefficients)
- New API endpoints: /enriched, /lifecycle, /regional/:tech

DB growth: 89 → 1,168 transceivers, 0 → 416 prices, 6 vendors
Qdrant: 1,162 products embedded with nomic-embed-text

Research: Norton-Bass model, standards-to-market timelines, hype signals
2026-03-28 02:30:19 +13:00

204 lines
6.5 KiB
TypeScript

/**
* Hype Cycle API routes
*
* GET /api/hype-cycle — All technologies with current phase
* GET /api/hype-cycle/enriched — All technologies with data-driven metrics
* GET /api/hype-cycle/lifecycle — Revenue lifecycle predictions for all speeds
* GET /api/hype-cycle/regional/:tech — Regional adoption model for a technology
* GET /api/hype-cycle/:tech — Specific technology with 5-year forecast
*/
import { Router, Request, Response } from "express";
import {
computeAllHypeCycles,
computeHypeCycle,
findTechnology,
TECH_GENERATIONS,
SPECIAL_TECHS,
} from "../hype-cycle/norton-bass";
import {
getDataDrivenOverrides,
getSpeedClassMetrics,
computeRevenueLifecycle,
computeRegionalAdoption,
} from "../hype-cycle/data-enrichment";
export const hypeCycleRouter = Router();
const q = (p: string, req: Request): string | undefined =>
req.query[p] ? String(req.query[p]) : undefined;
// GET /api/hype-cycle — All technologies (model-only, fast)
hypeCycleRouter.get("/", (_req: Request, res: Response) => {
const yearParam = q("year", _req);
const year = yearParam ? parseInt(yearParam) : new Date().getFullYear();
const results = computeAllHypeCycles(year);
const sorted = [...results].sort((a, b) => a.positionPct - b.positionPct);
res.json({
success: true,
year,
model: "Norton-Bass Multigenerational Diffusion",
technologies: sorted.map((r) => ({
technology: r.technology,
phase: r.phaseLabel,
positionPct: r.positionPct,
adoptionPct: r.adoptionPct,
compositeScore: r.compositeScore,
peakYear: r.forecast.peakShipmentYear,
yearsToPlateauFromNow: r.forecast.yearsToPlateauFromNow,
})),
});
});
// GET /api/hype-cycle/enriched — Data-driven enrichment from scraped data
hypeCycleRouter.get("/enriched", async (_req: Request, res: Response) => {
try {
const yearParam = q("year", _req);
const year = yearParam ? parseInt(yearParam) : new Date().getFullYear();
const overridesMap = await getDataDrivenOverrides();
const allTechs = [...TECH_GENERATIONS, ...SPECIAL_TECHS];
const results = allTechs.map((tech) => {
const overrides = overridesMap.get(tech.speedGbps);
return computeHypeCycle(tech, year, overrides);
});
const sorted = [...results].sort((a, b) => a.positionPct - b.positionPct);
// Also include raw metrics for transparency
const speedMetrics = await getSpeedClassMetrics();
res.json({
success: true,
year,
model: "Norton-Bass + Data-Driven Enrichment",
dataSource: {
totalTransceivers: speedMetrics.reduce((s, m) => s + m.skuCount, 0),
totalPricePoints: speedMetrics.reduce((s, m) => s + m.priceCount, 0),
speedClasses: speedMetrics.length,
},
technologies: sorted.map((r) => ({
technology: r.technology,
phase: r.phaseLabel,
positionPct: r.positionPct,
adoptionPct: r.adoptionPct,
compositeScore: r.compositeScore,
peakYear: r.forecast.peakShipmentYear,
yearsToPlateauFromNow: r.forecast.yearsToPlateauFromNow,
metrics: r.metrics,
fiveYearForecast: r.forecast.fiveYearProjection,
})),
rawSpeedMetrics: speedMetrics.map((m) => ({
speedGbps: m.speedGbps,
vendorCount: m.vendorCount,
skuCount: m.skuCount,
avgPrice: m.avgPrice ? Math.round(m.avgPrice * 100) / 100 : null,
minPrice: m.minPrice ? Math.round(m.minPrice * 100) / 100 : null,
maxPrice: m.maxPrice ? Math.round(m.maxPrice * 100) / 100 : null,
formFactors: m.formFactors,
reachVariants: m.reachVariants,
})),
});
} catch (err) {
console.error("Enriched hype cycle error:", err);
res.status(500).json({ success: false, error: "Failed to compute enriched hype cycle" });
}
});
// GET /api/hype-cycle/lifecycle — Revenue lifecycle predictions
hypeCycleRouter.get("/lifecycle", async (_req: Request, res: Response) => {
try {
const currentYear = new Date().getFullYear();
const speedMetrics = await getSpeedClassMetrics();
const priceMap = new Map(speedMetrics.map((m) => [m.speedGbps, m.avgPrice]));
const allTechs = [...TECH_GENERATIONS, ...SPECIAL_TECHS];
const lifecycles = allTechs.map((tech) =>
computeRevenueLifecycle(
tech.speedGbps,
tech.name,
tech.introYear,
tech.peakYear,
currentYear,
priceMap.get(tech.speedGbps),
)
);
// Sort by revenue index (highest current revenue first)
const sorted = [...lifecycles].sort((a, b) => b.revenueIndex - a.revenueIndex);
res.json({
success: true,
currentYear,
lifecycles: sorted,
});
} catch (err) {
console.error("Lifecycle error:", err);
res.status(500).json({ success: false, error: "Failed to compute lifecycles" });
}
});
// GET /api/hype-cycle/regional/:tech — Regional adoption by technology
hypeCycleRouter.get("/regional/:tech", (req: Request, res: Response) => {
const techQuery = req.params.tech;
const currentYear = new Date().getFullYear();
const tech = findTechnology(techQuery);
if (!tech) {
res.status(404).json({
success: false,
error: `Technology "${techQuery}" not found. Available: 1G, 10G, 25G, 40G, 100G, 400G, 800G, 1.6T, CPO, LPO, 400ZR`,
});
return;
}
const regions = computeRegionalAdoption(tech.peakYear, currentYear, tech.name);
res.json({
success: true,
technology: tech.name,
speedGbps: tech.speedGbps,
globalPeakYear: tech.peakYear,
regions,
});
});
// GET /api/hype-cycle/:tech — Specific technology detail (must be last!)
hypeCycleRouter.get("/:tech", (req: Request, res: Response) => {
const techQuery = req.params.tech;
const yearParam = q("year", req);
const year = yearParam ? parseInt(yearParam) : new Date().getFullYear();
const tech = findTechnology(techQuery);
if (!tech) {
res.status(404).json({
success: false,
error: `Technology "${techQuery}" not found. Available: 1G, 10G, 25G, 40G, 100G, 400G, 800G, 1.6T, CPO, LPO, 400ZR`,
});
return;
}
const result = computeHypeCycle(tech, year);
// Add regional data
const regions = computeRegionalAdoption(tech.peakYear, year, tech.name);
// Add revenue lifecycle
const lifecycle = computeRevenueLifecycle(
tech.speedGbps,
tech.name,
tech.introYear,
tech.peakYear,
year,
);
res.json({
success: true,
...result,
regionalAdoption: regions,
revenueLifecycle: lifecycle,
});
});