transceiver-db/packages/scraper/src/utils/hype-cycle-engine.ts

533 lines
20 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

/**
* TIP Hype Cycle Engine — Norton-Bass Diffusion Model
*
* Fits a Bass diffusion model to historical optical transceiver shipment share data,
* maps the resulting adoption curve to a Gartner Hype Cycle phase, and stores the
* analysis in hype_cycle_analysis table.
*
* Bass model (single-generation):
* cumulative adopters: F(t) = [1 - e^(-(p+q)*t)] / [1 + (q/p)*e^(-(p+q)*t)]
* rate of adoption: f(t) = dF/dt = (p + q*F(t)) * (1 - F(t)) * M
*
* Parameters:
* p — coefficient of innovation (typical: 0.0010.03)
* q — coefficient of imitation (typical: 0.100.50)
* M — market potential as fraction of total port shipments (01)
*
* Gartner phase mapping via position on the f(t) curve relative to f(t_peak):
* < 5% of M cumulative → Innovation Trigger
* 530% rising phase → Peak of Inflated Expectations
* past peak, f(t) declining, F<60% → Trough of Disillusionment
* F 6085% → Slope of Enlightenment
* F > 85% → Plateau of Productivity
*
* Sources for seed data:
* Dell'Oro Group (public summaries), LightCounting quarterly (free tier),
* CIR Research, company earnings calls (public).
*/
import { pool } from "./db";
// ── Historical seed data ───────────────────────────────────────────────────────
// Approximate shipment share (% of total optical transceiver port volume) per year.
// Derived from publicly available Dell'Oro / LightCounting industry summaries.
// These are best estimates — the engine will fit curves to these points.
interface HistoricalPoint {
year: number;
shipmentShare: number; // 0.0 1.0 fraction of total port shipments
aspUsd?: number; // OEM ASP at that year (optional, for price trajectory)
}
const HISTORICAL_DATA: Record<string, HistoricalPoint[]> = {
"10G-SFP+": [
{ year: 2010, shipmentShare: 0.05, aspUsd: 800 },
{ year: 2011, shipmentShare: 0.12, aspUsd: 600 },
{ year: 2012, shipmentShare: 0.22, aspUsd: 450 },
{ year: 2013, shipmentShare: 0.32, aspUsd: 320 },
{ year: 2014, shipmentShare: 0.42, aspUsd: 240 },
{ year: 2015, shipmentShare: 0.52, aspUsd: 180 },
{ year: 2016, shipmentShare: 0.60, aspUsd: 130 },
{ year: 2017, shipmentShare: 0.62, aspUsd: 90 },
{ year: 2018, shipmentShare: 0.58, aspUsd: 60 },
{ year: 2019, shipmentShare: 0.50, aspUsd: 40 },
{ year: 2020, shipmentShare: 0.42, aspUsd: 28 },
{ year: 2021, shipmentShare: 0.36, aspUsd: 20 },
{ year: 2022, shipmentShare: 0.30, aspUsd: 14 },
{ year: 2023, shipmentShare: 0.24, aspUsd: 10 },
{ year: 2024, shipmentShare: 0.18, aspUsd: 7 },
{ year: 2025, shipmentShare: 0.13, aspUsd: 5 },
],
"100G-QSFP28": [
{ year: 2016, shipmentShare: 0.02, aspUsd: 3200 },
{ year: 2017, shipmentShare: 0.08, aspUsd: 2200 },
{ year: 2018, shipmentShare: 0.18, aspUsd: 1400 },
{ year: 2019, shipmentShare: 0.28, aspUsd: 850 },
{ year: 2020, shipmentShare: 0.36, aspUsd: 480 },
{ year: 2021, shipmentShare: 0.42, aspUsd: 280 },
{ year: 2022, shipmentShare: 0.44, aspUsd: 160 },
{ year: 2023, shipmentShare: 0.42, aspUsd: 95 },
{ year: 2024, shipmentShare: 0.38, aspUsd: 60 },
{ year: 2025, shipmentShare: 0.34, aspUsd: 40 },
],
"400G-QSFP-DD": [
{ year: 2020, shipmentShare: 0.01, aspUsd: 12000 },
{ year: 2021, shipmentShare: 0.04, aspUsd: 7500 },
{ year: 2022, shipmentShare: 0.09, aspUsd: 4200 },
{ year: 2023, shipmentShare: 0.18, aspUsd: 2200 },
{ year: 2024, shipmentShare: 0.28, aspUsd: 1100 },
{ year: 2025, shipmentShare: 0.36, aspUsd: 600 },
],
"800G-OSFP": [
{ year: 2023, shipmentShare: 0.005, aspUsd: 24000 },
{ year: 2024, shipmentShare: 0.025, aspUsd: 16000 },
{ year: 2025, shipmentShare: 0.07, aspUsd: 10000 },
],
"400G-ZR": [
{ year: 2021, shipmentShare: 0.005, aspUsd: 8500 },
{ year: 2022, shipmentShare: 0.015, aspUsd: 5500 },
{ year: 2023, shipmentShare: 0.030, aspUsd: 3800 },
{ year: 2024, shipmentShare: 0.050, aspUsd: 2800 },
{ year: 2025, shipmentShare: 0.075, aspUsd: 2100 },
],
"1.6T-OSFP": [
{ year: 2025, shipmentShare: 0.001, aspUsd: 40000 },
],
};
// Year the technology first entered commercial availability (t=0 for Bass model)
const FIRST_YEAR: Record<string, number> = {
"10G-SFP+": 2009,
"100G-QSFP28": 2015,
"400G-QSFP-DD": 2019,
"800G-OSFP": 2022,
"400G-ZR": 2020,
"1.6T-OSFP": 2024,
};
// Estimated peak market share (M) — as fraction of all optical transceiver ports at peak
// Technologies plateau at different shares because of market stratification
const MARKET_POTENTIAL: Record<string, number> = {
"10G-SFP+": 0.65, // Was dominant, now declining
"100G-QSFP28": 0.48, // Plateaued, being replaced by 400G
"400G-QSFP-DD": 0.42, // Growing, will peak ~2027
"800G-OSFP": 0.35, // Just starting, will peak ~2028-2029
"400G-ZR": 0.15, // Coherent niche, smaller market
"1.6T-OSFP": 0.30, // Future generation, estimate only
};
// ── Bass model math ────────────────────────────────────────────────────────────
/** Bass cumulative adoption fraction F(t) — 0..1 relative to M */
function bassCumulative(p: number, q: number, t: number): number {
if (t <= 0) return 0;
const pq = p + q;
const num = 1 - Math.exp(-pq * t);
const den = 1 + (q / p) * Math.exp(-pq * t);
return num / den;
}
/** Bass instantaneous adoption rate f(t) = dF/dt * M */
function bassRate(p: number, q: number, M: number, t: number): number {
const F = bassCumulative(p, q, t);
return (p + q * F) * (1 - F) * M;
}
/** Year at which Bass adoption rate peaks: t_peak = ln(q/p) / (p+q) */
function bassPeakTime(p: number, q: number): number {
if (q <= p) return 0; // no peak (monotonically declining from t=0, unusual)
return Math.log(q / p) / (p + q);
}
// ── Least-squares Bass fitting ────────────────────────────────────────────────
interface BassParams { p: number; q: number; M: number; rSquared: number }
/**
* Fit Bass parameters (p, q) to historical shipment share data via gradient descent.
* M is fixed (from MARKET_POTENTIAL) to reduce the search space.
*/
function fitBass(
data: HistoricalPoint[],
firstYear: number,
M: number
): BassParams {
if (data.length < 2) {
// Not enough data — return reasonable defaults
return { p: 0.01, q: 0.25, M, rSquared: 0 };
}
// Convert to t (years since first year) and normalised share (relative to M)
const points = data.map((d) => ({
t: d.year - firstYear,
F: Math.min(d.shipmentShare / M, 0.999), // cumulative fraction of M
}));
// Grid search over reasonable p/q ranges, then refine
let bestP = 0.01;
let bestQ = 0.25;
let bestSse = Infinity;
const pRange = [0.001, 0.003, 0.005, 0.008, 0.01, 0.015, 0.02, 0.03];
const qRange = [0.05, 0.10, 0.15, 0.20, 0.25, 0.30, 0.35, 0.40, 0.50];
for (const p of pRange) {
for (const q of qRange) {
let sse = 0;
for (const pt of points) {
const predicted = bassCumulative(p, q, pt.t);
sse += (predicted - pt.F) ** 2;
}
if (sse < bestSse) {
bestSse = sse;
bestP = p;
bestQ = q;
}
}
}
// Fine-grained refinement around best grid point
const step = 0.001;
for (let dp = -5; dp <= 5; dp++) {
for (let dq = -10; dq <= 10; dq++) {
const p = Math.max(0.001, bestP + dp * step);
const q = Math.max(0.01, bestQ + dq * step);
let sse = 0;
for (const pt of points) {
const predicted = bassCumulative(p, q, pt.t);
sse += (predicted - pt.F) ** 2;
}
if (sse < bestSse) {
bestSse = sse;
bestP = p;
bestQ = q;
}
}
}
// Compute R²
const meanF = points.reduce((s, pt) => s + pt.F, 0) / points.length;
const ssTot = points.reduce((s, pt) => s + (pt.F - meanF) ** 2, 0);
const rSquared = ssTot > 0 ? Math.max(0, 1 - bestSse / ssTot) : 0;
return { p: bestP, q: bestQ, M, rSquared };
}
// ── Gartner Hype Cycle phase detection ───────────────────────────────────────
type HypePhase =
| "innovation_trigger"
| "peak_inflated_expectations"
| "trough_disillusionment"
| "slope_enlightenment"
| "plateau_productivity";
interface PhaseResult {
phase: HypePhase;
hypeScore: number; // 0100 position on Gartner curve
yearsToNext: number; // estimated years until next phase
phaseSinceYear: number; // approximate calendar year current phase started
}
/**
* Map Bass model position to Gartner Hype Cycle phase.
* Uses cumulative adoption F(t) and whether we're before/after the peak.
*/
function detectPhase(
p: number,
q: number,
M: number,
currentT: number,
firstYear: number
): PhaseResult {
const tPeak = bassPeakTime(p, q);
const F = bassCumulative(p, q, currentT);
const currentShare = F * M;
const peakF = bassCumulative(p, q, tPeak);
// Normalised position: 0 = start, 1 = peak rate (not peak cumulative)
const prePeakNorm = tPeak > 0 ? Math.min(currentT / tPeak, 1) : 1;
const isPrePeak = currentT < tPeak;
let phase: HypePhase;
let hypeScore: number;
let yearsToNext: number;
let phaseSinceYear: number;
if (currentShare < 0.03 * M) {
// < 3% of market potential — innovation trigger
phase = "innovation_trigger";
hypeScore = Math.round(prePeakNorm * 25);
// Estimate time to reach 5% share
let tTo5pct = currentT + 1;
while (bassCumulative(p, q, tTo5pct) * M < 0.05 * M && tTo5pct < 30) tTo5pct++;
yearsToNext = Math.max(0.5, tTo5pct - currentT);
phaseSinceYear = firstYear;
} else if (isPrePeak || F < peakF * 0.85) {
// Rising phase — Peak of Inflated Expectations
phase = "peak_inflated_expectations";
// Score 2575 based on how close we are to the rate peak
hypeScore = Math.round(25 + prePeakNorm * 50);
yearsToNext = Math.max(0.5, tPeak - currentT + 1);
// Phase started when share crossed 5%
let tStart = 0;
while (bassCumulative(p, q, tStart) * M < 0.03 * M && tStart < tPeak) tStart += 0.5;
phaseSinceYear = Math.round(firstYear + tStart);
} else if (F < 0.50) {
// Post-peak, still < 50% adoption — Trough of Disillusionment
phase = "trough_disillusionment";
// Score 75 → 45 (declining from peak)
const postPeakProgress = (F - peakF * 0.85) / (0.50 - peakF * 0.85);
hypeScore = Math.round(75 - postPeakProgress * 30);
// Time to reach 50% share
let tTo50 = currentT + 0.5;
while (bassCumulative(p, q, tTo50) < 0.50 && tTo50 < 30) tTo50 += 0.5;
yearsToNext = Math.max(0.5, tTo50 - currentT);
phaseSinceYear = Math.round(firstYear + tPeak);
} else if (F < 0.85) {
// 5085% cumulative — Slope of Enlightenment
phase = "slope_enlightenment";
const enlightProgress = (F - 0.50) / (0.85 - 0.50);
hypeScore = Math.round(45 + enlightProgress * 30);
// Time to reach 85% share
let tTo85 = currentT + 0.5;
while (bassCumulative(p, q, tTo85) < 0.85 && tTo85 < 30) tTo85 += 0.5;
yearsToNext = Math.max(0.5, tTo85 - currentT);
let tStart50 = currentT;
while (tStart50 > 0 && bassCumulative(p, q, tStart50 - 0.5) >= 0.50) tStart50 -= 0.5;
phaseSinceYear = Math.round(firstYear + tStart50);
} else {
// > 85% adoption — Plateau of Productivity
phase = "plateau_productivity";
hypeScore = Math.min(100, Math.round(75 + (F - 0.85) / (1 - 0.85) * 25));
yearsToNext = 0; // no next phase
let tStart85 = currentT;
while (tStart85 > 0 && bassCumulative(p, q, tStart85 - 0.5) < 0.85) tStart85 += 0.5;
phaseSinceYear = Math.round(firstYear + tStart85);
}
return { phase, hypeScore, yearsToNext, phaseSinceYear };
}
// ── ASP projection ────────────────────────────────────────────────────────────
/**
* Project ASP decline based on historical data (log-linear regression).
* Returns: current ASP (estimated) + % decline expected over 3 years.
*/
function projectAsp(data: HistoricalPoint[]): { currentAsp: number | null; declinePct3y: number | null } {
const withAsp = data.filter((d) => d.aspUsd != null && d.aspUsd > 0);
if (withAsp.length < 2) return { currentAsp: withAsp[0]?.aspUsd ?? null, declinePct3y: null };
// Log-linear fit: ln(asp) = a + b * year
const n = withAsp.length;
const sumX = withAsp.reduce((s, d) => s + d.year, 0);
const sumY = withAsp.reduce((s, d) => s + Math.log(d.aspUsd!), 0);
const sumXY = withAsp.reduce((s, d) => s + d.year * Math.log(d.aspUsd!), 0);
const sumX2 = withAsp.reduce((s, d) => s + d.year * d.year, 0);
const b = (n * sumXY - sumX * sumY) / (n * sumX2 - sumX * sumX);
const a = (sumY - b * sumX) / n;
const currentYear = new Date().getFullYear();
const currentAsp = Math.exp(a + b * currentYear);
const futureAsp = Math.exp(a + b * (currentYear + 3));
const declinePct3y = ((currentAsp - futureAsp) / currentAsp) * 100;
return {
currentAsp: Math.round(currentAsp),
declinePct3y: Math.round(declinePct3y),
};
}
// ── Seed market_metrics with historical data ──────────────────────────────────
async function seedMarketMetrics(): Promise<void> {
const currentYear = new Date().getFullYear();
for (const [tech, points] of Object.entries(HISTORICAL_DATA)) {
for (const pt of points) {
if (pt.year > currentYear) continue; // Don't seed future projections
// shipment_share
await pool.query(
`INSERT INTO market_metrics (time, technology, metric_type, value, source, notes)
VALUES ($1, $2, 'shipment_share', $3, $4, $5)
ON CONFLICT DO NOTHING`,
[
new Date(pt.year, 6, 1).toISOString(), // mid-year timestamp
tech,
pt.shipmentShare,
"Dell\'Oro/LightCounting (public summary)",
`Annual shipment share ${pt.year}`,
]
);
// asp_usd
if (pt.aspUsd) {
await pool.query(
`INSERT INTO market_metrics (time, technology, metric_type, value, source, notes)
VALUES ($1, $2, 'asp_usd', $3, $4, $5)
ON CONFLICT DO NOTHING`,
[
new Date(pt.year, 6, 1).toISOString(),
tech,
pt.aspUsd,
"Dell\'Oro/LightCounting (public summary)",
`OEM ASP estimate ${pt.year} USD`,
]
);
}
}
}
}
// ── Main compute function ─────────────────────────────────────────────────────
export async function computeHypeCycle(): Promise<void> {
console.log("=== Hype Cycle Engine — Norton-Bass Model ===\n");
// Seed historical data if not present
const { rows: existing } = await pool.query(
`SELECT COUNT(*) AS n FROM market_metrics WHERE metric_type = 'shipment_share'`
);
if (parseInt(existing[0].n, 10) < 10) {
console.log(" Seeding historical market_metrics data...");
await seedMarketMetrics();
console.log(" Seed complete.\n");
}
const currentYear = new Date().getFullYear();
const results: Array<{
technology: string;
phase: HypePhase;
hypeScore: number;
currentShare: number;
aspCurrent: number | null;
aspDecline3y: number | null;
}> = [];
for (const [tech, data] of Object.entries(HISTORICAL_DATA)) {
const firstYear = FIRST_YEAR[tech] ?? (data[0]?.year ?? 2020);
const M = MARKET_POTENTIAL[tech] ?? 0.30;
const currentT = currentYear - firstYear;
// Fit Bass model
const { p, q, rSquared } = fitBass(data, firstYear, M);
const tPeak = bassPeakTime(p, q);
const F = bassCumulative(p, q, currentT);
const currentShare = F * M;
// Detect phase
const { phase, hypeScore, yearsToNext, phaseSinceYear } = detectPhase(
p, q, M, currentT, firstYear
);
// Project 1y and 3y adoption
const F1y = bassCumulative(p, q, currentT + 1);
const F3y = bassCumulative(p, q, currentT + 3);
// ASP projection
const { currentAsp, declinePct3y } = projectAsp(data);
// Store result in hype_cycle_analysis
await pool.query(
`INSERT INTO hype_cycle_analysis (
computed_at, technology,
bass_p, bass_q, bass_m,
t_peak_year, current_t, current_share,
projected_share_1y, projected_share_3y,
hype_phase, hype_score, phase_since_year, years_to_next_phase,
asp_current_usd, asp_decline_pct_3y,
r_squared, data_points, notes
) VALUES (
NOW(), $1,
$2, $3, $4,
$5, $6, $7,
$8, $9,
$10, $11, $12, $13,
$14, $15,
$16, $17, $18
)`,
[
tech,
p, q, M,
Math.round((firstYear + tPeak) * 10) / 10,
currentT,
Math.round(currentShare * 1000) / 1000,
Math.round(F1y * M * 1000) / 1000,
Math.round(F3y * M * 1000) / 1000,
phase,
hypeScore,
phaseSinceYear,
Math.round(yearsToNext * 10) / 10,
currentAsp,
declinePct3y,
Math.round(rSquared * 1000) / 1000,
data.length,
`Fitted via grid search + refinement. Data points: ${data[0]?.year}${data[data.length - 1]?.year}`,
]
);
results.push({ technology: tech, phase, hypeScore, currentShare, aspCurrent: currentAsp, aspDecline3y: declinePct3y });
const phaseEmoji: Record<HypePhase, string> = {
innovation_trigger: "🔬",
peak_inflated_expectations: "📈",
trough_disillusionment: "📉",
slope_enlightenment: "🔄",
plateau_productivity: "✅",
};
console.log(
` ${phaseEmoji[phase]} ${tech.padEnd(18)} ` +
`phase=${phase.padEnd(30)} ` +
`score=${hypeScore.toString().padStart(3)} ` +
`share=${(currentShare * 100).toFixed(1).padStart(5)}% ` +
`R²=${rSquared.toFixed(2)} ` +
`ASP=${currentAsp ? `$${currentAsp.toLocaleString()}` : "n/a"}`
);
}
// Update hype_score in market_metrics for API consumers
for (const r of results) {
await pool.query(
`INSERT INTO market_metrics (time, technology, metric_type, value, source, notes)
VALUES (NOW(), $1, 'hype_score', $2, 'hype-cycle-engine', $3)`,
[r.technology, r.hypeScore, r.phase]
);
}
// Cleanup old analysis rows
await pool.query(`SELECT cleanup_hype_cycle_analysis()`);
console.log(`\n=== Hype Cycle Engine Complete — ${results.length} technologies analyzed ===\n`);
}
// ── Export for API endpoint ───────────────────────────────────────────────────
/** Fetch latest hype cycle analysis for all or a specific technology */
export async function getHypeCycleAnalysis(technology?: string): Promise<unknown[]> {
const sql = technology
? `SELECT * FROM hype_cycle_analysis WHERE technology = $1 ORDER BY computed_at DESC LIMIT 1`
: `SELECT DISTINCT ON (technology) * FROM hype_cycle_analysis ORDER BY technology, computed_at DESC`;
const { rows } = await pool.query(sql, technology ? [technology] : []);
return rows;
}
// ── CLI entry point ───────────────────────────────────────────────────────────
if (require.main === module) {
computeHypeCycle()
.then(() => pool.end())
.catch((err: unknown) => {
console.error("Fatal:", err);
pool.end();
process.exit(1);
});
}