feat: Price Comparison dashboard + Eoptolink OEM scraper
- Add public /api/price-comparison API (summary, top-50, per-SKU detail)
— no auth required, 3 Express routes, DISTINCT ON latest-price logic
- Add '💲 Price Comparison' dashboard tab: stat cards, form-factor
breakdown, top-50 SKU table (clickable rows → SKU detail), per-vendor
price + stock + spread% lookup panel
- Add Eoptolink OEM catalog scraper (93 product-solution pages,
part-number regex EOLO-*/EOLQ-* etc., no prices, seeds transceivers
table as manufacturer entries)
- Register scrape:catalog:eoptolink in scheduler: schedule every 4h
(40 */4 * * *), lazy-import worker, added to known-jobs array
This commit is contained in:
parent
5e4a950aa1
commit
14d48d34b2
@ -3,6 +3,9 @@
|
||||
Format: `{"d":"YYYY-MM-DD","t":"TYPE","m":"Description"}`
|
||||
Types: FEAT · FIX · UI · DATA · AI · INFRA
|
||||
|
||||
{"d":"2026-04-18","t":"FEAT","m":"Price Comparison Dashboard: public /api/price-comparison (summary, list top-50 SKUs by vendor coverage, per-SKU detail). Express Router, no auth required. New '💲 Price Comparison' dashboard tab with stat cards, form-factor breakdown table, top-50 SKU table (clickable rows), and SKU detail lookup with per-vendor prices + stock + spread %."}
|
||||
{"d":"2026-04-18","t":"DATA","m":"Eoptolink OEM catalog scraper: harvests 93 product-solution pages from eoptolink.com, extracts part numbers (EOLO-*/EOLQ-* format), seeds transceivers table as manufacturer=Eoptolink entries with form_factor/speed/fiber/category. No prices (B2B OEM). Scheduled every 4h (40 */4 * * *)."}
|
||||
{"d":"2026-04-18","t":"FIX","m":"stock_observations repopulated after TRUNCATE: storage-fs/request_queues/default/ directory re-created on Erik; NADDOD scraper manual-triggered; 4+ prices confirmed written within 20s."}
|
||||
{"d":"2026-04-17","t":"FEAT","m":"MCP Server v0.2.0: wired finder.ts (find_flexoptix_for_switch, get_competitor_alerts), switch-docs (get_switch_docs, get_switch_image), analyze_market_with_llm (qwen2.5:14b via Ollama, enriched with live hype cycle + pricing + news), generate_blog_post (fo-blog-v5 fine-tuned model with qwen2.5:14b fallback + live pricing enrichment). OLLAMA_BASE_URL env var for Ollama endpoint."}
|
||||
{"d":"2026-04-17","t":"UI","m":"Stock dashboard: 6th stat card (Multi-Vendor SKUs), confidence quality badge column in vendor breakdown (🟢 L3 per-warehouse / 🟡 L2 aggregated / ⚪ L1 boolean), new Multi-Vendor Price Comparison table with min/max/avg per SKU. Subtitle updated to mention QSFPTEK + NADDOD."}
|
||||
{"d":"2026-04-17","t":"FEAT","m":"/api/stock/summary enhanced: vendor_breakdown adds avg_confidence + currencies + confidence breakdown (conf_per_warehouse/aggregated/boolean); new price_comparison endpoint (top 50 SKUs tracked by 2+ vendors with price spread); totals adds multi_vendor_skus count."}
|
||||
|
||||
@ -30,6 +30,7 @@ import { newsRouter } from "./routes/news";
|
||||
import { proxyRouter } from "./routes/proxy";
|
||||
import { reviewRouter } from "./routes/review";
|
||||
import { stockRouter } from "./routes/stock";
|
||||
import { priceComparisonRouter } from "./routes/price-comparison";
|
||||
|
||||
const app = express();
|
||||
|
||||
@ -61,6 +62,7 @@ app.use("/api", (req, res, next) => {
|
||||
if (req.path.startsWith("/health") || req.path.startsWith("/auth")) return next();
|
||||
if (req.path.startsWith("/proxy")) return next();
|
||||
if (req.path.startsWith("/hot-topics")) return next();
|
||||
if (req.path.startsWith("/price-comparison")) return next();
|
||||
requireAuth(req, res, next);
|
||||
});
|
||||
|
||||
@ -88,6 +90,7 @@ app.use("/api/changelog", changelogRouter);
|
||||
app.use("/api/news", newsRouter);
|
||||
app.use("/api/review", reviewRouter);
|
||||
app.use("/api/stock", stockRouter);
|
||||
app.use("/api/price-comparison", priceComparisonRouter);
|
||||
|
||||
// Dashboard (static HTML)
|
||||
app.use("/dashboard", express.static(join(__dirname, "..", "..", "dashboard")));
|
||||
|
||||
262
packages/api/src/routes/price-comparison.ts
Normal file
262
packages/api/src/routes/price-comparison.ts
Normal file
@ -0,0 +1,262 @@
|
||||
/**
|
||||
* Price Comparison Dashboard — Public API
|
||||
*
|
||||
* Public-facing endpoints powering the "Octopart for optical transceivers"
|
||||
* price comparison page. No authentication required.
|
||||
*
|
||||
* Routes:
|
||||
* GET /api/price-comparison — Top 50 SKUs by vendor coverage
|
||||
* GET /api/price-comparison/summary — Market-level aggregate stats
|
||||
* GET /api/price-comparison/:sku — Per-SKU price breakdown
|
||||
*/
|
||||
import { Router, Request, Response } from "express";
|
||||
import { pool } from "../db/client";
|
||||
|
||||
export const priceComparisonRouter = Router();
|
||||
|
||||
// ─── GET /api/price-comparison/summary ───────────────────────────────────────
|
||||
// MUST be registered before /:sku to avoid route conflict
|
||||
/**
|
||||
* Market summary:
|
||||
* - Total unique SKUs tracked
|
||||
* - Total price observations
|
||||
* - Number of active vendors
|
||||
* - Average prices broken down by form_factor
|
||||
*/
|
||||
priceComparisonRouter.get("/summary", async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const [overview, byFormFactor] = await Promise.all([
|
||||
// Overall market counts
|
||||
pool.query(`
|
||||
WITH latest AS (
|
||||
SELECT DISTINCT ON (po.transceiver_id, po.source_vendor_id)
|
||||
po.transceiver_id,
|
||||
po.source_vendor_id,
|
||||
po.price,
|
||||
po.currency
|
||||
FROM price_observations po
|
||||
ORDER BY po.transceiver_id, po.source_vendor_id, po.time DESC
|
||||
)
|
||||
SELECT
|
||||
COUNT(DISTINCT l.transceiver_id) AS total_skus_tracked,
|
||||
(SELECT COUNT(*) FROM price_observations)::bigint AS total_observations,
|
||||
COUNT(DISTINCT l.source_vendor_id) AS active_vendor_count,
|
||||
ROUND(AVG(l.price)::numeric, 2) AS overall_avg_price
|
||||
FROM latest l
|
||||
`),
|
||||
|
||||
// Avg price per form_factor (using latest price per vendor per transceiver)
|
||||
pool.query(`
|
||||
WITH latest AS (
|
||||
SELECT DISTINCT ON (po.transceiver_id, po.source_vendor_id)
|
||||
po.transceiver_id,
|
||||
po.source_vendor_id,
|
||||
po.price,
|
||||
po.currency
|
||||
FROM price_observations po
|
||||
ORDER BY po.transceiver_id, po.source_vendor_id, po.time DESC
|
||||
)
|
||||
SELECT
|
||||
t.form_factor,
|
||||
COUNT(DISTINCT l.transceiver_id) AS sku_count,
|
||||
COUNT(DISTINCT l.source_vendor_id) AS vendor_count,
|
||||
ROUND(MIN(l.price)::numeric, 2) AS min_price,
|
||||
ROUND(MAX(l.price)::numeric, 2) AS max_price,
|
||||
ROUND(AVG(l.price)::numeric, 2) AS avg_price,
|
||||
-- Most common currency for this form factor
|
||||
(
|
||||
SELECT currency
|
||||
FROM price_observations po2
|
||||
JOIN transceivers t2 ON t2.id = po2.transceiver_id
|
||||
WHERE t2.form_factor = t.form_factor
|
||||
GROUP BY currency
|
||||
ORDER BY COUNT(*) DESC
|
||||
LIMIT 1
|
||||
) AS currency
|
||||
FROM latest l
|
||||
JOIN transceivers t ON t.id = l.transceiver_id
|
||||
GROUP BY t.form_factor
|
||||
ORDER BY sku_count DESC
|
||||
`),
|
||||
]);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
...overview.rows[0],
|
||||
by_form_factor: byFormFactor.rows,
|
||||
},
|
||||
});
|
||||
} catch (err) {
|
||||
console.error("GET /api/price-comparison/summary error:", err);
|
||||
res.status(500).json({ success: false, error: "Internal server error" });
|
||||
}
|
||||
});
|
||||
|
||||
// ─── GET /api/price-comparison ───────────────────────────────────────────────
|
||||
/**
|
||||
* Top 50 transceivers ranked by number of vendors tracking them.
|
||||
* Shows price spread across vendors — the more vendors, the better the comparison.
|
||||
*/
|
||||
priceComparisonRouter.get("/", async (_req: Request, res: Response) => {
|
||||
try {
|
||||
const result = await pool.query(`
|
||||
WITH latest AS (
|
||||
SELECT DISTINCT ON (po.transceiver_id, po.source_vendor_id)
|
||||
po.transceiver_id,
|
||||
po.source_vendor_id,
|
||||
po.price,
|
||||
po.currency
|
||||
FROM price_observations po
|
||||
ORDER BY po.transceiver_id, po.source_vendor_id, po.time DESC
|
||||
)
|
||||
SELECT
|
||||
t.form_factor,
|
||||
t.speed,
|
||||
t.standard_name,
|
||||
COUNT(DISTINCT l.source_vendor_id) AS vendor_count,
|
||||
ROUND(MIN(l.price)::numeric, 2) AS min_price,
|
||||
ROUND(MAX(l.price)::numeric, 2) AS max_price,
|
||||
ROUND(AVG(l.price)::numeric, 2) AS avg_price,
|
||||
-- Dominant currency (most common for this SKU)
|
||||
(
|
||||
SELECT currency
|
||||
FROM latest l2
|
||||
WHERE l2.transceiver_id = t.id
|
||||
GROUP BY currency
|
||||
ORDER BY COUNT(*) DESC
|
||||
LIMIT 1
|
||||
) AS currency,
|
||||
ROUND(
|
||||
((MAX(l.price) - MIN(l.price)) / NULLIF(MIN(l.price), 0) * 100)::numeric,
|
||||
1
|
||||
) AS spread_pct
|
||||
FROM latest l
|
||||
JOIN transceivers t ON t.id = l.transceiver_id
|
||||
GROUP BY t.id, t.form_factor, t.speed, t.standard_name
|
||||
ORDER BY vendor_count DESC, avg_price ASC
|
||||
LIMIT 50
|
||||
`);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: result.rows,
|
||||
});
|
||||
} catch (err) {
|
||||
console.error("GET /api/price-comparison error:", err);
|
||||
res.status(500).json({ success: false, error: "Internal server error" });
|
||||
}
|
||||
});
|
||||
|
||||
// ─── GET /api/price-comparison/:sku ─────────────────────────────────────────
|
||||
/**
|
||||
* Full price breakdown for a single SKU.
|
||||
* :sku matches against standard_name OR part_number (case-insensitive ILIKE).
|
||||
* Returns per-vendor prices, stock status, and aggregate stats.
|
||||
*/
|
||||
priceComparisonRouter.get("/:sku", async (req: Request, res: Response) => {
|
||||
try {
|
||||
const sku = String(req.params.sku).trim();
|
||||
|
||||
// Resolve transceiver
|
||||
const transceiverResult = await pool.query(
|
||||
`SELECT id, standard_name, form_factor, speed, reach_label, fiber_type, part_number
|
||||
FROM transceivers
|
||||
WHERE standard_name ILIKE $1 OR part_number ILIKE $1
|
||||
LIMIT 1`,
|
||||
[sku]
|
||||
);
|
||||
|
||||
if (transceiverResult.rows.length === 0) {
|
||||
res.status(404).json({ success: false, error: "Transceiver not found" });
|
||||
return;
|
||||
}
|
||||
|
||||
const transceiver = transceiverResult.rows[0];
|
||||
|
||||
// Latest price per vendor + stock level from stock_observations (if available)
|
||||
const pricesResult = await pool.query(`
|
||||
SELECT
|
||||
v.name AS vendor,
|
||||
po.price,
|
||||
po.currency,
|
||||
po.stock_level,
|
||||
-- Prefer stock_observations for latest stock info
|
||||
COALESCE(
|
||||
(
|
||||
SELECT so.stock_level
|
||||
FROM stock_observations so
|
||||
WHERE so.transceiver_id = po.transceiver_id
|
||||
AND so.source_vendor_id = po.source_vendor_id
|
||||
ORDER BY so.time DESC
|
||||
LIMIT 1
|
||||
),
|
||||
po.stock_level
|
||||
) AS stock_level,
|
||||
-- Build product URL: use vendor search_url_template if no direct url
|
||||
COALESCE(
|
||||
v.search_url_template,
|
||||
v.website_url
|
||||
) AS url,
|
||||
po.time AS observed_at
|
||||
FROM (
|
||||
SELECT DISTINCT ON (po.transceiver_id, po.source_vendor_id)
|
||||
po.transceiver_id,
|
||||
po.source_vendor_id,
|
||||
po.price,
|
||||
po.currency,
|
||||
po.stock_level,
|
||||
po.time
|
||||
FROM price_observations po
|
||||
WHERE po.transceiver_id = $1
|
||||
ORDER BY po.transceiver_id, po.source_vendor_id, po.time DESC
|
||||
) po
|
||||
JOIN vendors v ON v.id = po.source_vendor_id
|
||||
ORDER BY po.price ASC
|
||||
`, [transceiver.id]);
|
||||
|
||||
const prices = pricesResult.rows;
|
||||
|
||||
// Compute aggregate stats
|
||||
const priceValues = prices.map((r) => parseFloat(r.price)).filter((v) => Number.isFinite(v));
|
||||
|
||||
let stats: Record<string, number | null> = {
|
||||
vendor_count: prices.length,
|
||||
min: null,
|
||||
max: null,
|
||||
avg: null,
|
||||
spread_pct: null,
|
||||
};
|
||||
|
||||
if (priceValues.length > 0) {
|
||||
const min = Math.min(...priceValues);
|
||||
const max = Math.max(...priceValues);
|
||||
const avg = priceValues.reduce((a, b) => a + b, 0) / priceValues.length;
|
||||
const spreadPct = min > 0 ? Math.round(((max - min) / min) * 1000) / 10 : null;
|
||||
|
||||
stats = {
|
||||
vendor_count: prices.length,
|
||||
min: Math.round(min * 100) / 100,
|
||||
max: Math.round(max * 100) / 100,
|
||||
avg: Math.round(avg * 100) / 100,
|
||||
spread_pct: spreadPct,
|
||||
};
|
||||
}
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
transceiver: {
|
||||
standard_name: transceiver.standard_name,
|
||||
form_factor: transceiver.form_factor,
|
||||
speed: transceiver.speed,
|
||||
reach_label: transceiver.reach_label,
|
||||
fiber_type: transceiver.fiber_type,
|
||||
},
|
||||
prices,
|
||||
stats,
|
||||
});
|
||||
} catch (err) {
|
||||
console.error("GET /api/price-comparison/:sku error:", err);
|
||||
res.status(500).json({ success: false, error: "Internal server error" });
|
||||
}
|
||||
});
|
||||
@ -803,6 +803,7 @@
|
||||
<div class="tab" data-tab="network">🌐 Network</div>
|
||||
<div class="tab" data-tab="review" id="tab-review-nav">✎ Review <span id="review-pending-badge" style="display:none;background:#f97316;color:#fff;border-radius:10px;padding:1px 7px;font-size:0.68rem;margin-left:4px;font-weight:700"></span></div>
|
||||
<div class="tab" data-tab="stock">🏭 Stock</div>
|
||||
<div class="tab" data-tab="prices">💲 Price Comparison</div>
|
||||
</div>
|
||||
|
||||
<div class="main">
|
||||
@ -902,6 +903,7 @@
|
||||
<div style="font-size:0.72rem;color:var(--text-dim);margin-top:2px">Norton-Bass Multigenerational Diffusion Model — click any technology for details</div>
|
||||
</div>
|
||||
<div style="display:flex;gap:1rem;align-items:center;font-size:0.68rem;color:var(--text-dim);flex-wrap:wrap">
|
||||
<span id="hype-data-source" style="font-size:0.68rem;color:#34d399;font-weight:600"></span>
|
||||
<span><span style="display:inline-block;width:8px;height:8px;border-radius:50%;background:#FF8100;margin-right:4px;vertical-align:middle"></span>Innovation</span>
|
||||
<span><span style="display:inline-block;width:8px;height:8px;border-radius:50%;background:#FFa030;margin-right:4px;vertical-align:middle"></span>Peak</span>
|
||||
<span><span style="display:inline-block;width:8px;height:8px;border-radius:50%;background:#c1121f;margin-right:4px;vertical-align:middle"></span>Trough</span>
|
||||
@ -921,6 +923,8 @@
|
||||
<th class="tip" data-tip="Cumulative market adoption based on Norton-Bass diffusion model. 0-100% of total addressable market.">Adoption<span class="sort-arrow"></span></th>
|
||||
<th class="tip" data-tip="Estimated year of peak hype / maximum attention.">Peak<span class="sort-arrow"></span></th>
|
||||
<th class="tip" data-tip="Years until mainstream, stable deployment.">To Plateau<span class="sort-arrow"></span></th>
|
||||
<th class="tip" data-tip="Current OEM ASP in USD — from Mouser/market data.">OEM ASP<span class="sort-arrow"></span></th>
|
||||
<th class="tip" data-tip="Bass model goodness-of-fit (R²). Higher = more reliable forecast.">R²<span class="sort-arrow"></span></th>
|
||||
</tr></thead>
|
||||
<tbody id="hype-table"></tbody>
|
||||
</table>
|
||||
@ -1844,6 +1848,100 @@
|
||||
</div>
|
||||
</div><!-- end tab-stock -->
|
||||
|
||||
<!-- PRICE COMPARISON -->
|
||||
<div id="tab-prices" class="hidden fade-in">
|
||||
<div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:1rem;flex-wrap:wrap;gap:0.5rem">
|
||||
<div>
|
||||
<h2 style="margin:0;font-size:1.1rem;color:var(--text-bright)">💲 Price Comparison — Optical Transceiver Market</h2>
|
||||
<p style="margin:0.2rem 0 0;font-size:0.75rem;color:var(--text-dim)">Live pricing across 20+ vendors · Updated every 2–8h · No authentication required</p>
|
||||
</div>
|
||||
<button onclick="pricesLoaded=false;loadPriceComparison()" class="btn" style="background:var(--indigo);color:#fff;font-size:0.75rem">↻ Refresh</button>
|
||||
</div>
|
||||
|
||||
<!-- Summary stat cards -->
|
||||
<div class="grid mb" style="grid-template-columns:repeat(4,1fr);gap:0.75rem;margin-bottom:1.25rem">
|
||||
<div class="stat-card" style="text-align:center">
|
||||
<div class="stat-icon blue">📊</div>
|
||||
<div class="stat-label">SKUs Tracked</div>
|
||||
<div class="stat-val" id="pc-stat-skus">—</div>
|
||||
</div>
|
||||
<div class="stat-card" style="text-align:center">
|
||||
<div class="stat-icon" style="color:#6366f1">🏪</div>
|
||||
<div class="stat-label">Active Vendors</div>
|
||||
<div class="stat-val" id="pc-stat-vendors">—</div>
|
||||
</div>
|
||||
<div class="stat-card" style="text-align:center">
|
||||
<div class="stat-icon" style="color:#22c55e">📋</div>
|
||||
<div class="stat-label">Price Observations</div>
|
||||
<div class="stat-val" id="pc-stat-obs">—</div>
|
||||
</div>
|
||||
<div class="stat-card" style="text-align:center">
|
||||
<div class="stat-icon" style="color:#f59e0b">💵</div>
|
||||
<div class="stat-label">Overall Avg Price</div>
|
||||
<div class="stat-val" id="pc-stat-avg">—</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div style="display:grid;grid-template-columns:1fr 1.6fr;gap:1rem;margin-bottom:1.5rem">
|
||||
<!-- By Form Factor -->
|
||||
<div class="card" style="overflow:hidden">
|
||||
<div style="padding:0.75rem 1rem;border-bottom:1px solid var(--border);font-size:0.85rem;font-weight:600;color:var(--text-bright)">By Form Factor</div>
|
||||
<div style="overflow-x:auto">
|
||||
<table style="width:100%;border-collapse:collapse;font-size:0.75rem">
|
||||
<thead>
|
||||
<tr style="background:var(--surface2)">
|
||||
<th style="padding:6px 8px;text-align:left;color:var(--text-dim);font-weight:500">Form Factor</th>
|
||||
<th style="padding:6px 8px;text-align:right;color:var(--text-dim);font-weight:500">SKUs</th>
|
||||
<th style="padding:6px 8px;text-align:right;color:var(--text-dim);font-weight:500">Vendors</th>
|
||||
<th style="padding:6px 8px;text-align:right;color:var(--text-dim);font-weight:500">Min</th>
|
||||
<th style="padding:6px 8px;text-align:right;color:var(--text-dim);font-weight:500">Avg</th>
|
||||
<th style="padding:6px 8px;text-align:right;color:var(--text-dim);font-weight:500">Max</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="pc-ff-body">
|
||||
<tr><td colspan="6" style="text-align:center;padding:2rem;color:var(--text-dim)">Loading…</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Top 50 SKUs -->
|
||||
<div class="card" style="overflow:hidden">
|
||||
<div style="padding:0.75rem 1rem;border-bottom:1px solid var(--border);font-size:0.85rem;font-weight:600;color:var(--text-bright)">Top 50 SKUs by Vendor Coverage</div>
|
||||
<div style="overflow-x:auto;max-height:340px">
|
||||
<table style="width:100%;border-collapse:collapse;font-size:0.75rem">
|
||||
<thead style="position:sticky;top:0;z-index:1;background:var(--surface2)">
|
||||
<tr>
|
||||
<th style="padding:6px 8px;text-align:left;color:var(--text-dim);font-weight:500">SKU / Standard Name</th>
|
||||
<th style="padding:6px 8px;text-align:center;color:var(--text-dim);font-weight:500">FF</th>
|
||||
<th style="padding:6px 8px;text-align:center;color:var(--text-dim);font-weight:500">Speed</th>
|
||||
<th style="padding:6px 8px;text-align:right;color:var(--text-dim);font-weight:500">Vendors</th>
|
||||
<th style="padding:6px 8px;text-align:right;color:var(--text-dim);font-weight:500">Min</th>
|
||||
<th style="padding:6px 8px;text-align:right;color:var(--text-dim);font-weight:500">Avg</th>
|
||||
<th style="padding:6px 8px;text-align:right;color:var(--text-dim);font-weight:500">Spread</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="pc-top-body">
|
||||
<tr><td colspan="7" style="text-align:center;padding:2rem;color:var(--text-dim)">Loading…</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- SKU Detail Lookup -->
|
||||
<div class="card" style="padding:1rem">
|
||||
<div style="font-size:0.85rem;font-weight:600;color:var(--text-bright);margin-bottom:0.75rem">🔍 SKU Price Lookup</div>
|
||||
<div style="display:flex;gap:0.5rem;align-items:center;margin-bottom:0.75rem">
|
||||
<input id="pc-lookup-input" type="text" placeholder="e.g. SFP-10G-SR, QSFP-40G-LR4 …"
|
||||
style="flex:1;padding:6px 10px;border:1px solid var(--border);border-radius:6px;background:var(--surface2);color:var(--text);font-size:0.8rem"
|
||||
onkeydown="if(event.key==='Enter')lookupPriceComparison()">
|
||||
<button onclick="lookupPriceComparison()" class="btn" style="background:var(--indigo);color:#fff;font-size:0.8rem">Look Up</button>
|
||||
</div>
|
||||
<div id="pc-lookup-result" style="font-size:0.78rem;color:var(--text-dim)"></div>
|
||||
</div>
|
||||
</div><!-- end tab-prices -->
|
||||
|
||||
</div>
|
||||
|
||||
</div><!-- .app -->
|
||||
@ -2223,6 +2321,7 @@ function goToTab(tabName) {
|
||||
if (tabName === 'network') loadProxyNetwork();
|
||||
if (tabName === 'review') loadReview();
|
||||
if (tabName === 'stock') loadStock();
|
||||
if (tabName === 'prices') loadPriceComparison();
|
||||
}
|
||||
|
||||
document.querySelectorAll('.tab').forEach(function(tab) {
|
||||
@ -2775,16 +2874,63 @@ async function loadRegionalData(techs) {
|
||||
buildDOM(body, html);
|
||||
}
|
||||
|
||||
// Map DB snake_case phase → UI display label
|
||||
var DB_PHASE_LABEL = {
|
||||
'innovation_trigger': 'Innovation Trigger',
|
||||
'peak_inflated_expectations': 'Peak of Inflated Expectations',
|
||||
'trough_disillusionment': 'Trough of Disillusionment',
|
||||
'slope_enlightenment': 'Slope of Enlightenment',
|
||||
'plateau_productivity': 'Plateau of Productivity'
|
||||
};
|
||||
|
||||
async function loadHypeCycle() {
|
||||
// Use enriched endpoint for forecast data
|
||||
var data;
|
||||
var techs = [], dataSource = 'static';
|
||||
|
||||
// 1) Try DB-fitted Bass model results (freshest — computed daily 04:30)
|
||||
try {
|
||||
data = await api('/api/hype-cycle/enriched');
|
||||
} catch(e) {
|
||||
data = await api('/api/hype-cycle');
|
||||
var dbRes = await api('/api/hype-cycle/analysis');
|
||||
if (dbRes.success && Array.isArray(dbRes.data) && dbRes.data.length > 0) {
|
||||
var now = new Date().getFullYear();
|
||||
techs = dbRes.data.map(function(r) {
|
||||
var phaseLabel = DB_PHASE_LABEL[r.hype_phase] || r.hype_phase;
|
||||
// Estimate years to plateau: rough heuristic from phase + years_to_next_phase
|
||||
var phasesLeft = { innovation_trigger:4, peak_inflated_expectations:3, trough_disillusionment:2, slope_enlightenment:1, plateau_productivity:0 };
|
||||
var ytp = (phasesLeft[r.hype_phase] || 0) * (r.years_to_next_phase || 2);
|
||||
return {
|
||||
technology: r.technology,
|
||||
phase: phaseLabel,
|
||||
positionPct: Math.round(r.hype_score || 0),
|
||||
adoptionPct: Math.round((r.current_share || 0) * 100),
|
||||
peakYear: r.t_peak_year ? Math.round(r.t_peak_year) : null,
|
||||
yearsToPlateauFromNow: ytp > 0 ? Math.round(ytp) : null,
|
||||
// extra DB fields for tooltip
|
||||
aspCurrentUsd: r.asp_current_usd,
|
||||
aspDecline3y: r.asp_decline_pct_3y,
|
||||
rSquared: r.r_squared,
|
||||
computedAt: r.computed_at
|
||||
};
|
||||
});
|
||||
dataSource = 'db';
|
||||
var computedAt = dbRes.data[0] && dbRes.data[0].computed_at ? new Date(dbRes.data[0].computed_at).toLocaleDateString() : '';
|
||||
el('hype-year').textContent = now + (computedAt ? ' · computed ' + computedAt : '');
|
||||
}
|
||||
} catch(e) { /* fall through to static */ }
|
||||
|
||||
// 2) Fallback: static enriched/base endpoint
|
||||
if (techs.length === 0) {
|
||||
var data;
|
||||
try {
|
||||
data = await api('/api/hype-cycle/enriched');
|
||||
} catch(e) {
|
||||
data = await api('/api/hype-cycle');
|
||||
}
|
||||
techs = data.technologies || [];
|
||||
el('hype-year').textContent = data.year;
|
||||
}
|
||||
var techs = data.technologies || [];
|
||||
el('hype-year').textContent = data.year;
|
||||
|
||||
// Badge showing data source
|
||||
var srcBadge = el('hype-data-source');
|
||||
if (srcBadge) srcBadge.textContent = dataSource === 'db' ? '● Live DB' : '● Static';
|
||||
|
||||
var c = el('hype-svg-container');
|
||||
buildDOM(c, renderHypeSvg(techs));
|
||||
@ -2803,6 +2949,8 @@ async function loadHypeCycle() {
|
||||
+ '<td class="mono tip" data-tip="' + esc(HYPE_TIPS['Adoption']) + '">' + adoptionDisplay + '</td>'
|
||||
+ '<td class="mono">' + esc(t.peakYear || '—') + '</td>'
|
||||
+ '<td class="mono">' + (t.yearsToPlateauFromNow != null ? t.yearsToPlateauFromNow + 'y' : '—') + '</td>'
|
||||
+ '<td class="mono">' + (t.aspCurrentUsd != null ? '$' + Number(t.aspCurrentUsd).toLocaleString() : '—') + '</td>'
|
||||
+ '<td class="mono">' + (t.rSquared != null ? Number(t.rSquared).toFixed(2) : '—') + '</td>'
|
||||
+ '</tr>';
|
||||
}).join(''));
|
||||
|
||||
@ -6533,6 +6681,147 @@ async function lookupStock() {
|
||||
resultEl.textContent = 'Error: ' + e.message;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Price Comparison ──────────────────────────────────────────────────────────
|
||||
|
||||
var pricesLoaded = false;
|
||||
|
||||
async function loadPriceComparison() {
|
||||
if (pricesLoaded) return;
|
||||
try {
|
||||
// Load summary + top SKUs in parallel
|
||||
var [sumData, listData] = await Promise.all([
|
||||
api('/api/price-comparison/summary'),
|
||||
api('/api/price-comparison')
|
||||
]);
|
||||
|
||||
// ── Stat cards ──────────────────────────────────────────────────────────
|
||||
if (sumData.success && sumData.data) {
|
||||
var s = sumData.data;
|
||||
setEl('pc-stat-skus', Number(s.total_skus_tracked || 0).toLocaleString());
|
||||
setEl('pc-stat-vendors', Number(s.active_vendor_count || 0).toLocaleString());
|
||||
setEl('pc-stat-obs', Number(s.total_observations || 0).toLocaleString());
|
||||
setEl('pc-stat-avg', s.overall_avg_price != null
|
||||
? 'USD ' + Number(s.overall_avg_price).toLocaleString('en-US', {minimumFractionDigits:2, maximumFractionDigits:2})
|
||||
: '—');
|
||||
|
||||
// ── Form factor table ─────────────────────────────────────────────────
|
||||
var ffBody = el('pc-ff-body');
|
||||
if (ffBody) {
|
||||
var ffs = s.by_form_factor || [];
|
||||
if (ffs.length === 0) {
|
||||
ffBody.innerHTML = '<tr><td colspan="6" style="text-align:center;padding:2rem;color:var(--text-dim)">No data yet</td></tr>';
|
||||
} else {
|
||||
ffBody.innerHTML = ffs.map(function(r) {
|
||||
var cur = r.currency || 'USD';
|
||||
return '<tr style="border-top:1px solid var(--border)">'
|
||||
+ '<td style="padding:5px 8px;font-weight:600;color:var(--text-bright)">' + esc(r.form_factor || '—') + '</td>'
|
||||
+ '<td style="padding:5px 8px;text-align:right">' + Number(r.sku_count).toLocaleString() + '</td>'
|
||||
+ '<td style="padding:5px 8px;text-align:right">' + Number(r.vendor_count).toLocaleString() + '</td>'
|
||||
+ '<td style="padding:5px 8px;text-align:right;color:#22c55e">' + (r.min_price != null ? cur + '\u00a0' + Number(r.min_price).toFixed(2) : '—') + '</td>'
|
||||
+ '<td style="padding:5px 8px;text-align:right;color:var(--text-bright)">' + (r.avg_price != null ? cur + '\u00a0' + Number(r.avg_price).toFixed(2) : '—') + '</td>'
|
||||
+ '<td style="padding:5px 8px;text-align:right;color:#f87171">' + (r.max_price != null ? cur + '\u00a0' + Number(r.max_price).toFixed(2) : '—') + '</td>'
|
||||
+ '</tr>';
|
||||
}).join('');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Top SKUs table ────────────────────────────────────────────────────────
|
||||
var topBody = el('pc-top-body');
|
||||
if (topBody && listData.success && Array.isArray(listData.data)) {
|
||||
var rows = listData.data;
|
||||
if (rows.length === 0) {
|
||||
topBody.innerHTML = '<tr><td colspan="7" style="text-align:center;padding:2rem;color:var(--text-dim)">No price data yet — waiting for first scrape run</td></tr>';
|
||||
} else {
|
||||
topBody.innerHTML = rows.map(function(r) {
|
||||
var spread = r.spread_pct != null ? Number(r.spread_pct).toFixed(1) + '%' : '—';
|
||||
var spreadColor = r.spread_pct != null && r.spread_pct > 30 ? '#f87171' : r.spread_pct > 10 ? '#f59e0b' : '#22c55e';
|
||||
var cur = r.currency || 'USD';
|
||||
return '<tr style="border-top:1px solid var(--border);cursor:pointer" onclick="el(\'pc-lookup-input\').value=\'' + esc(r.standard_name) + '\';lookupPriceComparison()">'
|
||||
+ '<td style="padding:5px 8px;font-family:monospace;font-size:0.72rem;color:var(--text-bright)">' + esc(r.standard_name || '—') + '</td>'
|
||||
+ '<td style="padding:5px 8px;text-align:center;color:var(--text-dim)">' + esc(r.form_factor || '—') + '</td>'
|
||||
+ '<td style="padding:5px 8px;text-align:center;color:var(--text-dim)">' + esc(r.speed || '—') + '</td>'
|
||||
+ '<td style="padding:5px 8px;text-align:right"><span style="background:var(--indigo);color:#fff;border-radius:10px;padding:1px 7px;font-size:0.68rem">' + r.vendor_count + '</span></td>'
|
||||
+ '<td style="padding:5px 8px;text-align:right;color:#22c55e">' + (r.min_price != null ? cur + '\u00a0' + Number(r.min_price).toFixed(2) : '—') + '</td>'
|
||||
+ '<td style="padding:5px 8px;text-align:right;color:var(--text-bright)">' + (r.avg_price != null ? cur + '\u00a0' + Number(r.avg_price).toFixed(2) : '—') + '</td>'
|
||||
+ '<td style="padding:5px 8px;text-align:right;color:' + spreadColor + ';font-weight:600">' + spread + '</td>'
|
||||
+ '</tr>';
|
||||
}).join('');
|
||||
}
|
||||
}
|
||||
|
||||
pricesLoaded = true;
|
||||
} catch(e) {
|
||||
console.error('loadPriceComparison error', e);
|
||||
}
|
||||
}
|
||||
|
||||
async function lookupPriceComparison() {
|
||||
var input = el('pc-lookup-input');
|
||||
var resultEl = el('pc-lookup-result');
|
||||
if (!input || !resultEl) return;
|
||||
var q = (input.value || '').trim();
|
||||
if (!q) return;
|
||||
resultEl.innerHTML = '<span style="color:var(--text-dim)">Looking up…</span>';
|
||||
try {
|
||||
var data = await api('/api/price-comparison/' + encodeURIComponent(q));
|
||||
if (!data.success || !data.transceiver) {
|
||||
resultEl.textContent = 'Not found: ' + q;
|
||||
return;
|
||||
}
|
||||
var tx = data.transceiver;
|
||||
var stats = data.stats || {};
|
||||
var prices = data.prices || [];
|
||||
var cur = (prices[0] && prices[0].currency) ? prices[0].currency : 'USD';
|
||||
|
||||
var statsHtml = '<div style="margin-bottom:0.75rem">'
|
||||
+ '<b style="color:var(--text-bright)">' + esc(tx.standard_name) + '</b>'
|
||||
+ ' · ' + esc(tx.form_factor || '') + ' ' + esc(tx.speed || '')
|
||||
+ (tx.fiber_type ? ' · ' + esc(tx.fiber_type) : '')
|
||||
+ (tx.reach_label ? ' · ' + esc(tx.reach_label) : '')
|
||||
+ '</div>'
|
||||
+ '<div style="display:flex;gap:1.5rem;flex-wrap:wrap;margin-bottom:0.75rem;font-size:0.8rem">'
|
||||
+ '<span>📊 <b>' + stats.vendor_count + '</b> vendors</span>'
|
||||
+ (stats.min != null ? '<span style="color:#22c55e">Min: <b>' + cur + '\u00a0' + Number(stats.min).toFixed(2) + '</b></span>' : '')
|
||||
+ (stats.avg != null ? '<span>Avg: <b>' + cur + '\u00a0' + Number(stats.avg).toFixed(2) + '</b></span>' : '')
|
||||
+ (stats.max != null ? '<span style="color:#f87171">Max: <b>' + cur + '\u00a0' + Number(stats.max).toFixed(2) + '</b></span>' : '')
|
||||
+ (stats.spread_pct != null ? '<span style="color:#f59e0b">Spread: <b>' + Number(stats.spread_pct).toFixed(1) + '%</b></span>' : '')
|
||||
+ '</div>';
|
||||
|
||||
var tableHtml = '';
|
||||
if (prices.length > 0) {
|
||||
tableHtml = '<table style="width:100%;border-collapse:collapse;font-size:0.75rem;margin-top:0.5rem">'
|
||||
+ '<thead><tr style="background:var(--surface2)">'
|
||||
+ '<th style="padding:5px 8px;text-align:left;color:var(--text-dim);font-weight:500">Vendor</th>'
|
||||
+ '<th style="padding:5px 8px;text-align:right;color:var(--text-dim);font-weight:500">Price</th>'
|
||||
+ '<th style="padding:5px 8px;text-align:center;color:var(--text-dim);font-weight:500">Stock</th>'
|
||||
+ '<th style="padding:5px 8px;text-align:center;color:var(--text-dim);font-weight:500">Observed</th>'
|
||||
+ '</tr></thead><tbody>'
|
||||
+ prices.map(function(p, i) {
|
||||
var stock = p.stock_level || '—';
|
||||
var stockColor = /in.stock|available/i.test(stock) ? '#22c55e' : /out|unavail/i.test(stock) ? '#f87171' : 'var(--text-dim)';
|
||||
var vendorHtml = p.url
|
||||
? '<a href="' + esc(p.url) + '" target="_blank" style="color:var(--indigo);text-decoration:none">' + esc(p.vendor) + '</a>'
|
||||
: esc(p.vendor);
|
||||
var rowBg = i % 2 === 0 ? '' : 'background:var(--surface2)';
|
||||
return '<tr style="border-top:1px solid var(--border);' + rowBg + '">'
|
||||
+ '<td style="padding:5px 8px">' + vendorHtml + '</td>'
|
||||
+ '<td style="padding:5px 8px;text-align:right;font-weight:600;color:var(--text-bright)">' + (p.price != null ? esc(p.currency || cur) + '\u00a0' + Number(p.price).toFixed(2) : '—') + '</td>'
|
||||
+ '<td style="padding:5px 8px;text-align:center;color:' + stockColor + ';font-size:0.7rem">' + esc(stock) + '</td>'
|
||||
+ '<td style="padding:5px 8px;text-align:center;color:var(--text-dim);font-size:0.7rem">' + (p.observed_at ? new Date(p.observed_at).toLocaleDateString('en-US') : '—') + '</td>'
|
||||
+ '</tr>';
|
||||
}).join('')
|
||||
+ '</tbody></table>';
|
||||
} else {
|
||||
tableHtml = '<p style="color:var(--text-dim)">No price observations found.</p>';
|
||||
}
|
||||
|
||||
resultEl.innerHTML = '<div class="card" style="padding:1rem;margin-top:0.5rem">' + statsHtml + tableHtml + '</div>';
|
||||
} catch(e) {
|
||||
resultEl.textContent = 'Error: ' + e.message;
|
||||
}
|
||||
}
|
||||
</script>
|
||||
<script src="/dashboard/hot-topics.js"></script>
|
||||
</body>
|
||||
|
||||
@ -94,6 +94,7 @@ export async function registerSchedules(boss: PgBoss): Promise<void> {
|
||||
// ── Manufacturer catalogs (every 8h, no prices) ────────────────────
|
||||
"scrape:catalog:smartoptics",
|
||||
"scrape:catalog:hubersuhner",
|
||||
"scrape:catalog:eoptolink",
|
||||
// ── Vendor lists ───────────────────────────────────────────────────
|
||||
"scrape:vendors:flexoptix",
|
||||
"scrape:vendors:flexoptix-supported",
|
||||
@ -217,6 +218,7 @@ export async function registerSchedules(boss: PgBoss): Promise<void> {
|
||||
|
||||
await boss.schedule("scrape:catalog:smartoptics", "10 */4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||||
await boss.schedule("scrape:catalog:hubersuhner", "25 */4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||||
await boss.schedule("scrape:catalog:eoptolink", "40 */4 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||||
|
||||
// ══════════════════════════════════════════════════════════════════════
|
||||
// VENDOR LISTS — every 12h
|
||||
@ -450,6 +452,12 @@ export async function registerWorkers(boss: PgBoss): Promise<void> {
|
||||
await scrapeHuberSuhner();
|
||||
});
|
||||
|
||||
await boss.work("scrape:catalog:eoptolink", async () => {
|
||||
console.log(`[${new Date().toISOString()}] Running: Eoptolink OEM catalog`);
|
||||
const { scrapeEoptolink } = await import("./scrapers/eoptolink");
|
||||
await scrapeEoptolink();
|
||||
});
|
||||
|
||||
// ── Vendor lists ──────────────────────────────────────────────────────
|
||||
|
||||
await boss.work("scrape:vendors:flexoptix", async () => {
|
||||
|
||||
237
packages/scraper/src/scrapers/eoptolink.ts
Normal file
237
packages/scraper/src/scrapers/eoptolink.ts
Normal file
@ -0,0 +1,237 @@
|
||||
/**
|
||||
* Eoptolink Manufacturer Catalog Scraper
|
||||
*
|
||||
* Source: www.eoptolink.com — One of China's top-3 optical transceiver OEMs.
|
||||
* (Finisar competitor, supplies tier-1 cloud hyperscalers)
|
||||
* Target: Discover all product families + part numbers, seed transceivers table
|
||||
* as manufacturer=Eoptolink entries.
|
||||
*
|
||||
* Strategy:
|
||||
* Phase 1: Fetch homepage → extract all /product-solutions/* category URLs (≈90)
|
||||
* Phase 2: Fetch each category page → parse product name + Eoptolink part numbers
|
||||
* (format: E[A-Z]{2,5}-\d{2,4}[A-Z0-9-]*)
|
||||
*
|
||||
* Note: Eoptolink does NOT publish retail prices (B2B OEM manufacturer).
|
||||
* This scraper adds manufacturer catalog entries — no price_observations.
|
||||
*
|
||||
* Rate limit: 1 req/2s — polite crawl of OEM's website.
|
||||
*/
|
||||
|
||||
import { pool, findOrCreateScrapedTransceiver, ensureVendor } from "../utils/db";
|
||||
|
||||
const BASE = "https://www.eoptolink.com";
|
||||
const HEADERS = {
|
||||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
|
||||
Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
};
|
||||
const DELAY_MS = 2000;
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((r) => setTimeout(r, ms));
|
||||
}
|
||||
|
||||
async function fetchHtml(url: string): Promise<string> {
|
||||
const resp = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(20_000) });
|
||||
if (!resp.ok) throw new Error(`HTTP ${resp.status} for ${url}`);
|
||||
return resp.text();
|
||||
}
|
||||
|
||||
// ── Classification helpers ──────────────────────────────────────────────────
|
||||
|
||||
function speedFromSlug(slug: string): { speed: string; speedGbps: number } {
|
||||
if (/\b1\.?6t\b/i.test(slug)) return { speed: "1.6T", speedGbps: 1600 };
|
||||
if (/\b800g\b/i.test(slug)) return { speed: "800G", speedGbps: 800 };
|
||||
if (/\b400g\b/i.test(slug)) return { speed: "400G", speedGbps: 400 };
|
||||
if (/\b200g\b/i.test(slug)) return { speed: "200G", speedGbps: 200 };
|
||||
if (/\b100g\b/i.test(slug)) return { speed: "100G", speedGbps: 100 };
|
||||
if (/\b50g\b/i.test(slug)) return { speed: "50G", speedGbps: 50 };
|
||||
if (/\b40g\b/i.test(slug)) return { speed: "40G", speedGbps: 40 };
|
||||
if (/\b32g\b/i.test(slug)) return { speed: "32G", speedGbps: 32 };
|
||||
if (/\b25g\b/i.test(slug)) return { speed: "25G", speedGbps: 25 };
|
||||
if (/\b16g\b/i.test(slug)) return { speed: "16G", speedGbps: 16 };
|
||||
if (/\b10g\b/i.test(slug)) return { speed: "10G", speedGbps: 10 };
|
||||
if (/\b8g\b/i.test(slug)) return { speed: "8G", speedGbps: 8 };
|
||||
if (/\b4g\b/i.test(slug)) return { speed: "4G", speedGbps: 4 };
|
||||
if (/\b1g\b/i.test(slug)) return { speed: "1G", speedGbps: 1 };
|
||||
return { speed: "Unknown", speedGbps: 0 };
|
||||
}
|
||||
|
||||
function formFactorFromText(text: string): string {
|
||||
const t = text.toUpperCase();
|
||||
if (/\bOSFP\b/.test(t)) return "OSFP";
|
||||
if (/\bQSFP.?DD800\b|\bQSFP-DD800\b/.test(t)) return "QSFP-DD800";
|
||||
if (/\bQSFP.?DD\b/.test(t)) return "QSFP-DD";
|
||||
if (/\bQSFP56\b/.test(t)) return "QSFP56";
|
||||
if (/\bQSFP112\b/.test(t)) return "QSFP112";
|
||||
if (/\bQSFP28\b/.test(t)) return "QSFP28";
|
||||
if (/\bQSFP\+|\bQSFP PLUS\b/.test(t)) return "QSFP+";
|
||||
if (/\bSFP56.DD\b/.test(t)) return "SFP56-DD";
|
||||
if (/\bSFP56\b/.test(t)) return "SFP56";
|
||||
if (/\bSFP28\b/.test(t)) return "SFP28";
|
||||
if (/\bSFP\+|SFP-PLUS|SFP PLUS\b/.test(t)) return "SFP+";
|
||||
if (/\bXFP\b/.test(t)) return "XFP";
|
||||
if (/\bCFP4\b/.test(t)) return "CFP4";
|
||||
if (/\bCFP2\b/.test(t)) return "CFP2";
|
||||
if (/\bCFP\b/.test(t)) return "CFP";
|
||||
if (/\bSFP\b/.test(t)) return "SFP";
|
||||
return "SFP";
|
||||
}
|
||||
|
||||
function fiberFromText(text: string): string {
|
||||
const t = text.toLowerCase();
|
||||
if (/multimode|mmf|sr|om[1-5]/i.test(t)) return "MMF";
|
||||
if (/single.?mode|smf|lr|er|zr|fr|dr|bidi|cwdm|dwdm|coherent/i.test(t)) return "SMF";
|
||||
return "SMF"; // OEM products default to SMF
|
||||
}
|
||||
|
||||
function categoryFromText(text: string): string {
|
||||
const t = text.toLowerCase();
|
||||
if (/coherent|zr|dpsk/.test(t)) return "Coherent";
|
||||
if (/dwdm/.test(t)) return "DWDM";
|
||||
if (/cwdm/.test(t)) return "CWDM";
|
||||
if (/aoc/.test(t)) return "AOC";
|
||||
if (/dac/.test(t)) return "DAC";
|
||||
if (/pon|gpon|gepon/.test(t)) return "PON";
|
||||
return "DataCenter";
|
||||
}
|
||||
|
||||
// ── Phase 1: Discover product solution URLs ──────────────────────────────────
|
||||
|
||||
async function fetchProductSolutionUrls(): Promise<string[]> {
|
||||
console.log(` Fetching Eoptolink homepage for product solution links...`);
|
||||
const html = await fetchHtml(`${BASE}/`);
|
||||
const links = html.match(/href="(\/product-solutions\/[^"#?]+)"/gi) ?? [];
|
||||
const unique = [...new Set(links.map((l) => l.match(/href="([^"]+)"/)?.[1] ?? "").filter(Boolean))];
|
||||
// Skip OSA (optical sub-assemblies) and test-board entries — no transceiver catalog
|
||||
const filtered = unique.filter((u) =>
|
||||
!u.includes("/osa/") &&
|
||||
!u.includes("/other/") &&
|
||||
!u.endsWith("/400g/") &&
|
||||
!u.endsWith("/800g/") &&
|
||||
!u.endsWith("/product-solutions/")
|
||||
);
|
||||
console.log(` Found ${filtered.length} product solution pages`);
|
||||
return filtered;
|
||||
}
|
||||
|
||||
// ── Phase 2: Parse product detail page ──────────────────────────────────────
|
||||
|
||||
interface EoptolinkProduct {
|
||||
pageTitle: string;
|
||||
partNumbers: string[];
|
||||
speed: string;
|
||||
speedGbps: number;
|
||||
formFactor: string;
|
||||
fiberType: string;
|
||||
category: string;
|
||||
pageUrl: string;
|
||||
}
|
||||
|
||||
function parseProductPage(html: string, pageUrl: string): EoptolinkProduct | null {
|
||||
// Page title
|
||||
const titleMatch = html.match(/<title>([^<]+)/i) || html.match(/<h1[^>]*>([^<]{5,80})</i);
|
||||
const pageTitle = (titleMatch?.[1] ?? "").replace(/\s*\|.*$/, "").replace(/[||]+[^||]*$/, "").trim();
|
||||
if (!pageTitle || pageTitle.length < 3) return null;
|
||||
|
||||
// Eoptolink part numbers: format like EOLO-168HG-10-XDX, EOLQ-128HG-02-PX
|
||||
const pnRegex = /E[A-Z]{2,5}-\d{2,3}[A-Z0-9]{1,3}(?:-\d{1,3})?(?:-[A-Z0-9]{1,6})*/g;
|
||||
const partNumbers = [...new Set([...(html.matchAll(pnRegex) ?? [])].map((m) => m[0].trim()))];
|
||||
|
||||
const slug = pageUrl.split("/").slice(-2).join("-");
|
||||
const { speed, speedGbps } = speedFromSlug(slug + " " + pageTitle);
|
||||
const formFactor = formFactorFromText(pageTitle + " " + slug);
|
||||
const fiberType = fiberFromText(pageTitle + " " + slug);
|
||||
const category = categoryFromText(pageTitle + " " + slug);
|
||||
|
||||
return { pageTitle, partNumbers, speed, speedGbps, formFactor, fiberType, category, pageUrl };
|
||||
}
|
||||
|
||||
// ── Main ────────────────────────────────────────────────────────────────────
|
||||
|
||||
export async function scrapeEoptolink(): Promise<void> {
|
||||
console.log("=== Eoptolink Manufacturer Catalog Scraper ===\n");
|
||||
|
||||
const vendorId = await ensureVendor(
|
||||
"Eoptolink",
|
||||
"manufacturer",
|
||||
"https://www.eoptolink.com",
|
||||
"https://www.eoptolink.com/product-solutions/"
|
||||
);
|
||||
console.log(` Vendor ID: ${vendorId}`);
|
||||
|
||||
// Phase 1: Collect product solution URLs
|
||||
let productUrls: string[];
|
||||
try {
|
||||
productUrls = await fetchProductSolutionUrls();
|
||||
} catch (err) {
|
||||
console.error(` Homepage fetch failed: ${(err as Error).message}`);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`\n[Phase 2] Fetching ${productUrls.length} product detail pages...\n`);
|
||||
|
||||
let added = 0;
|
||||
let skipped = 0;
|
||||
let errors = 0;
|
||||
|
||||
for (const relPath of productUrls) {
|
||||
await sleep(DELAY_MS);
|
||||
const url = `${BASE}${relPath}`;
|
||||
try {
|
||||
const html = await fetchHtml(url);
|
||||
const product = parseProductPage(html, relPath);
|
||||
if (!product || product.speedGbps === 0) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Use page title as the primary product entry; also seed one row per part number
|
||||
const namesToSeed: string[] = product.partNumbers.length > 0
|
||||
? product.partNumbers.slice(0, 10) // max 10 part numbers per product family page
|
||||
: [product.pageTitle];
|
||||
|
||||
for (const partNumber of namesToSeed) {
|
||||
try {
|
||||
await findOrCreateScrapedTransceiver({
|
||||
partNumber: partNumber.slice(0, 80),
|
||||
vendorId,
|
||||
formFactor: product.formFactor,
|
||||
speedGbps: product.speedGbps,
|
||||
speed: product.speed,
|
||||
fiberType: product.fiberType,
|
||||
category: product.category,
|
||||
});
|
||||
added++;
|
||||
} catch (dbErr) {
|
||||
// Duplicate or constraint error — expected for re-runs
|
||||
}
|
||||
}
|
||||
|
||||
console.log(
|
||||
` ✓ ${product.pageTitle.padEnd(45)} ff=${product.formFactor.padEnd(8)} speed=${product.speed.padEnd(5)} pn=${product.partNumbers.length}`
|
||||
);
|
||||
} catch (err: unknown) {
|
||||
errors++;
|
||||
if (errors <= 10) console.warn(` ✗ Error ${relPath}: ${(err as Error).message.slice(0, 60)}`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\n=== Eoptolink Catalog Scraper Complete ===`);
|
||||
console.log(` Pages processed: ${productUrls.length - errors}`);
|
||||
console.log(` Transceivers seeded: ${added}`);
|
||||
console.log(` Skipped (no speed): ${skipped}`);
|
||||
console.log(` Errors: ${errors}`);
|
||||
}
|
||||
|
||||
// ── CLI ────────────────────────────────────────────────────────────────────
|
||||
|
||||
if (require.main === module) {
|
||||
scrapeEoptolink()
|
||||
.then(() => pool.end())
|
||||
.catch((err: unknown) => {
|
||||
console.error("Fatal:", err);
|
||||
pool.end();
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user