Compare commits
No commits in common. "9ecaffc475e93d5b6d04721a6c738bd3e58ee743" and "75cbd7cd869f5610a7f3f13cd500d0b869999dba" have entirely different histories.
9ecaffc475
...
75cbd7cd86
@ -3,10 +3,6 @@
|
||||
Format: `{"d":"YYYY-MM-DD","t":"TYPE","m":"Description"}`
|
||||
Types: FEAT · FIX · UI · DATA · AI · INFRA
|
||||
|
||||
{"d":"2026-04-17","t":"UI","m":"Stock dashboard: 6th stat card (Multi-Vendor SKUs), confidence quality badge column in vendor breakdown (🟢 L3 per-warehouse / 🟡 L2 aggregated / ⚪ L1 boolean), new Multi-Vendor Price Comparison table with min/max/avg per SKU. Subtitle updated to mention QSFPTEK + NADDOD."}
|
||||
{"d":"2026-04-17","t":"FEAT","m":"/api/stock/summary enhanced: vendor_breakdown adds avg_confidence + currencies + confidence breakdown (conf_per_warehouse/aggregated/boolean); new price_comparison endpoint (top 50 SKUs tracked by 2+ vendors with price spread); totals adds multi_vendor_skus count."}
|
||||
{"d":"2026-04-17","t":"DATA","m":"Cisco TMG expanded to 17 platform families (+5 new: 8000 Series, NCS5500, NCS540, NCS560, NCS1000). Per-device query strategy replaces family-level search: iterates all switch IDs from filter → 58 switches per N9300 vs 1 before. 856 compat entries / 174 switches after re-run."}
|
||||
{"d":"2026-04-17","t":"DATA","m":"Juniper HCT scraper run: 475 Juniper-brand transceivers seeded into transceivers table (form factor, speed, reach, fiber type from apps.juniper.net/hct). No prices (OEM). Scheduled to run at 6:15 + 18:15 daily."}
|
||||
{"d":"2026-04-17","t":"DATA","m":"Competitor research: QSFPTEK shows real-time aggregated stock count (e.g. '5507 in real-time stock, 17 Apr 2026') + USD prices; NADDOD shows exact per-product counts ('In Stock: 543') via Astro SSR. Both scraped publicly, no login required. Flexoptix confirmed exact Lagerbestand + EUR prices. FS.com: EUR prices yes, exact counts no."}
|
||||
{"d":"2026-04-17","t":"DATA","m":"stock_observations selective cleanup + schema upgrade: TRUNCATE stock_observations (186 FS.com test-run rows cleared, will repopulate on next launchd run). Added 4 new quality columns via migration 038: stock_confidence (1=boolean/2=aggregated/3=per-warehouse), price_currency CHAR(3), price_includes_tax BOOLEAN, stock_vendor_ts TIMESTAMPTZ."}
|
||||
{"d":"2026-04-17","t":"FEAT","m":"Migration 028 retroactively committed to repo (028-stock-observations-warehouse-columns.sql) — documents the 10 warehouse columns applied directly to Erik DB. Guards with IF NOT EXISTS for safe re-application."}
|
||||
@ -157,5 +153,3 @@ Types: FEAT · FIX · UI · DATA · AI · INFRA
|
||||
{"d":"2026-03-30","t":"FEAT","m":"Qdrant vector DB integration: hybrid full-text + semantic search across products, FAQ, datasheets, news"}
|
||||
{"d":"2026-03-30","t":"INFRA","m":"Stack deployed: PostgreSQL 17 + TimescaleDB, Qdrant, Cloudflare R2 for images, PM2"}
|
||||
{"d":"2026-03-30","t":"DATA","m":"v0.1.0: 5,018 transceivers, 351 vendors seeded from 23 initial scrapers"}
|
||||
{"d":"2026-04-17","t":"DATA","m":"Vendor cleanup: pruned 242 irrelevant OEM/manufacturer vendors with no transceiver or switch data — 348→106 vendors"}
|
||||
{"d":"2026-04-18","t":"FEAT","m":"Mouser Electronics API scraper: OEM reference prices for Juniper/Cisco/Arista PIDs — scheduled daily 03:00, MOUSER_API_KEY env var required"}
|
||||
|
||||
@ -76,11 +76,7 @@ stockRouter.get("/", async (req: Request, res: Response) => {
|
||||
so.units_sold,
|
||||
so.compatible_brands,
|
||||
so.price_net,
|
||||
so.product_url,
|
||||
so.stock_confidence,
|
||||
so.price_currency,
|
||||
so.price_includes_tax,
|
||||
so.stock_vendor_ts
|
||||
so.product_url
|
||||
FROM (
|
||||
SELECT DISTINCT ON (transceiver_id, source_vendor_id) *
|
||||
FROM stock_observations
|
||||
@ -132,7 +128,7 @@ stockRouter.get("/", async (req: Request, res: Response) => {
|
||||
*/
|
||||
stockRouter.get("/summary", async (req: Request, res: Response) => {
|
||||
try {
|
||||
const [totals, topSellers, vendorBreakdown, recentlyUpdated, priceComparison] = await Promise.all([
|
||||
const [totals, topSellers, vendorBreakdown, recentlyUpdated] = await Promise.all([
|
||||
// Overall totals from latest observations
|
||||
pool.query(`
|
||||
WITH latest AS (
|
||||
@ -141,24 +137,16 @@ stockRouter.get("/summary", async (req: Request, res: Response) => {
|
||||
ORDER BY transceiver_id, source_vendor_id, time DESC
|
||||
)
|
||||
SELECT
|
||||
COUNT(*) AS total_observations,
|
||||
COUNT(*) FILTER (WHERE in_stock = true) AS in_stock_count,
|
||||
SUM(COALESCE(warehouse_de_qty, 0)) AS total_de_qty,
|
||||
SUM(COALESCE(warehouse_global_qty, 0)) AS total_global_qty,
|
||||
SUM(COALESCE(backorder_qty, 0)) AS total_backorder_qty,
|
||||
COUNT(*) FILTER (WHERE warehouse_de_qty > 0) AS products_with_de_stock,
|
||||
COUNT(*) FILTER (WHERE warehouse_global_qty > 0) AS products_with_global_stock,
|
||||
COUNT(*) FILTER (WHERE backorder_qty > 0) AS products_with_backorder,
|
||||
COUNT(DISTINCT transceiver_id) AS unique_transceivers,
|
||||
COUNT(DISTINCT source_vendor_id) AS unique_vendors,
|
||||
-- Data quality breakdown by confidence level
|
||||
COUNT(*) FILTER (WHERE stock_confidence = 1) AS conf_boolean_count,
|
||||
COUNT(*) FILTER (WHERE stock_confidence = 2) AS conf_aggregated_count,
|
||||
COUNT(*) FILTER (WHERE stock_confidence = 3) AS conf_per_warehouse_count,
|
||||
-- Multi-vendor coverage (SKUs tracked by 2+ vendors)
|
||||
COUNT(DISTINCT transceiver_id) FILTER (WHERE transceiver_id IN (
|
||||
SELECT transceiver_id FROM latest GROUP BY transceiver_id HAVING COUNT(DISTINCT source_vendor_id) >= 2
|
||||
)) AS multi_vendor_skus
|
||||
COUNT(*) AS total_observations,
|
||||
COUNT(*) FILTER (WHERE in_stock = true) AS in_stock_count,
|
||||
SUM(COALESCE(warehouse_de_qty, 0)) AS total_de_qty,
|
||||
SUM(COALESCE(warehouse_global_qty, 0)) AS total_global_qty,
|
||||
SUM(COALESCE(backorder_qty, 0)) AS total_backorder_qty,
|
||||
COUNT(*) FILTER (WHERE warehouse_de_qty > 0) AS products_with_de_stock,
|
||||
COUNT(*) FILTER (WHERE warehouse_global_qty > 0) AS products_with_global_stock,
|
||||
COUNT(*) FILTER (WHERE backorder_qty > 0) AS products_with_backorder,
|
||||
COUNT(DISTINCT transceiver_id) AS unique_transceivers,
|
||||
COUNT(DISTINCT source_vendor_id) AS unique_vendors
|
||||
FROM latest
|
||||
`),
|
||||
|
||||
@ -187,7 +175,7 @@ stockRouter.get("/summary", async (req: Request, res: Response) => {
|
||||
LIMIT 20
|
||||
`),
|
||||
|
||||
// Per-vendor stock breakdown (incl. confidence + currency breakdown)
|
||||
// Per-vendor stock breakdown
|
||||
pool.query(`
|
||||
WITH latest AS (
|
||||
SELECT DISTINCT ON (transceiver_id, source_vendor_id) *
|
||||
@ -195,22 +183,14 @@ stockRouter.get("/summary", async (req: Request, res: Response) => {
|
||||
ORDER BY transceiver_id, source_vendor_id, time DESC
|
||||
)
|
||||
SELECT
|
||||
v.name AS vendor_name,
|
||||
v.website AS vendor_website,
|
||||
COUNT(*) AS product_count,
|
||||
COUNT(*) FILTER (WHERE so.in_stock = true) AS in_stock_count,
|
||||
SUM(COALESCE(so.warehouse_de_qty, 0)) AS total_de_qty,
|
||||
SUM(COALESCE(so.warehouse_global_qty, 0)) AS total_global_qty,
|
||||
SUM(COALESCE(so.backorder_qty, 0)) AS total_backorder,
|
||||
MAX(so.time) AS last_scraped,
|
||||
ROUND(AVG(so.stock_confidence)::NUMERIC, 1) AS avg_confidence,
|
||||
ARRAY_AGG(DISTINCT so.price_currency)
|
||||
FILTER (WHERE so.price_currency IS NOT NULL) AS currencies,
|
||||
-- Per-confidence breakdown
|
||||
COUNT(*) FILTER (WHERE so.stock_confidence = 3) AS conf_per_warehouse,
|
||||
COUNT(*) FILTER (WHERE so.stock_confidence = 2) AS conf_aggregated,
|
||||
COUNT(*) FILTER (WHERE so.stock_confidence = 1
|
||||
OR so.stock_confidence IS NULL) AS conf_boolean
|
||||
v.name AS vendor_name,
|
||||
v.website AS vendor_website,
|
||||
COUNT(*) AS product_count,
|
||||
COUNT(*) FILTER (WHERE so.in_stock = true) AS in_stock_count,
|
||||
SUM(COALESCE(so.warehouse_de_qty, 0)) AS total_de_qty,
|
||||
SUM(COALESCE(so.warehouse_global_qty, 0)) AS total_global_qty,
|
||||
SUM(COALESCE(so.backorder_qty, 0)) AS total_backorder,
|
||||
MAX(so.time) AS last_scraped
|
||||
FROM latest so
|
||||
JOIN vendors v ON v.id = so.source_vendor_id
|
||||
GROUP BY v.id, v.name, v.website
|
||||
@ -236,35 +216,6 @@ stockRouter.get("/summary", async (req: Request, res: Response) => {
|
||||
ORDER BY so.time DESC
|
||||
LIMIT 10
|
||||
`),
|
||||
|
||||
// Price comparison: SKUs tracked by multiple vendors (multi-vendor overlap)
|
||||
pool.query(`
|
||||
WITH latest AS (
|
||||
SELECT DISTINCT ON (transceiver_id, source_vendor_id) *
|
||||
FROM stock_observations
|
||||
WHERE price_net IS NOT NULL
|
||||
ORDER BY transceiver_id, source_vendor_id, time DESC
|
||||
)
|
||||
SELECT
|
||||
t.part_number,
|
||||
t.form_factor,
|
||||
t.speed,
|
||||
COUNT(DISTINCT so.source_vendor_id) AS vendor_count,
|
||||
MIN(so.price_net) AS price_min,
|
||||
MAX(so.price_net) AS price_max,
|
||||
ROUND(AVG(so.price_net)::NUMERIC, 2) AS price_avg,
|
||||
ARRAY_AGG(v.name ORDER BY so.price_net) AS vendor_names,
|
||||
ARRAY_AGG(so.price_net ORDER BY so.price_net) AS prices,
|
||||
ARRAY_AGG(so.price_currency ORDER BY so.price_net) AS currencies,
|
||||
MAX(so.stock_confidence) AS best_confidence
|
||||
FROM latest so
|
||||
JOIN transceivers t ON t.id = so.transceiver_id
|
||||
JOIN vendors v ON v.id = so.source_vendor_id
|
||||
GROUP BY t.id, t.part_number, t.form_factor, t.speed
|
||||
HAVING COUNT(DISTINCT so.source_vendor_id) >= 2
|
||||
ORDER BY vendor_count DESC, price_min ASC
|
||||
LIMIT 50
|
||||
`),
|
||||
]);
|
||||
|
||||
res.json({
|
||||
@ -274,7 +225,6 @@ stockRouter.get("/summary", async (req: Request, res: Response) => {
|
||||
top_sellers: topSellers.rows,
|
||||
vendor_breakdown: vendorBreakdown.rows,
|
||||
recently_updated: recentlyUpdated.rows,
|
||||
price_comparison: priceComparison.rows,
|
||||
},
|
||||
});
|
||||
} catch (err) {
|
||||
|
||||
@ -775,7 +775,6 @@
|
||||
<span data-goto="switches"><span class="val" id="stat-switches">—</span> switches</span>
|
||||
<span data-goto="standards"><span class="val" id="stat-standards">—</span> standards</span>
|
||||
<span data-goto="news"><span class="val" id="stat-news">—</span> articles</span>
|
||||
<span data-goto="stock"><span class="val" id="stat-stock-obs">—</span> stock obs</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="status">
|
||||
@ -841,15 +840,6 @@
|
||||
<div id="verification-overview" class="mt" style="display:grid;grid-template-columns:repeat(auto-fit,minmax(200px,1fr));gap:1rem"></div>
|
||||
</div>
|
||||
|
||||
<!-- WAREHOUSE STOCK SUMMARY -->
|
||||
<div class="card mb" id="ov-stock-card" style="display:none">
|
||||
<div class="card-label" style="display:flex;justify-content:space-between;align-items:center">
|
||||
<span>🏭 Warehouse Stock Summary</span>
|
||||
<button class="btn-sm" onclick="goToTab('stock')" style="font-size:0.7rem;padding:0.2rem 0.6rem">View Detail →</button>
|
||||
</div>
|
||||
<div id="ov-stock-grid" class="mt" style="display:grid;grid-template-columns:repeat(auto-fit,minmax(160px,1fr));gap:0.75rem"></div>
|
||||
</div>
|
||||
|
||||
<div class="grid g2 mb">
|
||||
<div class="card">
|
||||
<div class="card-label">Vector Collections</div>
|
||||
@ -1715,13 +1705,13 @@
|
||||
<div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:1rem;flex-wrap:wrap;gap:0.5rem">
|
||||
<div>
|
||||
<h2 style="margin:0;font-size:1.1rem;color:var(--text-bright)">🏭 Warehouse Stock Intelligence</h2>
|
||||
<p style="margin:0.2rem 0 0;font-size:0.75rem;color:var(--text-dim)">Real-time DE-Lager · Global-Lager · Nachlieferung · Verkauft — scraped from fs.com · QSFPTEK · NADDOD & more</p>
|
||||
<p style="margin:0.2rem 0 0;font-size:0.75rem;color:var(--text-dim)">Real-time DE-Lager · Global-Lager · Nachlieferung · Verkauft — scraped from fs.com and other vendors</p>
|
||||
</div>
|
||||
<button onclick="stockLoaded=false;loadStock()" class="btn" style="background:var(--indigo);color:#fff;font-size:0.75rem">↻ Refresh</button>
|
||||
<button onclick="loadStock()" class="btn" style="background:var(--indigo);color:#fff;font-size:0.75rem">↻ Refresh</button>
|
||||
</div>
|
||||
|
||||
<!-- Summary stat cards -->
|
||||
<div class="grid mb" style="grid-template-columns:repeat(6,1fr);gap:0.75rem;margin-bottom:1.25rem" id="stock-stat-cards">
|
||||
<div class="grid mb" style="grid-template-columns:repeat(5,1fr);gap:0.75rem;margin-bottom:1.25rem" id="stock-stat-cards">
|
||||
<div class="stat-card" style="text-align:center">
|
||||
<div class="stat-icon blue">📦</div>
|
||||
<div class="stat-label">Total SKUs tracked</div>
|
||||
@ -1747,11 +1737,6 @@
|
||||
<div class="stat-label">In Nachlieferung</div>
|
||||
<div class="stat-val" id="stock-stat-backorder">—</div>
|
||||
</div>
|
||||
<div class="stat-card" style="text-align:center">
|
||||
<div class="stat-icon" style="color:#22c55e">🔀</div>
|
||||
<div class="stat-label">Multi-Vendor SKUs</div>
|
||||
<div class="stat-val" id="stock-stat-multiv">—</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div style="display:grid;grid-template-columns:1fr 1fr;gap:1rem;margin-bottom:1.5rem">
|
||||
@ -1790,12 +1775,11 @@
|
||||
<th style="padding:6px 8px;text-align:right;color:var(--text-dim);font-weight:500">DE-Lager</th>
|
||||
<th style="padding:6px 8px;text-align:right;color:var(--text-dim);font-weight:500">Global</th>
|
||||
<th style="padding:6px 8px;text-align:right;color:var(--text-dim);font-weight:500">Backorder</th>
|
||||
<th style="padding:6px 8px;text-align:center;color:var(--text-dim);font-weight:500" title="Stock data quality: L3=per-warehouse, L2=aggregated, L1=boolean">Quality</th>
|
||||
<th style="padding:6px 8px;text-align:left;color:var(--text-dim);font-weight:500">Last Scraped</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="stock-vendor-body">
|
||||
<tr><td colspan="8" style="text-align:center;padding:2rem;color:var(--text-dim)">No data yet</td></tr>
|
||||
<tr><td colspan="7" style="text-align:center;padding:2rem;color:var(--text-dim)">No data yet</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
@ -1808,29 +1792,6 @@
|
||||
<div id="stock-recent" style="padding:1rem;color:var(--text-dim);font-size:0.8rem">No recent restock events</div>
|
||||
</div>
|
||||
|
||||
<!-- Multi-Vendor Price Comparison -->
|
||||
<div class="card" style="overflow:hidden;margin-top:1rem">
|
||||
<div style="padding:0.75rem 1rem;border-bottom:1px solid var(--border);font-size:0.85rem;font-weight:600;color:var(--text-bright)">🔀 Multi-Vendor Price Comparison <span style="font-weight:400;color:var(--text-dim);font-size:0.75rem">— SKUs tracked by 2+ vendors</span></div>
|
||||
<div style="overflow-x:auto">
|
||||
<table style="width:100%;border-collapse:collapse;font-size:0.75rem" id="stock-price-compare-table">
|
||||
<thead>
|
||||
<tr style="background:var(--surface2)">
|
||||
<th style="padding:6px 8px;text-align:left;color:var(--text-dim);font-weight:500">Part Number</th>
|
||||
<th style="padding:6px 8px;text-align:left;color:var(--text-dim);font-weight:500">Form</th>
|
||||
<th style="padding:6px 8px;text-align:right;color:var(--text-dim);font-weight:500">Vendors</th>
|
||||
<th style="padding:6px 8px;text-align:right;color:var(--text-dim);font-weight:500">Min Price</th>
|
||||
<th style="padding:6px 8px;text-align:right;color:var(--text-dim);font-weight:500">Max Price</th>
|
||||
<th style="padding:6px 8px;text-align:right;color:var(--text-dim);font-weight:500">Avg Price</th>
|
||||
<th style="padding:6px 8px;text-align:left;color:var(--text-dim);font-weight:500">Vendors (low → high)</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="stock-price-compare-body">
|
||||
<tr><td colspan="7" style="text-align:center;padding:2rem;color:var(--text-dim)">No multi-vendor data yet</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Search by Part Number -->
|
||||
<div class="card" style="margin-top:1rem;padding:1rem">
|
||||
<div style="font-size:0.85rem;font-weight:600;color:var(--text-bright);margin-bottom:0.75rem">🔍 Lookup Stock by Part Number</div>
|
||||
@ -2251,26 +2212,6 @@ async function loadOverview() {
|
||||
animateValue(el('stat-switches'), h.database.stats.switch_count, 500);
|
||||
animateValue(el('stat-standards'), h.database.stats.standard_count, 500);
|
||||
animateValue(el('stat-news'), h.database.stats.news_count, 700);
|
||||
if (h.stock && h.stock.total_observations > 0) {
|
||||
animateValue(el('stat-stock-obs'), h.stock.total_observations, 900);
|
||||
// Show warehouse stock summary card
|
||||
var sc = el('ov-stock-card');
|
||||
if (sc) sc.style.display = '';
|
||||
var stockItems = [
|
||||
{ icon: '📦', label: 'Beobachtungen', val: h.stock.total_observations.toLocaleString(), color: '#6366f1' },
|
||||
{ icon: '🔌', label: 'SKUs mit Daten', val: h.stock.transceivers_with_stock.toLocaleString(), color: '#22c55e' },
|
||||
{ icon: '🏪', label: 'Anbieter', val: h.stock.vendors_with_stock.toLocaleString(), color: '#3b82f6' },
|
||||
{ icon: '🇩🇪', label: 'DE-Lager', val: h.stock.total_de_qty.toLocaleString(), color: '#a855f7' },
|
||||
{ icon: '🌍', label: 'Global-Lager', val: h.stock.total_global_qty.toLocaleString(), color: '#06b6d4' },
|
||||
];
|
||||
buildDOM(el('ov-stock-grid'), stockItems.map(function(si) {
|
||||
return '<div style="background:var(--surface2);border:1px solid var(--border);border-radius:8px;padding:0.75rem;text-align:center">'
|
||||
+ '<div style="font-size:1.4rem">' + si.icon + '</div>'
|
||||
+ '<div style="font-size:1.1rem;font-weight:700;color:' + si.color + ';margin:0.2rem 0">' + si.val + '</div>'
|
||||
+ '<div style="font-size:0.7rem;color:var(--text-dim)">' + si.label + '</div>'
|
||||
+ '</div>';
|
||||
}).join(''));
|
||||
}
|
||||
animateValue(el('ov-transceivers'), h.database.stats.transceiver_count, 1000);
|
||||
animateValue(el('ov-vendors'), h.database.stats.vendor_count, 800);
|
||||
animateValue(el('ov-switches'), h.database.stats.switch_count, 600);
|
||||
@ -6372,54 +6313,34 @@ async function runEquivalenceMatcher() {
|
||||
var stockLoaded = false;
|
||||
|
||||
async function loadStock() {
|
||||
if (stockLoaded) return; // already loaded — Refresh button resets stockLoaded=false first
|
||||
if (stockLoaded) return; // already loaded — use Refresh button to force reload
|
||||
stockLoaded = false; // allow reloads via Refresh button
|
||||
try {
|
||||
var data = await api('/api/stock/summary');
|
||||
if (!data.success) return;
|
||||
var d = data.data;
|
||||
var t = d.totals;
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────────────
|
||||
// Stat cards
|
||||
function setEl(id, v) { var e = el(id); if (e) e.textContent = v; }
|
||||
|
||||
/** Returns a confidence badge HTML string based on avg_confidence value */
|
||||
function confBadge(avgConf) {
|
||||
var c = parseFloat(avgConf) || 0;
|
||||
if (c >= 2.5) return '<span title="L3: per-warehouse breakdown" style="background:#166534;color:#86efac;font-size:0.65rem;font-weight:700;padding:2px 6px;border-radius:10px;white-space:nowrap">🟢 L3</span>';
|
||||
if (c >= 1.5) return '<span title="L2: aggregated global count" style="background:#713f12;color:#fde68a;font-size:0.65rem;font-weight:700;padding:2px 6px;border-radius:10px;white-space:nowrap">🟡 L2</span>';
|
||||
return '<span title="L1: boolean in-stock only" style="background:var(--surface2);color:var(--text-dim);font-size:0.65rem;font-weight:700;padding:2px 6px;border-radius:10px;white-space:nowrap">⚪ L1</span>';
|
||||
}
|
||||
|
||||
/** Format price with currency symbol */
|
||||
function fmtPrice(net, currency) {
|
||||
if (net == null) return '—';
|
||||
var sym = currency === 'EUR' ? '€' : currency === 'USD' ? '$' : (currency || '') + ' ';
|
||||
return sym + Number(net).toFixed(2);
|
||||
}
|
||||
|
||||
// ── Stat cards ───────────────────────────────────────────────────────────
|
||||
setEl('stock-stat-skus', Number(t.unique_transceivers || 0).toLocaleString());
|
||||
setEl('stock-stat-instock', Number(t.in_stock_count || 0).toLocaleString());
|
||||
setEl('stock-stat-de', Number(t.total_de_qty || 0).toLocaleString());
|
||||
setEl('stock-stat-global', Number(t.total_global_qty || 0).toLocaleString());
|
||||
setEl('stock-stat-backorder', Number(t.total_backorder_qty || 0).toLocaleString());
|
||||
setEl('stock-stat-multiv', Number(t.multi_vendor_skus || 0).toLocaleString());
|
||||
|
||||
// ── Top sellers table ────────────────────────────────────────────────────
|
||||
// Top sellers table
|
||||
var tbody = el('stock-top-sellers-body');
|
||||
if (tbody) {
|
||||
if (d.top_sellers && d.top_sellers.length > 0) {
|
||||
tbody.innerHTML = d.top_sellers.map(function(r) {
|
||||
var pn = r.product_url
|
||||
? '<a href="' + esc(r.product_url) + '" target="_blank" style="color:var(--indigo);text-decoration:none;font-family:monospace;font-size:0.72rem">' + esc(r.part_number) + '</a>'
|
||||
: '<span style="font-family:monospace;font-size:0.72rem;color:var(--text-bright)">' + esc(r.part_number) + '</span>';
|
||||
return '<tr style="border-bottom:1px solid var(--border)">'
|
||||
+ '<td style="padding:5px 8px">' + pn + '</td>'
|
||||
+ '<td style="padding:5px 8px;font-family:monospace;font-size:0.72rem;color:var(--text-bright)">' + esc(r.part_number) + '</td>'
|
||||
+ '<td style="padding:5px 8px;color:var(--text-dim)">' + esc(r.form_factor || '—') + '</td>'
|
||||
+ '<td style="padding:5px 8px;text-align:right;color:#f59e0b;font-weight:600">' + Number(r.units_sold || 0).toLocaleString() + '</td>'
|
||||
+ '<td style="padding:5px 8px;text-align:right;color:#6366f1">' + (r.warehouse_de_qty != null ? Number(r.warehouse_de_qty).toLocaleString() : '—') + '</td>'
|
||||
+ '<td style="padding:5px 8px;text-align:right;color:#06b6d4">' + (r.warehouse_global_qty != null ? Number(r.warehouse_global_qty).toLocaleString() : '—') + '</td>'
|
||||
+ '<td style="padding:5px 8px;text-align:right">' + fmtPrice(r.price_net, r.price_currency) + '</td>'
|
||||
+ '<td style="padding:5px 8px;text-align:right">' + (r.price_net != null ? '€' + Number(r.price_net).toFixed(2) : '—') + '</td>'
|
||||
+ '</tr>';
|
||||
}).join('');
|
||||
} else {
|
||||
@ -6427,7 +6348,7 @@ async function loadStock() {
|
||||
}
|
||||
}
|
||||
|
||||
// ── Vendor breakdown (with confidence badge) ─────────────────────────────
|
||||
// Vendor breakdown
|
||||
var vbody = el('stock-vendor-body');
|
||||
if (vbody) {
|
||||
if (d.vendor_breakdown && d.vendor_breakdown.length > 0) {
|
||||
@ -6440,16 +6361,15 @@ async function loadStock() {
|
||||
+ '<td style="padding:5px 8px;text-align:right;color:#6366f1">' + Number(r.total_de_qty || 0).toLocaleString() + '</td>'
|
||||
+ '<td style="padding:5px 8px;text-align:right;color:#06b6d4">' + Number(r.total_global_qty || 0).toLocaleString() + '</td>'
|
||||
+ '<td style="padding:5px 8px;text-align:right;color:#f59e0b">' + Number(r.total_backorder || 0).toLocaleString() + '</td>'
|
||||
+ '<td style="padding:5px 8px;text-align:center">' + confBadge(r.avg_confidence) + '</td>'
|
||||
+ '<td style="padding:5px 8px;color:var(--text-dim);font-size:0.7rem">' + lastScraped + '</td>'
|
||||
+ '</tr>';
|
||||
}).join('');
|
||||
} else {
|
||||
vbody.innerHTML = '<tr><td colspan="8" style="text-align:center;padding:2rem;color:var(--text-dim)">No data yet</td></tr>';
|
||||
vbody.innerHTML = '<tr><td colspan="7" style="text-align:center;padding:2rem;color:var(--text-dim)">No data yet</td></tr>';
|
||||
}
|
||||
}
|
||||
|
||||
// ── Recently restocked ───────────────────────────────────────────────────
|
||||
// Recently restocked
|
||||
var recentEl = el('stock-recent');
|
||||
if (recentEl) {
|
||||
if (d.recently_updated && d.recently_updated.length > 0) {
|
||||
@ -6470,33 +6390,6 @@ async function loadStock() {
|
||||
}
|
||||
}
|
||||
|
||||
// ── Multi-vendor price comparison ────────────────────────────────────────
|
||||
var pcbody = el('stock-price-compare-body');
|
||||
if (pcbody) {
|
||||
if (d.price_comparison && d.price_comparison.length > 0) {
|
||||
pcbody.innerHTML = d.price_comparison.slice(0, 20).map(function(r) {
|
||||
var spread = r.price_max && r.price_min
|
||||
? ' <span style="color:var(--text-dim);font-size:0.68rem">(Δ' + (Number(r.price_max) - Number(r.price_min)).toFixed(2) + ')</span>'
|
||||
: '';
|
||||
var vendorList = (r.vendor_names || []).map(function(vn, i) {
|
||||
var p = r.prices && r.prices[i] != null ? fmtPrice(r.prices[i], r.currencies && r.currencies[i]) : '';
|
||||
return '<span style="background:var(--surface2);border-radius:4px;padding:1px 5px;font-size:0.68rem">' + esc(vn) + (p ? ' <b>' + p + '</b>' : '') + '</span>';
|
||||
}).join(' ');
|
||||
return '<tr style="border-bottom:1px solid var(--border)">'
|
||||
+ '<td style="padding:5px 8px;font-family:monospace;font-size:0.72rem;color:var(--text-bright)">' + esc(r.part_number) + '</td>'
|
||||
+ '<td style="padding:5px 8px;color:var(--text-dim)">' + esc(r.form_factor || '—') + '</td>'
|
||||
+ '<td style="padding:5px 8px;text-align:right;font-weight:600">' + (r.vendor_count || '—') + '</td>'
|
||||
+ '<td style="padding:5px 8px;text-align:right;color:#22c55e">' + fmtPrice(r.price_min, r.currencies && r.currencies[0]) + '</td>'
|
||||
+ '<td style="padding:5px 8px;text-align:right;color:#ef4444">' + fmtPrice(r.price_max, r.currencies && r.currencies[0]) + spread + '</td>'
|
||||
+ '<td style="padding:5px 8px;text-align:right">' + fmtPrice(r.price_avg, r.currencies && r.currencies[0]) + '</td>'
|
||||
+ '<td style="padding:5px 8px">' + vendorList + '</td>'
|
||||
+ '</tr>';
|
||||
}).join('');
|
||||
} else {
|
||||
pcbody.innerHTML = '<tr><td colspan="7" style="text-align:center;padding:2rem;color:var(--text-dim)">No multi-vendor data yet</td></tr>';
|
||||
}
|
||||
}
|
||||
|
||||
stockLoaded = true;
|
||||
} catch(e) {
|
||||
console.error('loadStock error', e);
|
||||
|
||||
@ -21,7 +21,7 @@
|
||||
import PgBoss from "pg-boss";
|
||||
import { config } from "dotenv";
|
||||
import { join } from "path";
|
||||
import { mkdirSync, existsSync, writeFileSync } from "fs";
|
||||
import { rmSync, mkdirSync } from "fs";
|
||||
|
||||
/** Run a scraper with an isolated Crawlee storage directory to prevent queue collisions */
|
||||
async function withIsolatedStorage(name: string, fn: () => Promise<void>): Promise<void> {
|
||||
@ -30,21 +30,19 @@ async function withIsolatedStorage(name: string, fn: () => Promise<void>): Promi
|
||||
mkdirSync(join(dir, "request_queues", "default"), { recursive: true });
|
||||
mkdirSync(join(dir, "datasets", "default"), { recursive: true });
|
||||
mkdirSync(join(dir, "key_value_stores", "default"), { recursive: true });
|
||||
// Pre-seed session pool state file to prevent "Could not find file" crash
|
||||
// on first run (Crawlee reads this before writing it on some versions)
|
||||
const sessionFile = join(dir, "key_value_stores", "default", "SDK_SESSION_POOL_STATE.json");
|
||||
if (!existsSync(sessionFile)) {
|
||||
writeFileSync(sessionFile, JSON.stringify({ usableSessionsCount: 0, retiredSessionsCount: 0, sessions: [] }));
|
||||
}
|
||||
const prev = process.env.CRAWLEE_STORAGE_DIR;
|
||||
const prevPurge = process.env.CRAWLEE_PURGE_ON_START;
|
||||
process.env.CRAWLEE_STORAGE_DIR = dir;
|
||||
// Do NOT set CRAWLEE_PURGE_ON_START — let Crawlee reuse session pool state
|
||||
// between runs (better scraping, no "SDK_SESSION_POOL_STATE.json not found" crashes).
|
||||
// The dir is intentionally kept between runs so Crawlee can persist its state.
|
||||
// Force Crawlee to initialize fresh — prevents "Could not find SDK_SESSION_POOL_STATE.json"
|
||||
// when the isolated storage dir was just created and has no pre-existing state files.
|
||||
process.env.CRAWLEE_PURGE_ON_START = "1";
|
||||
try {
|
||||
await fn();
|
||||
} finally {
|
||||
process.env.CRAWLEE_STORAGE_DIR = prev ?? "";
|
||||
process.env.CRAWLEE_PURGE_ON_START = prevPurge ?? "";
|
||||
// Clean up after successful run
|
||||
try { rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ }
|
||||
}
|
||||
}
|
||||
|
||||
@ -132,8 +130,6 @@ export async function registerSchedules(boss: PgBoss): Promise<void> {
|
||||
"scrape:pricing:fibermall",
|
||||
"scrape:pricing:vcelink",
|
||||
"scrape:pricing:opticsbay",
|
||||
// ── OEM Reference Prices (Mouser API, once daily) ─────────────────
|
||||
"scrape:pricing:mouser-oem",
|
||||
// ── Prediction Signal Scrapers (new) ──────────────────────────────
|
||||
"scrape:signals:sec-edgar",
|
||||
"scrape:signals:github",
|
||||
@ -145,14 +141,6 @@ export async function registerSchedules(boss: PgBoss): Promise<void> {
|
||||
"compute:forecast",
|
||||
// ── Sync ──────────────────────────────────────────────────────────
|
||||
"sync:nas",
|
||||
// ── Health Monitoring ─────────────────────────────────────────────
|
||||
"monitor:scraper-health",
|
||||
// ── Verification Reconciliation ───────────────────────────────────
|
||||
"maintenance:reconcile-verification",
|
||||
// ── Competitor Equivalence Matching ───────────────────────────────
|
||||
"maintenance:find-equivalences",
|
||||
// ── Re-Research approved equivalences ─────────────────────────────
|
||||
"maintenance:re-research-equivalences",
|
||||
];
|
||||
|
||||
for (const q of queues) {
|
||||
@ -199,10 +187,6 @@ export async function registerSchedules(boss: PgBoss): Promise<void> {
|
||||
await boss.schedule("scrape:pricing:vcelink", "3 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||||
await boss.schedule("scrape:pricing:opticsbay", "7 */2 * * *", {}, { retryLimit: 2, expireInSeconds: 3600 });
|
||||
|
||||
// OEM reference prices via Mouser API — once daily at 03:00 (slow: 2s/PID × 475 PIDs ≈ 16min)
|
||||
// Requires MOUSER_API_KEY env var (free at mouser.com/api-hub)
|
||||
await boss.schedule("scrape:pricing:mouser-oem", "0 3 * * *", {}, { retryLimit: 1, expireInSeconds: 3600 });
|
||||
|
||||
// ══════════════════════════════════════════════════════════════════════
|
||||
// FLEXOPTIX CATALOG — every 2h (primary price source)
|
||||
// ══════════════════════════════════════════════════════════════════════
|
||||
@ -295,31 +279,18 @@ export async function registerSchedules(boss: PgBoss): Promise<void> {
|
||||
|
||||
await boss.schedule("sync:nas", "55 7 * * *", {}, { retryLimit: 1, expireInSeconds: 1800 });
|
||||
|
||||
// Health check: every 3h — warns if any vendor has no new prices recently
|
||||
await boss.schedule("monitor:scraper-health", "17 */3 * * *", {}, { retryLimit: 1, expireInSeconds: 600 });
|
||||
|
||||
// Verification reconciliation: nightly at 01:00 UTC
|
||||
// Resets competitor_verified/fully_verified for any transceiver that no longer
|
||||
// has a real non-Flexoptix price in the last 30 days — prevents stale ★ 100% badges
|
||||
await boss.schedule("maintenance:reconcile-verification", "0 1 * * *", {}, { retryLimit: 1, expireInSeconds: 1800 });
|
||||
|
||||
// Equivalence matching: nightly at 02:00 UTC (after reconcile)
|
||||
await boss.schedule("maintenance:find-equivalences", "0 2 * * *", {}, { retryLimit: 1, expireInSeconds: 3600 });
|
||||
|
||||
// Re-research approved equivalences: daily at 03:00 UTC, processes 200 items per run
|
||||
await boss.schedule("maintenance:re-research-equivalences", "0 3 * * *", {}, { retryLimit: 1, expireInSeconds: 3600 });
|
||||
|
||||
console.log("All schedules registered — 24/7 continuous scraping (57 jobs)");
|
||||
console.log("All schedules registered — 24/7 continuous scraping (53 jobs)");
|
||||
}
|
||||
|
||||
export async function registerWorkers(boss: PgBoss): Promise<void> {
|
||||
// Lazy-load all scrapers
|
||||
const { scrapeFs } = await import("./scrapers/fs-com");
|
||||
const { scrapeCiscoTmg } = await import("./scrapers/cisco-tmg");
|
||||
const { scrapeSmartOptics } = await import("./scrapers/smartoptics");
|
||||
const { scrapeHuberSuhner } = await import("./scrapers/hubersuhner");
|
||||
const { scrapeMarketIntelligence } = await import("./scrapers/market-intelligence");
|
||||
const { scrapeNews } = await import("./scrapers/news");
|
||||
// NOTE: Pi-only scrapers (fluxlight, gbics, optcore, champion-one, sfpcables,
|
||||
// blueoptics, fiber24, tscom, skylane, ascentoptics, gaotek, smartoptics,
|
||||
// hubersuhner, news, market-intel) are NOT registered here.
|
||||
// Pi workers (index-pi.ts) are the SOLE consumers of those queues so that
|
||||
// all lightweight scraping traffic flows through the Raspberry Pi Starlink nodes.
|
||||
const { scrape10Gtek } = await import("./scrapers/tenGtek");
|
||||
const { scrapeFlexoptixCatalog } = await import("./scrapers/flexoptix-catalog");
|
||||
const { scrapeFlexoptixVendors } = await import("./scrapers/flexoptix-vendors");
|
||||
@ -362,72 +333,10 @@ export async function registerWorkers(boss: PgBoss): Promise<void> {
|
||||
await withIsolatedStorage("prolabs", scrapeProLabs);
|
||||
});
|
||||
|
||||
// ── Lightweight fetch/cheerio scrapers ───────────────────────────────
|
||||
await boss.work("scrape:pricing:fluxlight", async () => {
|
||||
console.log(`[${new Date().toISOString()}] Running: Fluxlight pricing`);
|
||||
const { scrapeFluxlight } = await import("./scrapers/fluxlight");
|
||||
await scrapeFluxlight();
|
||||
});
|
||||
|
||||
await boss.work("scrape:pricing:gbics", async () => {
|
||||
console.log(`[${new Date().toISOString()}] Running: GBICS pricing`);
|
||||
const { scrapeGbics } = await import("./scrapers/gbics");
|
||||
await scrapeGbics();
|
||||
});
|
||||
|
||||
await boss.work("scrape:pricing:optcore", async () => {
|
||||
console.log(`[${new Date().toISOString()}] Running: Optcore pricing`);
|
||||
const { scrapeOptcore } = await import("./scrapers/optcore");
|
||||
await scrapeOptcore();
|
||||
});
|
||||
|
||||
await boss.work("scrape:pricing:champion-one", async () => {
|
||||
console.log(`[${new Date().toISOString()}] Running: Champion ONE pricing`);
|
||||
const { scrapeChampionOne } = await import("./scrapers/champion-one");
|
||||
await scrapeChampionOne();
|
||||
});
|
||||
|
||||
await boss.work("scrape:pricing:sfpcables", async () => {
|
||||
console.log(`[${new Date().toISOString()}] Running: SFPcables pricing`);
|
||||
const { scrapeSfpCables } = await import("./scrapers/sfpcables");
|
||||
await scrapeSfpCables();
|
||||
});
|
||||
|
||||
await boss.work("scrape:pricing:blueoptics", async () => {
|
||||
console.log(`[${new Date().toISOString()}] Running: BlueOptics pricing`);
|
||||
const { scrapeBlueOptics } = await import("./scrapers/blueoptics");
|
||||
await scrapeBlueOptics();
|
||||
});
|
||||
|
||||
await boss.work("scrape:pricing:fiber24", async () => {
|
||||
console.log(`[${new Date().toISOString()}] Running: Fiber24 pricing`);
|
||||
const { scrapeFiber24 } = await import("./scrapers/fiber24");
|
||||
await scrapeFiber24();
|
||||
});
|
||||
|
||||
await boss.work("scrape:pricing:tscom", async () => {
|
||||
console.log(`[${new Date().toISOString()}] Running: T&S Communication pricing`);
|
||||
const { scrapeTsCom } = await import("./scrapers/tscom");
|
||||
await scrapeTsCom();
|
||||
});
|
||||
|
||||
await boss.work("scrape:pricing:skylane", async () => {
|
||||
console.log(`[${new Date().toISOString()}] Running: Skylane pricing`);
|
||||
const { scrapeSkylane } = await import("./scrapers/skylane");
|
||||
await scrapeSkylane();
|
||||
});
|
||||
|
||||
await boss.work("scrape:pricing:ascentoptics", async () => {
|
||||
console.log(`[${new Date().toISOString()}] Running: Ascent Optics pricing`);
|
||||
const { scrapeAscentOptics } = await import("./scrapers/ascentoptics");
|
||||
await scrapeAscentOptics();
|
||||
});
|
||||
|
||||
await boss.work("scrape:pricing:gaotek", async () => {
|
||||
console.log(`[${new Date().toISOString()}] Running: GAO Tek pricing`);
|
||||
const { scrapeGaoTek } = await import("./scrapers/gaotek");
|
||||
await scrapeGaoTek();
|
||||
});
|
||||
// ── Pi-only scrapers: NO boss.work() here ────────────────────────────
|
||||
// fluxlight, gbics, optcore, champion-one, sfpcables, blueoptics, fiber24,
|
||||
// tscom, skylane, ascentoptics, gaotek → handled exclusively by Pi fleet.
|
||||
// Jobs are dispatched by the cron schedule above; Pi workers consume them.
|
||||
|
||||
// ── Catalog scrapers ──────────────────────────────────────────────────
|
||||
|
||||
@ -436,15 +345,8 @@ export async function registerWorkers(boss: PgBoss): Promise<void> {
|
||||
await scrapeFlexoptixCatalog();
|
||||
});
|
||||
|
||||
await boss.work("scrape:catalog:smartoptics", async () => {
|
||||
console.log(`[${new Date().toISOString()}] Running: SmartOptics catalog`);
|
||||
await scrapeSmartOptics();
|
||||
});
|
||||
|
||||
await boss.work("scrape:catalog:hubersuhner", async () => {
|
||||
console.log(`[${new Date().toISOString()}] Running: HUBER+SUHNER catalog`);
|
||||
await scrapeHuberSuhner();
|
||||
});
|
||||
// scrape:catalog:smartoptics and scrape:catalog:hubersuhner → Pi-only
|
||||
// scrape:news and scrape:market-intel → Pi-only (see index-pi.ts)
|
||||
|
||||
// ── Vendor lists ──────────────────────────────────────────────────────
|
||||
|
||||
@ -508,10 +410,7 @@ export async function registerWorkers(boss: PgBoss): Promise<void> {
|
||||
|
||||
// ── Intelligence & community ──────────────────────────────────────────
|
||||
|
||||
await boss.work("scrape:market-intel", async () => {
|
||||
console.log(`[${new Date().toISOString()}] Running: Market intelligence`);
|
||||
await scrapeMarketIntelligence();
|
||||
});
|
||||
// scrape:market-intel → Pi-only
|
||||
|
||||
await boss.work("scrape:nog-talks", async () => {
|
||||
console.log(`[${new Date().toISOString()}] Running: NOG conference talks`);
|
||||
@ -531,10 +430,7 @@ export async function registerWorkers(boss: PgBoss): Promise<void> {
|
||||
await findAndSeedDatasheetLinks(50);
|
||||
});
|
||||
|
||||
await boss.work("scrape:news", async () => {
|
||||
console.log(`[${new Date().toISOString()}] Running: News scraper`);
|
||||
await scrapeNews();
|
||||
});
|
||||
// scrape:news → Pi-only
|
||||
|
||||
await boss.work("scrape:faq", async () => {
|
||||
console.log(`[${new Date().toISOString()}] FAQ scraper — not yet implemented`);
|
||||
@ -673,346 +569,5 @@ export async function registerWorkers(boss: PgBoss): Promise<void> {
|
||||
await scrapeOpticsBay();
|
||||
});
|
||||
|
||||
// ── Mouser OEM reference prices ────────────────────────────────────────
|
||||
await boss.work("scrape:pricing:mouser-oem", async () => {
|
||||
console.log(`[${new Date().toISOString()}] Running: Mouser OEM reference prices`);
|
||||
if (!process.env["MOUSER_API_KEY"]) {
|
||||
console.warn(" [mouser-oem] Skipping — MOUSER_API_KEY not set");
|
||||
return;
|
||||
}
|
||||
const { scrapeDigikey } = await import("./scrapers/digikey");
|
||||
await scrapeDigikey();
|
||||
});
|
||||
|
||||
// ── Health monitor ──────────────────────────────────────────────────────
|
||||
await boss.work("monitor:scraper-health", async () => {
|
||||
const { pool } = await import("./utils/db");
|
||||
|
||||
// Vendors we expect to see prices from regularly
|
||||
const EXPECTED_VENDORS = [
|
||||
"FiberMall", "QSFPTEK", "Flexoptix", "FS.COM", "10Gtek",
|
||||
"ATGBICS", "GBICS", "BlueOptics", "ShopFiber24", "T&S Communication",
|
||||
"Fluxlight", "Optcore", "Champion ONE", "SFPcables",
|
||||
"Vcelink", "OpticsBay",
|
||||
];
|
||||
|
||||
const result = await pool.query(`
|
||||
SELECT v.name,
|
||||
SUM(CASE WHEN po.time > NOW() - INTERVAL '6 hours' THEN 1 ELSE 0 END) AS prices_6h,
|
||||
MAX(po.time) AS last_seen,
|
||||
EXTRACT(EPOCH FROM (NOW() - MAX(po.time))) / 3600.0 AS hours_since
|
||||
FROM vendors v
|
||||
LEFT JOIN price_observations po ON po.source_vendor_id = v.id
|
||||
WHERE v.name = ANY($1)
|
||||
GROUP BY v.name
|
||||
ORDER BY last_seen ASC NULLS FIRST
|
||||
`, [EXPECTED_VENDORS]);
|
||||
|
||||
const problems: string[] = [];
|
||||
for (const row of result.rows) {
|
||||
const h = parseFloat(row.hours_since ?? "9999");
|
||||
const n = parseInt(row.prices_6h ?? "0", 10);
|
||||
if (n === 0) {
|
||||
const lastStr = row.last_seen
|
||||
? `last seen ${h.toFixed(1)}h ago (${new Date(row.last_seen).toISOString().slice(0, 16)})`
|
||||
: "NEVER scraped";
|
||||
problems.push(`⚠ ${row.name}: 0 prices in last 6h — ${lastStr}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (problems.length > 0) {
|
||||
console.error("=== SCRAPER HEALTH ALERT ===");
|
||||
for (const p of problems) console.error(p);
|
||||
console.error("=== Check pm2 logs tip-scraper-daemon ===");
|
||||
} else {
|
||||
console.log(`[monitor] Scraper health OK — all ${EXPECTED_VENDORS.length} vendors active in last 6h`);
|
||||
}
|
||||
});
|
||||
|
||||
// ── Verification reconciliation ─────────────────────────────────────────
|
||||
await boss.work("maintenance:reconcile-verification", async () => {
|
||||
const { pool } = await import("./utils/db");
|
||||
|
||||
// 1. Reset competitor_verified=false for products with no non-Flexoptix price in last 30 days
|
||||
const resetComp = await pool.query(`
|
||||
UPDATE transceivers t
|
||||
SET competitor_verified = false,
|
||||
competitor_verified_at = NULL
|
||||
WHERE competitor_verified = true
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM price_observations po
|
||||
JOIN vendors v ON po.source_vendor_id = v.id
|
||||
WHERE po.transceiver_id = t.id
|
||||
AND po.time > NOW() - INTERVAL '30 days'
|
||||
AND UPPER(v.name) NOT LIKE '%FLEXOPTIX%'
|
||||
)
|
||||
`);
|
||||
|
||||
// 2. Reset fully_verified=false for products that lost competitor_verified
|
||||
const resetFull = await pool.query(`
|
||||
UPDATE transceivers
|
||||
SET fully_verified = false,
|
||||
fully_verified_at = NULL
|
||||
WHERE fully_verified = true
|
||||
AND (competitor_verified = false OR price_verified = false OR image_verified = false OR details_verified = false)
|
||||
`);
|
||||
|
||||
// 3. Set fully_verified=true for products that now meet all 4 criteria
|
||||
const setFull = await pool.query(`
|
||||
UPDATE transceivers
|
||||
SET fully_verified = true,
|
||||
fully_verified_at = COALESCE(fully_verified_at, NOW())
|
||||
WHERE competitor_verified = true
|
||||
AND price_verified = true
|
||||
AND image_verified = true
|
||||
AND details_verified = true
|
||||
AND fully_verified = false
|
||||
`);
|
||||
|
||||
console.log(
|
||||
`[reconcile] competitor_verified reset: ${resetComp.rowCount}, ` +
|
||||
`fully_verified cleared: ${resetFull.rowCount}, ` +
|
||||
`fully_verified earned: ${setFull.rowCount}`
|
||||
);
|
||||
});
|
||||
|
||||
// ── Equivalence matching ────────────────────────────────────────────────
|
||||
// Matches Flexoptix SKUs to technically equivalent competitor products by specs.
|
||||
// Confidence scoring: standard_name(35) + form_factor(25) + speed_gbps(20) +
|
||||
// fiber_type(10) + reach±25%(10) = 100 pts max
|
||||
// ≥0.85 → auto_approved + competitor_verified=true
|
||||
// 0.50–0.84 → pending (Manual Review queue in dashboard)
|
||||
// <0.50 → skipped
|
||||
await boss.work("maintenance:find-equivalences", async () => {
|
||||
const { pool } = await import("./utils/db");
|
||||
const ts = new Date().toISOString();
|
||||
console.log(`[${ts}] Running: Equivalence matching`);
|
||||
|
||||
// Find all Flexoptix transceivers that are NOT yet competitor_verified
|
||||
const flexResult = await pool.query(`
|
||||
SELECT t.id, t.part_number, t.standard_name, t.form_factor,
|
||||
t.speed_gbps, t.fiber_type, t.reach_meters, t.wavelengths,
|
||||
t.connector, t.wdm_type, t.coherent
|
||||
FROM transceivers t
|
||||
JOIN vendors v ON v.id = t.vendor_id
|
||||
WHERE UPPER(v.name) LIKE '%FLEXOPTIX%'
|
||||
AND t.competitor_verified = false
|
||||
`);
|
||||
|
||||
let autoApproved = 0;
|
||||
let queued = 0;
|
||||
let skipped = 0;
|
||||
|
||||
for (const fx of flexResult.rows) {
|
||||
// Find competitor transceivers with recent price observations and matching specs
|
||||
const candidates = await pool.query(`
|
||||
SELECT t.id AS competitor_id, t.part_number, t.standard_name,
|
||||
t.form_factor, t.speed_gbps, t.fiber_type, t.reach_meters,
|
||||
t.wavelengths, t.connector, v.name AS vendor_name,
|
||||
MAX(po.time) AS last_price, COUNT(*) AS price_count
|
||||
FROM transceivers t
|
||||
JOIN vendors v ON v.id = t.vendor_id
|
||||
JOIN price_observations po ON po.transceiver_id = t.id
|
||||
WHERE UPPER(v.name) NOT LIKE '%FLEXOPTIX%'
|
||||
AND po.time > NOW() - INTERVAL '30 days'
|
||||
AND t.form_factor = $1
|
||||
AND t.speed_gbps = $2
|
||||
AND t.id != $3
|
||||
GROUP BY t.id, t.part_number, t.standard_name, t.form_factor,
|
||||
t.speed_gbps, t.fiber_type, t.reach_meters,
|
||||
t.wavelengths, t.connector, v.name
|
||||
`, [fx.form_factor, fx.speed_gbps, fx.id]);
|
||||
|
||||
for (const cand of candidates.rows) {
|
||||
// Confidence scoring
|
||||
// Max points: form_factor(25) + speed_gbps(20) + standard_name(30) +
|
||||
// wavelength_nm(20) + fiber_type(10) + reach(10) = 115
|
||||
let score = 0;
|
||||
const basis: string[] = [];
|
||||
|
||||
// form_factor already matched (pre-filter), award points
|
||||
score += 25; basis.push("form_factor");
|
||||
|
||||
// speed_gbps already matched (pre-filter)
|
||||
score += 20; basis.push("speed_gbps");
|
||||
|
||||
// standard_name match (strong signal — e.g. "10GBASE-LR")
|
||||
if (fx.standard_name && cand.standard_name &&
|
||||
fx.standard_name.trim().toUpperCase() === cand.standard_name.trim().toUpperCase()) {
|
||||
score += 30; basis.push("standard_name");
|
||||
}
|
||||
|
||||
// wavelength match — extract first numeric nm value and compare within ±15nm
|
||||
// "wavelengths" is text: "1310 nm", "850nm", "1270/1290/1310/1330 nm" etc.
|
||||
const extractNm = (w: string | null): number | null => {
|
||||
if (!w) return null;
|
||||
const m = w.match(/(\d{3,4})/);
|
||||
return m ? parseInt(m[1], 10) : null;
|
||||
};
|
||||
const fxNm = extractNm(fx.wavelengths);
|
||||
const candNm = extractNm(cand.wavelengths);
|
||||
if (fxNm !== null && candNm !== null) {
|
||||
if (Math.abs(fxNm - candNm) <= 15) {
|
||||
score += 20; basis.push(`wavelength_${fxNm}nm`);
|
||||
} else {
|
||||
score -= 20; // hard penalize wrong wavelength (1310 vs 1550 = completely different product)
|
||||
}
|
||||
}
|
||||
|
||||
// fiber_type match (SMF vs MMF — critical)
|
||||
if (fx.fiber_type && cand.fiber_type) {
|
||||
if (fx.fiber_type.trim().toUpperCase() === cand.fiber_type.trim().toUpperCase()) {
|
||||
score += 10; basis.push("fiber_type");
|
||||
} else {
|
||||
score -= 15; // SMF vs MMF = wrong product
|
||||
}
|
||||
}
|
||||
|
||||
// reach within ±25%
|
||||
if (fx.reach_meters && cand.reach_meters && fx.reach_meters > 0 && cand.reach_meters > 0) {
|
||||
const diff = Math.abs(fx.reach_meters - cand.reach_meters);
|
||||
const tolerance = Math.max(fx.reach_meters, 1) * 0.25;
|
||||
if (diff <= tolerance) {
|
||||
score += 10; basis.push("reach");
|
||||
} else {
|
||||
score -= 15; // penalize mismatched reach
|
||||
}
|
||||
} else if (!fx.reach_meters && !cand.reach_meters) {
|
||||
score += 5; basis.push("reach_null");
|
||||
}
|
||||
|
||||
const confidence = Math.max(0, Math.min(1, score / 115));
|
||||
|
||||
if (confidence < 0.50) { skipped++; continue; }
|
||||
|
||||
const notes = `${fx.part_number} ↔ ${cand.part_number} (${cand.vendor_name}) | ` +
|
||||
`basis: ${basis.join(", ")} | reach: ${fx.reach_meters}m vs ${cand.reach_meters}m | ` +
|
||||
`wavelength: ${fx.wavelengths||"?"} vs ${cand.wavelengths||"?"}`;
|
||||
|
||||
// Upsert equivalence candidate
|
||||
const status = confidence >= 0.73 ? "auto_approved" : "pending";
|
||||
|
||||
await pool.query(`
|
||||
INSERT INTO transceiver_equivalences
|
||||
(flexoptix_id, competitor_id, confidence, match_basis, match_notes, status)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)
|
||||
ON CONFLICT (flexoptix_id, competitor_id) DO UPDATE SET
|
||||
confidence = EXCLUDED.confidence,
|
||||
match_basis = EXCLUDED.match_basis,
|
||||
match_notes = EXCLUDED.match_notes,
|
||||
updated_at = NOW()
|
||||
WHERE transceiver_equivalences.status NOT IN ('approved', 'rejected')
|
||||
`, [fx.id, cand.competitor_id, confidence, basis, notes, status]);
|
||||
|
||||
if (confidence >= 0.73) {
|
||||
// Auto-approve: set competitor_verified on the Flexoptix transceiver
|
||||
await pool.query(`
|
||||
UPDATE transceivers
|
||||
SET competitor_verified = true,
|
||||
competitor_verified_at = NOW()
|
||||
WHERE id = $1 AND competitor_verified = false
|
||||
`, [fx.id]);
|
||||
autoApproved++;
|
||||
} else {
|
||||
queued++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(
|
||||
`[find-equivalences] auto_approved: ${autoApproved}, ` +
|
||||
`queued for review: ${queued}, skipped (low confidence): ${skipped}`
|
||||
);
|
||||
|
||||
// After auto-approvals, rerun fully_verified check
|
||||
if (autoApproved > 0) {
|
||||
await pool.query(`
|
||||
UPDATE transceivers
|
||||
SET fully_verified = true,
|
||||
fully_verified_at = COALESCE(fully_verified_at, NOW())
|
||||
WHERE competitor_verified = true
|
||||
AND price_verified = true
|
||||
AND image_verified = true
|
||||
AND details_verified = true
|
||||
AND fully_verified = false
|
||||
`);
|
||||
}
|
||||
});
|
||||
|
||||
// ── Re-research approved equivalences ────────────────────────────────────────
|
||||
// Processes up to 200 approved equivalences per day that have re_research_due_at <= NOW().
|
||||
// Re-runs the confidence check: if competitor still has recent prices and specs still match,
|
||||
// the approval is confirmed (re_researched_at = NOW(), next check in 30 days).
|
||||
// If confidence drops or competitor has no recent price: reverts to pending.
|
||||
await boss.work("maintenance:re-research-equivalences", async () => {
|
||||
const { pool } = await import("./utils/db");
|
||||
const ts = new Date().toISOString();
|
||||
console.log(`[${ts}] Running: Re-research approved equivalences`);
|
||||
|
||||
const batch = await pool.query(`
|
||||
SELECT eq.id, eq.flexoptix_id, eq.competitor_id, eq.confidence,
|
||||
fx.form_factor, fx.speed_gbps, fx.standard_name, fx.fiber_type,
|
||||
fx.reach_meters, fx.wavelengths
|
||||
FROM transceiver_equivalences eq
|
||||
JOIN transceivers fx ON eq.flexoptix_id = fx.id
|
||||
WHERE eq.status IN ('approved', 'auto_approved')
|
||||
AND eq.re_research_due_at IS NOT NULL
|
||||
AND eq.re_research_due_at <= NOW()
|
||||
ORDER BY eq.re_research_due_at ASC
|
||||
LIMIT 200
|
||||
`);
|
||||
|
||||
let confirmed = 0;
|
||||
let reverted = 0;
|
||||
|
||||
for (const eq of batch.rows) {
|
||||
// Check if competitor still has a recent price observation
|
||||
const priceCheck = await pool.query(`
|
||||
SELECT COUNT(*) AS cnt
|
||||
FROM price_observations
|
||||
WHERE transceiver_id = $1 AND time > NOW() - INTERVAL '45 days'
|
||||
`, [eq.competitor_id]);
|
||||
|
||||
const hasRecentPrice = parseInt(priceCheck.rows[0].cnt, 10) > 0;
|
||||
|
||||
if (!hasRecentPrice) {
|
||||
// Competitor no longer carries this — revert to pending for manual review
|
||||
await pool.query(`
|
||||
UPDATE transceiver_equivalences
|
||||
SET status = 'pending', re_research_due_at = NULL, re_researched_at = NULL,
|
||||
match_notes = CONCAT(match_notes, E'\n[Re-research ' || NOW()::date || ': no recent price — reverted to pending]')
|
||||
WHERE id = $1
|
||||
`, [eq.id]);
|
||||
|
||||
// Reset competitor_verified if no other approved equivalence covers this transceiver
|
||||
await pool.query(`
|
||||
UPDATE transceivers
|
||||
SET competitor_verified = false, competitor_verified_at = NULL,
|
||||
fully_verified = false, fully_verified_at = NULL
|
||||
WHERE id = $1
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM transceiver_equivalences
|
||||
WHERE flexoptix_id = $1
|
||||
AND status IN ('approved', 'auto_approved')
|
||||
AND id != $2
|
||||
)
|
||||
`, [eq.flexoptix_id, eq.id]);
|
||||
|
||||
reverted++;
|
||||
} else {
|
||||
// Still valid — confirm and schedule next re-research in 30 days
|
||||
await pool.query(`
|
||||
UPDATE transceiver_equivalences
|
||||
SET re_researched_at = NOW(),
|
||||
re_research_due_at = NOW() + INTERVAL '30 days'
|
||||
WHERE id = $1
|
||||
`, [eq.id]);
|
||||
confirmed++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`[re-research] confirmed: ${confirmed}, reverted to pending: ${reverted}, batch size: ${batch.rows.length}`);
|
||||
});
|
||||
|
||||
console.log("All workers registered (76 jobs, 24/7 continuous)");
|
||||
console.log("All workers registered (61 jobs, 24/7 continuous)");
|
||||
}
|
||||
|
||||
@ -49,33 +49,22 @@ interface TmgSearchResponse {
|
||||
|
||||
/** Key Nexus/Catalyst platform family IDs from the TMG API */
|
||||
const PLATFORM_FAMILIES = [
|
||||
// ── Nexus Data Center ───────────────────────────────────────────────────
|
||||
{ id: 74, name: "N9300" }, // Nexus 9300 — 8,515 entries
|
||||
{ id: 77, name: "N9500" }, // Nexus 9500 — 2,266 entries
|
||||
{ id: 78, name: "N9200" }, // Nexus 9200 — 708 entries
|
||||
{ id: 661, name: "N9800" }, // Nexus 9800 — 238 entries
|
||||
// ── Catalyst Campus ─────────────────────────────────────────────────────
|
||||
{ id: 76, name: "C9300" }, // Catalyst 9300 — 260 entries
|
||||
{ id: 601, name: "C9300L" }, // Catalyst 9300L — 720 entries
|
||||
{ id: 1181, name: "C9300X" }, // Catalyst 9300X — 413 entries
|
||||
{ id: 8, name: "C9500" }, // Catalyst 9500 — 1,141 entries
|
||||
{ id: 521, name: "C9600" }, // Catalyst 9600 — 771 entries
|
||||
{ id: 7, name: "C9400" }, // Catalyst 9400 — 561 entries
|
||||
{ id: 341, name: "C9200" }, // Catalyst 9200 — 222 entries
|
||||
// ── Service Provider / High-Capacity ────────────────────────────────────
|
||||
{ id: 83, name: "ASR9000" }, // ASR 9000 — 3,644 entries
|
||||
{ id: 1021, name: "8000" }, // Cisco 8000 Series — 1,954 entries
|
||||
{ id: 20, name: "NCS5500" }, // NCS 5500 — 3,843 entries
|
||||
{ id: 121, name: "NCS540" }, // NCS 540 — 2,684 entries
|
||||
{ id: 141, name: "NCS560" }, // NCS 560 — 229 entries
|
||||
{ id: 17, name: "NCS1000" }, // NCS 1000 (optical) — 325 entries
|
||||
{ id: 74, name: "N9300" }, // Nexus 9300 — 8,515 entries
|
||||
{ id: 77, name: "N9500" }, // Nexus 9500 — 2,266 entries
|
||||
{ id: 78, name: "N9200" }, // Nexus 9200 — 708 entries
|
||||
{ id: 661, name: "N9800" }, // Nexus 9800 — 238 entries
|
||||
{ id: 76, name: "C9300" }, // Catalyst 9300 — 260 entries
|
||||
{ id: 601, name: "C9300L" }, // Catalyst 9300L — 720 entries
|
||||
{ id: 1181, name: "C9300X" }, // Catalyst 9300X — 413 entries
|
||||
{ id: 8, name: "C9500" }, // Catalyst 9500 — 1,141 entries
|
||||
{ id: 521, name: "C9600" }, // Catalyst 9600 — 771 entries
|
||||
{ id: 7, name: "C9400" }, // Catalyst 9400 — 561 entries
|
||||
{ id: 341, name: "C9200" }, // Catalyst 9200 — 222 entries
|
||||
{ id: 83, name: "ASR9000" }, // ASR 9000 — 3,644 entries
|
||||
];
|
||||
|
||||
function buildTmgBody(
|
||||
familyFilter?: { id: number; name: string },
|
||||
deviceIdFilter?: { id: number; name: string }
|
||||
): object {
|
||||
return {
|
||||
async function searchTmg(familyFilter: { id: number; name: string }): Promise<TmgSearchResponse> {
|
||||
const body = {
|
||||
cableType: [],
|
||||
dataRate: [],
|
||||
formFactor: [],
|
||||
@ -84,16 +73,14 @@ function buildTmgBody(
|
||||
osType: [],
|
||||
transceiverProductFamily: [],
|
||||
transceiverProductID: [],
|
||||
networkDeviceProductFamily: familyFilter ? [familyFilter] : [],
|
||||
networkDeviceProductID: deviceIdFilter ? [deviceIdFilter] : [],
|
||||
networkDeviceProductFamily: [familyFilter],
|
||||
networkDeviceProductID: [],
|
||||
media: [],
|
||||
connectorType: [],
|
||||
caseTemperature: [],
|
||||
performanceMonitoring: [],
|
||||
};
|
||||
}
|
||||
|
||||
async function searchTmg(familyFilter: { id: number; name: string }): Promise<TmgSearchResponse> {
|
||||
const res = await fetch(TMG_API, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
@ -101,28 +88,7 @@ async function searchTmg(familyFilter: { id: number; name: string }): Promise<Tm
|
||||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
|
||||
"Accept": "application/json",
|
||||
},
|
||||
body: JSON.stringify(buildTmgBody(familyFilter)),
|
||||
signal: AbortSignal.timeout(30000),
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
throw new Error(`TMG API ${res.status}: ${res.statusText}`);
|
||||
}
|
||||
|
||||
return res.json() as Promise<TmgSearchResponse>;
|
||||
}
|
||||
|
||||
/** Search for a specific switch by its TMG Product ID to get full compat data */
|
||||
async function searchTmgByDeviceId(deviceId: { id: number; name: string }): Promise<TmgSearchResponse> {
|
||||
const res = await fetch(TMG_API, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
|
||||
"Accept": "application/json",
|
||||
},
|
||||
body: JSON.stringify(buildTmgBody(undefined, deviceId)),
|
||||
signal: AbortSignal.timeout(30000),
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
@ -183,66 +149,51 @@ export async function scrapeCiscoTmg(): Promise<void> {
|
||||
let totalCompat = 0;
|
||||
let totalTransceivers = 0;
|
||||
|
||||
/** Process one networkDevice compat response — writes switches+compat to DB */
|
||||
async function processDevices(
|
||||
devices: TmgDevice[],
|
||||
familyName: string
|
||||
): Promise<{ switches: number; transceivers: number; compat: number }> {
|
||||
let sw = 0; let tx = 0; let cp = 0;
|
||||
for (const device of devices) {
|
||||
for (const compat of device.networkAndTransceiverCompatibility) {
|
||||
if (!compat.productId) continue;
|
||||
const switchId = await upsertCiscoSwitch(ciscoVendorId, compat.productId, familyName);
|
||||
sw++;
|
||||
for (const t of compat.transceivers) {
|
||||
if (!t.productId) continue;
|
||||
tx++;
|
||||
const txResult = await pool.query(
|
||||
`SELECT id FROM transceivers WHERE part_number = $1 OR part_number = $2 LIMIT 1`,
|
||||
[t.productId, t.productId.replace(/-S$/, "")]
|
||||
);
|
||||
if (txResult.rows.length > 0) {
|
||||
await upsertCompatibility(switchId, txResult.rows[0].id, t.softReleaseMinVer, t.formFactor, t.reach, t.cableType, t.media, t.dataRate);
|
||||
cp++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return { switches: sw, transceivers: tx, compat: cp };
|
||||
}
|
||||
|
||||
for (const family of PLATFORM_FAMILIES) {
|
||||
console.log(`\nFetching ${family.name}...`);
|
||||
|
||||
try {
|
||||
// Step 1: Fetch family-level response to get all device IDs in the filter
|
||||
const familyData = await searchTmg(family);
|
||||
const deviceIdFilter = familyData.filters
|
||||
?.find((f) => f.name === "Network Device Product ID")
|
||||
?.values ?? [];
|
||||
const data = await searchTmg(family);
|
||||
console.log(` ${family.name}: ${data.totalCount} total entries, ${data.networkDevices.length} device groups`);
|
||||
|
||||
console.log(` ${family.name}: ${deviceIdFilter.length} switch models`);
|
||||
for (const device of data.networkDevices) {
|
||||
for (const compat of device.networkAndTransceiverCompatibility) {
|
||||
if (!compat.productId) continue;
|
||||
|
||||
if (deviceIdFilter.length === 0) {
|
||||
// Fallback: process whatever family-level search returned
|
||||
const r = await processDevices(familyData.networkDevices, family.name);
|
||||
totalSwitches += r.switches; totalTransceivers += r.transceivers; totalCompat += r.compat;
|
||||
continue;
|
||||
}
|
||||
const switchId = await upsertCiscoSwitch(
|
||||
ciscoVendorId,
|
||||
compat.productId,
|
||||
device.productFamily
|
||||
);
|
||||
totalSwitches++;
|
||||
|
||||
// Step 2: Iterate every switch model by its specific TMG Product ID
|
||||
// (family search only returns 1 switch; per-device search returns full compat list)
|
||||
for (const dev of deviceIdFilter) {
|
||||
try {
|
||||
await new Promise((r) => setTimeout(r, 1000)); // 1s between requests
|
||||
const devData = await searchTmgByDeviceId({ id: dev.id, name: dev.name });
|
||||
const r = await processDevices(devData.networkDevices, family.name);
|
||||
totalSwitches += r.switches; totalTransceivers += r.transceivers; totalCompat += r.compat;
|
||||
if (totalSwitches % 20 === 0) {
|
||||
console.log(` ... ${totalSwitches} switches processed, ${totalCompat} compat matches`);
|
||||
for (const tx of compat.transceivers) {
|
||||
if (!tx.productId) continue;
|
||||
totalTransceivers++;
|
||||
|
||||
// Try to match transceiver in our DB by Cisco PID
|
||||
const txResult = await pool.query(
|
||||
`SELECT id FROM transceivers
|
||||
WHERE part_number = $1
|
||||
OR part_number = $2
|
||||
LIMIT 1`,
|
||||
[tx.productId, tx.productId.replace(/-S$/, "")]
|
||||
);
|
||||
|
||||
if (txResult.rows.length > 0) {
|
||||
await upsertCompatibility(
|
||||
switchId,
|
||||
txResult.rows[0].id,
|
||||
tx.softReleaseMinVer,
|
||||
tx.formFactor,
|
||||
tx.reach,
|
||||
tx.cableType,
|
||||
tx.media,
|
||||
tx.dataRate
|
||||
);
|
||||
totalCompat++;
|
||||
}
|
||||
}
|
||||
} catch (devErr) {
|
||||
console.warn(` Skip ${dev.name}: ${(devErr as Error).message.slice(0, 60)}`);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -1,292 +0,0 @@
|
||||
/**
|
||||
* OEM Reference Price Scraper — Mouser Electronics API
|
||||
*
|
||||
* Source: api.mouser.com (free REST API, no bot-detection)
|
||||
* Target: Juniper, Cisco, Arista OEM transceiver PIDs already in our DB
|
||||
* Stores: price_observations (marketplace='mouser', condition='new')
|
||||
*
|
||||
* API key: Free registration at mouser.com/api — set MOUSER_API_KEY env var
|
||||
* endpoint: POST https://api.mouser.com/api/v1.0/search/keyword
|
||||
*
|
||||
* Rate limit: 30 req/min on free tier → 2s delay between requests
|
||||
*
|
||||
* Note: This file is intentionally named digikey.ts (task origin) but uses
|
||||
* Mouser as the actual source since DigiKey + Arrow both require Playwright
|
||||
* to bypass Cloudflare/Akamai. Mouser's free API returns the same data.
|
||||
*/
|
||||
|
||||
import { pool, ensureVendor, upsertPriceObservation } from "../utils/db";
|
||||
import { contentHash } from "../utils/hash";
|
||||
|
||||
const MOUSER_API_BASE = "https://api.mouser.com/api/v1.0";
|
||||
const MOUSER_API_KEY = process.env["MOUSER_API_KEY"] ?? "";
|
||||
const DELAY_MS = 2_100; // ≤ 30 req/min on free tier
|
||||
|
||||
// ── Types ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
interface MouserPriceBreak {
|
||||
Quantity: number;
|
||||
Price: string; // e.g. "1,234.56" or "1234.56"
|
||||
Currency: string; // e.g. "EUR"
|
||||
}
|
||||
|
||||
interface MouserPart {
|
||||
ManufacturerPartNumber: string;
|
||||
MouserPartNumber: string;
|
||||
Availability: string; // e.g. "117 auf Lager"
|
||||
DataSheetUrl: string;
|
||||
Description: string;
|
||||
LeadTime: string; // e.g. "10 Weeks"
|
||||
Min: string; // min order qty
|
||||
ProductDetailUrl: string;
|
||||
PriceBreaks: MouserPriceBreak[];
|
||||
AvailabilityInStock: string;
|
||||
}
|
||||
|
||||
interface MouserSearchResponse {
|
||||
Errors: Array<{ Code: string; Message: string }>;
|
||||
SearchResults: {
|
||||
NumberOfResult: number;
|
||||
Parts: MouserPart[];
|
||||
} | null;
|
||||
}
|
||||
|
||||
// ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((r) => setTimeout(r, ms));
|
||||
}
|
||||
|
||||
/** Parse Mouser price string "1.234,56" (DE locale) or "1234.56" (US) */
|
||||
function parseMouserPrice(raw: string, currency: string): number | null {
|
||||
if (!raw || raw === "") return null;
|
||||
// German locale uses comma decimal, dot thousands → "1.234,56"
|
||||
// US locale uses dot decimal → "1234.56"
|
||||
const cleaned = currency === "EUR"
|
||||
? raw.replace(/\./g, "").replace(",", ".") // "1.234,56" → "1234.56"
|
||||
: raw.replace(/,/g, ""); // "1,234.56" → "1234.56"
|
||||
const n = parseFloat(cleaned);
|
||||
return Number.isFinite(n) && n > 0 ? n : null;
|
||||
}
|
||||
|
||||
/** Extract quantity from availability string like "117 auf Lager" or "117 In Stock" */
|
||||
function parseAvailability(avail: string): { qty: number; stockLevel: string } {
|
||||
if (!avail) return { qty: 0, stockLevel: "out_of_stock" };
|
||||
const lower = avail.toLowerCase();
|
||||
|
||||
// Check for discontinued / not available
|
||||
if (lower.includes("nicht verfügbar") || lower.includes("not available") || lower.includes("obsolete")) {
|
||||
return { qty: 0, stockLevel: "discontinued" };
|
||||
}
|
||||
|
||||
// Extract number
|
||||
const match = avail.match(/(\d[\d,.]*)(?:\s|$)/);
|
||||
const qty = match ? parseInt(match[1].replace(/[,.]/, ""), 10) : 0;
|
||||
|
||||
if (qty === 0) return { qty: 0, stockLevel: "out_of_stock" };
|
||||
if (qty < 10) return { qty, stockLevel: "low_stock" };
|
||||
return { qty, stockLevel: "in_stock" };
|
||||
}
|
||||
|
||||
/** Get the 1-unit price (or lowest break price) in EUR */
|
||||
function extractPrice(part: MouserPart): { price: number; currency: string } | null {
|
||||
const breaks = part.PriceBreaks;
|
||||
if (!breaks || breaks.length === 0) return null;
|
||||
|
||||
// Sort by quantity ascending, take qty=1 or first available
|
||||
const sorted = [...breaks].sort((a, b) => a.Quantity - b.Quantity);
|
||||
const first = sorted[0];
|
||||
if (!first) return null;
|
||||
|
||||
const currency = (first.Currency ?? "EUR").toUpperCase();
|
||||
const price = parseMouserPrice(first.Price, currency);
|
||||
if (price === null) return null;
|
||||
|
||||
return { price, currency };
|
||||
}
|
||||
|
||||
// ── API call ──────────────────────────────────────────────────────────────────
|
||||
|
||||
async function searchMouser(partNumber: string): Promise<MouserPart | null> {
|
||||
if (!MOUSER_API_KEY) return null;
|
||||
|
||||
const url = `${MOUSER_API_BASE}/search/keyword?apiKey=${MOUSER_API_KEY}&langId=1&searchWithSapnningRows=false`;
|
||||
|
||||
let resp: Response;
|
||||
try {
|
||||
resp = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Accept: "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
SearchByKeywordRequest: {
|
||||
keyword: partNumber,
|
||||
records: 5,
|
||||
startingRecord: 0,
|
||||
searchOptions: "1", // Exact match preferred
|
||||
searchWithSapnningRows: false,
|
||||
},
|
||||
}),
|
||||
signal: AbortSignal.timeout(15_000),
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
console.warn(` [Mouser] Fetch error for ${partNumber}: ${(err as Error).message.slice(0, 60)}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!resp.ok) {
|
||||
if (resp.status === 429) {
|
||||
console.warn(` [Mouser] Rate limited — backing off 30s`);
|
||||
await sleep(30_000);
|
||||
} else {
|
||||
console.warn(` [Mouser] HTTP ${resp.status} for ${partNumber}`);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
const data = (await resp.json()) as MouserSearchResponse;
|
||||
|
||||
if (data.Errors && data.Errors.length > 0) {
|
||||
const errMsg = data.Errors.map((e) => e.Message).join("; ");
|
||||
console.warn(` [Mouser] API error for ${partNumber}: ${errMsg.slice(0, 80)}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
const parts = data.SearchResults?.Parts ?? [];
|
||||
if (parts.length === 0) return null;
|
||||
|
||||
const norm = partNumber.toUpperCase().trim();
|
||||
|
||||
// Prefer exact MPN match
|
||||
const exact = parts.find((p) => (p.ManufacturerPartNumber ?? "").toUpperCase().trim() === norm);
|
||||
return exact ?? parts[0] ?? null;
|
||||
}
|
||||
|
||||
// ── Main ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
export async function scrapeDigikey(): Promise<void> {
|
||||
console.log("=== OEM Reference Price Scraper (Mouser Electronics API) ===\n");
|
||||
|
||||
if (!MOUSER_API_KEY) {
|
||||
console.error(
|
||||
" ERROR: MOUSER_API_KEY not set.\n" +
|
||||
" Register free at https://www.mouser.com/api-hub/ → get API key → set env var.\n" +
|
||||
" Free tier: 1000 queries/month — enough for 475 Juniper PIDs."
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Register Mouser as a vendor (distributor = reseller type)
|
||||
const vendorId = await ensureVendor(
|
||||
"Mouser Electronics",
|
||||
"reseller",
|
||||
"https://www.mouser.de",
|
||||
"https://www.mouser.de/Search/Refine?Keyword="
|
||||
);
|
||||
console.log(` Vendor ID: ${vendorId}`);
|
||||
|
||||
// Load OEM transceiver PIDs
|
||||
const TARGET_VENDORS = ["Juniper Networks", "Cisco Systems", "Arista Networks", "FS.COM", "SmartOptics"];
|
||||
const { rows: transceivers } = await pool.query<{
|
||||
id: string;
|
||||
part_number: string;
|
||||
form_factor: string;
|
||||
speed: string;
|
||||
vendor_name: string;
|
||||
}>(
|
||||
`SELECT t.id, t.part_number, t.form_factor, t.speed, v.name AS vendor_name
|
||||
FROM transceivers t
|
||||
JOIN vendors v ON v.id = t.vendor_id
|
||||
WHERE v.name = ANY($1)
|
||||
AND t.part_number IS NOT NULL
|
||||
AND t.part_number NOT ILIKE '%Transceiver%'
|
||||
AND t.part_number NOT ILIKE '%-Transceivers'
|
||||
AND LENGTH(t.part_number) BETWEEN 4 AND 35
|
||||
ORDER BY v.name, t.part_number`,
|
||||
[TARGET_VENDORS]
|
||||
);
|
||||
|
||||
console.log(` Found ${transceivers.length} OEM PIDs to price-check\n`);
|
||||
|
||||
let found = 0;
|
||||
let notFound = 0;
|
||||
let errors = 0;
|
||||
|
||||
for (let i = 0; i < transceivers.length; i++) {
|
||||
const tx = transceivers[i];
|
||||
|
||||
if (i > 0 && i % 20 === 0) {
|
||||
console.log(
|
||||
` [${i}/${transceivers.length}] found=${found} not_found=${notFound} errors=${errors}`
|
||||
);
|
||||
}
|
||||
|
||||
const part = await searchMouser(tx.part_number);
|
||||
|
||||
if (!part) {
|
||||
notFound++;
|
||||
await sleep(DELAY_MS);
|
||||
continue;
|
||||
}
|
||||
|
||||
const priceData = extractPrice(part);
|
||||
if (!priceData) {
|
||||
notFound++;
|
||||
await sleep(DELAY_MS);
|
||||
continue;
|
||||
}
|
||||
|
||||
const { qty, stockLevel } = parseAvailability(part.Availability);
|
||||
const productUrl = part.ProductDetailUrl
|
||||
? `https://www.mouser.de${part.ProductDetailUrl}`
|
||||
: `https://www.mouser.de/Search/Refine?Keyword=${encodeURIComponent(tx.part_number)}`;
|
||||
|
||||
const hash = contentHash(
|
||||
`mouser:${tx.id}:${priceData.price}:${priceData.currency}:${stockLevel}`
|
||||
);
|
||||
|
||||
try {
|
||||
await upsertPriceObservation({
|
||||
transceiverId: tx.id,
|
||||
sourceVendorId: vendorId,
|
||||
price: priceData.price,
|
||||
currency: priceData.currency,
|
||||
stockLevel,
|
||||
quantityAvailable: qty,
|
||||
url: productUrl,
|
||||
contentHash: hash,
|
||||
});
|
||||
found++;
|
||||
console.log(
|
||||
` ✓ ${tx.part_number.padEnd(32)} ${priceData.currency} ${priceData.price.toFixed(2).padStart(9)} ${stockLevel.padEnd(13)} qty=${qty}`
|
||||
);
|
||||
} catch (err: unknown) {
|
||||
errors++;
|
||||
console.warn(
|
||||
` ✗ DB error ${tx.part_number}: ${(err as Error).message.slice(0, 60)}`
|
||||
);
|
||||
}
|
||||
|
||||
await sleep(DELAY_MS);
|
||||
}
|
||||
|
||||
console.log(`\n=== Mouser OEM Scraper Complete ===`);
|
||||
console.log(` Processed: ${transceivers.length}`);
|
||||
console.log(` Found: ${found}`);
|
||||
console.log(` Not found: ${notFound}`);
|
||||
console.log(` DB errors: ${errors}\n`);
|
||||
}
|
||||
|
||||
// ── CLI ───────────────────────────────────────────────────────────────────────
|
||||
|
||||
if (require.main === module) {
|
||||
scrapeDigikey()
|
||||
.then(() => pool.end())
|
||||
.catch((err: unknown) => {
|
||||
console.error("Fatal:", err);
|
||||
pool.end();
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user