feat(naddod-scraper): extract per-warehouse stock breakdown + write warehouse_global_qty

Adds parseWarehouseStock() to decode the HTML-entity-encoded warehouse_stock JSON
(us/nl/sg/cn per-region array). When the static page has warehouse data, writes:
  warehouse_de_qty   ← nl (EU-closest warehouse)
  warehouse_global_qty ← sum(us+nl+sg+cn), or falls back to quantity_available
  stock_confidence   ← 3 (L3) when warehouse breakdown available, else 2

Note: per-warehouse quantities require JS execution to populate (API-loaded);
static HTML has [0,0] placeholders. The fallback ensures NADDOD global totals
appear in the competitor-by-tech dashboard comparison.
This commit is contained in:
Rene Fichtmueller 2026-06-06 16:34:02 +00:00
parent 9bf7da3fda
commit 03fdfa7d51

View File

@ -193,6 +193,45 @@ function parseStockText(html: string): { qty?: number; confidence: 1 | 2 } | nul
return { confidence: 1 }; // fallback: boolean
}
/**
* Parse per-warehouse stock breakdown from NADDOD HTML.
* The data lives as HTML-entity-encoded JSON in a hydration payload:
* "warehouse_stock":[0,{"us":[0,543],"nl":[0,211],"sg":[0,0],"cn":[0,0]}]
* Format per region: [signalBit, quantity]
* Mapping: us US, nl NL/EU (closest to DE), sg APAC, cn CN
* We use nl as DE-equivalent (EU warehouse) and sum all for global.
*/
function parseWarehouseStock(html: string): {
eu: number | null; // nl warehouse → DE-equivalent
global: number | null; // us+nl+sg+cn total
} | null {
// Strip HTML entities so we can JSON.parse
const decoded = html
.replace(/"/g, '"')
.replace(/&/g, '&')
.replace(/'/g, "'");
const m = decoded.match(/"warehouse_stock":\[0,(\{[^}]+\})\]/);
if (!m) return null;
try {
const ws = JSON.parse(m[1]) as Record<string, unknown>;
function qty(key: string): number {
const v = ws[key];
if (Array.isArray(v) && v.length >= 2 && typeof v[1] === 'number') return Math.max(0, v[1]);
return 0;
}
const us = qty('us'), nl = qty('nl'), sg = qty('sg'), cn = qty('cn');
const total = us + nl + sg + cn;
// Only return data if at least one warehouse has stock
if (total === 0 && nl === 0) return null; // no warehouse breakdown available
return { eu: nl, global: total };
} catch {
return null;
}
}
// ── HTTP helpers ────────────────────────────────────────────────────────────
async function fetchText(url: string): Promise<string> {
@ -458,16 +497,20 @@ export async function scrapeNaddod(): Promise<void> {
if (isNew) priceUpdates++;
}
// Stock observation
// Stock observation — enhanced with per-warehouse breakdown
if (stock !== null) {
const stockLevel = stock.qty !== undefined ? (stock.qty > 0 ? "in_stock" : "out_of_stock") : "in_stock";
const warehouseData = parseWarehouseStock(html);
const isNew = await upsertStockObservation({
transceiverId: txId,
sourceVendorId: vendorId,
stockLevel,
quantityAvailable: stock.qty !== undefined && stock.qty > 0 ? stock.qty : undefined,
// NL warehouse ≈ EU/DE equivalent; sum of all warehouses = global
warehouseDeQty: warehouseData?.eu ?? undefined,
warehouseGlobalQty: warehouseData?.global ?? (stock.qty !== undefined && stock.qty > 0 ? stock.qty : undefined),
productUrl: url,
stockConfidence: stock.confidence,
stockConfidence: warehouseData ? 3 : stock.confidence, // L3 when per-warehouse available
priceCurrency: "USD",
priceIncludesTax: false,
});