feat(naddod-scraper): extract per-warehouse stock breakdown + write warehouse_global_qty
Adds parseWarehouseStock() to decode the HTML-entity-encoded warehouse_stock JSON (us/nl/sg/cn per-region array). When the static page has warehouse data, writes: warehouse_de_qty ← nl (EU-closest warehouse) warehouse_global_qty ← sum(us+nl+sg+cn), or falls back to quantity_available stock_confidence ← 3 (L3) when warehouse breakdown available, else 2 Note: per-warehouse quantities require JS execution to populate (API-loaded); static HTML has [0,0] placeholders. The fallback ensures NADDOD global totals appear in the competitor-by-tech dashboard comparison.
This commit is contained in:
parent
9bf7da3fda
commit
03fdfa7d51
@ -193,6 +193,45 @@ function parseStockText(html: string): { qty?: number; confidence: 1 | 2 } | nul
|
||||
return { confidence: 1 }; // fallback: boolean
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Parse per-warehouse stock breakdown from NADDOD HTML.
|
||||
* The data lives as HTML-entity-encoded JSON in a hydration payload:
|
||||
* "warehouse_stock":[0,{"us":[0,543],"nl":[0,211],"sg":[0,0],"cn":[0,0]}]
|
||||
* Format per region: [signalBit, quantity]
|
||||
* Mapping: us → US, nl → NL/EU (closest to DE), sg → APAC, cn → CN
|
||||
* We use nl as DE-equivalent (EU warehouse) and sum all for global.
|
||||
*/
|
||||
function parseWarehouseStock(html: string): {
|
||||
eu: number | null; // nl warehouse → DE-equivalent
|
||||
global: number | null; // us+nl+sg+cn total
|
||||
} | null {
|
||||
// Strip HTML entities so we can JSON.parse
|
||||
const decoded = html
|
||||
.replace(/"/g, '"')
|
||||
.replace(/&/g, '&')
|
||||
.replace(/'/g, "'");
|
||||
|
||||
const m = decoded.match(/"warehouse_stock":\[0,(\{[^}]+\})\]/);
|
||||
if (!m) return null;
|
||||
|
||||
try {
|
||||
const ws = JSON.parse(m[1]) as Record<string, unknown>;
|
||||
function qty(key: string): number {
|
||||
const v = ws[key];
|
||||
if (Array.isArray(v) && v.length >= 2 && typeof v[1] === 'number') return Math.max(0, v[1]);
|
||||
return 0;
|
||||
}
|
||||
const us = qty('us'), nl = qty('nl'), sg = qty('sg'), cn = qty('cn');
|
||||
const total = us + nl + sg + cn;
|
||||
// Only return data if at least one warehouse has stock
|
||||
if (total === 0 && nl === 0) return null; // no warehouse breakdown available
|
||||
return { eu: nl, global: total };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ── HTTP helpers ────────────────────────────────────────────────────────────
|
||||
|
||||
async function fetchText(url: string): Promise<string> {
|
||||
@ -458,16 +497,20 @@ export async function scrapeNaddod(): Promise<void> {
|
||||
if (isNew) priceUpdates++;
|
||||
}
|
||||
|
||||
// Stock observation
|
||||
// Stock observation — enhanced with per-warehouse breakdown
|
||||
if (stock !== null) {
|
||||
const stockLevel = stock.qty !== undefined ? (stock.qty > 0 ? "in_stock" : "out_of_stock") : "in_stock";
|
||||
const warehouseData = parseWarehouseStock(html);
|
||||
const isNew = await upsertStockObservation({
|
||||
transceiverId: txId,
|
||||
sourceVendorId: vendorId,
|
||||
stockLevel,
|
||||
quantityAvailable: stock.qty !== undefined && stock.qty > 0 ? stock.qty : undefined,
|
||||
// NL warehouse ≈ EU/DE equivalent; sum of all warehouses = global
|
||||
warehouseDeQty: warehouseData?.eu ?? undefined,
|
||||
warehouseGlobalQty: warehouseData?.global ?? (stock.qty !== undefined && stock.qty > 0 ? stock.qty : undefined),
|
||||
productUrl: url,
|
||||
stockConfidence: stock.confidence,
|
||||
stockConfidence: warehouseData ? 3 : stock.confidence, // L3 when per-warehouse available
|
||||
priceCurrency: "USD",
|
||||
priceIncludesTax: false,
|
||||
});
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user