feat(naddod-scraper): extract per-warehouse stock breakdown + write warehouse_global_qty
Adds parseWarehouseStock() to decode the HTML-entity-encoded warehouse_stock JSON (us/nl/sg/cn per-region array). When the static page has warehouse data, writes: warehouse_de_qty ← nl (EU-closest warehouse) warehouse_global_qty ← sum(us+nl+sg+cn), or falls back to quantity_available stock_confidence ← 3 (L3) when warehouse breakdown available, else 2 Note: per-warehouse quantities require JS execution to populate (API-loaded); static HTML has [0,0] placeholders. The fallback ensures NADDOD global totals appear in the competitor-by-tech dashboard comparison.
This commit is contained in:
parent
9bf7da3fda
commit
03fdfa7d51
@ -193,6 +193,45 @@ function parseStockText(html: string): { qty?: number; confidence: 1 | 2 } | nul
|
|||||||
return { confidence: 1 }; // fallback: boolean
|
return { confidence: 1 }; // fallback: boolean
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse per-warehouse stock breakdown from NADDOD HTML.
|
||||||
|
* The data lives as HTML-entity-encoded JSON in a hydration payload:
|
||||||
|
* "warehouse_stock":[0,{"us":[0,543],"nl":[0,211],"sg":[0,0],"cn":[0,0]}]
|
||||||
|
* Format per region: [signalBit, quantity]
|
||||||
|
* Mapping: us → US, nl → NL/EU (closest to DE), sg → APAC, cn → CN
|
||||||
|
* We use nl as DE-equivalent (EU warehouse) and sum all for global.
|
||||||
|
*/
|
||||||
|
function parseWarehouseStock(html: string): {
|
||||||
|
eu: number | null; // nl warehouse → DE-equivalent
|
||||||
|
global: number | null; // us+nl+sg+cn total
|
||||||
|
} | null {
|
||||||
|
// Strip HTML entities so we can JSON.parse
|
||||||
|
const decoded = html
|
||||||
|
.replace(/"/g, '"')
|
||||||
|
.replace(/&/g, '&')
|
||||||
|
.replace(/'/g, "'");
|
||||||
|
|
||||||
|
const m = decoded.match(/"warehouse_stock":\[0,(\{[^}]+\})\]/);
|
||||||
|
if (!m) return null;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const ws = JSON.parse(m[1]) as Record<string, unknown>;
|
||||||
|
function qty(key: string): number {
|
||||||
|
const v = ws[key];
|
||||||
|
if (Array.isArray(v) && v.length >= 2 && typeof v[1] === 'number') return Math.max(0, v[1]);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
const us = qty('us'), nl = qty('nl'), sg = qty('sg'), cn = qty('cn');
|
||||||
|
const total = us + nl + sg + cn;
|
||||||
|
// Only return data if at least one warehouse has stock
|
||||||
|
if (total === 0 && nl === 0) return null; // no warehouse breakdown available
|
||||||
|
return { eu: nl, global: total };
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ── HTTP helpers ────────────────────────────────────────────────────────────
|
// ── HTTP helpers ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
async function fetchText(url: string): Promise<string> {
|
async function fetchText(url: string): Promise<string> {
|
||||||
@ -458,16 +497,20 @@ export async function scrapeNaddod(): Promise<void> {
|
|||||||
if (isNew) priceUpdates++;
|
if (isNew) priceUpdates++;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stock observation
|
// Stock observation — enhanced with per-warehouse breakdown
|
||||||
if (stock !== null) {
|
if (stock !== null) {
|
||||||
const stockLevel = stock.qty !== undefined ? (stock.qty > 0 ? "in_stock" : "out_of_stock") : "in_stock";
|
const stockLevel = stock.qty !== undefined ? (stock.qty > 0 ? "in_stock" : "out_of_stock") : "in_stock";
|
||||||
|
const warehouseData = parseWarehouseStock(html);
|
||||||
const isNew = await upsertStockObservation({
|
const isNew = await upsertStockObservation({
|
||||||
transceiverId: txId,
|
transceiverId: txId,
|
||||||
sourceVendorId: vendorId,
|
sourceVendorId: vendorId,
|
||||||
stockLevel,
|
stockLevel,
|
||||||
quantityAvailable: stock.qty !== undefined && stock.qty > 0 ? stock.qty : undefined,
|
quantityAvailable: stock.qty !== undefined && stock.qty > 0 ? stock.qty : undefined,
|
||||||
|
// NL warehouse ≈ EU/DE equivalent; sum of all warehouses = global
|
||||||
|
warehouseDeQty: warehouseData?.eu ?? undefined,
|
||||||
|
warehouseGlobalQty: warehouseData?.global ?? (stock.qty !== undefined && stock.qty > 0 ? stock.qty : undefined),
|
||||||
productUrl: url,
|
productUrl: url,
|
||||||
stockConfidence: stock.confidence,
|
stockConfidence: warehouseData ? 3 : stock.confidence, // L3 when per-warehouse available
|
||||||
priceCurrency: "USD",
|
priceCurrency: "USD",
|
||||||
priceIncludesTax: false,
|
priceIncludesTax: false,
|
||||||
});
|
});
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user