From 4299e24e13f458e7c298c6e8dbf0696cb5205d59 Mon Sep 17 00:00:00 2001 From: Rene Fichtmueller Date: Sat, 4 Jul 2026 22:37:05 +0200 Subject: [PATCH] fix(scraper): parse FS.com delivery dates + lead_time; fix units_sold K/M inflation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FS.com renders per-warehouse availability as a bare word-date on each line ("2 Stk. im DE-Lager, 8. Juli 2026 Lieferbar"). The old regex only matched a "Lieferung:"/"erwartet:" prefix or numeric dd.mm.yyyy, so warehouse_de/global delivery_date + backorder_estimated_date were 0% populated across 85k rows, and lead_time_days was never written at all. - new fs-com-parse.ts: pure, unit-tested parsers (13 node:test cases). - extractDeliveryDate finds the bare word/numeric date per warehouse, bounded to its own line so a date never bleeds to a neighbouring warehouse. - computeLeadTimeDays = min(delivery_date - today), floored at 0; threaded into upsertStockObservation (now writes lead_time_days) and upsertPriceObservation. - parseGermanQty: K/M abbreviations now treat '.' as a decimal ("4.8K"=4800, "1.4M"=1.4M) instead of stripping it (was "1.4M"->14,000,000 — the outlier). - units_sold sanity gate at ingest (FS_MAX_UNITS_SOLD, default 2,000,000). - sql/122: purge impossible historical units_sold, re-document columns as collected. --- .../scraper/src/scrapers/fs-com-parse.test.ts | 138 +++++++++ packages/scraper/src/scrapers/fs-com-parse.ts | 290 ++++++++++++++++++ packages/scraper/src/scrapers/fs-com.ts | 164 ++-------- packages/scraper/src/utils/db.ts | 9 +- packages/scraper/tsconfig.json | 2 +- ...com-delivery-dates-and-units-sold-gate.sql | 54 ++++ 6 files changed, 518 insertions(+), 139 deletions(-) create mode 100644 packages/scraper/src/scrapers/fs-com-parse.test.ts create mode 100644 packages/scraper/src/scrapers/fs-com-parse.ts create mode 100644 sql/122-fs-com-delivery-dates-and-units-sold-gate.sql diff --git a/packages/scraper/src/scrapers/fs-com-parse.test.ts b/packages/scraper/src/scrapers/fs-com-parse.test.ts new file mode 100644 index 0000000..03947a4 --- /dev/null +++ b/packages/scraper/src/scrapers/fs-com-parse.test.ts @@ -0,0 +1,138 @@ +/** + * Unit tests for the FS.com pure parsers. Run with: + * npx tsx --test src/scrapers/fs-com-parse.test.ts + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + parseGermanQty, + parseGermanDate, + parseGermanPrice, + extractDeliveryDate, + parseWarehouseStock, + computeLeadTimeDays, + sanitizeUnitsSold, +} from "./fs-com-parse"; + +// ── parseGermanQty — the K/M decimal bug ───────────────────────────────────── + +test("parseGermanQty: plain integers use '.'/',' as thousands", () => { + assert.equal(parseGermanQty("4.895"), 4895); + assert.equal(parseGermanQty("1.587"), 1587); + assert.equal(parseGermanQty("1.501"), 1501); + assert.equal(parseGermanQty("2"), 2); + assert.equal(parseGermanQty("1627"), 1627); +}); + +test("parseGermanQty: K/M abbreviations treat '.' as a DECIMAL (bug fix)", () => { + // Previously "4.8K" -> 48000 and "1.4M" -> 14000000 (the garbage outlier). + assert.equal(parseGermanQty("4.8K"), 4800); + assert.equal(parseGermanQty("210.9K"), 210900); + assert.equal(parseGermanQty("504.4K"), 504400); + assert.equal(parseGermanQty("1.4M"), 1_400_000); + assert.equal(parseGermanQty("1.2M"), 1_200_000); + assert.equal(parseGermanQty("63K"), 63000); // no decimal — unchanged + assert.equal(parseGermanQty("14M"), 14_000_000); // caught later by the gate +}); + +test("parseGermanQty: K/M with German comma decimal still works", () => { + assert.equal(parseGermanQty("4,8K"), 4800); + assert.equal(parseGermanQty("1,4M"), 1_400_000); +}); + +// ── parseGermanDate ────────────────────────────────────────────────────────── + +test("parseGermanDate: word and numeric forms", () => { + assert.equal(parseGermanDate("8. Juli 2026"), "2026-07-08"); + assert.equal(parseGermanDate("17. Juli 2026"), "2026-07-17"); + assert.equal(parseGermanDate("20 Apr., 2026"), "2026-04-20"); + assert.equal(parseGermanDate("28. März 2026"), "2026-03-28"); + assert.equal(parseGermanDate("20.04.2026"), "2026-04-20"); + assert.equal(parseGermanDate("nonsense"), undefined); +}); + +// ── extractDeliveryDate ────────────────────────────────────────────────────── + +test("extractDeliveryDate: finds bare word-date, no keyword needed", () => { + assert.equal(extractDeliveryDate("2 Stk. im DE-Lager, 8. Juli 2026 Lieferbar"), "2026-07-08"); + assert.equal(extractDeliveryDate("1.627 Stk. im Global-Lager, 17. Juli 2026"), "2026-07-17"); +}); + +test("extractDeliveryDate: rejects stray/garbage years and empty", () => { + assert.equal(extractDeliveryDate("© 2019 FS.COM"), undefined); + assert.equal(extractDeliveryDate("auf Lager, sofort verfügbar"), undefined); +}); + +// ── parseWarehouseStock — the real screenshot block ────────────────────────── + +const SCREENSHOT_BLOCK = [ + "2 Stk. im DE-Lager, 8. Juli 2026 Lieferbar", + "1.627 Stk. im Global-Lager, 17. Juli 2026", + "1 in Nachlieferung, 28. Juli 2026", + "4.8K verkauft", +].join("\n"); + +test("parseWarehouseStock: full block yields qtys + per-warehouse dates + sold", () => { + const s = parseWarehouseStock(SCREENSHOT_BLOCK); + assert.equal(s.deQty, 2); + assert.equal(s.deDeliveryDate, "2026-07-08"); + assert.equal(s.globalQty, 1627); + assert.equal(s.globalDeliveryDate, "2026-07-17"); + assert.equal(s.backorderQty, 1); + assert.equal(s.backorderDate, "2026-07-28"); + assert.equal(s.unitsSold, 4800); // 4.8K, correctly de-abbreviated +}); + +test("parseWarehouseStock: a warehouse with no date does not steal a neighbour's date", () => { + const block = [ + "5 Stk. im DE-Lager", // no date — available now + "800 Stk. im Global-Lager, 17. Juli 2026", + ].join("\n"); + const s = parseWarehouseStock(block); + assert.equal(s.deQty, 5); + assert.equal(s.deDeliveryDate, undefined); // must NOT be 2026-07-17 + assert.equal(s.globalQty, 800); + assert.equal(s.globalDeliveryDate, "2026-07-17"); +}); + +test("parseWarehouseStock: gates the 1.4M -> 14M units_sold garbage", () => { + const block = "12 Stk. im DE-Lager, 8. Juli 2026 Lieferbar\n14M verkauft"; + const s = parseWarehouseStock(block); + assert.equal(s.deQty, 12); + assert.equal(s.unitsSold, undefined); // 14M > cap → dropped as unknown +}); + +test("parseWarehouseStock: empty when no warehouse text present", () => { + assert.deepEqual(parseWarehouseStock("Just a product description, nothing else."), {}); +}); + +// ── sanitizeUnitsSold ──────────────────────────────────────────────────────── + +test("sanitizeUnitsSold: keeps plausible, drops implausible/negative", () => { + assert.equal(sanitizeUnitsSold(4800), 4800); + assert.equal(sanitizeUnitsSold(504400), 504400); + assert.equal(sanitizeUnitsSold(14_000_000), undefined); + assert.equal(sanitizeUnitsSold(-5), undefined); + assert.equal(sanitizeUnitsSold(undefined), undefined); + assert.equal(sanitizeUnitsSold(2_500_000, 2_000_000), undefined); + assert.equal(sanitizeUnitsSold(1_500_000, 2_000_000), 1_500_000); +}); + +// ── computeLeadTimeDays ────────────────────────────────────────────────────── + +test("computeLeadTimeDays: min across dates, floored at 0, undefined when empty", () => { + const now = new Date("2026-07-04T09:00:00"); + assert.equal(computeLeadTimeDays(["2026-07-08", "2026-07-17", "2026-07-28"], now), 4); + assert.equal(computeLeadTimeDays([undefined, null, "2026-07-05"], now), 1); + assert.equal(computeLeadTimeDays(["2026-07-01"], now), 0); // past date → 0, not negative + assert.equal(computeLeadTimeDays([], now), undefined); + assert.equal(computeLeadTimeDays([null, undefined], now), undefined); +}); + +// ── parseGermanPrice (unchanged behaviour, regression guard) ───────────────── + +test("parseGermanPrice: net EUR parsing", () => { + assert.equal(parseGermanPrice("42,50"), 42.5); + assert.equal(parseGermanPrice("1.063,02 €"), 1063.02); + assert.equal(parseGermanPrice("5,10 € ohne MwSt."), 5.1); +}); diff --git a/packages/scraper/src/scrapers/fs-com-parse.ts b/packages/scraper/src/scrapers/fs-com-parse.ts new file mode 100644 index 0000000..a2d6b81 --- /dev/null +++ b/packages/scraper/src/scrapers/fs-com-parse.ts @@ -0,0 +1,290 @@ +/** + * FS.com parsing helpers — pure, side-effect-free, unit-tested. + * + * Extracted from fs-com.ts so the number/date/warehouse parsing can be tested + * in isolation (fs-com.ts pulls in Crawlee + Playwright + a pg Pool, which we + * do not want to load in a unit test). + * + * The FS.com German storefront (www.fs.com/de/) renders per-warehouse + * availability as a short line per warehouse, e.g.: + * + * 2 Stk. im DE-Lager, 8. Juli 2026 Lieferbar + * 1.627 Stk. im Global-Lager, 17. Juli 2026 + * 1 in Nachlieferung, 28. Juli 2026 + * 4.8K verkauft + * + * The quantity part was already parsed correctly; the delivery-date part was + * not, because the previous regex only looked for a "Lieferung:"/"erwartet:" + * keyword prefix or a numeric dd.mm.yyyy date — neither of which matches the + * bare word-date ("8. Juli 2026") that FS.com actually renders. + */ + +// ── Constants ──────────────────────────────────────────────────────────────── + +const GERMAN_MONTHS: Record = { + jan: "01", feb: "02", mär: "03", mar: "03", + apr: "04", mai: "05", may: "05", jun: "06", + jul: "07", aug: "08", sep: "09", okt: "10", + oct: "10", nov: "11", dez: "12", dec: "12", +}; + +/** + * Upper bound for a plausible per-SKU lifetime "verkauft" (units-sold) counter. + * Values above this are treated as unknown (dropped), not stored. FS.com's + * biggest sellers are in the hundreds of thousands; multi-million values are + * a mis-parse (see parseGermanQty K/M bug) or vendor marketing inflation. + * Override with FS_MAX_UNITS_SOLD. + */ +export const DEFAULT_MAX_UNITS_SOLD = 2_000_000; + +/** How far past a warehouse line to look for its delivery date (characters). */ +const DATE_WINDOW = 200; + +// ── Number parsing ─────────────────────────────────────────────────────────── + +/** + * Parse the mantissa of a K/M-abbreviated number ("4.8" in "4.8K"). + * + * K/M abbreviations carry a small decimal, never a grouped thousands value, so + * a lone "." is the decimal point ("4.8K" = 4 800). German pages may instead + * use "," as the decimal ("4,8K"); a "." alongside a "," is then a thousands + * grouping and is stripped. + */ +function parseAbbrevMantissa(s: string): number { + if (s.includes(",")) return parseFloat(s.replace(/\./g, "").replace(",", ".")); + return parseFloat(s); // "." is the decimal separator +} + +/** + * Parse a German-formatted quantity string. + * "4.895" → 4895 (period = thousands separator in a plain integer) + * "1.587" → 1587 + * "4.8K" → 4800 (period = decimal in a K/M abbreviation) + * "210.9K" → 210900 + * "1.4M" → 1400000 + * "63K" → 63000 + */ +export function parseGermanQty(text: string): number | undefined { + const t = text.trim().replace(/\s/g, ""); + if (!t) return undefined; + + const kMatch = t.match(/^([\d.,]+)[Kk]$/); + if (kMatch) { + const n = parseAbbrevMantissa(kMatch[1]); + return isNaN(n) ? undefined : Math.round(n * 1_000); + } + + const mMatch = t.match(/^([\d.,]+)[Mm]$/); + if (mMatch) { + const n = parseAbbrevMantissa(mMatch[1]); + return isNaN(n) ? undefined : Math.round(n * 1_000_000); + } + + // Plain integer: "." and "," are both thousands separators here. + const n = parseInt(t.replace(/\./g, "").replace(/,/g, ""), 10); + return isNaN(n) ? undefined : n; +} + +/** + * Parse a German date to ISO "YYYY-MM-DD". + * "8. Juli 2026" → "2026-07-08" + * "20 Apr., 2026" → "2026-04-20" + * "20.04.2026" → "2026-04-20" + */ +export function parseGermanDate(text: string): string | undefined { + const numericMatch = text.match(/(\d{1,2})\.(\d{1,2})\.(\d{4})/); + if (numericMatch) { + const [, d, m, y] = numericMatch; + return `${y}-${m.padStart(2, "0")}-${d.padStart(2, "0")}`; + } + const wordMatch = text.match(/(\d{1,2})\.?\s+([A-Za-zÄÖÜäöüß]+)\.?,?\s*(\d{4})/); + if (!wordMatch) return undefined; + const day = wordMatch[1].padStart(2, "0"); + const monthRaw = wordMatch[2] + .toLowerCase() + .replace(/ä/g, "a").replace(/ö/g, "o").replace(/ü/g, "u") + .slice(0, 3); + const month = GERMAN_MONTHS[monthRaw]; + if (!month) return undefined; + return `${wordMatch[3]}-${month}-${day}`; +} + +/** + * Parse a German price to a EUR float. + * "42,50" → 42.50 + * "1.063,02" → 1063.02 + */ +export function parseGermanPrice(raw: string): number | undefined { + const cleaned = raw.replace(/[^0-9.,]/g, "").trim(); + if (!cleaned) return undefined; + let normalized: string; + if (/\d+\.\d{3},\d{2}/.test(cleaned)) { + normalized = cleaned.replace(/\./g, "").replace(",", "."); + } else if (cleaned.includes(",")) { + normalized = cleaned.replace(",", "."); + } else { + normalized = cleaned; + } + const n = parseFloat(normalized); + return isNaN(n) || n <= 0 ? undefined : n; +} + +// ── Units-sold sanity gate ─────────────────────────────────────────────────── + +/** + * Clamp an implausible units-sold value to `undefined` (unknown). Returns the + * value unchanged when it is a finite, non-negative number at or below `cap`. + */ +export function sanitizeUnitsSold( + raw: number | undefined, + cap: number = DEFAULT_MAX_UNITS_SOLD +): number | undefined { + if (raw === undefined) return undefined; + if (!Number.isFinite(raw) || raw < 0) return undefined; + if (raw > cap) return undefined; + return raw; +} + +// ── Delivery-date extraction ───────────────────────────────────────────────── + +/** dd.mm.yyyy OR "8. Juli 2026" / "20 Apr., 2026" (day + month word + year). */ +const DELIVERY_DATE_RE = + /(\d{1,2}\.\d{1,2}\.\d{4})|(\d{1,2}\.?\s+[A-Za-zÄÖÜäöüß]{3,}\.?,?\s*\d{4})/; + +/** + * Find the first plausible calendar date inside a bounded text block (the slice + * that belongs to one warehouse line) and return it as ISO "YYYY-MM-DD". + * The date can appear with or without a "Lieferbar"/"Lieferung" keyword — FS.com + * renders it bare ("…DE-Lager, 8. Juli 2026 Lieferbar"), so no keyword is required. + */ +export function extractDeliveryDate(block: string): string | undefined { + const m = block.match(DELIVERY_DATE_RE); + if (!m) return undefined; + const iso = parseGermanDate(m[0]); + if (!iso) return undefined; + const year = Number(iso.slice(0, 4)); + if (year < 2024 || year > 2100) return undefined; // reject stray/garbage years + return iso; +} + +// ── Warehouse block parsing ────────────────────────────────────────────────── + +export interface WarehouseStock { + deQty?: number; + deDeliveryDate?: string; + globalQty?: number; + globalDeliveryDate?: string; + backorderQty?: number; + backorderDate?: string; + unitsSold?: number; +} + +const DE_QTY_RE: RegExp[] = [ + /(\d[\d.,KkMm]*)\s*Stk\.\s*im\s*DE[- ]Lager/i, + /(\d[\d.,KkMm]*)\s*im\s*DE[- ]?Lager/i, +]; +const GLOBAL_QTY_RE: RegExp[] = [ + /(\d[\d.,KkMm]*)\s*Stk\.\s*im\s*Global[- ]Lager/i, + /(\d[\d.,KkMm]*)\s*im\s*Global[- ]?(?:Lager|Warehouse)/i, + /(\d[\d.,KkMm]*)\s*in\s+Global\s+Warehouse/i, +]; +const BACKORDER_QTY_RE: RegExp[] = [ + /(\d[\d.,KkMm]*)\s*(?:Stk\.)?\s*in\s+Nachlieferung/i, + /Nachlieferung[:\s]*(\d[\d.,KkMm]*)/i, +]; +const UNITS_SOLD_RE: RegExp[] = [ + /(\d[\d.,KkMm]*)\s*(?:[Mm]al\s+)?[Vv]erkauft/, + /([\d.,KkMm]+)\+?\s*sold/i, +]; + +interface SectionMatch { value: string; index: number; } + +function firstMatch(text: string, res: RegExp[]): SectionMatch | undefined { + for (const re of res) { + const m = text.match(re); + if (m && m[1] != null && m.index != null) return { value: m[1], index: m.index }; + } + return undefined; +} + +/** + * Parse the full warehouse-availability block out of a product page's body + * text: DE-Lager, Global-Lager and Nachlieferung quantities, each with the + * delivery date rendered on its line, plus the "verkauft"/"sold" counter. + * + * A section's delivery date is looked for only in the window between that + * section's quantity and the start of the next section, so a warehouse with no + * date never picks up a neighbouring warehouse's date. + */ +export function parseWarehouseStock( + bodyText: string, + unitsSoldCap: number = DEFAULT_MAX_UNITS_SOLD +): WarehouseStock { + const t = bodyText; + const de = firstMatch(t, DE_QTY_RE); + const gl = firstMatch(t, GLOBAL_QTY_RE); + const bo = firstMatch(t, BACKORDER_QTY_RE); + const sold = firstMatch(t, UNITS_SOLD_RE); + + const starts = [de, gl, bo, sold] + .filter((s): s is SectionMatch => s !== undefined) + .map((s) => s.index); + + const dateFor = (sec: SectionMatch | undefined): string | undefined => { + if (!sec) return undefined; + const later = starts.filter((b) => b > sec.index); + const end = later.length ? Math.min(sec.index + DATE_WINDOW, ...later) : sec.index + DATE_WINDOW; + return extractDeliveryDate(t.slice(sec.index, end)); + }; + + const result: WarehouseStock = {}; + + if (de) { + const q = parseGermanQty(de.value); + if (q !== undefined) result.deQty = q; + const d = dateFor(de); + if (d) result.deDeliveryDate = d; + } + if (gl) { + const q = parseGermanQty(gl.value); + if (q !== undefined) result.globalQty = q; + const d = dateFor(gl); + if (d) result.globalDeliveryDate = d; + } + if (bo) { + const q = parseGermanQty(bo.value); + if (q !== undefined) result.backorderQty = q; + const d = dateFor(bo); + if (d) result.backorderDate = d; + } + if (sold) { + const s = sanitizeUnitsSold(parseGermanQty(sold.value), unitsSoldCap); + if (s !== undefined) result.unitsSold = s; + } + + return result; +} + +// ── Lead-time derivation ───────────────────────────────────────────────────── + +/** + * Derive lead-time in days as the soonest availability across the warehouses + * that quote a delivery date: min(date − today), floored at 0. Returns + * `undefined` when no date is known (never fabricate a 0). + */ +export function computeLeadTimeDays( + dates: Array, + now: Date = new Date() +): number | undefined { + const todayUtc = Date.UTC(now.getFullYear(), now.getMonth(), now.getDate()); + let min: number | undefined; + for (const d of dates) { + if (!d) continue; + const m = d.match(/^(\d{4})-(\d{2})-(\d{2})$/); + if (!m) continue; + const dUtc = Date.UTC(Number(m[1]), Number(m[2]) - 1, Number(m[3])); + const days = Math.max(0, Math.round((dUtc - todayUtc) / 86_400_000)); + if (min === undefined || days < min) min = days; + } + return min; +} diff --git a/packages/scraper/src/scrapers/fs-com.ts b/packages/scraper/src/scrapers/fs-com.ts index b377c8c..30ba2e6 100644 --- a/packages/scraper/src/scrapers/fs-com.ts +++ b/packages/scraper/src/scrapers/fs-com.ts @@ -63,6 +63,12 @@ import { } from "../utils/db"; import { contentHash } from "../utils/hash"; import { updateVerifiedSpecs, parseSpecTable } from "../utils/spec-updater"; +import { + parseGermanPrice, + parseWarehouseStock, + computeLeadTimeDays, + DEFAULT_MAX_UNITS_SOLD, +} from "./fs-com-parse"; // ── Constants ────────────────────────────────────────────────────────────────── @@ -75,6 +81,8 @@ const FORCE_REVALIDATE = process.env["TIP_FORCE_REVALIDATE"] === "1"; const ONLY_MISSING_IMAGES = process.env["FS_ONLY_MISSING_IMAGES"] === "1"; const DB_DETAIL_ONLY = process.env["FS_DB_DETAIL_ONLY"] === "1"; const URL_DISCOVERY_ONLY = process.env["FS_URL_DISCOVERY_ONLY"] === "1"; +// Drop implausible "verkauft" (units-sold) counters at ingest (see fs-com-parse). +const FS_MAX_UNITS_SOLD = parseInt(process.env["FS_MAX_UNITS_SOLD"] ?? String(DEFAULT_MAX_UNITS_SOLD), 10); const PROXY_URLS = (process.env["PROXY_URLS"] ?? "") .split(",") @@ -113,83 +121,7 @@ const DE_COOKIES = [ { name: "country", value: "DE", domain: ".fs.com", path: "/" }, ]; -// ── German locale parsers ────────────────────────────────────────────────────── - -const GERMAN_MONTHS: Record = { - jan: "01", feb: "02", mär: "03", mar: "03", - apr: "04", mai: "05", may: "05", jun: "06", - jul: "07", aug: "08", sep: "09", okt: "10", - oct: "10", nov: "11", dez: "12", dec: "12", -}; - -/** - * Parse German-formatted quantity string. - * "4.895" → 4895 (period = thousands separator in German) - * "210.9K" → 210900 - * "1.2M" → 1200000 - */ -function parseGermanQty(text: string): number | undefined { - const t = text.trim().replace(/\s/g, ""); - if (!t) return undefined; - - const kMatch = t.match(/^([\d.,]+)[Kk]$/); - if (kMatch) { - const n = parseFloat(kMatch[1].replace(/\./g, "").replace(",", ".")); - return isNaN(n) ? undefined : Math.round(n * 1_000); - } - - const mMatch = t.match(/^([\d.,]+)[Mm]$/); - if (mMatch) { - const n = parseFloat(mMatch[1].replace(/\./g, "").replace(",", ".")); - return isNaN(n) ? undefined : Math.round(n * 1_000_000); - } - - const n = parseInt(t.replace(/\./g, "").replace(/,/g, ""), 10); - return isNaN(n) ? undefined : n; -} - -/** - * Parse German date to ISO "YYYY-MM-DD". - * "20 Apr., 2026" → "2026-04-20" - * "20.04.2026" → "2026-04-20" - */ -function parseGermanDate(text: string): string | undefined { - const numericMatch = text.match(/(\d{1,2})\.(\d{1,2})\.(\d{4})/); - if (numericMatch) { - const [, d, m, y] = numericMatch; - return `${y}-${m.padStart(2, "0")}-${d.padStart(2, "0")}`; - } - const wordMatch = text.match(/(\d{1,2})\.?\s+([A-Za-zÄÖÜäöüß]+)\.?,?\s*(\d{4})/); - if (!wordMatch) return undefined; - const day = wordMatch[1].padStart(2, "0"); - const monthRaw = wordMatch[2] - .toLowerCase() - .replace(/ä/g, "a").replace(/ö/g, "o").replace(/ü/g, "u") - .slice(0, 3); - const month = GERMAN_MONTHS[monthRaw]; - if (!month) return undefined; - return `${wordMatch[3]}-${month}-${day}`; -} - -/** - * Parse German price to EUR float. - * "42,50" → 42.50 - * "1.063,02" → 1063.02 - */ -function parseGermanPrice(raw: string): number | undefined { - const cleaned = raw.replace(/[^0-9.,]/g, "").trim(); - if (!cleaned) return undefined; - let normalized: string; - if (/\d+\.\d{3},\d{2}/.test(cleaned)) { - normalized = cleaned.replace(/\./g, "").replace(",", "."); - } else if (cleaned.includes(",")) { - normalized = cleaned.replace(",", "."); - } else { - normalized = cleaned; - } - const n = parseFloat(normalized); - return isNaN(n) || n <= 0 ? undefined : n; -} +// German number/date/warehouse parsers live in ./fs-com-parse (pure + unit-tested). // ── Stock level helper ───────────────────────────────────────────────────────── @@ -648,64 +580,18 @@ async function scrapeProductDetails( } } - // ── DE-Lager ─────────────────────────────────────────────────────────── - let deQty: number | undefined; - let deDeliveryDate: string | undefined; - const deM = - t.match(/(\d[\d.,KkMm]*)\s*Stk\.\s*im\s*DE[- ]Lager/i) ?? - t.match(/(\d[\d.,KkMm]*)\s*im\s*DE[- ]?Lager/i); - if (deM?.[1]) { - deQty = parseGermanQty(deM[1]); - const idx = t.indexOf(deM[0]); - const ctx = t.slice(idx, idx + 300); - const dm = - ctx.match(/Lieferung[:\s]+([0-9]{1,2}\.?\s+\w+\.?,?\s*\d{4})/i) ?? - ctx.match(/erwartet[:\s]+([0-9]{1,2}\.?\s+\w+\.?,?\s*\d{4})/i) ?? - ctx.match(/([0-9]{1,2}\.[0-9]{1,2}\.[0-9]{4})/); - if (dm?.[1]) deDeliveryDate = parseGermanDate(dm[1]); - } - - // ── Global-Lager ─────────────────────────────────────────────────────── - let globalQty: number | undefined; - let globalDeliveryDate: string | undefined; - const glM = - t.match(/(\d[\d.,KkMm]*)\s*Stk\.\s*im\s*Global[- ]Lager/i) ?? - t.match(/(\d[\d.,KkMm]*)\s*im\s*Global[- ]?(?:Lager|Warehouse)/i) ?? - t.match(/(\d[\d.,KkMm]*)\s*in\s+Global\s+Warehouse/i); - if (glM?.[1]) { - globalQty = parseGermanQty(glM[1]); - const idx = t.indexOf(glM[0]); - const ctx = t.slice(idx, idx + 300); - const dm = - ctx.match(/Lieferung[:\s]+([0-9]{1,2}\.?\s+\w+\.?,?\s*\d{4})/i) ?? - ctx.match(/erwartet[:\s]+([0-9]{1,2}\.?\s+\w+\.?,?\s*\d{4})/i) ?? - ctx.match(/([0-9]{1,2}\.[0-9]{1,2}\.[0-9]{4})/); - if (dm?.[1]) globalDeliveryDate = parseGermanDate(dm[1]); - } - - // ── Nachlieferung ────────────────────────────────────────────────────── - let backorderQty: number | undefined; - let backorderDate: string | undefined; - const boM = - t.match(/(\d[\d.,KkMm]*)\s*(?:Stk\.)?\s*in\s+Nachlieferung/i) ?? - t.match(/Nachlieferung[:\s]*(\d[\d.,KkMm]*)/i); - if (boM?.[1]) { - backorderQty = parseGermanQty(boM[1]); - const idx = t.indexOf(boM[0]); - const ctx = t.slice(idx, idx + 300); - const dm = - ctx.match(/[Ee]rwartet[:\s]+([0-9]{1,2}\.?\s+\w+\.?,?\s*\d{4})/i) ?? - ctx.match(/Lieferung[:\s]+([0-9]{1,2}\.?\s+\w+\.?,?\s*\d{4})/i) ?? - ctx.match(/([0-9]{1,2}\.[0-9]{1,2}\.[0-9]{4})/); - if (dm?.[1]) backorderDate = parseGermanDate(dm[1]); - } - - // ── Units sold ───────────────────────────────────────────────────────── - let unitsSold: number | undefined; - const soldM = - t.match(/(\d[\d.,KkMm]*)\s*(?:[Mm]al\s+)?[Vv]erkauft/) ?? - t.match(/([\d.,KkMm]+)\+?\s*sold/i); - if (soldM?.[1]) unitsSold = parseGermanQty(soldM[1]); + // ── Warehouse availability (qty + per-warehouse delivery date + units sold) ── + // FS.com renders each warehouse on its own line, e.g. + // "2 Stk. im DE-Lager, 8. Juli 2026 Lieferbar". parseWarehouseStock + // extracts the bare word-date per warehouse and gates units_sold. + const wh = parseWarehouseStock(t, FS_MAX_UNITS_SOLD); + const deQty = wh.deQty; + const deDeliveryDate = wh.deDeliveryDate; + const globalQty = wh.globalQty; + const globalDeliveryDate = wh.globalDeliveryDate; + const backorderQty = wh.backorderQty; + const backorderDate = wh.backorderDate; + const unitsSold = wh.unitsSold; // ── Part number refinement ───────────────────────────────────────────── let partNumber = listingPn; @@ -1010,6 +896,12 @@ export async function scrapeFs(): Promise { const stockLevel = deriveStockLevel(detail.deQty, detail.globalQty, detail.backorderQty); const totalQty = (detail.deQty ?? 0) + (detail.globalQty ?? 0); + // Soonest availability across the warehouses that quote a delivery date. + const leadTimeDays = computeLeadTimeDays([ + detail.deDeliveryDate, + detail.globalDeliveryDate, + detail.backorderDate, + ]); if (detail.priceNet && detail.priceNet > 0) { const hash = contentHash({ @@ -1024,6 +916,7 @@ export async function scrapeFs(): Promise { currency: "EUR", stockLevel, quantityAvailable: totalQty > 0 ? totalQty : undefined, + leadTimeDays, url: detail.url, contentHash: hash, }); @@ -1042,6 +935,7 @@ export async function scrapeFs(): Promise { backorderQty: detail.backorderQty, backorderEstimatedDate: detail.backorderDate ?? null, unitsSold: detail.unitsSold, + leadTimeDays, compatibleBrands: detail.compatibleBrands, priceNet: detail.priceNet, productUrl: detail.url, diff --git a/packages/scraper/src/utils/db.ts b/packages/scraper/src/utils/db.ts index e23cd69..c0cd439 100644 --- a/packages/scraper/src/utils/db.ts +++ b/packages/scraper/src/utils/db.ts @@ -394,6 +394,8 @@ export async function upsertStockObservation(params: { backorderQty?: number; backorderEstimatedDate?: string | null; unitsSold?: number; + /** Soonest availability in days across warehouses that quote a delivery date. */ + leadTimeDays?: number; compatibleBrands?: string[]; priceNet?: number; productUrl?: string; @@ -444,7 +446,7 @@ export async function upsertStockObservation(params: { warehouse_de_qty, warehouse_de_delivery_date, warehouse_global_qty, warehouse_global_delivery_date, backorder_qty, backorder_estimated_date, - units_sold, compatible_brands, price_net, product_url, + units_sold, lead_time_days, compatible_brands, price_net, product_url, stock_confidence, price_currency, price_includes_tax, stock_vendor_ts ) VALUES ( NOW(), $1, $2, @@ -452,8 +454,8 @@ export async function upsertStockObservation(params: { $5, $6::date, $7, $8::date, $9, $10::date, - $11, $12, $13, $14, - $15, $16, $17, $18 + $11, $12, $13, $14, $15, + $16, $17, $18, $19 )`, [ params.transceiverId, @@ -467,6 +469,7 @@ export async function upsertStockObservation(params: { params.backorderQty ?? null, params.backorderEstimatedDate ?? null, params.unitsSold ?? null, + params.leadTimeDays ?? null, params.compatibleBrands?.length ? params.compatibleBrands : null, params.priceNet ?? null, params.productUrl ?? null, diff --git a/packages/scraper/tsconfig.json b/packages/scraper/tsconfig.json index 8c1b141..9bbd3e1 100644 --- a/packages/scraper/tsconfig.json +++ b/packages/scraper/tsconfig.json @@ -15,5 +15,5 @@ "incremental": true }, "include": ["src/**/*"], - "exclude": ["node_modules", "dist"] + "exclude": ["node_modules", "dist", "src/**/*.test.ts"] } diff --git a/sql/122-fs-com-delivery-dates-and-units-sold-gate.sql b/sql/122-fs-com-delivery-dates-and-units-sold-gate.sql new file mode 100644 index 0000000..48b7730 --- /dev/null +++ b/sql/122-fs-com-delivery-dates-and-units-sold-gate.sql @@ -0,0 +1,54 @@ +-- Migration 122: FS.com delivery dates + lead_time now collected; purge units_sold garbage +-- ───────────────────────────────────────────────────────────────────────────── +-- Follow-up to migration 121, which documented lead_time_days as NOT COLLECTED and +-- flagged that populating it was a scraper-side task. That task is now done: +-- +-- * fs-com.ts (via fs-com-parse.ts) parses the bare word-date FS.com renders on +-- each warehouse line ("… im DE-Lager, 8. Juli 2026 Lieferbar") into +-- warehouse_de_delivery_date / warehouse_global_delivery_date / +-- backorder_estimated_date, and derives lead_time_days = min(date − today). +-- * upsertStockObservation now writes lead_time_days. +-- +-- Historical rows stay NULL until the next FS.com run re-scrapes each SKU; NULL +-- therefore still means "unknown / not yet captured", not "zero lead time". +-- +-- Second fix: units_sold was ~10× inflated for decimal-abbreviated values +-- ("4.8K"→48000, "1.4M"→14000000) because the K/M parser stripped the decimal +-- point. The parser is fixed AND ingest now drops values above a sanity cap. +-- This migration removes the clearly-impossible historical values (the multi- +-- million "outliers") so no consumer ingests them before the re-scrape. Sub-cap +-- values that may still be mildly inflated self-correct on the next FS.com run +-- (a changed value writes a fresh observation; consumers read the latest). +-- ───────────────────────────────────────────────────────────────────────────── + +-- 1. Purge impossible units_sold (matches the ingest gate default, FS_MAX_UNITS_SOLD=2,000,000). +UPDATE stock_observations +SET units_sold = NULL +WHERE units_sold IS NOT NULL + AND units_sold > 2000000; + +-- 2. Re-document the columns as collected-by-scraper (reversing migration 121's note). +COMMENT ON COLUMN stock_observations.warehouse_de_delivery_date IS + 'FS.com DE-Lager availability date (collected 2026-07-04+ by fs-com-parse). ' + 'NULL = warehouse in stock now, vendor did not quote a date, or SKU not yet re-scraped.'; + +COMMENT ON COLUMN stock_observations.warehouse_global_delivery_date IS + 'FS.com Global-Lager availability date (collected 2026-07-04+ by fs-com-parse). ' + 'NULL semantics as warehouse_de_delivery_date.'; + +COMMENT ON COLUMN stock_observations.backorder_estimated_date IS + 'FS.com Nachlieferung (backorder) estimated date (collected 2026-07-04+ by fs-com-parse). ' + 'NULL = no backorder date quoted or SKU not yet re-scraped.'; + +COMMENT ON COLUMN stock_observations.lead_time_days IS + 'Soonest availability in days = min(delivery_date − today) across warehouses that quote a ' + 'date. Collected 2026-07-04+ by the FS.com scraper; other vendors do not expose a lead time. ' + 'Treat NULL as "unknown / not captured" (do NOT default to 0).'; + +COMMENT ON COLUMN stock_observations.units_sold IS + 'FS.com "verkauft" lifetime sell-through counter. Parser de-abbreviates K/M correctly and ' + 'ingest drops values > 2,000,000 as implausible. FS.com only; NULL elsewhere.'; + +INSERT INTO _migrations (filename) +VALUES ('122-fs-com-delivery-dates-and-units-sold-gate.sql') +ON CONFLICT (filename) DO NOTHING;