fix(scraper): parse FS.com delivery dates + lead_time; fix units_sold K/M inflation

FS.com renders per-warehouse availability as a bare word-date on each line
("2 Stk. im DE-Lager, 8. Juli 2026 Lieferbar"). The old regex only matched a
"Lieferung:"/"erwartet:" prefix or numeric dd.mm.yyyy, so warehouse_de/global
delivery_date + backorder_estimated_date were 0% populated across 85k rows, and
lead_time_days was never written at all.

- new fs-com-parse.ts: pure, unit-tested parsers (13 node:test cases).
- extractDeliveryDate finds the bare word/numeric date per warehouse, bounded to
  its own line so a date never bleeds to a neighbouring warehouse.
- computeLeadTimeDays = min(delivery_date - today), floored at 0; threaded into
  upsertStockObservation (now writes lead_time_days) and upsertPriceObservation.
- parseGermanQty: K/M abbreviations now treat '.' as a decimal ("4.8K"=4800,
  "1.4M"=1.4M) instead of stripping it (was "1.4M"->14,000,000 — the outlier).
- units_sold sanity gate at ingest (FS_MAX_UNITS_SOLD, default 2,000,000).
- sql/122: purge impossible historical units_sold, re-document columns as collected.
This commit is contained in:
Rene Fichtmueller 2026-07-04 22:37:05 +02:00
parent dd2c917f05
commit 4299e24e13
6 changed files with 518 additions and 139 deletions

View File

@ -0,0 +1,138 @@
/**
* Unit tests for the FS.com pure parsers. Run with:
* npx tsx --test src/scrapers/fs-com-parse.test.ts
*/
import { test } from "node:test";
import assert from "node:assert/strict";
import {
parseGermanQty,
parseGermanDate,
parseGermanPrice,
extractDeliveryDate,
parseWarehouseStock,
computeLeadTimeDays,
sanitizeUnitsSold,
} from "./fs-com-parse";
// ── parseGermanQty — the K/M decimal bug ─────────────────────────────────────
test("parseGermanQty: plain integers use '.'/',' as thousands", () => {
assert.equal(parseGermanQty("4.895"), 4895);
assert.equal(parseGermanQty("1.587"), 1587);
assert.equal(parseGermanQty("1.501"), 1501);
assert.equal(parseGermanQty("2"), 2);
assert.equal(parseGermanQty("1627"), 1627);
});
test("parseGermanQty: K/M abbreviations treat '.' as a DECIMAL (bug fix)", () => {
// Previously "4.8K" -> 48000 and "1.4M" -> 14000000 (the garbage outlier).
assert.equal(parseGermanQty("4.8K"), 4800);
assert.equal(parseGermanQty("210.9K"), 210900);
assert.equal(parseGermanQty("504.4K"), 504400);
assert.equal(parseGermanQty("1.4M"), 1_400_000);
assert.equal(parseGermanQty("1.2M"), 1_200_000);
assert.equal(parseGermanQty("63K"), 63000); // no decimal — unchanged
assert.equal(parseGermanQty("14M"), 14_000_000); // caught later by the gate
});
test("parseGermanQty: K/M with German comma decimal still works", () => {
assert.equal(parseGermanQty("4,8K"), 4800);
assert.equal(parseGermanQty("1,4M"), 1_400_000);
});
// ── parseGermanDate ──────────────────────────────────────────────────────────
test("parseGermanDate: word and numeric forms", () => {
assert.equal(parseGermanDate("8. Juli 2026"), "2026-07-08");
assert.equal(parseGermanDate("17. Juli 2026"), "2026-07-17");
assert.equal(parseGermanDate("20 Apr., 2026"), "2026-04-20");
assert.equal(parseGermanDate("28. März 2026"), "2026-03-28");
assert.equal(parseGermanDate("20.04.2026"), "2026-04-20");
assert.equal(parseGermanDate("nonsense"), undefined);
});
// ── extractDeliveryDate ──────────────────────────────────────────────────────
test("extractDeliveryDate: finds bare word-date, no keyword needed", () => {
assert.equal(extractDeliveryDate("2 Stk. im DE-Lager, 8. Juli 2026 Lieferbar"), "2026-07-08");
assert.equal(extractDeliveryDate("1.627 Stk. im Global-Lager, 17. Juli 2026"), "2026-07-17");
});
test("extractDeliveryDate: rejects stray/garbage years and empty", () => {
assert.equal(extractDeliveryDate("© 2019 FS.COM"), undefined);
assert.equal(extractDeliveryDate("auf Lager, sofort verfügbar"), undefined);
});
// ── parseWarehouseStock — the real screenshot block ──────────────────────────
const SCREENSHOT_BLOCK = [
"2 Stk. im DE-Lager, 8. Juli 2026 Lieferbar",
"1.627 Stk. im Global-Lager, 17. Juli 2026",
"1 in Nachlieferung, 28. Juli 2026",
"4.8K verkauft",
].join("\n");
test("parseWarehouseStock: full block yields qtys + per-warehouse dates + sold", () => {
const s = parseWarehouseStock(SCREENSHOT_BLOCK);
assert.equal(s.deQty, 2);
assert.equal(s.deDeliveryDate, "2026-07-08");
assert.equal(s.globalQty, 1627);
assert.equal(s.globalDeliveryDate, "2026-07-17");
assert.equal(s.backorderQty, 1);
assert.equal(s.backorderDate, "2026-07-28");
assert.equal(s.unitsSold, 4800); // 4.8K, correctly de-abbreviated
});
test("parseWarehouseStock: a warehouse with no date does not steal a neighbour's date", () => {
const block = [
"5 Stk. im DE-Lager", // no date — available now
"800 Stk. im Global-Lager, 17. Juli 2026",
].join("\n");
const s = parseWarehouseStock(block);
assert.equal(s.deQty, 5);
assert.equal(s.deDeliveryDate, undefined); // must NOT be 2026-07-17
assert.equal(s.globalQty, 800);
assert.equal(s.globalDeliveryDate, "2026-07-17");
});
test("parseWarehouseStock: gates the 1.4M -> 14M units_sold garbage", () => {
const block = "12 Stk. im DE-Lager, 8. Juli 2026 Lieferbar\n14M verkauft";
const s = parseWarehouseStock(block);
assert.equal(s.deQty, 12);
assert.equal(s.unitsSold, undefined); // 14M > cap → dropped as unknown
});
test("parseWarehouseStock: empty when no warehouse text present", () => {
assert.deepEqual(parseWarehouseStock("Just a product description, nothing else."), {});
});
// ── sanitizeUnitsSold ────────────────────────────────────────────────────────
test("sanitizeUnitsSold: keeps plausible, drops implausible/negative", () => {
assert.equal(sanitizeUnitsSold(4800), 4800);
assert.equal(sanitizeUnitsSold(504400), 504400);
assert.equal(sanitizeUnitsSold(14_000_000), undefined);
assert.equal(sanitizeUnitsSold(-5), undefined);
assert.equal(sanitizeUnitsSold(undefined), undefined);
assert.equal(sanitizeUnitsSold(2_500_000, 2_000_000), undefined);
assert.equal(sanitizeUnitsSold(1_500_000, 2_000_000), 1_500_000);
});
// ── computeLeadTimeDays ──────────────────────────────────────────────────────
test("computeLeadTimeDays: min across dates, floored at 0, undefined when empty", () => {
const now = new Date("2026-07-04T09:00:00");
assert.equal(computeLeadTimeDays(["2026-07-08", "2026-07-17", "2026-07-28"], now), 4);
assert.equal(computeLeadTimeDays([undefined, null, "2026-07-05"], now), 1);
assert.equal(computeLeadTimeDays(["2026-07-01"], now), 0); // past date → 0, not negative
assert.equal(computeLeadTimeDays([], now), undefined);
assert.equal(computeLeadTimeDays([null, undefined], now), undefined);
});
// ── parseGermanPrice (unchanged behaviour, regression guard) ─────────────────
test("parseGermanPrice: net EUR parsing", () => {
assert.equal(parseGermanPrice("42,50"), 42.5);
assert.equal(parseGermanPrice("1.063,02 €"), 1063.02);
assert.equal(parseGermanPrice("5,10 € ohne MwSt."), 5.1);
});

View File

@ -0,0 +1,290 @@
/**
* FS.com parsing helpers pure, side-effect-free, unit-tested.
*
* Extracted from fs-com.ts so the number/date/warehouse parsing can be tested
* in isolation (fs-com.ts pulls in Crawlee + Playwright + a pg Pool, which we
* do not want to load in a unit test).
*
* The FS.com German storefront (www.fs.com/de/) renders per-warehouse
* availability as a short line per warehouse, e.g.:
*
* 2 Stk. im DE-Lager, 8. Juli 2026 Lieferbar
* 1.627 Stk. im Global-Lager, 17. Juli 2026
* 1 in Nachlieferung, 28. Juli 2026
* 4.8K verkauft
*
* The quantity part was already parsed correctly; the delivery-date part was
* not, because the previous regex only looked for a "Lieferung:"/"erwartet:"
* keyword prefix or a numeric dd.mm.yyyy date neither of which matches the
* bare word-date ("8. Juli 2026") that FS.com actually renders.
*/
// ── Constants ────────────────────────────────────────────────────────────────
const GERMAN_MONTHS: Record<string, string> = {
jan: "01", feb: "02", mär: "03", mar: "03",
apr: "04", mai: "05", may: "05", jun: "06",
jul: "07", aug: "08", sep: "09", okt: "10",
oct: "10", nov: "11", dez: "12", dec: "12",
};
/**
* Upper bound for a plausible per-SKU lifetime "verkauft" (units-sold) counter.
* Values above this are treated as unknown (dropped), not stored. FS.com's
* biggest sellers are in the hundreds of thousands; multi-million values are
* a mis-parse (see parseGermanQty K/M bug) or vendor marketing inflation.
* Override with FS_MAX_UNITS_SOLD.
*/
export const DEFAULT_MAX_UNITS_SOLD = 2_000_000;
/** How far past a warehouse line to look for its delivery date (characters). */
const DATE_WINDOW = 200;
// ── Number parsing ───────────────────────────────────────────────────────────
/**
* Parse the mantissa of a K/M-abbreviated number ("4.8" in "4.8K").
*
* K/M abbreviations carry a small decimal, never a grouped thousands value, so
* a lone "." is the decimal point ("4.8K" = 4 800). German pages may instead
* use "," as the decimal ("4,8K"); a "." alongside a "," is then a thousands
* grouping and is stripped.
*/
function parseAbbrevMantissa(s: string): number {
if (s.includes(",")) return parseFloat(s.replace(/\./g, "").replace(",", "."));
return parseFloat(s); // "." is the decimal separator
}
/**
* Parse a German-formatted quantity string.
* "4.895" 4895 (period = thousands separator in a plain integer)
* "1.587" 1587
* "4.8K" 4800 (period = decimal in a K/M abbreviation)
* "210.9K" 210900
* "1.4M" 1400000
* "63K" 63000
*/
export function parseGermanQty(text: string): number | undefined {
const t = text.trim().replace(/\s/g, "");
if (!t) return undefined;
const kMatch = t.match(/^([\d.,]+)[Kk]$/);
if (kMatch) {
const n = parseAbbrevMantissa(kMatch[1]);
return isNaN(n) ? undefined : Math.round(n * 1_000);
}
const mMatch = t.match(/^([\d.,]+)[Mm]$/);
if (mMatch) {
const n = parseAbbrevMantissa(mMatch[1]);
return isNaN(n) ? undefined : Math.round(n * 1_000_000);
}
// Plain integer: "." and "," are both thousands separators here.
const n = parseInt(t.replace(/\./g, "").replace(/,/g, ""), 10);
return isNaN(n) ? undefined : n;
}
/**
* Parse a German date to ISO "YYYY-MM-DD".
* "8. Juli 2026" "2026-07-08"
* "20 Apr., 2026" "2026-04-20"
* "20.04.2026" "2026-04-20"
*/
export function parseGermanDate(text: string): string | undefined {
const numericMatch = text.match(/(\d{1,2})\.(\d{1,2})\.(\d{4})/);
if (numericMatch) {
const [, d, m, y] = numericMatch;
return `${y}-${m.padStart(2, "0")}-${d.padStart(2, "0")}`;
}
const wordMatch = text.match(/(\d{1,2})\.?\s+([A-Za-zÄÖÜäöüß]+)\.?,?\s*(\d{4})/);
if (!wordMatch) return undefined;
const day = wordMatch[1].padStart(2, "0");
const monthRaw = wordMatch[2]
.toLowerCase()
.replace(/ä/g, "a").replace(/ö/g, "o").replace(/ü/g, "u")
.slice(0, 3);
const month = GERMAN_MONTHS[monthRaw];
if (!month) return undefined;
return `${wordMatch[3]}-${month}-${day}`;
}
/**
* Parse a German price to a EUR float.
* "42,50" 42.50
* "1.063,02" 1063.02
*/
export function parseGermanPrice(raw: string): number | undefined {
const cleaned = raw.replace(/[^0-9.,]/g, "").trim();
if (!cleaned) return undefined;
let normalized: string;
if (/\d+\.\d{3},\d{2}/.test(cleaned)) {
normalized = cleaned.replace(/\./g, "").replace(",", ".");
} else if (cleaned.includes(",")) {
normalized = cleaned.replace(",", ".");
} else {
normalized = cleaned;
}
const n = parseFloat(normalized);
return isNaN(n) || n <= 0 ? undefined : n;
}
// ── Units-sold sanity gate ───────────────────────────────────────────────────
/**
* Clamp an implausible units-sold value to `undefined` (unknown). Returns the
* value unchanged when it is a finite, non-negative number at or below `cap`.
*/
export function sanitizeUnitsSold(
raw: number | undefined,
cap: number = DEFAULT_MAX_UNITS_SOLD
): number | undefined {
if (raw === undefined) return undefined;
if (!Number.isFinite(raw) || raw < 0) return undefined;
if (raw > cap) return undefined;
return raw;
}
// ── Delivery-date extraction ─────────────────────────────────────────────────
/** dd.mm.yyyy OR "8. Juli 2026" / "20 Apr., 2026" (day + month word + year). */
const DELIVERY_DATE_RE =
/(\d{1,2}\.\d{1,2}\.\d{4})|(\d{1,2}\.?\s+[A-Za-zÄÖÜäöüß]{3,}\.?,?\s*\d{4})/;
/**
* Find the first plausible calendar date inside a bounded text block (the slice
* that belongs to one warehouse line) and return it as ISO "YYYY-MM-DD".
* The date can appear with or without a "Lieferbar"/"Lieferung" keyword FS.com
* renders it bare ("…DE-Lager, 8. Juli 2026 Lieferbar"), so no keyword is required.
*/
export function extractDeliveryDate(block: string): string | undefined {
const m = block.match(DELIVERY_DATE_RE);
if (!m) return undefined;
const iso = parseGermanDate(m[0]);
if (!iso) return undefined;
const year = Number(iso.slice(0, 4));
if (year < 2024 || year > 2100) return undefined; // reject stray/garbage years
return iso;
}
// ── Warehouse block parsing ──────────────────────────────────────────────────
export interface WarehouseStock {
deQty?: number;
deDeliveryDate?: string;
globalQty?: number;
globalDeliveryDate?: string;
backorderQty?: number;
backorderDate?: string;
unitsSold?: number;
}
const DE_QTY_RE: RegExp[] = [
/(\d[\d.,KkMm]*)\s*Stk\.\s*im\s*DE[- ]Lager/i,
/(\d[\d.,KkMm]*)\s*im\s*DE[- ]?Lager/i,
];
const GLOBAL_QTY_RE: RegExp[] = [
/(\d[\d.,KkMm]*)\s*Stk\.\s*im\s*Global[- ]Lager/i,
/(\d[\d.,KkMm]*)\s*im\s*Global[- ]?(?:Lager|Warehouse)/i,
/(\d[\d.,KkMm]*)\s*in\s+Global\s+Warehouse/i,
];
const BACKORDER_QTY_RE: RegExp[] = [
/(\d[\d.,KkMm]*)\s*(?:Stk\.)?\s*in\s+Nachlieferung/i,
/Nachlieferung[:\s]*(\d[\d.,KkMm]*)/i,
];
const UNITS_SOLD_RE: RegExp[] = [
/(\d[\d.,KkMm]*)\s*(?:[Mm]al\s+)?[Vv]erkauft/,
/([\d.,KkMm]+)\+?\s*sold/i,
];
interface SectionMatch { value: string; index: number; }
function firstMatch(text: string, res: RegExp[]): SectionMatch | undefined {
for (const re of res) {
const m = text.match(re);
if (m && m[1] != null && m.index != null) return { value: m[1], index: m.index };
}
return undefined;
}
/**
* Parse the full warehouse-availability block out of a product page's body
* text: DE-Lager, Global-Lager and Nachlieferung quantities, each with the
* delivery date rendered on its line, plus the "verkauft"/"sold" counter.
*
* A section's delivery date is looked for only in the window between that
* section's quantity and the start of the next section, so a warehouse with no
* date never picks up a neighbouring warehouse's date.
*/
export function parseWarehouseStock(
bodyText: string,
unitsSoldCap: number = DEFAULT_MAX_UNITS_SOLD
): WarehouseStock {
const t = bodyText;
const de = firstMatch(t, DE_QTY_RE);
const gl = firstMatch(t, GLOBAL_QTY_RE);
const bo = firstMatch(t, BACKORDER_QTY_RE);
const sold = firstMatch(t, UNITS_SOLD_RE);
const starts = [de, gl, bo, sold]
.filter((s): s is SectionMatch => s !== undefined)
.map((s) => s.index);
const dateFor = (sec: SectionMatch | undefined): string | undefined => {
if (!sec) return undefined;
const later = starts.filter((b) => b > sec.index);
const end = later.length ? Math.min(sec.index + DATE_WINDOW, ...later) : sec.index + DATE_WINDOW;
return extractDeliveryDate(t.slice(sec.index, end));
};
const result: WarehouseStock = {};
if (de) {
const q = parseGermanQty(de.value);
if (q !== undefined) result.deQty = q;
const d = dateFor(de);
if (d) result.deDeliveryDate = d;
}
if (gl) {
const q = parseGermanQty(gl.value);
if (q !== undefined) result.globalQty = q;
const d = dateFor(gl);
if (d) result.globalDeliveryDate = d;
}
if (bo) {
const q = parseGermanQty(bo.value);
if (q !== undefined) result.backorderQty = q;
const d = dateFor(bo);
if (d) result.backorderDate = d;
}
if (sold) {
const s = sanitizeUnitsSold(parseGermanQty(sold.value), unitsSoldCap);
if (s !== undefined) result.unitsSold = s;
}
return result;
}
// ── Lead-time derivation ─────────────────────────────────────────────────────
/**
* Derive lead-time in days as the soonest availability across the warehouses
* that quote a delivery date: min(date today), floored at 0. Returns
* `undefined` when no date is known (never fabricate a 0).
*/
export function computeLeadTimeDays(
dates: Array<string | null | undefined>,
now: Date = new Date()
): number | undefined {
const todayUtc = Date.UTC(now.getFullYear(), now.getMonth(), now.getDate());
let min: number | undefined;
for (const d of dates) {
if (!d) continue;
const m = d.match(/^(\d{4})-(\d{2})-(\d{2})$/);
if (!m) continue;
const dUtc = Date.UTC(Number(m[1]), Number(m[2]) - 1, Number(m[3]));
const days = Math.max(0, Math.round((dUtc - todayUtc) / 86_400_000));
if (min === undefined || days < min) min = days;
}
return min;
}

View File

@ -63,6 +63,12 @@ import {
} from "../utils/db";
import { contentHash } from "../utils/hash";
import { updateVerifiedSpecs, parseSpecTable } from "../utils/spec-updater";
import {
parseGermanPrice,
parseWarehouseStock,
computeLeadTimeDays,
DEFAULT_MAX_UNITS_SOLD,
} from "./fs-com-parse";
// ── Constants ──────────────────────────────────────────────────────────────────
@ -75,6 +81,8 @@ const FORCE_REVALIDATE = process.env["TIP_FORCE_REVALIDATE"] === "1";
const ONLY_MISSING_IMAGES = process.env["FS_ONLY_MISSING_IMAGES"] === "1";
const DB_DETAIL_ONLY = process.env["FS_DB_DETAIL_ONLY"] === "1";
const URL_DISCOVERY_ONLY = process.env["FS_URL_DISCOVERY_ONLY"] === "1";
// Drop implausible "verkauft" (units-sold) counters at ingest (see fs-com-parse).
const FS_MAX_UNITS_SOLD = parseInt(process.env["FS_MAX_UNITS_SOLD"] ?? String(DEFAULT_MAX_UNITS_SOLD), 10);
const PROXY_URLS = (process.env["PROXY_URLS"] ?? "")
.split(",")
@ -113,83 +121,7 @@ const DE_COOKIES = [
{ name: "country", value: "DE", domain: ".fs.com", path: "/" },
];
// ── German locale parsers ──────────────────────────────────────────────────────
const GERMAN_MONTHS: Record<string, string> = {
jan: "01", feb: "02", mär: "03", mar: "03",
apr: "04", mai: "05", may: "05", jun: "06",
jul: "07", aug: "08", sep: "09", okt: "10",
oct: "10", nov: "11", dez: "12", dec: "12",
};
/**
* Parse German-formatted quantity string.
* "4.895" 4895 (period = thousands separator in German)
* "210.9K" 210900
* "1.2M" 1200000
*/
function parseGermanQty(text: string): number | undefined {
const t = text.trim().replace(/\s/g, "");
if (!t) return undefined;
const kMatch = t.match(/^([\d.,]+)[Kk]$/);
if (kMatch) {
const n = parseFloat(kMatch[1].replace(/\./g, "").replace(",", "."));
return isNaN(n) ? undefined : Math.round(n * 1_000);
}
const mMatch = t.match(/^([\d.,]+)[Mm]$/);
if (mMatch) {
const n = parseFloat(mMatch[1].replace(/\./g, "").replace(",", "."));
return isNaN(n) ? undefined : Math.round(n * 1_000_000);
}
const n = parseInt(t.replace(/\./g, "").replace(/,/g, ""), 10);
return isNaN(n) ? undefined : n;
}
/**
* Parse German date to ISO "YYYY-MM-DD".
* "20 Apr., 2026" "2026-04-20"
* "20.04.2026" "2026-04-20"
*/
function parseGermanDate(text: string): string | undefined {
const numericMatch = text.match(/(\d{1,2})\.(\d{1,2})\.(\d{4})/);
if (numericMatch) {
const [, d, m, y] = numericMatch;
return `${y}-${m.padStart(2, "0")}-${d.padStart(2, "0")}`;
}
const wordMatch = text.match(/(\d{1,2})\.?\s+([A-Za-zÄÖÜäöüß]+)\.?,?\s*(\d{4})/);
if (!wordMatch) return undefined;
const day = wordMatch[1].padStart(2, "0");
const monthRaw = wordMatch[2]
.toLowerCase()
.replace(/ä/g, "a").replace(/ö/g, "o").replace(/ü/g, "u")
.slice(0, 3);
const month = GERMAN_MONTHS[monthRaw];
if (!month) return undefined;
return `${wordMatch[3]}-${month}-${day}`;
}
/**
* Parse German price to EUR float.
* "42,50" 42.50
* "1.063,02" 1063.02
*/
function parseGermanPrice(raw: string): number | undefined {
const cleaned = raw.replace(/[^0-9.,]/g, "").trim();
if (!cleaned) return undefined;
let normalized: string;
if (/\d+\.\d{3},\d{2}/.test(cleaned)) {
normalized = cleaned.replace(/\./g, "").replace(",", ".");
} else if (cleaned.includes(",")) {
normalized = cleaned.replace(",", ".");
} else {
normalized = cleaned;
}
const n = parseFloat(normalized);
return isNaN(n) || n <= 0 ? undefined : n;
}
// German number/date/warehouse parsers live in ./fs-com-parse (pure + unit-tested).
// ── Stock level helper ─────────────────────────────────────────────────────────
@ -648,64 +580,18 @@ async function scrapeProductDetails(
}
}
// ── DE-Lager ───────────────────────────────────────────────────────────
let deQty: number | undefined;
let deDeliveryDate: string | undefined;
const deM =
t.match(/(\d[\d.,KkMm]*)\s*Stk\.\s*im\s*DE[- ]Lager/i) ??
t.match(/(\d[\d.,KkMm]*)\s*im\s*DE[- ]?Lager/i);
if (deM?.[1]) {
deQty = parseGermanQty(deM[1]);
const idx = t.indexOf(deM[0]);
const ctx = t.slice(idx, idx + 300);
const dm =
ctx.match(/Lieferung[:\s]+([0-9]{1,2}\.?\s+\w+\.?,?\s*\d{4})/i) ??
ctx.match(/erwartet[:\s]+([0-9]{1,2}\.?\s+\w+\.?,?\s*\d{4})/i) ??
ctx.match(/([0-9]{1,2}\.[0-9]{1,2}\.[0-9]{4})/);
if (dm?.[1]) deDeliveryDate = parseGermanDate(dm[1]);
}
// ── Global-Lager ───────────────────────────────────────────────────────
let globalQty: number | undefined;
let globalDeliveryDate: string | undefined;
const glM =
t.match(/(\d[\d.,KkMm]*)\s*Stk\.\s*im\s*Global[- ]Lager/i) ??
t.match(/(\d[\d.,KkMm]*)\s*im\s*Global[- ]?(?:Lager|Warehouse)/i) ??
t.match(/(\d[\d.,KkMm]*)\s*in\s+Global\s+Warehouse/i);
if (glM?.[1]) {
globalQty = parseGermanQty(glM[1]);
const idx = t.indexOf(glM[0]);
const ctx = t.slice(idx, idx + 300);
const dm =
ctx.match(/Lieferung[:\s]+([0-9]{1,2}\.?\s+\w+\.?,?\s*\d{4})/i) ??
ctx.match(/erwartet[:\s]+([0-9]{1,2}\.?\s+\w+\.?,?\s*\d{4})/i) ??
ctx.match(/([0-9]{1,2}\.[0-9]{1,2}\.[0-9]{4})/);
if (dm?.[1]) globalDeliveryDate = parseGermanDate(dm[1]);
}
// ── Nachlieferung ──────────────────────────────────────────────────────
let backorderQty: number | undefined;
let backorderDate: string | undefined;
const boM =
t.match(/(\d[\d.,KkMm]*)\s*(?:Stk\.)?\s*in\s+Nachlieferung/i) ??
t.match(/Nachlieferung[:\s]*(\d[\d.,KkMm]*)/i);
if (boM?.[1]) {
backorderQty = parseGermanQty(boM[1]);
const idx = t.indexOf(boM[0]);
const ctx = t.slice(idx, idx + 300);
const dm =
ctx.match(/[Ee]rwartet[:\s]+([0-9]{1,2}\.?\s+\w+\.?,?\s*\d{4})/i) ??
ctx.match(/Lieferung[:\s]+([0-9]{1,2}\.?\s+\w+\.?,?\s*\d{4})/i) ??
ctx.match(/([0-9]{1,2}\.[0-9]{1,2}\.[0-9]{4})/);
if (dm?.[1]) backorderDate = parseGermanDate(dm[1]);
}
// ── Units sold ─────────────────────────────────────────────────────────
let unitsSold: number | undefined;
const soldM =
t.match(/(\d[\d.,KkMm]*)\s*(?:[Mm]al\s+)?[Vv]erkauft/) ??
t.match(/([\d.,KkMm]+)\+?\s*sold/i);
if (soldM?.[1]) unitsSold = parseGermanQty(soldM[1]);
// ── Warehouse availability (qty + per-warehouse delivery date + units sold) ──
// FS.com renders each warehouse on its own line, e.g.
// "2 Stk. im DE-Lager, 8. Juli 2026 Lieferbar". parseWarehouseStock
// extracts the bare word-date per warehouse and gates units_sold.
const wh = parseWarehouseStock(t, FS_MAX_UNITS_SOLD);
const deQty = wh.deQty;
const deDeliveryDate = wh.deDeliveryDate;
const globalQty = wh.globalQty;
const globalDeliveryDate = wh.globalDeliveryDate;
const backorderQty = wh.backorderQty;
const backorderDate = wh.backorderDate;
const unitsSold = wh.unitsSold;
// ── Part number refinement ─────────────────────────────────────────────
let partNumber = listingPn;
@ -1010,6 +896,12 @@ export async function scrapeFs(): Promise<void> {
const stockLevel = deriveStockLevel(detail.deQty, detail.globalQty, detail.backorderQty);
const totalQty = (detail.deQty ?? 0) + (detail.globalQty ?? 0);
// Soonest availability across the warehouses that quote a delivery date.
const leadTimeDays = computeLeadTimeDays([
detail.deDeliveryDate,
detail.globalDeliveryDate,
detail.backorderDate,
]);
if (detail.priceNet && detail.priceNet > 0) {
const hash = contentHash({
@ -1024,6 +916,7 @@ export async function scrapeFs(): Promise<void> {
currency: "EUR",
stockLevel,
quantityAvailable: totalQty > 0 ? totalQty : undefined,
leadTimeDays,
url: detail.url,
contentHash: hash,
});
@ -1042,6 +935,7 @@ export async function scrapeFs(): Promise<void> {
backorderQty: detail.backorderQty,
backorderEstimatedDate: detail.backorderDate ?? null,
unitsSold: detail.unitsSold,
leadTimeDays,
compatibleBrands: detail.compatibleBrands,
priceNet: detail.priceNet,
productUrl: detail.url,

View File

@ -394,6 +394,8 @@ export async function upsertStockObservation(params: {
backorderQty?: number;
backorderEstimatedDate?: string | null;
unitsSold?: number;
/** Soonest availability in days across warehouses that quote a delivery date. */
leadTimeDays?: number;
compatibleBrands?: string[];
priceNet?: number;
productUrl?: string;
@ -444,7 +446,7 @@ export async function upsertStockObservation(params: {
warehouse_de_qty, warehouse_de_delivery_date,
warehouse_global_qty, warehouse_global_delivery_date,
backorder_qty, backorder_estimated_date,
units_sold, compatible_brands, price_net, product_url,
units_sold, lead_time_days, compatible_brands, price_net, product_url,
stock_confidence, price_currency, price_includes_tax, stock_vendor_ts
) VALUES (
NOW(), $1, $2,
@ -452,8 +454,8 @@ export async function upsertStockObservation(params: {
$5, $6::date,
$7, $8::date,
$9, $10::date,
$11, $12, $13, $14,
$15, $16, $17, $18
$11, $12, $13, $14, $15,
$16, $17, $18, $19
)`,
[
params.transceiverId,
@ -467,6 +469,7 @@ export async function upsertStockObservation(params: {
params.backorderQty ?? null,
params.backorderEstimatedDate ?? null,
params.unitsSold ?? null,
params.leadTimeDays ?? null,
params.compatibleBrands?.length ? params.compatibleBrands : null,
params.priceNet ?? null,
params.productUrl ?? null,

View File

@ -15,5 +15,5 @@
"incremental": true
},
"include": ["src/**/*"],
"exclude": ["node_modules", "dist"]
"exclude": ["node_modules", "dist", "src/**/*.test.ts"]
}

View File

@ -0,0 +1,54 @@
-- Migration 122: FS.com delivery dates + lead_time now collected; purge units_sold garbage
-- ─────────────────────────────────────────────────────────────────────────────
-- Follow-up to migration 121, which documented lead_time_days as NOT COLLECTED and
-- flagged that populating it was a scraper-side task. That task is now done:
--
-- * fs-com.ts (via fs-com-parse.ts) parses the bare word-date FS.com renders on
-- each warehouse line ("… im DE-Lager, 8. Juli 2026 Lieferbar") into
-- warehouse_de_delivery_date / warehouse_global_delivery_date /
-- backorder_estimated_date, and derives lead_time_days = min(date today).
-- * upsertStockObservation now writes lead_time_days.
--
-- Historical rows stay NULL until the next FS.com run re-scrapes each SKU; NULL
-- therefore still means "unknown / not yet captured", not "zero lead time".
--
-- Second fix: units_sold was ~10× inflated for decimal-abbreviated values
-- ("4.8K"→48000, "1.4M"→14000000) because the K/M parser stripped the decimal
-- point. The parser is fixed AND ingest now drops values above a sanity cap.
-- This migration removes the clearly-impossible historical values (the multi-
-- million "outliers") so no consumer ingests them before the re-scrape. Sub-cap
-- values that may still be mildly inflated self-correct on the next FS.com run
-- (a changed value writes a fresh observation; consumers read the latest).
-- ─────────────────────────────────────────────────────────────────────────────
-- 1. Purge impossible units_sold (matches the ingest gate default, FS_MAX_UNITS_SOLD=2,000,000).
UPDATE stock_observations
SET units_sold = NULL
WHERE units_sold IS NOT NULL
AND units_sold > 2000000;
-- 2. Re-document the columns as collected-by-scraper (reversing migration 121's note).
COMMENT ON COLUMN stock_observations.warehouse_de_delivery_date IS
'FS.com DE-Lager availability date (collected 2026-07-04+ by fs-com-parse). '
'NULL = warehouse in stock now, vendor did not quote a date, or SKU not yet re-scraped.';
COMMENT ON COLUMN stock_observations.warehouse_global_delivery_date IS
'FS.com Global-Lager availability date (collected 2026-07-04+ by fs-com-parse). '
'NULL semantics as warehouse_de_delivery_date.';
COMMENT ON COLUMN stock_observations.backorder_estimated_date IS
'FS.com Nachlieferung (backorder) estimated date (collected 2026-07-04+ by fs-com-parse). '
'NULL = no backorder date quoted or SKU not yet re-scraped.';
COMMENT ON COLUMN stock_observations.lead_time_days IS
'Soonest availability in days = min(delivery_date today) across warehouses that quote a '
'date. Collected 2026-07-04+ by the FS.com scraper; other vendors do not expose a lead time. '
'Treat NULL as "unknown / not captured" (do NOT default to 0).';
COMMENT ON COLUMN stock_observations.units_sold IS
'FS.com "verkauft" lifetime sell-through counter. Parser de-abbreviates K/M correctly and '
'ingest drops values > 2,000,000 as implausible. FS.com only; NULL elsewhere.';
INSERT INTO _migrations (filename)
VALUES ('122-fs-com-delivery-dates-and-units-sold-gate.sql')
ON CONFLICT (filename) DO NOTHING;