fix(scraper): parse FS.com delivery dates + lead_time; fix units_sold K/M inflation
FS.com renders per-warehouse availability as a bare word-date on each line
("2 Stk. im DE-Lager, 8. Juli 2026 Lieferbar"). The old regex only matched a
"Lieferung:"/"erwartet:" prefix or numeric dd.mm.yyyy, so warehouse_de/global
delivery_date + backorder_estimated_date were 0% populated across 85k rows, and
lead_time_days was never written at all.
- new fs-com-parse.ts: pure, unit-tested parsers (13 node:test cases).
- extractDeliveryDate finds the bare word/numeric date per warehouse, bounded to
its own line so a date never bleeds to a neighbouring warehouse.
- computeLeadTimeDays = min(delivery_date - today), floored at 0; threaded into
upsertStockObservation (now writes lead_time_days) and upsertPriceObservation.
- parseGermanQty: K/M abbreviations now treat '.' as a decimal ("4.8K"=4800,
"1.4M"=1.4M) instead of stripping it (was "1.4M"->14,000,000 — the outlier).
- units_sold sanity gate at ingest (FS_MAX_UNITS_SOLD, default 2,000,000).
- sql/122: purge impossible historical units_sold, re-document columns as collected.
This commit is contained in:
parent
dd2c917f05
commit
4299e24e13
138
packages/scraper/src/scrapers/fs-com-parse.test.ts
Normal file
138
packages/scraper/src/scrapers/fs-com-parse.test.ts
Normal file
@ -0,0 +1,138 @@
|
||||
/**
|
||||
* Unit tests for the FS.com pure parsers. Run with:
|
||||
* npx tsx --test src/scrapers/fs-com-parse.test.ts
|
||||
*/
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
parseGermanQty,
|
||||
parseGermanDate,
|
||||
parseGermanPrice,
|
||||
extractDeliveryDate,
|
||||
parseWarehouseStock,
|
||||
computeLeadTimeDays,
|
||||
sanitizeUnitsSold,
|
||||
} from "./fs-com-parse";
|
||||
|
||||
// ── parseGermanQty — the K/M decimal bug ─────────────────────────────────────
|
||||
|
||||
test("parseGermanQty: plain integers use '.'/',' as thousands", () => {
|
||||
assert.equal(parseGermanQty("4.895"), 4895);
|
||||
assert.equal(parseGermanQty("1.587"), 1587);
|
||||
assert.equal(parseGermanQty("1.501"), 1501);
|
||||
assert.equal(parseGermanQty("2"), 2);
|
||||
assert.equal(parseGermanQty("1627"), 1627);
|
||||
});
|
||||
|
||||
test("parseGermanQty: K/M abbreviations treat '.' as a DECIMAL (bug fix)", () => {
|
||||
// Previously "4.8K" -> 48000 and "1.4M" -> 14000000 (the garbage outlier).
|
||||
assert.equal(parseGermanQty("4.8K"), 4800);
|
||||
assert.equal(parseGermanQty("210.9K"), 210900);
|
||||
assert.equal(parseGermanQty("504.4K"), 504400);
|
||||
assert.equal(parseGermanQty("1.4M"), 1_400_000);
|
||||
assert.equal(parseGermanQty("1.2M"), 1_200_000);
|
||||
assert.equal(parseGermanQty("63K"), 63000); // no decimal — unchanged
|
||||
assert.equal(parseGermanQty("14M"), 14_000_000); // caught later by the gate
|
||||
});
|
||||
|
||||
test("parseGermanQty: K/M with German comma decimal still works", () => {
|
||||
assert.equal(parseGermanQty("4,8K"), 4800);
|
||||
assert.equal(parseGermanQty("1,4M"), 1_400_000);
|
||||
});
|
||||
|
||||
// ── parseGermanDate ──────────────────────────────────────────────────────────
|
||||
|
||||
test("parseGermanDate: word and numeric forms", () => {
|
||||
assert.equal(parseGermanDate("8. Juli 2026"), "2026-07-08");
|
||||
assert.equal(parseGermanDate("17. Juli 2026"), "2026-07-17");
|
||||
assert.equal(parseGermanDate("20 Apr., 2026"), "2026-04-20");
|
||||
assert.equal(parseGermanDate("28. März 2026"), "2026-03-28");
|
||||
assert.equal(parseGermanDate("20.04.2026"), "2026-04-20");
|
||||
assert.equal(parseGermanDate("nonsense"), undefined);
|
||||
});
|
||||
|
||||
// ── extractDeliveryDate ──────────────────────────────────────────────────────
|
||||
|
||||
test("extractDeliveryDate: finds bare word-date, no keyword needed", () => {
|
||||
assert.equal(extractDeliveryDate("2 Stk. im DE-Lager, 8. Juli 2026 Lieferbar"), "2026-07-08");
|
||||
assert.equal(extractDeliveryDate("1.627 Stk. im Global-Lager, 17. Juli 2026"), "2026-07-17");
|
||||
});
|
||||
|
||||
test("extractDeliveryDate: rejects stray/garbage years and empty", () => {
|
||||
assert.equal(extractDeliveryDate("© 2019 FS.COM"), undefined);
|
||||
assert.equal(extractDeliveryDate("auf Lager, sofort verfügbar"), undefined);
|
||||
});
|
||||
|
||||
// ── parseWarehouseStock — the real screenshot block ──────────────────────────
|
||||
|
||||
const SCREENSHOT_BLOCK = [
|
||||
"2 Stk. im DE-Lager, 8. Juli 2026 Lieferbar",
|
||||
"1.627 Stk. im Global-Lager, 17. Juli 2026",
|
||||
"1 in Nachlieferung, 28. Juli 2026",
|
||||
"4.8K verkauft",
|
||||
].join("\n");
|
||||
|
||||
test("parseWarehouseStock: full block yields qtys + per-warehouse dates + sold", () => {
|
||||
const s = parseWarehouseStock(SCREENSHOT_BLOCK);
|
||||
assert.equal(s.deQty, 2);
|
||||
assert.equal(s.deDeliveryDate, "2026-07-08");
|
||||
assert.equal(s.globalQty, 1627);
|
||||
assert.equal(s.globalDeliveryDate, "2026-07-17");
|
||||
assert.equal(s.backorderQty, 1);
|
||||
assert.equal(s.backorderDate, "2026-07-28");
|
||||
assert.equal(s.unitsSold, 4800); // 4.8K, correctly de-abbreviated
|
||||
});
|
||||
|
||||
test("parseWarehouseStock: a warehouse with no date does not steal a neighbour's date", () => {
|
||||
const block = [
|
||||
"5 Stk. im DE-Lager", // no date — available now
|
||||
"800 Stk. im Global-Lager, 17. Juli 2026",
|
||||
].join("\n");
|
||||
const s = parseWarehouseStock(block);
|
||||
assert.equal(s.deQty, 5);
|
||||
assert.equal(s.deDeliveryDate, undefined); // must NOT be 2026-07-17
|
||||
assert.equal(s.globalQty, 800);
|
||||
assert.equal(s.globalDeliveryDate, "2026-07-17");
|
||||
});
|
||||
|
||||
test("parseWarehouseStock: gates the 1.4M -> 14M units_sold garbage", () => {
|
||||
const block = "12 Stk. im DE-Lager, 8. Juli 2026 Lieferbar\n14M verkauft";
|
||||
const s = parseWarehouseStock(block);
|
||||
assert.equal(s.deQty, 12);
|
||||
assert.equal(s.unitsSold, undefined); // 14M > cap → dropped as unknown
|
||||
});
|
||||
|
||||
test("parseWarehouseStock: empty when no warehouse text present", () => {
|
||||
assert.deepEqual(parseWarehouseStock("Just a product description, nothing else."), {});
|
||||
});
|
||||
|
||||
// ── sanitizeUnitsSold ────────────────────────────────────────────────────────
|
||||
|
||||
test("sanitizeUnitsSold: keeps plausible, drops implausible/negative", () => {
|
||||
assert.equal(sanitizeUnitsSold(4800), 4800);
|
||||
assert.equal(sanitizeUnitsSold(504400), 504400);
|
||||
assert.equal(sanitizeUnitsSold(14_000_000), undefined);
|
||||
assert.equal(sanitizeUnitsSold(-5), undefined);
|
||||
assert.equal(sanitizeUnitsSold(undefined), undefined);
|
||||
assert.equal(sanitizeUnitsSold(2_500_000, 2_000_000), undefined);
|
||||
assert.equal(sanitizeUnitsSold(1_500_000, 2_000_000), 1_500_000);
|
||||
});
|
||||
|
||||
// ── computeLeadTimeDays ──────────────────────────────────────────────────────
|
||||
|
||||
test("computeLeadTimeDays: min across dates, floored at 0, undefined when empty", () => {
|
||||
const now = new Date("2026-07-04T09:00:00");
|
||||
assert.equal(computeLeadTimeDays(["2026-07-08", "2026-07-17", "2026-07-28"], now), 4);
|
||||
assert.equal(computeLeadTimeDays([undefined, null, "2026-07-05"], now), 1);
|
||||
assert.equal(computeLeadTimeDays(["2026-07-01"], now), 0); // past date → 0, not negative
|
||||
assert.equal(computeLeadTimeDays([], now), undefined);
|
||||
assert.equal(computeLeadTimeDays([null, undefined], now), undefined);
|
||||
});
|
||||
|
||||
// ── parseGermanPrice (unchanged behaviour, regression guard) ─────────────────
|
||||
|
||||
test("parseGermanPrice: net EUR parsing", () => {
|
||||
assert.equal(parseGermanPrice("42,50"), 42.5);
|
||||
assert.equal(parseGermanPrice("1.063,02 €"), 1063.02);
|
||||
assert.equal(parseGermanPrice("5,10 € ohne MwSt."), 5.1);
|
||||
});
|
||||
290
packages/scraper/src/scrapers/fs-com-parse.ts
Normal file
290
packages/scraper/src/scrapers/fs-com-parse.ts
Normal file
@ -0,0 +1,290 @@
|
||||
/**
|
||||
* FS.com parsing helpers — pure, side-effect-free, unit-tested.
|
||||
*
|
||||
* Extracted from fs-com.ts so the number/date/warehouse parsing can be tested
|
||||
* in isolation (fs-com.ts pulls in Crawlee + Playwright + a pg Pool, which we
|
||||
* do not want to load in a unit test).
|
||||
*
|
||||
* The FS.com German storefront (www.fs.com/de/) renders per-warehouse
|
||||
* availability as a short line per warehouse, e.g.:
|
||||
*
|
||||
* 2 Stk. im DE-Lager, 8. Juli 2026 Lieferbar
|
||||
* 1.627 Stk. im Global-Lager, 17. Juli 2026
|
||||
* 1 in Nachlieferung, 28. Juli 2026
|
||||
* 4.8K verkauft
|
||||
*
|
||||
* The quantity part was already parsed correctly; the delivery-date part was
|
||||
* not, because the previous regex only looked for a "Lieferung:"/"erwartet:"
|
||||
* keyword prefix or a numeric dd.mm.yyyy date — neither of which matches the
|
||||
* bare word-date ("8. Juli 2026") that FS.com actually renders.
|
||||
*/
|
||||
|
||||
// ── Constants ────────────────────────────────────────────────────────────────
|
||||
|
||||
const GERMAN_MONTHS: Record<string, string> = {
|
||||
jan: "01", feb: "02", mär: "03", mar: "03",
|
||||
apr: "04", mai: "05", may: "05", jun: "06",
|
||||
jul: "07", aug: "08", sep: "09", okt: "10",
|
||||
oct: "10", nov: "11", dez: "12", dec: "12",
|
||||
};
|
||||
|
||||
/**
|
||||
* Upper bound for a plausible per-SKU lifetime "verkauft" (units-sold) counter.
|
||||
* Values above this are treated as unknown (dropped), not stored. FS.com's
|
||||
* biggest sellers are in the hundreds of thousands; multi-million values are
|
||||
* a mis-parse (see parseGermanQty K/M bug) or vendor marketing inflation.
|
||||
* Override with FS_MAX_UNITS_SOLD.
|
||||
*/
|
||||
export const DEFAULT_MAX_UNITS_SOLD = 2_000_000;
|
||||
|
||||
/** How far past a warehouse line to look for its delivery date (characters). */
|
||||
const DATE_WINDOW = 200;
|
||||
|
||||
// ── Number parsing ───────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Parse the mantissa of a K/M-abbreviated number ("4.8" in "4.8K").
|
||||
*
|
||||
* K/M abbreviations carry a small decimal, never a grouped thousands value, so
|
||||
* a lone "." is the decimal point ("4.8K" = 4 800). German pages may instead
|
||||
* use "," as the decimal ("4,8K"); a "." alongside a "," is then a thousands
|
||||
* grouping and is stripped.
|
||||
*/
|
||||
function parseAbbrevMantissa(s: string): number {
|
||||
if (s.includes(",")) return parseFloat(s.replace(/\./g, "").replace(",", "."));
|
||||
return parseFloat(s); // "." is the decimal separator
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a German-formatted quantity string.
|
||||
* "4.895" → 4895 (period = thousands separator in a plain integer)
|
||||
* "1.587" → 1587
|
||||
* "4.8K" → 4800 (period = decimal in a K/M abbreviation)
|
||||
* "210.9K" → 210900
|
||||
* "1.4M" → 1400000
|
||||
* "63K" → 63000
|
||||
*/
|
||||
export function parseGermanQty(text: string): number | undefined {
|
||||
const t = text.trim().replace(/\s/g, "");
|
||||
if (!t) return undefined;
|
||||
|
||||
const kMatch = t.match(/^([\d.,]+)[Kk]$/);
|
||||
if (kMatch) {
|
||||
const n = parseAbbrevMantissa(kMatch[1]);
|
||||
return isNaN(n) ? undefined : Math.round(n * 1_000);
|
||||
}
|
||||
|
||||
const mMatch = t.match(/^([\d.,]+)[Mm]$/);
|
||||
if (mMatch) {
|
||||
const n = parseAbbrevMantissa(mMatch[1]);
|
||||
return isNaN(n) ? undefined : Math.round(n * 1_000_000);
|
||||
}
|
||||
|
||||
// Plain integer: "." and "," are both thousands separators here.
|
||||
const n = parseInt(t.replace(/\./g, "").replace(/,/g, ""), 10);
|
||||
return isNaN(n) ? undefined : n;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a German date to ISO "YYYY-MM-DD".
|
||||
* "8. Juli 2026" → "2026-07-08"
|
||||
* "20 Apr., 2026" → "2026-04-20"
|
||||
* "20.04.2026" → "2026-04-20"
|
||||
*/
|
||||
export function parseGermanDate(text: string): string | undefined {
|
||||
const numericMatch = text.match(/(\d{1,2})\.(\d{1,2})\.(\d{4})/);
|
||||
if (numericMatch) {
|
||||
const [, d, m, y] = numericMatch;
|
||||
return `${y}-${m.padStart(2, "0")}-${d.padStart(2, "0")}`;
|
||||
}
|
||||
const wordMatch = text.match(/(\d{1,2})\.?\s+([A-Za-zÄÖÜäöüß]+)\.?,?\s*(\d{4})/);
|
||||
if (!wordMatch) return undefined;
|
||||
const day = wordMatch[1].padStart(2, "0");
|
||||
const monthRaw = wordMatch[2]
|
||||
.toLowerCase()
|
||||
.replace(/ä/g, "a").replace(/ö/g, "o").replace(/ü/g, "u")
|
||||
.slice(0, 3);
|
||||
const month = GERMAN_MONTHS[monthRaw];
|
||||
if (!month) return undefined;
|
||||
return `${wordMatch[3]}-${month}-${day}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a German price to a EUR float.
|
||||
* "42,50" → 42.50
|
||||
* "1.063,02" → 1063.02
|
||||
*/
|
||||
export function parseGermanPrice(raw: string): number | undefined {
|
||||
const cleaned = raw.replace(/[^0-9.,]/g, "").trim();
|
||||
if (!cleaned) return undefined;
|
||||
let normalized: string;
|
||||
if (/\d+\.\d{3},\d{2}/.test(cleaned)) {
|
||||
normalized = cleaned.replace(/\./g, "").replace(",", ".");
|
||||
} else if (cleaned.includes(",")) {
|
||||
normalized = cleaned.replace(",", ".");
|
||||
} else {
|
||||
normalized = cleaned;
|
||||
}
|
||||
const n = parseFloat(normalized);
|
||||
return isNaN(n) || n <= 0 ? undefined : n;
|
||||
}
|
||||
|
||||
// ── Units-sold sanity gate ───────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Clamp an implausible units-sold value to `undefined` (unknown). Returns the
|
||||
* value unchanged when it is a finite, non-negative number at or below `cap`.
|
||||
*/
|
||||
export function sanitizeUnitsSold(
|
||||
raw: number | undefined,
|
||||
cap: number = DEFAULT_MAX_UNITS_SOLD
|
||||
): number | undefined {
|
||||
if (raw === undefined) return undefined;
|
||||
if (!Number.isFinite(raw) || raw < 0) return undefined;
|
||||
if (raw > cap) return undefined;
|
||||
return raw;
|
||||
}
|
||||
|
||||
// ── Delivery-date extraction ─────────────────────────────────────────────────
|
||||
|
||||
/** dd.mm.yyyy OR "8. Juli 2026" / "20 Apr., 2026" (day + month word + year). */
|
||||
const DELIVERY_DATE_RE =
|
||||
/(\d{1,2}\.\d{1,2}\.\d{4})|(\d{1,2}\.?\s+[A-Za-zÄÖÜäöüß]{3,}\.?,?\s*\d{4})/;
|
||||
|
||||
/**
|
||||
* Find the first plausible calendar date inside a bounded text block (the slice
|
||||
* that belongs to one warehouse line) and return it as ISO "YYYY-MM-DD".
|
||||
* The date can appear with or without a "Lieferbar"/"Lieferung" keyword — FS.com
|
||||
* renders it bare ("…DE-Lager, 8. Juli 2026 Lieferbar"), so no keyword is required.
|
||||
*/
|
||||
export function extractDeliveryDate(block: string): string | undefined {
|
||||
const m = block.match(DELIVERY_DATE_RE);
|
||||
if (!m) return undefined;
|
||||
const iso = parseGermanDate(m[0]);
|
||||
if (!iso) return undefined;
|
||||
const year = Number(iso.slice(0, 4));
|
||||
if (year < 2024 || year > 2100) return undefined; // reject stray/garbage years
|
||||
return iso;
|
||||
}
|
||||
|
||||
// ── Warehouse block parsing ──────────────────────────────────────────────────
|
||||
|
||||
export interface WarehouseStock {
|
||||
deQty?: number;
|
||||
deDeliveryDate?: string;
|
||||
globalQty?: number;
|
||||
globalDeliveryDate?: string;
|
||||
backorderQty?: number;
|
||||
backorderDate?: string;
|
||||
unitsSold?: number;
|
||||
}
|
||||
|
||||
const DE_QTY_RE: RegExp[] = [
|
||||
/(\d[\d.,KkMm]*)\s*Stk\.\s*im\s*DE[- ]Lager/i,
|
||||
/(\d[\d.,KkMm]*)\s*im\s*DE[- ]?Lager/i,
|
||||
];
|
||||
const GLOBAL_QTY_RE: RegExp[] = [
|
||||
/(\d[\d.,KkMm]*)\s*Stk\.\s*im\s*Global[- ]Lager/i,
|
||||
/(\d[\d.,KkMm]*)\s*im\s*Global[- ]?(?:Lager|Warehouse)/i,
|
||||
/(\d[\d.,KkMm]*)\s*in\s+Global\s+Warehouse/i,
|
||||
];
|
||||
const BACKORDER_QTY_RE: RegExp[] = [
|
||||
/(\d[\d.,KkMm]*)\s*(?:Stk\.)?\s*in\s+Nachlieferung/i,
|
||||
/Nachlieferung[:\s]*(\d[\d.,KkMm]*)/i,
|
||||
];
|
||||
const UNITS_SOLD_RE: RegExp[] = [
|
||||
/(\d[\d.,KkMm]*)\s*(?:[Mm]al\s+)?[Vv]erkauft/,
|
||||
/([\d.,KkMm]+)\+?\s*sold/i,
|
||||
];
|
||||
|
||||
interface SectionMatch { value: string; index: number; }
|
||||
|
||||
function firstMatch(text: string, res: RegExp[]): SectionMatch | undefined {
|
||||
for (const re of res) {
|
||||
const m = text.match(re);
|
||||
if (m && m[1] != null && m.index != null) return { value: m[1], index: m.index };
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the full warehouse-availability block out of a product page's body
|
||||
* text: DE-Lager, Global-Lager and Nachlieferung quantities, each with the
|
||||
* delivery date rendered on its line, plus the "verkauft"/"sold" counter.
|
||||
*
|
||||
* A section's delivery date is looked for only in the window between that
|
||||
* section's quantity and the start of the next section, so a warehouse with no
|
||||
* date never picks up a neighbouring warehouse's date.
|
||||
*/
|
||||
export function parseWarehouseStock(
|
||||
bodyText: string,
|
||||
unitsSoldCap: number = DEFAULT_MAX_UNITS_SOLD
|
||||
): WarehouseStock {
|
||||
const t = bodyText;
|
||||
const de = firstMatch(t, DE_QTY_RE);
|
||||
const gl = firstMatch(t, GLOBAL_QTY_RE);
|
||||
const bo = firstMatch(t, BACKORDER_QTY_RE);
|
||||
const sold = firstMatch(t, UNITS_SOLD_RE);
|
||||
|
||||
const starts = [de, gl, bo, sold]
|
||||
.filter((s): s is SectionMatch => s !== undefined)
|
||||
.map((s) => s.index);
|
||||
|
||||
const dateFor = (sec: SectionMatch | undefined): string | undefined => {
|
||||
if (!sec) return undefined;
|
||||
const later = starts.filter((b) => b > sec.index);
|
||||
const end = later.length ? Math.min(sec.index + DATE_WINDOW, ...later) : sec.index + DATE_WINDOW;
|
||||
return extractDeliveryDate(t.slice(sec.index, end));
|
||||
};
|
||||
|
||||
const result: WarehouseStock = {};
|
||||
|
||||
if (de) {
|
||||
const q = parseGermanQty(de.value);
|
||||
if (q !== undefined) result.deQty = q;
|
||||
const d = dateFor(de);
|
||||
if (d) result.deDeliveryDate = d;
|
||||
}
|
||||
if (gl) {
|
||||
const q = parseGermanQty(gl.value);
|
||||
if (q !== undefined) result.globalQty = q;
|
||||
const d = dateFor(gl);
|
||||
if (d) result.globalDeliveryDate = d;
|
||||
}
|
||||
if (bo) {
|
||||
const q = parseGermanQty(bo.value);
|
||||
if (q !== undefined) result.backorderQty = q;
|
||||
const d = dateFor(bo);
|
||||
if (d) result.backorderDate = d;
|
||||
}
|
||||
if (sold) {
|
||||
const s = sanitizeUnitsSold(parseGermanQty(sold.value), unitsSoldCap);
|
||||
if (s !== undefined) result.unitsSold = s;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// ── Lead-time derivation ─────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Derive lead-time in days as the soonest availability across the warehouses
|
||||
* that quote a delivery date: min(date − today), floored at 0. Returns
|
||||
* `undefined` when no date is known (never fabricate a 0).
|
||||
*/
|
||||
export function computeLeadTimeDays(
|
||||
dates: Array<string | null | undefined>,
|
||||
now: Date = new Date()
|
||||
): number | undefined {
|
||||
const todayUtc = Date.UTC(now.getFullYear(), now.getMonth(), now.getDate());
|
||||
let min: number | undefined;
|
||||
for (const d of dates) {
|
||||
if (!d) continue;
|
||||
const m = d.match(/^(\d{4})-(\d{2})-(\d{2})$/);
|
||||
if (!m) continue;
|
||||
const dUtc = Date.UTC(Number(m[1]), Number(m[2]) - 1, Number(m[3]));
|
||||
const days = Math.max(0, Math.round((dUtc - todayUtc) / 86_400_000));
|
||||
if (min === undefined || days < min) min = days;
|
||||
}
|
||||
return min;
|
||||
}
|
||||
@ -63,6 +63,12 @@ import {
|
||||
} from "../utils/db";
|
||||
import { contentHash } from "../utils/hash";
|
||||
import { updateVerifiedSpecs, parseSpecTable } from "../utils/spec-updater";
|
||||
import {
|
||||
parseGermanPrice,
|
||||
parseWarehouseStock,
|
||||
computeLeadTimeDays,
|
||||
DEFAULT_MAX_UNITS_SOLD,
|
||||
} from "./fs-com-parse";
|
||||
|
||||
// ── Constants ──────────────────────────────────────────────────────────────────
|
||||
|
||||
@ -75,6 +81,8 @@ const FORCE_REVALIDATE = process.env["TIP_FORCE_REVALIDATE"] === "1";
|
||||
const ONLY_MISSING_IMAGES = process.env["FS_ONLY_MISSING_IMAGES"] === "1";
|
||||
const DB_DETAIL_ONLY = process.env["FS_DB_DETAIL_ONLY"] === "1";
|
||||
const URL_DISCOVERY_ONLY = process.env["FS_URL_DISCOVERY_ONLY"] === "1";
|
||||
// Drop implausible "verkauft" (units-sold) counters at ingest (see fs-com-parse).
|
||||
const FS_MAX_UNITS_SOLD = parseInt(process.env["FS_MAX_UNITS_SOLD"] ?? String(DEFAULT_MAX_UNITS_SOLD), 10);
|
||||
|
||||
const PROXY_URLS = (process.env["PROXY_URLS"] ?? "")
|
||||
.split(",")
|
||||
@ -113,83 +121,7 @@ const DE_COOKIES = [
|
||||
{ name: "country", value: "DE", domain: ".fs.com", path: "/" },
|
||||
];
|
||||
|
||||
// ── German locale parsers ──────────────────────────────────────────────────────
|
||||
|
||||
const GERMAN_MONTHS: Record<string, string> = {
|
||||
jan: "01", feb: "02", mär: "03", mar: "03",
|
||||
apr: "04", mai: "05", may: "05", jun: "06",
|
||||
jul: "07", aug: "08", sep: "09", okt: "10",
|
||||
oct: "10", nov: "11", dez: "12", dec: "12",
|
||||
};
|
||||
|
||||
/**
|
||||
* Parse German-formatted quantity string.
|
||||
* "4.895" → 4895 (period = thousands separator in German)
|
||||
* "210.9K" → 210900
|
||||
* "1.2M" → 1200000
|
||||
*/
|
||||
function parseGermanQty(text: string): number | undefined {
|
||||
const t = text.trim().replace(/\s/g, "");
|
||||
if (!t) return undefined;
|
||||
|
||||
const kMatch = t.match(/^([\d.,]+)[Kk]$/);
|
||||
if (kMatch) {
|
||||
const n = parseFloat(kMatch[1].replace(/\./g, "").replace(",", "."));
|
||||
return isNaN(n) ? undefined : Math.round(n * 1_000);
|
||||
}
|
||||
|
||||
const mMatch = t.match(/^([\d.,]+)[Mm]$/);
|
||||
if (mMatch) {
|
||||
const n = parseFloat(mMatch[1].replace(/\./g, "").replace(",", "."));
|
||||
return isNaN(n) ? undefined : Math.round(n * 1_000_000);
|
||||
}
|
||||
|
||||
const n = parseInt(t.replace(/\./g, "").replace(/,/g, ""), 10);
|
||||
return isNaN(n) ? undefined : n;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse German date to ISO "YYYY-MM-DD".
|
||||
* "20 Apr., 2026" → "2026-04-20"
|
||||
* "20.04.2026" → "2026-04-20"
|
||||
*/
|
||||
function parseGermanDate(text: string): string | undefined {
|
||||
const numericMatch = text.match(/(\d{1,2})\.(\d{1,2})\.(\d{4})/);
|
||||
if (numericMatch) {
|
||||
const [, d, m, y] = numericMatch;
|
||||
return `${y}-${m.padStart(2, "0")}-${d.padStart(2, "0")}`;
|
||||
}
|
||||
const wordMatch = text.match(/(\d{1,2})\.?\s+([A-Za-zÄÖÜäöüß]+)\.?,?\s*(\d{4})/);
|
||||
if (!wordMatch) return undefined;
|
||||
const day = wordMatch[1].padStart(2, "0");
|
||||
const monthRaw = wordMatch[2]
|
||||
.toLowerCase()
|
||||
.replace(/ä/g, "a").replace(/ö/g, "o").replace(/ü/g, "u")
|
||||
.slice(0, 3);
|
||||
const month = GERMAN_MONTHS[monthRaw];
|
||||
if (!month) return undefined;
|
||||
return `${wordMatch[3]}-${month}-${day}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse German price to EUR float.
|
||||
* "42,50" → 42.50
|
||||
* "1.063,02" → 1063.02
|
||||
*/
|
||||
function parseGermanPrice(raw: string): number | undefined {
|
||||
const cleaned = raw.replace(/[^0-9.,]/g, "").trim();
|
||||
if (!cleaned) return undefined;
|
||||
let normalized: string;
|
||||
if (/\d+\.\d{3},\d{2}/.test(cleaned)) {
|
||||
normalized = cleaned.replace(/\./g, "").replace(",", ".");
|
||||
} else if (cleaned.includes(",")) {
|
||||
normalized = cleaned.replace(",", ".");
|
||||
} else {
|
||||
normalized = cleaned;
|
||||
}
|
||||
const n = parseFloat(normalized);
|
||||
return isNaN(n) || n <= 0 ? undefined : n;
|
||||
}
|
||||
// German number/date/warehouse parsers live in ./fs-com-parse (pure + unit-tested).
|
||||
|
||||
// ── Stock level helper ─────────────────────────────────────────────────────────
|
||||
|
||||
@ -648,64 +580,18 @@ async function scrapeProductDetails(
|
||||
}
|
||||
}
|
||||
|
||||
// ── DE-Lager ───────────────────────────────────────────────────────────
|
||||
let deQty: number | undefined;
|
||||
let deDeliveryDate: string | undefined;
|
||||
const deM =
|
||||
t.match(/(\d[\d.,KkMm]*)\s*Stk\.\s*im\s*DE[- ]Lager/i) ??
|
||||
t.match(/(\d[\d.,KkMm]*)\s*im\s*DE[- ]?Lager/i);
|
||||
if (deM?.[1]) {
|
||||
deQty = parseGermanQty(deM[1]);
|
||||
const idx = t.indexOf(deM[0]);
|
||||
const ctx = t.slice(idx, idx + 300);
|
||||
const dm =
|
||||
ctx.match(/Lieferung[:\s]+([0-9]{1,2}\.?\s+\w+\.?,?\s*\d{4})/i) ??
|
||||
ctx.match(/erwartet[:\s]+([0-9]{1,2}\.?\s+\w+\.?,?\s*\d{4})/i) ??
|
||||
ctx.match(/([0-9]{1,2}\.[0-9]{1,2}\.[0-9]{4})/);
|
||||
if (dm?.[1]) deDeliveryDate = parseGermanDate(dm[1]);
|
||||
}
|
||||
|
||||
// ── Global-Lager ───────────────────────────────────────────────────────
|
||||
let globalQty: number | undefined;
|
||||
let globalDeliveryDate: string | undefined;
|
||||
const glM =
|
||||
t.match(/(\d[\d.,KkMm]*)\s*Stk\.\s*im\s*Global[- ]Lager/i) ??
|
||||
t.match(/(\d[\d.,KkMm]*)\s*im\s*Global[- ]?(?:Lager|Warehouse)/i) ??
|
||||
t.match(/(\d[\d.,KkMm]*)\s*in\s+Global\s+Warehouse/i);
|
||||
if (glM?.[1]) {
|
||||
globalQty = parseGermanQty(glM[1]);
|
||||
const idx = t.indexOf(glM[0]);
|
||||
const ctx = t.slice(idx, idx + 300);
|
||||
const dm =
|
||||
ctx.match(/Lieferung[:\s]+([0-9]{1,2}\.?\s+\w+\.?,?\s*\d{4})/i) ??
|
||||
ctx.match(/erwartet[:\s]+([0-9]{1,2}\.?\s+\w+\.?,?\s*\d{4})/i) ??
|
||||
ctx.match(/([0-9]{1,2}\.[0-9]{1,2}\.[0-9]{4})/);
|
||||
if (dm?.[1]) globalDeliveryDate = parseGermanDate(dm[1]);
|
||||
}
|
||||
|
||||
// ── Nachlieferung ──────────────────────────────────────────────────────
|
||||
let backorderQty: number | undefined;
|
||||
let backorderDate: string | undefined;
|
||||
const boM =
|
||||
t.match(/(\d[\d.,KkMm]*)\s*(?:Stk\.)?\s*in\s+Nachlieferung/i) ??
|
||||
t.match(/Nachlieferung[:\s]*(\d[\d.,KkMm]*)/i);
|
||||
if (boM?.[1]) {
|
||||
backorderQty = parseGermanQty(boM[1]);
|
||||
const idx = t.indexOf(boM[0]);
|
||||
const ctx = t.slice(idx, idx + 300);
|
||||
const dm =
|
||||
ctx.match(/[Ee]rwartet[:\s]+([0-9]{1,2}\.?\s+\w+\.?,?\s*\d{4})/i) ??
|
||||
ctx.match(/Lieferung[:\s]+([0-9]{1,2}\.?\s+\w+\.?,?\s*\d{4})/i) ??
|
||||
ctx.match(/([0-9]{1,2}\.[0-9]{1,2}\.[0-9]{4})/);
|
||||
if (dm?.[1]) backorderDate = parseGermanDate(dm[1]);
|
||||
}
|
||||
|
||||
// ── Units sold ─────────────────────────────────────────────────────────
|
||||
let unitsSold: number | undefined;
|
||||
const soldM =
|
||||
t.match(/(\d[\d.,KkMm]*)\s*(?:[Mm]al\s+)?[Vv]erkauft/) ??
|
||||
t.match(/([\d.,KkMm]+)\+?\s*sold/i);
|
||||
if (soldM?.[1]) unitsSold = parseGermanQty(soldM[1]);
|
||||
// ── Warehouse availability (qty + per-warehouse delivery date + units sold) ──
|
||||
// FS.com renders each warehouse on its own line, e.g.
|
||||
// "2 Stk. im DE-Lager, 8. Juli 2026 Lieferbar". parseWarehouseStock
|
||||
// extracts the bare word-date per warehouse and gates units_sold.
|
||||
const wh = parseWarehouseStock(t, FS_MAX_UNITS_SOLD);
|
||||
const deQty = wh.deQty;
|
||||
const deDeliveryDate = wh.deDeliveryDate;
|
||||
const globalQty = wh.globalQty;
|
||||
const globalDeliveryDate = wh.globalDeliveryDate;
|
||||
const backorderQty = wh.backorderQty;
|
||||
const backorderDate = wh.backorderDate;
|
||||
const unitsSold = wh.unitsSold;
|
||||
|
||||
// ── Part number refinement ─────────────────────────────────────────────
|
||||
let partNumber = listingPn;
|
||||
@ -1010,6 +896,12 @@ export async function scrapeFs(): Promise<void> {
|
||||
|
||||
const stockLevel = deriveStockLevel(detail.deQty, detail.globalQty, detail.backorderQty);
|
||||
const totalQty = (detail.deQty ?? 0) + (detail.globalQty ?? 0);
|
||||
// Soonest availability across the warehouses that quote a delivery date.
|
||||
const leadTimeDays = computeLeadTimeDays([
|
||||
detail.deDeliveryDate,
|
||||
detail.globalDeliveryDate,
|
||||
detail.backorderDate,
|
||||
]);
|
||||
|
||||
if (detail.priceNet && detail.priceNet > 0) {
|
||||
const hash = contentHash({
|
||||
@ -1024,6 +916,7 @@ export async function scrapeFs(): Promise<void> {
|
||||
currency: "EUR",
|
||||
stockLevel,
|
||||
quantityAvailable: totalQty > 0 ? totalQty : undefined,
|
||||
leadTimeDays,
|
||||
url: detail.url,
|
||||
contentHash: hash,
|
||||
});
|
||||
@ -1042,6 +935,7 @@ export async function scrapeFs(): Promise<void> {
|
||||
backorderQty: detail.backorderQty,
|
||||
backorderEstimatedDate: detail.backorderDate ?? null,
|
||||
unitsSold: detail.unitsSold,
|
||||
leadTimeDays,
|
||||
compatibleBrands: detail.compatibleBrands,
|
||||
priceNet: detail.priceNet,
|
||||
productUrl: detail.url,
|
||||
|
||||
@ -394,6 +394,8 @@ export async function upsertStockObservation(params: {
|
||||
backorderQty?: number;
|
||||
backorderEstimatedDate?: string | null;
|
||||
unitsSold?: number;
|
||||
/** Soonest availability in days across warehouses that quote a delivery date. */
|
||||
leadTimeDays?: number;
|
||||
compatibleBrands?: string[];
|
||||
priceNet?: number;
|
||||
productUrl?: string;
|
||||
@ -444,7 +446,7 @@ export async function upsertStockObservation(params: {
|
||||
warehouse_de_qty, warehouse_de_delivery_date,
|
||||
warehouse_global_qty, warehouse_global_delivery_date,
|
||||
backorder_qty, backorder_estimated_date,
|
||||
units_sold, compatible_brands, price_net, product_url,
|
||||
units_sold, lead_time_days, compatible_brands, price_net, product_url,
|
||||
stock_confidence, price_currency, price_includes_tax, stock_vendor_ts
|
||||
) VALUES (
|
||||
NOW(), $1, $2,
|
||||
@ -452,8 +454,8 @@ export async function upsertStockObservation(params: {
|
||||
$5, $6::date,
|
||||
$7, $8::date,
|
||||
$9, $10::date,
|
||||
$11, $12, $13, $14,
|
||||
$15, $16, $17, $18
|
||||
$11, $12, $13, $14, $15,
|
||||
$16, $17, $18, $19
|
||||
)`,
|
||||
[
|
||||
params.transceiverId,
|
||||
@ -467,6 +469,7 @@ export async function upsertStockObservation(params: {
|
||||
params.backorderQty ?? null,
|
||||
params.backorderEstimatedDate ?? null,
|
||||
params.unitsSold ?? null,
|
||||
params.leadTimeDays ?? null,
|
||||
params.compatibleBrands?.length ? params.compatibleBrands : null,
|
||||
params.priceNet ?? null,
|
||||
params.productUrl ?? null,
|
||||
|
||||
@ -15,5 +15,5 @@
|
||||
"incremental": true
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
"exclude": ["node_modules", "dist", "src/**/*.test.ts"]
|
||||
}
|
||||
|
||||
54
sql/122-fs-com-delivery-dates-and-units-sold-gate.sql
Normal file
54
sql/122-fs-com-delivery-dates-and-units-sold-gate.sql
Normal file
@ -0,0 +1,54 @@
|
||||
-- Migration 122: FS.com delivery dates + lead_time now collected; purge units_sold garbage
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
-- Follow-up to migration 121, which documented lead_time_days as NOT COLLECTED and
|
||||
-- flagged that populating it was a scraper-side task. That task is now done:
|
||||
--
|
||||
-- * fs-com.ts (via fs-com-parse.ts) parses the bare word-date FS.com renders on
|
||||
-- each warehouse line ("… im DE-Lager, 8. Juli 2026 Lieferbar") into
|
||||
-- warehouse_de_delivery_date / warehouse_global_delivery_date /
|
||||
-- backorder_estimated_date, and derives lead_time_days = min(date − today).
|
||||
-- * upsertStockObservation now writes lead_time_days.
|
||||
--
|
||||
-- Historical rows stay NULL until the next FS.com run re-scrapes each SKU; NULL
|
||||
-- therefore still means "unknown / not yet captured", not "zero lead time".
|
||||
--
|
||||
-- Second fix: units_sold was ~10× inflated for decimal-abbreviated values
|
||||
-- ("4.8K"→48000, "1.4M"→14000000) because the K/M parser stripped the decimal
|
||||
-- point. The parser is fixed AND ingest now drops values above a sanity cap.
|
||||
-- This migration removes the clearly-impossible historical values (the multi-
|
||||
-- million "outliers") so no consumer ingests them before the re-scrape. Sub-cap
|
||||
-- values that may still be mildly inflated self-correct on the next FS.com run
|
||||
-- (a changed value writes a fresh observation; consumers read the latest).
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
-- 1. Purge impossible units_sold (matches the ingest gate default, FS_MAX_UNITS_SOLD=2,000,000).
|
||||
UPDATE stock_observations
|
||||
SET units_sold = NULL
|
||||
WHERE units_sold IS NOT NULL
|
||||
AND units_sold > 2000000;
|
||||
|
||||
-- 2. Re-document the columns as collected-by-scraper (reversing migration 121's note).
|
||||
COMMENT ON COLUMN stock_observations.warehouse_de_delivery_date IS
|
||||
'FS.com DE-Lager availability date (collected 2026-07-04+ by fs-com-parse). '
|
||||
'NULL = warehouse in stock now, vendor did not quote a date, or SKU not yet re-scraped.';
|
||||
|
||||
COMMENT ON COLUMN stock_observations.warehouse_global_delivery_date IS
|
||||
'FS.com Global-Lager availability date (collected 2026-07-04+ by fs-com-parse). '
|
||||
'NULL semantics as warehouse_de_delivery_date.';
|
||||
|
||||
COMMENT ON COLUMN stock_observations.backorder_estimated_date IS
|
||||
'FS.com Nachlieferung (backorder) estimated date (collected 2026-07-04+ by fs-com-parse). '
|
||||
'NULL = no backorder date quoted or SKU not yet re-scraped.';
|
||||
|
||||
COMMENT ON COLUMN stock_observations.lead_time_days IS
|
||||
'Soonest availability in days = min(delivery_date − today) across warehouses that quote a '
|
||||
'date. Collected 2026-07-04+ by the FS.com scraper; other vendors do not expose a lead time. '
|
||||
'Treat NULL as "unknown / not captured" (do NOT default to 0).';
|
||||
|
||||
COMMENT ON COLUMN stock_observations.units_sold IS
|
||||
'FS.com "verkauft" lifetime sell-through counter. Parser de-abbreviates K/M correctly and '
|
||||
'ingest drops values > 2,000,000 as implausible. FS.com only; NULL elsewhere.';
|
||||
|
||||
INSERT INTO _migrations (filename)
|
||||
VALUES ('122-fs-com-delivery-dates-and-units-sold-gate.sql')
|
||||
ON CONFLICT (filename) DO NOTHING;
|
||||
Loading…
x
Reference in New Issue
Block a user