4299e24e13
FS.com renders per-warehouse availability as a bare word-date on each line
("2 Stk. im DE-Lager, 8. Juli 2026 Lieferbar"). The old regex only matched a
"Lieferung:"/"erwartet:" prefix or numeric dd.mm.yyyy, so warehouse_de/global
delivery_date + backorder_estimated_date were 0% populated across 85k rows, and
lead_time_days was never written at all.
- new fs-com-parse.ts: pure, unit-tested parsers (13 node:test cases).
- extractDeliveryDate finds the bare word/numeric date per warehouse, bounded to
its own line so a date never bleeds to a neighbouring warehouse.
- computeLeadTimeDays = min(delivery_date - today), floored at 0; threaded into
upsertStockObservation (now writes lead_time_days) and upsertPriceObservation.
- parseGermanQty: K/M abbreviations now treat '.' as a decimal ("4.8K"=4800,
"1.4M"=1.4M) instead of stripping it (was "1.4M"->14,000,000 — the outlier).
- units_sold sanity gate at ingest (FS_MAX_UNITS_SOLD, default 2,000,000).
- sql/122: purge impossible historical units_sold, re-document columns as collected.
139 lines
6.3 KiB
TypeScript
139 lines
6.3 KiB
TypeScript
/**
|
|
* Unit tests for the FS.com pure parsers. Run with:
|
|
* npx tsx --test src/scrapers/fs-com-parse.test.ts
|
|
*/
|
|
import { test } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import {
|
|
parseGermanQty,
|
|
parseGermanDate,
|
|
parseGermanPrice,
|
|
extractDeliveryDate,
|
|
parseWarehouseStock,
|
|
computeLeadTimeDays,
|
|
sanitizeUnitsSold,
|
|
} from "./fs-com-parse";
|
|
|
|
// ── parseGermanQty — the K/M decimal bug ─────────────────────────────────────
|
|
|
|
test("parseGermanQty: plain integers use '.'/',' as thousands", () => {
|
|
assert.equal(parseGermanQty("4.895"), 4895);
|
|
assert.equal(parseGermanQty("1.587"), 1587);
|
|
assert.equal(parseGermanQty("1.501"), 1501);
|
|
assert.equal(parseGermanQty("2"), 2);
|
|
assert.equal(parseGermanQty("1627"), 1627);
|
|
});
|
|
|
|
test("parseGermanQty: K/M abbreviations treat '.' as a DECIMAL (bug fix)", () => {
|
|
// Previously "4.8K" -> 48000 and "1.4M" -> 14000000 (the garbage outlier).
|
|
assert.equal(parseGermanQty("4.8K"), 4800);
|
|
assert.equal(parseGermanQty("210.9K"), 210900);
|
|
assert.equal(parseGermanQty("504.4K"), 504400);
|
|
assert.equal(parseGermanQty("1.4M"), 1_400_000);
|
|
assert.equal(parseGermanQty("1.2M"), 1_200_000);
|
|
assert.equal(parseGermanQty("63K"), 63000); // no decimal — unchanged
|
|
assert.equal(parseGermanQty("14M"), 14_000_000); // caught later by the gate
|
|
});
|
|
|
|
test("parseGermanQty: K/M with German comma decimal still works", () => {
|
|
assert.equal(parseGermanQty("4,8K"), 4800);
|
|
assert.equal(parseGermanQty("1,4M"), 1_400_000);
|
|
});
|
|
|
|
// ── parseGermanDate ──────────────────────────────────────────────────────────
|
|
|
|
test("parseGermanDate: word and numeric forms", () => {
|
|
assert.equal(parseGermanDate("8. Juli 2026"), "2026-07-08");
|
|
assert.equal(parseGermanDate("17. Juli 2026"), "2026-07-17");
|
|
assert.equal(parseGermanDate("20 Apr., 2026"), "2026-04-20");
|
|
assert.equal(parseGermanDate("28. März 2026"), "2026-03-28");
|
|
assert.equal(parseGermanDate("20.04.2026"), "2026-04-20");
|
|
assert.equal(parseGermanDate("nonsense"), undefined);
|
|
});
|
|
|
|
// ── extractDeliveryDate ──────────────────────────────────────────────────────
|
|
|
|
test("extractDeliveryDate: finds bare word-date, no keyword needed", () => {
|
|
assert.equal(extractDeliveryDate("2 Stk. im DE-Lager, 8. Juli 2026 Lieferbar"), "2026-07-08");
|
|
assert.equal(extractDeliveryDate("1.627 Stk. im Global-Lager, 17. Juli 2026"), "2026-07-17");
|
|
});
|
|
|
|
test("extractDeliveryDate: rejects stray/garbage years and empty", () => {
|
|
assert.equal(extractDeliveryDate("© 2019 FS.COM"), undefined);
|
|
assert.equal(extractDeliveryDate("auf Lager, sofort verfügbar"), undefined);
|
|
});
|
|
|
|
// ── parseWarehouseStock — the real screenshot block ──────────────────────────
|
|
|
|
const SCREENSHOT_BLOCK = [
|
|
"2 Stk. im DE-Lager, 8. Juli 2026 Lieferbar",
|
|
"1.627 Stk. im Global-Lager, 17. Juli 2026",
|
|
"1 in Nachlieferung, 28. Juli 2026",
|
|
"4.8K verkauft",
|
|
].join("\n");
|
|
|
|
test("parseWarehouseStock: full block yields qtys + per-warehouse dates + sold", () => {
|
|
const s = parseWarehouseStock(SCREENSHOT_BLOCK);
|
|
assert.equal(s.deQty, 2);
|
|
assert.equal(s.deDeliveryDate, "2026-07-08");
|
|
assert.equal(s.globalQty, 1627);
|
|
assert.equal(s.globalDeliveryDate, "2026-07-17");
|
|
assert.equal(s.backorderQty, 1);
|
|
assert.equal(s.backorderDate, "2026-07-28");
|
|
assert.equal(s.unitsSold, 4800); // 4.8K, correctly de-abbreviated
|
|
});
|
|
|
|
test("parseWarehouseStock: a warehouse with no date does not steal a neighbour's date", () => {
|
|
const block = [
|
|
"5 Stk. im DE-Lager", // no date — available now
|
|
"800 Stk. im Global-Lager, 17. Juli 2026",
|
|
].join("\n");
|
|
const s = parseWarehouseStock(block);
|
|
assert.equal(s.deQty, 5);
|
|
assert.equal(s.deDeliveryDate, undefined); // must NOT be 2026-07-17
|
|
assert.equal(s.globalQty, 800);
|
|
assert.equal(s.globalDeliveryDate, "2026-07-17");
|
|
});
|
|
|
|
test("parseWarehouseStock: gates the 1.4M -> 14M units_sold garbage", () => {
|
|
const block = "12 Stk. im DE-Lager, 8. Juli 2026 Lieferbar\n14M verkauft";
|
|
const s = parseWarehouseStock(block);
|
|
assert.equal(s.deQty, 12);
|
|
assert.equal(s.unitsSold, undefined); // 14M > cap → dropped as unknown
|
|
});
|
|
|
|
test("parseWarehouseStock: empty when no warehouse text present", () => {
|
|
assert.deepEqual(parseWarehouseStock("Just a product description, nothing else."), {});
|
|
});
|
|
|
|
// ── sanitizeUnitsSold ────────────────────────────────────────────────────────
|
|
|
|
test("sanitizeUnitsSold: keeps plausible, drops implausible/negative", () => {
|
|
assert.equal(sanitizeUnitsSold(4800), 4800);
|
|
assert.equal(sanitizeUnitsSold(504400), 504400);
|
|
assert.equal(sanitizeUnitsSold(14_000_000), undefined);
|
|
assert.equal(sanitizeUnitsSold(-5), undefined);
|
|
assert.equal(sanitizeUnitsSold(undefined), undefined);
|
|
assert.equal(sanitizeUnitsSold(2_500_000, 2_000_000), undefined);
|
|
assert.equal(sanitizeUnitsSold(1_500_000, 2_000_000), 1_500_000);
|
|
});
|
|
|
|
// ── computeLeadTimeDays ──────────────────────────────────────────────────────
|
|
|
|
test("computeLeadTimeDays: min across dates, floored at 0, undefined when empty", () => {
|
|
const now = new Date("2026-07-04T09:00:00");
|
|
assert.equal(computeLeadTimeDays(["2026-07-08", "2026-07-17", "2026-07-28"], now), 4);
|
|
assert.equal(computeLeadTimeDays([undefined, null, "2026-07-05"], now), 1);
|
|
assert.equal(computeLeadTimeDays(["2026-07-01"], now), 0); // past date → 0, not negative
|
|
assert.equal(computeLeadTimeDays([], now), undefined);
|
|
assert.equal(computeLeadTimeDays([null, undefined], now), undefined);
|
|
});
|
|
|
|
// ── parseGermanPrice (unchanged behaviour, regression guard) ─────────────────
|
|
|
|
test("parseGermanPrice: net EUR parsing", () => {
|
|
assert.equal(parseGermanPrice("42,50"), 42.5);
|
|
assert.equal(parseGermanPrice("1.063,02 €"), 1063.02);
|
|
assert.equal(parseGermanPrice("5,10 € ohne MwSt."), 5.1);
|
|
});
|