transceiver-db/packages/scraper/src/utils/verify-product-page-prices.ts

import { pool, upsertPriceObservation } from "./db";
import { contentHash } from "./hash";

type Candidate = {
  id: string;
  vendorId: string;
  vendorName: string;
  partNumber: string;
  productUrl: string;
};

type PriceResult = {
  price: number;
  currency: string;
};

const HEADERS = {
  "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
  Accept: "text/html,application/xhtml+xml",
  "Accept-Language": "de-DE,de;q=0.9,en;q=0.8",
};

function sleep(ms: number): Promise<void> {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

function parseNumericPrice(value: string | undefined): number | undefined {
  if (!value) return undefined;
  const normalized = value
    .replace(/\s/g, "")
    .replace(/[^\d,.]/g, "")
    .replace(/\.(?=\d{3}(?:\D|$))/g, "")
    .replace(",", ".");
  const price = parseFloat(normalized);
  return Number.isFinite(price) && price > 0 && price < 50000 ? price : undefined;
}

function parseSfpCables(html: string): PriceResult | undefined {
  const patterns = [
    /<meta[^>]+property=["']product:price:amount["'][^>]+content=["']([\d,.]+)["']/i,
    /<meta[^>]+itemprop=["']price["'][^>]+content=["']([\d,.]+)["']/i,
    /<div[^>]+class=["'][^"']*price-box[^"']*["'][\s\S]{0,1500}?<span[^>]+class=["']price["'][^>]*>\s*US?\$?\s*([\d,.]+)/i,
    /<span[^>]+class=["']price["'][^>]*>\s*US?\$?\s*([\d,.]+)/i,
  ];
  for (const pattern of patterns) {
    const price = parseNumericPrice(html.match(pattern)?.[1]);
    if (price) return { price, currency: "USD" };
  }
  return undefined;
}

function parseShopFiber24(html: string): PriceResult | undefined {
  const price = parseNumericPrice(html.match(/itemprop=["']price["'][^>]+content=["']([\d,.]+)["']/i)?.[1])
    || parseNumericPrice(html.match(/([\d.]+,\d{2})\s*€/i)?.[1]);
  if (!price) return undefined;
  return { price, currency: "EUR" };
}

function parsePrice(vendorName: string, html: string): PriceResult | undefined {
  if (vendorName === "SFPcables") return parseSfpCables(html);
  if (vendorName === "ShopFiber24") return parseShopFiber24(html);
  return undefined;
}

function parseAtgbicsProductJson(jsonText: string): PriceResult | undefined {
  const parsed = JSON.parse(jsonText) as { price?: number; variants?: Array<{ price?: number | string }> };
  const rawPrice = parsed.price ?? parsed.variants?.[0]?.price;
  const numeric = typeof rawPrice === "string" ? parseFloat(rawPrice) : rawPrice;
  if (!Number.isFinite(numeric) || !numeric || numeric <= 0) return undefined;
  return { price: numeric / 100, currency: "GBP" };
}

async function fetchPage(url: string): Promise<string> {
  const resp = await fetch(url, {
    headers: HEADERS,
    signal: AbortSignal.timeout(30000),
  });
  if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
  return resp.text();
}

async function fetchVendorPrice(row: Candidate): Promise<PriceResult | undefined> {
  if (row.vendorName === "ATGBICS") {
    const jsonUrl = `${row.productUrl.replace(/\/$/, "")}.js`;
    return parseAtgbicsProductJson(await fetchPage(jsonUrl));
  }
  return parsePrice(row.vendorName, await fetchPage(row.productUrl));
}

async function main(): Promise<void> {
  const vendorFilter = (process.env["PRODUCT_PRICE_VENDOR"] || "").trim();
  const limit = parseInt(process.env["PRODUCT_PRICE_LIMIT"] || "100", 10);
  const apply = process.env["PRODUCT_PRICE_APPLY"] === "1";
  const vendorNames = vendorFilter
    ? vendorFilter.split(",").map((v) => v.trim()).filter(Boolean)
    : ["SFPcables", "ShopFiber24", "ATGBICS"];

  const result = await pool.query<Candidate>(
    `SELECT t.id,
            t.part_number AS "partNumber",
            t.product_page_url AS "productUrl",
            v.id AS "vendorId",
            v.name AS "vendorName"
     FROM transceivers t
     JOIN vendors v ON v.id = t.vendor_id
     WHERE v.name = ANY($1)
       AND COALESCE(t.category, '') != 'NonTransceiver'
       AND COALESCE(t.price_verified, false) = false
       AND COALESCE(t.price_status, 'needs_research') IN ('unknown', 'needs_research', 'ambiguous')
       AND COALESCE(t.product_page_url, '') != ''
     ORDER BY (COALESCE(t.speed_gbps, 0) >= 100) DESC,   -- 100G+ zuerst (Lupo-Fokus)
       v.name, t.part_number
     LIMIT $2`,
    [vendorNames, limit],
  );

  let prices = 0;
  let skipped = 0;
  let errors = 0;

  console.log("=== Product page price verifier ===", { vendorNames, limit, apply, count: result.rows.length });

  for (const row of result.rows) {
    await sleep(800);
    try {
      const parsed = await fetchVendorPrice(row);
      if (!parsed) {
        skipped++;
        continue;
      }

      if (apply) {
        const hash = contentHash({ price: parsed.price, currency: parsed.currency, part: row.partNumber });
        await upsertPriceObservation({
          transceiverId: row.id,
          sourceVendorId: row.vendorId,
          price: parsed.price,
          currency: parsed.currency,
          stockLevel: "in_stock",
          url: row.productUrl,
          contentHash: hash,
        });
      }

      console.log("price verified", {
        vendor: row.vendorName,
        partNumber: row.partNumber,
        price: parsed.price,
        currency: parsed.currency,
        apply,
      });
      prices++;
    } catch (err) {
      errors++;
      console.warn("price page failed", {
        vendor: row.vendorName,
        partNumber: row.partNumber,
        error: (err as Error).message,
      });
    }
  }

  console.log("Product page price verifier complete", { prices, skipped, errors, apply });
}

if (require.main === module) {
  main()
    .then(() => pool.end())
    .catch((err) => {
      console.error("Fatal:", err);
      pool.end();
      process.exit(1);
    });
}