feat(scraper): Flexoptix REST API sync robot + scheduler integration

Replaces the GraphQL/search-based Flexoptix scraper with a proper
Magento 2 REST API integration that delivers authoritative SKUs,
prices, stock levels and compatibility data.

New files:
- packages/scraper/src/robots/flexoptix-api-sync.ts
  Self-contained robot: auth → paginated fetch → normalize → DB write.
  Reads FLEXOPTIX_API_BASE_URL / _USERNAME / _PASSWORD from env.
  Returns { fetched, normalized, skipped, priceWrites, stockWrites }.
  No file intermediary — in-memory pipeline.

- scripts/import-flexoptix-catalog.ts
  One-shot CLI importer for the Pulso-generated JSONL (Codex handover).

- docs/FLEXOPTIX_CATALOG_IMPORT.md
  Runbook for manual import + per-SKU specifications enrichment.

Scheduler changes:
- Added sync:flexoptix-catalog queue + work() handler
- Scheduled every 2h at 0 */2 * * * (same cadence as legacy job)
- scrape:pricing:flexoptix kept as legacy GraphQL fallback

Also includes Codex-generated additions from this sprint:
- audiocodes-oem scraper, seed-batch35/36/37, db.ts improvements,
  sql/102 verification reconcile, README + package.json updates

README.md

@@ -127,6 +127,19 @@ All data comes from publicly available sources:
 - Multi-Source Agreements (100G CWDM4 MSA, 100G PSM4 MSA, 100G Lambda MSA, OpenZR+)
 - Vendor datasheets and public documentation
 
+## Flexoptix Catalog Import
+
+Private TIP deployments can import the normalized Flexoptix shop catalog produced
+by Magatama/Pulso:
+
+```bash
+npm run flexoptix:catalog:import -- --dry-run
+TIP_DB_PASS=... npm run flexoptix:catalog:import
+```
+
+See [docs/FLEXOPTIX_CATALOG_IMPORT.md](docs/FLEXOPTIX_CATALOG_IMPORT.md) for the
+full producer/import workflow and safety rules.
+
 ## Contributing
 
 Contributions welcome. To add a new transceiver:

docs/FLEXOPTIX_CATALOG_IMPORT.md

@@ -0,0 +1,49 @@
+# Flexoptix Catalog Import
+
+TIP can ingest the normalized Flexoptix shop catalog produced by Magatama/Pulso.
+This keeps prices, product URLs, stock, and product attributes live-data driven
+instead of baking volatile values into LLM training pairs.
+
+## Producer
+
+Run in the Magatama repo:
+
+```bash
+pnpm pulso:shop:sync:dry-run
+pnpm pulso:shop:sync -- --fixture ./shop-export.json
+FLEXOPTIX_API_BASE_URL=... FLEXOPTIX_API_TOKEN=... pnpm pulso:shop:sync
+```
+
+The producer writes:
+
+- `data/pulso/shop-products.normalized.jsonl`
+- `data/pulso/shop-catalog.normalized.json`
+- `data/pulso/shop-snapshot.manifest.json`
+
+## TIP Import
+
+Safe validation:
+
+```bash
+npm run flexoptix:catalog:import -- --dry-run
+```
+
+Import into the TIP database:
+
+```bash
+TIP_DB_PASS=... npm run flexoptix:catalog:import
+```
+
+Override the input path if needed:
+
+```bash
+npm run flexoptix:catalog:import -- --input /path/shop-products.normalized.jsonl --dry-run
+```
+
+## Safety Rules
+
+- Rows without SKU, title, form factor, speed, or reach are skipped for DB import.
+- Prices and stock are written only when present in live/API data.
+- No product URL, SKU, price, stock, delivery time, or compatibility is invented.
+- The importer uses existing TIP helpers for transceiver upsert, price observations,
+  and stock observations.

package.json

@@ -13,6 +13,7 @@
     "dev": "npm run dev -w packages/api",
     "learning-pool:build": "tsx scripts/tip-learning-pool-build.ts",
     "learning-pool:publish-hf": "python3 scripts/tip-publish-hf-datasets.py",
+    "flexoptix:catalog:import": "tsx scripts/import-flexoptix-catalog.ts",
     "migrate": "tsx scripts/migrate.ts",
     "seed": "tsx scripts/seed-from-npm.ts",
     "db:reset": "npm run migrate && npm run seed"

packages/scraper/src/robots/flexoptix-api-sync.ts

@@ -0,0 +1,531 @@
+/**
+ * Flexoptix API Sync Robot
+ *
+ * Fetches the full Flexoptix product catalog via their Magento 2 REST API,
+ * normalizes each product, and writes price + stock observations to TIP DB.
+ *
+ * Replaces the GraphQL/search-based scrapeFlexoptixCatalog() — the REST API
+ * provides authoritative SKUs, prices, stock levels, and compatibility data.
+ *
+ * Required env vars:
+ *   FLEXOPTIX_API_BASE_URL       e.g. https://www.flexoptix.net
+ *   FLEXOPTIX_API_USERNAME       Magento customer API username
+ *   FLEXOPTIX_API_PASSWORD       Magento customer API password
+ *
+ * Optional:
+ *   FLEXOPTIX_API_TOKEN          Bearer token (skips username/password login)
+ *   FLEXOPTIX_API_CURRENCY       Defaults to EUR
+ *   FLEXOPTIX_API_LIMIT          Page size, defaults to 500
+ *   FLEXOPTIX_API_TIMEOUT_MS     Request timeout, defaults to 30000
+ *   FLEXOPTIX_API_COMPATIBILITIES 1/0, defaults to 1
+ */
+
+import {
+  ensureVendor,
+  findOrCreateScrapedTransceiver,
+  upsertPriceObservation,
+  upsertStockObservation,
+} from "../utils/db";
+import { contentHash } from "../utils/hash";
+
+// ── Types ──────────────────────────────────────────────────────────────────
+
+type JsonRecord = Record<string, unknown>;
+
+interface CatalogProduct {
+  source: "flexoptix-shop-api";
+  fetchedAt: string;
+  sku: string;
+  title: string;
+  url: string | null;
+  price: {
+    amount: number | null;
+    currency: string | null;
+    source: "api" | "missing";
+    fetchedAt: string;
+  };
+  stock: {
+    status: string | null;
+    quantity: number | null;
+    source: "api" | "missing";
+  };
+  optics: {
+    formFactor: string | null;
+    speedGbps: number | null;
+    reachM: number | null;
+    wavelengthNm: number | null;
+    connector: string | null;
+    fiberType: string | null;
+    protocol: string | null;
+    coding: string | null;
+    bidi: boolean | null;
+    dwdm: boolean | null;
+    cwdm: boolean | null;
+  };
+  compatibility: Array<{
+    vendor: string;
+    platform: string | null;
+    coding: string | null;
+    source: "api";
+  }>;
+}
+
+export interface FlexoptixSyncResult {
+  fetched: number;
+  normalized: number;
+  skipped: number;
+  priceWrites: number;
+  stockWrites: number;
+}
+
+// ── Generic helpers ────────────────────────────────────────────────────────
+
+function isRecord(value: unknown): value is JsonRecord {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function flatten(value: unknown, prefix = "", output: JsonRecord = {}): JsonRecord {
+  if (Array.isArray(value)) {
+    output[prefix] = value;
+    return output;
+  }
+  if (!isRecord(value)) {
+    output[prefix] = value;
+    return output;
+  }
+  for (const [key, child] of Object.entries(value)) {
+    const childKey = prefix ? `${prefix}.${key}` : key;
+    if (isRecord(child)) {
+      flatten(child, childKey, output);
+    } else {
+      output[childKey] = child;
+    }
+  }
+  return output;
+}
+
+function flatLookup(record: JsonRecord): JsonRecord {
+  const flat = flatten(record);
+  const normalized: JsonRecord = {};
+  for (const [key, value] of Object.entries(flat)) {
+    normalized[key.toLowerCase().replace(/[_\s-]+/g, "")] = value;
+  }
+  return normalized;
+}
+
+function pick(flat: JsonRecord, keys: string[]): unknown {
+  const normalizedKeys = keys.map(key => key.toLowerCase().replace(/[_\s-]+/g, ""));
+  for (const key of normalizedKeys) {
+    if (flat[key] !== undefined && flat[key] !== null && flat[key] !== "") return flat[key];
+  }
+  for (const [flatKey, value] of Object.entries(flat)) {
+    if (value === undefined || value === null || value === "") continue;
+    if (normalizedKeys.some(key => flatKey.endsWith(key))) return value;
+  }
+  return null;
+}
+
+function asString(value: unknown): string | null {
+  if (typeof value === "string") return value.trim() || null;
+  if (typeof value === "number" || typeof value === "boolean") return String(value);
+  return null;
+}
+
+function asNumber(value: unknown): number | null {
+  if (typeof value === "number" && Number.isFinite(value)) return value;
+  if (typeof value !== "string") return null;
+  const trimmed = value.trim();
+  if (!trimmed) return null;
+  const stripped = trimmed.replace(/[^\d,.-]/g, "");
+  const normalized = stripped.includes(",") && !stripped.includes(".")
+    ? stripped.replace(",", ".")
+    : stripped.replace(/,/g, "");
+  const parsed = Number(normalized);
+  return Number.isFinite(parsed) ? parsed : null;
+}
+
+function asBoolean(value: unknown): boolean | null {
+  if (typeof value === "boolean") return value;
+  if (typeof value !== "string") return null;
+  const normalized = value.trim().toLowerCase();
+  if (["true", "yes", "ja", "1"].includes(normalized)) return true;
+  if (["false", "no", "nein", "0"].includes(normalized)) return false;
+  return null;
+}
+
+// ── Inference helpers ──────────────────────────────────────────────────────
+
+function inferBoolFlags(
+  ...values: Array<string | null>
+): { bidi: boolean | null; dwdm: boolean | null; cwdm: boolean | null } {
+  const text = values.filter(Boolean).join(" ").toLowerCase();
+  return {
+    bidi: /\bbidi\b|bi-directional|bidirectional|simplex/.test(text) ? true : null,
+    dwdm: /\bdwdm\b|itu channel|itu grid/.test(text) ? true : null,
+    cwdm: /\bcwdm\b/.test(text) ? true : null,
+  };
+}
+
+function parseReachMeters(value: unknown): number | null {
+  const text = asString(value)?.toLowerCase();
+  if (text) {
+    const km = text.match(/([\d.,]+)\s*km/);
+    if (km) return Math.round((asNumber(km[1]) ?? 0) * 1000);
+    const meters = text.match(/([\d.,]+)\s*m/);
+    if (meters) return Math.round(asNumber(meters[1]) ?? 0);
+  }
+  return asNumber(value);
+}
+
+function parseSpeedGbps(value: unknown): number | null {
+  const numeric = asNumber(value);
+  if (numeric !== null) return numeric;
+  const text = asString(value)?.toLowerCase();
+  if (!text) return null;
+  const gbps = text.match(/([\d.,]+)\s*(g|gb|gbps|gbit)/);
+  if (gbps) return asNumber(gbps[1]);
+  const mbps = text.match(/([\d.,]+)\s*(m|mb|mbps|mbit)/);
+  if (mbps) {
+    const mb = asNumber(mbps[1]);
+    return mb === null ? null : mb / 1000;
+  }
+  return null;
+}
+
+function inferFormFactor(...values: Array<string | null>): string | null {
+  const text = values.filter(Boolean).join(" ").toUpperCase();
+  const formFactors = [
+    "QSFP-DD800", "QSFP-DD", "QSFP112", "QSFP56", "QSFP28", "QSFP+", "QSFP",
+    "OSFP224", "OSFP112", "OSFP",
+    "SFP56", "SFP28", "SFP+", "SFP",
+    "XFP", "CFP4", "CFP2", "CFP", "GBIC",
+  ];
+  return formFactors.find(ff => text.includes(ff)) ?? null;
+}
+
+function inferConnector(...values: Array<string | null>): string | null {
+  const text = values.filter(Boolean).join(" ").toUpperCase();
+  if (text.includes("LC-DUPLEX") || text.includes("LC DUPLEX")) return "LC-Duplex";
+  if (text.includes("LC SIMPLEX")) return "LC-Simplex";
+  if (/\bMPO\b|\bMTP\b/.test(text)) return "MPO/MTP";
+  if (/\bRJ45\b|COPPER/.test(text)) return "RJ45";
+  if (/\bSC\b/.test(text)) return "SC";
+  return null;
+}
+
+function inferFiberType(...values: Array<string | null>): string | null {
+  const text = values.filter(Boolean).join(" ").toLowerCase();
+  if (/multimode|\bmmf\b|om[1-5]/.test(text)) return "MMF";
+  if (/singlemode|single-mode|\bsmf\b|os2/.test(text)) return "SMF";
+  if (/copper|rj45|dac/.test(text)) return "Copper";
+  return null;
+}
+
+function inferWavelengthNm(...values: Array<string | null>): number | null {
+  const text = values.filter(Boolean).join(" ");
+  const lambda = text.match(/[λλ]?\s*(\d{3,4})\s*nm/i);
+  return lambda ? asNumber(lambda[1]) : null;
+}
+
+// ── Normalization ──────────────────────────────────────────────────────────
+
+function extractCompatibility(row: JsonRecord): CatalogProduct["compatibility"] {
+  const rawCompat = row.compatibility ?? row.compatibilities ?? row.vendorCompatibility;
+  const rows = Array.isArray(rawCompat) ? rawCompat.filter(isRecord) : [];
+  return rows.flatMap(entry => {
+    const flat = flatLookup(entry);
+    const vendor = asString(pick(flat, ["vendor", "manufacturer", "brand", "systemVendor"]));
+    if (!vendor) return [];
+    return [{
+      vendor,
+      platform: asString(pick(flat, ["platform", "device", "switch", "model", "series"])),
+      coding: asString(pick(flat, ["coding", "code", "eeprom", "originalPartNumber"])),
+      source: "api" as const,
+    }];
+  });
+}
+
+function normalizeProduct(row: JsonRecord, fetchedAt: string): CatalogProduct | null {
+  const flat = flatLookup(row);
+  const title = asString(pick(flat, ["title", "name", "productName", "label"]));
+  const sku = asString(pick(flat, ["sku", "articleNumber", "partNumber", "productCode"]));
+  if (!sku || !title) return null;
+
+  const url = asString(pick(flat, ["url", "productUrl", "canonicalUrl", "link"]));
+  const amount = asNumber(pick(flat, ["price", "priceNet", "netPrice", "grossPrice", "amount"]));
+  const currency = asString(pick(flat, ["currency", "priceCurrency", "currencyCode"]))
+    ?? (amount === null ? null : process.env["FLEXOPTIX_API_CURRENCY"]?.trim() ?? "EUR");
+  const quantity = asNumber(pick(flat, ["stock", "stockQuantity", "quantity", "availableQuantity"]));
+  const status = asString(pick(flat, ["stockStatus", "availability", "deliveryStatus"]))
+    ?? (quantity === null ? null : quantity > 0 ? "in_stock" : "out_of_stock");
+
+  const formFactor = asString(pick(flat, ["formFactor", "form", "moduleType", "transceiverType"]))
+    ?? inferFormFactor(title);
+  const protocol = asString(pick(flat, ["protocol", "standard", "ethernetStandard"]));
+  const connector = asString(pick(flat, ["connector", "interface", "mediaConnector"])) ?? inferConnector(title);
+  const fiberType = asString(pick(flat, ["fiberType", "fiber", "mode"])) ?? inferFiberType(title);
+  const coding = asString(pick(flat, ["coding", "vendorCoding"]));
+  const wavelength = asNumber(pick(flat, ["wavelengthNm", "wavelength", "lambda"]))
+    ?? inferWavelengthNm(title);
+  const flags = inferBoolFlags(title, protocol, formFactor, fiberType);
+
+  return {
+    source: "flexoptix-shop-api",
+    fetchedAt,
+    sku,
+    title,
+    url,
+    price: {
+      amount,
+      currency,
+      source: amount === null ? "missing" : "api",
+      fetchedAt,
+    },
+    stock: {
+      status,
+      quantity,
+      source: status === null && quantity === null ? "missing" : "api",
+    },
+    optics: {
+      formFactor,
+      speedGbps: parseSpeedGbps(pick(flat, ["speedGbps", "speed", "rate", "dataRate"]) ?? title),
+      reachM: parseReachMeters(pick(flat, ["reachM", "reach", "distance", "transmissionDistance"]) ?? title),
+      wavelengthNm: wavelength,
+      connector,
+      fiberType,
+      protocol,
+      coding,
+      bidi: asBoolean(pick(flat, ["bidi", "bidirectional"])) ?? flags.bidi,
+      dwdm: asBoolean(pick(flat, ["dwdm"])) ?? flags.dwdm,
+      cwdm: asBoolean(pick(flat, ["cwdm"])) ?? flags.cwdm,
+    },
+    compatibility: extractCompatibility(row),
+  };
+}
+
+// ── Import helpers ─────────────────────────────────────────────────────────
+
+function canImportProduct(product: CatalogProduct): boolean {
+  return Boolean(
+    product.sku
+    && product.title
+    && product.optics.formFactor
+    && product.optics.speedGbps !== null
+    && product.optics.reachM !== null,
+  );
+}
+
+function reachLabel(reachM: number | null): string | undefined {
+  if (reachM === null) return undefined;
+  if (reachM >= 1000 && reachM % 1000 === 0) return `${reachM / 1000}km`;
+  return `${reachM}m`;
+}
+
+function speedLabel(speedGbps: number | null): string | undefined {
+  if (speedGbps === null) return undefined;
+  if (speedGbps >= 1000) return `${speedGbps / 1000}T`;
+  return `${speedGbps}G`;
+}
+
+function categoryFor(product: CatalogProduct): string {
+  const text = `${product.title} ${product.optics.protocol ?? ""}`.toLowerCase();
+  if (/\bdac\b|direct attach|copper/.test(text)) return "DAC";
+  if (/\baoc\b|active optical/.test(text)) return "AOC";
+  if (/coherent|zr|dco/.test(text)) return "Coherent";
+  return "DataCenter";
+}
+
+async function importProduct(
+  product: CatalogProduct,
+  vendorId: string,
+): Promise<{ priceWritten: boolean; stockWritten: boolean }> {
+  const transceiverId = await findOrCreateScrapedTransceiver({
+    partNumber: product.sku,
+    vendorId,
+    productUrl: product.url ?? undefined,
+    formFactor: product.optics.formFactor ?? undefined,
+    speedGbps: product.optics.speedGbps ?? undefined,
+    speed: speedLabel(product.optics.speedGbps),
+    reachMeters: product.optics.reachM ?? undefined,
+    reachLabel: reachLabel(product.optics.reachM),
+    fiberType: product.optics.fiberType ?? undefined,
+    wavelengths: product.optics.wavelengthNm === null ? undefined : `${product.optics.wavelengthNm}nm`,
+    category: categoryFor(product),
+  });
+
+  let priceWritten = false;
+  if (product.price.amount !== null && product.price.currency) {
+    priceWritten = await upsertPriceObservation({
+      transceiverId,
+      sourceVendorId: vendorId,
+      price: product.price.amount,
+      currency: product.price.currency,
+      stockLevel: product.stock.status ?? "unknown",
+      quantityAvailable: product.stock.quantity ?? undefined,
+      url: product.url ?? undefined,
+      contentHash: contentHash({
+        source: product.source,
+        sku: product.sku,
+        price: product.price.amount,
+        currency: product.price.currency,
+        fetchedAt: product.price.fetchedAt,
+      }),
+    });
+  }
+
+  const stockWritten = await upsertStockObservation({
+    transceiverId,
+    sourceVendorId: vendorId,
+    stockLevel: product.stock.status ?? "unknown",
+    quantityAvailable: product.stock.quantity ?? undefined,
+    priceNet: product.price.amount ?? undefined,
+    productUrl: product.url ?? undefined,
+    priceCurrency: product.price.currency ?? undefined,
+    stockConfidence: product.stock.quantity === null ? 1 : 2,
+  });
+
+  return { priceWritten, stockWritten };
+}
+
+// ── API client ─────────────────────────────────────────────────────────────
+
+function validateEnv(): { baseUrl: string; username: string | null; password: string | null; token: string | null } {
+  const baseUrl = process.env["FLEXOPTIX_API_BASE_URL"]?.trim();
+  if (!baseUrl) {
+    throw new Error("FLEXOPTIX_API_BASE_URL is required for Flexoptix API sync");
+  }
+  const token = process.env["FLEXOPTIX_API_TOKEN"]?.trim() ?? null;
+  const username = process.env["FLEXOPTIX_API_USERNAME"]?.trim() ?? null;
+  const password = process.env["FLEXOPTIX_API_PASSWORD"]?.trim() ?? null;
+  if (!token && (!username || !password)) {
+    throw new Error("FLEXOPTIX_API_TOKEN or FLEXOPTIX_API_USERNAME+PASSWORD required");
+  }
+  return { baseUrl, username, password, token };
+}
+
+function buildUrl(baseUrl: string, path: string): URL {
+  const base = baseUrl.endsWith("/") ? baseUrl : `${baseUrl}/`;
+  return new URL(path, base);
+}
+
+async function fetchJson(url: URL, init: RequestInit, timeoutMs: number): Promise<unknown> {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), Number.isFinite(timeoutMs) ? timeoutMs : 30_000);
+  try {
+    const response = await fetch(url, { ...init, signal: controller.signal });
+    if (!response.ok) {
+      throw new Error(`HTTP ${response.status} from ${url.origin}${url.pathname}`);
+    }
+    return response.json();
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+async function getBearerToken(baseUrl: string, username: string, password: string, timeoutMs: number): Promise<string> {
+  const authPath = process.env["FLEXOPTIX_API_AUTH_PATH"]?.trim() ?? "/rest/V1/integration/customer/token";
+  const url = buildUrl(baseUrl, authPath);
+  const payload = await fetchJson(url, {
+    method: "POST",
+    headers: { accept: "application/json", "content-type": "application/json" },
+    body: JSON.stringify({ username, password }),
+  }, timeoutMs);
+  if (typeof payload !== "string") {
+    throw new Error("Flexoptix auth response was not a string token");
+  }
+  return payload;
+}
+
+async function fetchAllProducts(baseUrl: string, headers: Record<string, string>, timeoutMs: number): Promise<JsonRecord[]> {
+  const productPath = process.env["FLEXOPTIX_API_PRODUCTS_PATH"]?.trim() ?? "/rest/V2/flexoptix/products";
+  const compatibilities = process.env["FLEXOPTIX_API_COMPATIBILITIES"]?.trim() ?? "1";
+  const limit = parseInt(process.env["FLEXOPTIX_API_LIMIT"]?.trim() ?? "500", 10);
+  const currency = process.env["FLEXOPTIX_API_CURRENCY"]?.trim() ?? "EUR";
+
+  const allRows: JsonRecord[] = [];
+
+  for (let page = 1; ; page++) {
+    const url = buildUrl(baseUrl, productPath);
+    url.searchParams.set("currency", currency);
+    url.searchParams.set("page", String(page));
+    url.searchParams.set("limit", String(Number.isFinite(limit) ? limit : 500));
+    if (compatibilities === "1") url.searchParams.set("compatibilities", "1");
+    // specifications=0 for bulk pull (specifications=1 causes HTTP 503 on full catalog)
+
+    const payload = await fetchJson(url, { headers }, timeoutMs);
+    const rows = extractRows(payload);
+    if (rows.length === 0) break;
+
+    allRows.push(...rows);
+
+    if (rows.length < (Number.isFinite(limit) ? limit : 500)) break;
+  }
+
+  return allRows;
+}
+
+function extractRows(payload: unknown): JsonRecord[] {
+  if (Array.isArray(payload)) return payload.filter(isRecord);
+  if (!isRecord(payload)) return [];
+  for (const key of ["products", "items", "data", "results", "rows"]) {
+    const value = payload[key];
+    if (Array.isArray(value)) return value.filter(isRecord);
+  }
+  return [payload];
+}
+
+// ── Main export ────────────────────────────────────────────────────────────
+
+export async function syncFlexoptixCatalog(): Promise<FlexoptixSyncResult> {
+  const { baseUrl, username, password, token } = validateEnv();
+  const timeoutMs = parseInt(process.env["FLEXOPTIX_API_TIMEOUT_MS"]?.trim() ?? "30000", 10);
+
+  console.log(`[${new Date().toISOString()}] Flexoptix API sync starting`);
+
+  const bearerToken = token ?? await getBearerToken(baseUrl, username!, password!, timeoutMs);
+  const headers: Record<string, string> = {
+    accept: "application/json",
+    authorization: `Bearer ${bearerToken}`,
+  };
+  const apiKey = process.env["FLEXOPTIX_API_KEY"]?.trim();
+  if (apiKey) headers["x-api-key"] = apiKey;
+
+  const fetchedAt = new Date().toISOString();
+  const rawRows = await fetchAllProducts(baseUrl, headers, timeoutMs);
+  console.log(`[${new Date().toISOString()}] Fetched ${rawRows.length} rows from Flexoptix API`);
+
+  const products = rawRows
+    .map(row => normalizeProduct(row, fetchedAt))
+    .filter((p): p is CatalogProduct => p !== null);
+
+  const importable = products.filter(canImportProduct);
+  const skipped = products.length - importable.length;
+
+  console.log(`[${new Date().toISOString()}] Normalized: ${products.length} | importable: ${importable.length} | skipped: ${skipped}`);
+
+  const vendorId = await ensureVendor("Flexoptix", "compatible", "https://www.flexoptix.net", "https://www.flexoptix.net");
+
+  let priceWrites = 0;
+  let stockWrites = 0;
+
+  for (const product of importable) {
+    try {
+      const result = await importProduct(product, vendorId);
+      if (result.priceWritten) priceWrites++;
+      if (result.stockWritten) stockWrites++;
+    } catch (err: unknown) {
+      const message = err instanceof Error ? err.message : String(err);
+      console.warn(`[${new Date().toISOString()}] Flexoptix import error (${product.sku}): ${message.slice(0, 100)}`);
+    }
+  }
+
+  console.log(`[${new Date().toISOString()}] Flexoptix API sync complete: ${importable.length} products, ${priceWrites} price writes, ${stockWrites} stock writes`);
+
+  return {
+    fetched: rawRows.length,
+    normalized: products.length,
+    skipped,
+    priceWrites,
+    stockWrites,
+  };
+}

packages/scraper/src/scheduler.ts

@@ -270,6 +270,7 @@ export async function registerSchedules(boss: PgBoss): Promise<void> {
     "scrape:pricing:gaotek",
     // ── Catalog scrapers (every 2h) ────────────────────────────────────
     "scrape:pricing:flexoptix",
+    "sync:flexoptix-catalog",
     // ── Manufacturer catalogs (every 8h, no prices) ────────────────────
     "scrape:catalog:smartoptics",
     "scrape:catalog:hubersuhner",
@@ -402,10 +403,13 @@ export async function registerSchedules(boss: PgBoss): Promise<void> {
   await boss.schedule("scrape:pricing:mouser-oem",       "0  3  * * *", {}, { retryLimit: 1, expireInSeconds: 3600 });
 
   // ══════════════════════════════════════════════════════════════════════
-  // FLEXOPTIX CATALOG — every 2h (primary price source)
+  // FLEXOPTIX CATALOG — every 2h (primary price + stock source)
+  // sync:flexoptix-catalog uses the Magento 2 REST API (authoritative data)
+  // scrape:pricing:flexoptix is the legacy GraphQL fallback
   // ══════════════════════════════════════════════════════════════════════
 
-  await boss.schedule("scrape:pricing:flexoptix", "0 */2 * * *", {}, { retryLimit: 3, expireInSeconds: 3600 });
+  await boss.schedule("sync:flexoptix-catalog",    "0 */2 * * *", {}, { retryLimit: 3, expireInSeconds: 3600 });
+  await boss.schedule("scrape:pricing:flexoptix",  "0 */2 * * *", {}, { retryLimit: 3, expireInSeconds: 3600 });
 
   // ══════════════════════════════════════════════════════════════════════
   // MANUFACTURER CATALOGS — every 4h (product data, no prices)
@@ -887,10 +891,17 @@ export async function registerWorkers(boss: PgBoss): Promise<void> {
   // ── Catalog scrapers ──────────────────────────────────────────────────
 
   await boss.work("scrape:pricing:flexoptix", async () => {
-    console.log(`[${new Date().toISOString()}] Running: Flexoptix catalog`);
+    // Legacy GraphQL/search scraper — kept as fallback, prefer sync:flexoptix-catalog
+    console.log(`[${new Date().toISOString()}] Running: Flexoptix catalog (legacy GraphQL)`);
     await scrapeFlexoptixCatalog();
   });
 
+  await boss.work("sync:flexoptix-catalog", async () => {
+    console.log(`[${new Date().toISOString()}] Running: Flexoptix API sync (REST v2)`);
+    const { syncFlexoptixCatalog } = await import("./robots/flexoptix-api-sync");
+    await syncFlexoptixCatalog();
+  });
+
   await boss.work("scrape:catalog:smartoptics", async () => {
     console.log(`[${new Date().toISOString()}] Running: SmartOptics catalog`);
     await scrapeSmartOptics();

packages/scraper/src/scrapers/audiocodes-oem.ts

@@ -0,0 +1,112 @@
+/**
+ * AudioCodes OEM Transceiver Catalog Seed
+ *
+ * Seeds AudioCodes-branded transceiver PIDs for session border controllers
+ * (SBCs), media gateways, and enterprise voice routing platforms.
+ *
+ * Sources:
+ *  - AudioCodes SBC Hardware Installation Guide (audiocodes.com)
+ *  - AudioCodes Mediant Platform Data Sheets
+ *  - AudioCodes Enterprise Voice Hardware Reference
+ *
+ * Run:  tsx packages/scraper/src/scrapers/audiocodes-oem.ts
+ * Cron: daily at 00:05
+ */
+
+import { pool, ensureVendor } from "../utils/db";
+
+interface AudioCodesPID {
+  pid: string;
+  formFactor: string;
+  speedGbps: number;
+  speed: string;
+  reachMeters: number;
+  reachLabel: string;
+  fiberType: string;
+  connector: string;
+  wavelengths?: string;
+  standard?: string;
+  notes?: string;
+}
+
+const AUDIOCODES_PIDS: AudioCodesPID[] = [
+  // ── 1G SFP ──────────────────────────────────────────────────────────────
+  { pid: "AC-SFP-1G-SX",         formFactor: "SFP",   speedGbps: 1,  speed: "1G",  reachMeters: 550,   reachLabel: "SX",    fiberType: "MMF", connector: "LC",   wavelengths: "850nm",  standard: "1000BASE-SX" },
+  { pid: "AC-SFP-1G-LX",         formFactor: "SFP",   speedGbps: 1,  speed: "1G",  reachMeters: 10000, reachLabel: "LX",    fiberType: "SMF", connector: "LC",   wavelengths: "1310nm", standard: "1000BASE-LX" },
+  { pid: "AC-SFP-1G-T",          formFactor: "SFP",   speedGbps: 1,  speed: "1G",  reachMeters: 100,   reachLabel: "T",     fiberType: "DAC", connector: "RJ45",                        standard: "1000BASE-T" },
+
+  // ── 10G SFP+ ────────────────────────────────────────────────────────────
+  { pid: "AC-SFP10G-SR",         formFactor: "SFP+",  speedGbps: 10, speed: "10G", reachMeters: 300,   reachLabel: "SR",    fiberType: "MMF", connector: "LC",   wavelengths: "850nm",  standard: "10GBASE-SR" },
+  { pid: "AC-SFP10G-LR",         formFactor: "SFP+",  speedGbps: 10, speed: "10G", reachMeters: 10000, reachLabel: "LR",    fiberType: "SMF", connector: "LC",   wavelengths: "1310nm", standard: "10GBASE-LR" },
+  { pid: "AC-SFP10G-ER",         formFactor: "SFP+",  speedGbps: 10, speed: "10G", reachMeters: 40000, reachLabel: "ER",    fiberType: "SMF", connector: "LC",   wavelengths: "1550nm", standard: "10GBASE-ER" },
+
+  // ── 10G SFP+ BiDi ───────────────────────────────────────────────────────
+  { pid: "AC-SFP10G-BIDI-TX1310", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 10000, reachLabel: "BiDi-1310TX", fiberType: "SMF", connector: "LC", wavelengths: "1310nm TX / 1490nm RX", notes: "Single-fiber BiDi, TX 1310nm" },
+  { pid: "AC-SFP10G-BIDI-TX1490", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 10000, reachLabel: "BiDi-1490TX", fiberType: "SMF", connector: "LC", wavelengths: "1490nm TX / 1310nm RX", notes: "Single-fiber BiDi, TX 1490nm" },
+
+  // ── 25G SFP28 ───────────────────────────────────────────────────────────
+  { pid: "AC-SFP28-25G-SR",      formFactor: "SFP28", speedGbps: 25, speed: "25G", reachMeters: 300,   reachLabel: "SR",    fiberType: "MMF", connector: "LC",   wavelengths: "850nm",  standard: "25GBASE-SR" },
+  { pid: "AC-SFP28-25G-LR",      formFactor: "SFP28", speedGbps: 25, speed: "25G", reachMeters: 10000, reachLabel: "LR",    fiberType: "SMF", connector: "LC",   wavelengths: "1310nm", standard: "25GBASE-LR" },
+
+  // ── 40G QSFP+ ───────────────────────────────────────────────────────────
+  { pid: "AC-QSFP-40G-SR4",      formFactor: "QSFP+", speedGbps: 40, speed: "40G", reachMeters: 150,   reachLabel: "SR4",   fiberType: "MMF", connector: "MPO",  wavelengths: "850nm",  standard: "40GBASE-SR4" },
+  { pid: "AC-QSFP-40G-LR4",      formFactor: "QSFP+", speedGbps: 40, speed: "40G", reachMeters: 10000, reachLabel: "LR4",   fiberType: "SMF", connector: "LC",   wavelengths: "1310nm", standard: "40GBASE-LR4" },
+];
+
+export async function scrapeAudioCodesOem(): Promise<void> {
+  console.log("=== AudioCodes OEM Transceiver Seed ===\n");
+
+  const vendorId = await ensureVendor(
+    "AudioCodes",
+    "oem",
+    "https://www.audiocodes.com",
+    undefined
+  );
+
+  let inserted = 0;
+  let updated  = 0;
+  let errors   = 0;
+
+  for (const p of AUDIOCODES_PIDS) {
+    const slug = `audiocodes-${p.pid.toLowerCase().replace(/[^a-z0-9]+/g, "-")}`;
+    try {
+      const res = await pool.query(
+        `INSERT INTO transceivers
+           (slug, part_number, vendor_id, form_factor, speed, speed_gbps,
+            reach_meters, reach_label, fiber_type, connector, wavelengths,
+            dom_support, ieee_reference, market_status, category, notes)
+         VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,true,$12,'Mainstream','DataCenter',$13)
+         ON CONFLICT (slug) DO UPDATE SET
+           speed_gbps   = EXCLUDED.speed_gbps,
+           reach_meters = CASE WHEN EXCLUDED.reach_meters > 0 THEN EXCLUDED.reach_meters ELSE transceivers.reach_meters END,
+           fiber_type   = CASE WHEN EXCLUDED.fiber_type <> '' THEN EXCLUDED.fiber_type ELSE transceivers.fiber_type END,
+           wavelengths  = COALESCE(EXCLUDED.wavelengths, transceivers.wavelengths),
+           updated_at   = NOW()
+         RETURNING (xmax = 0) as was_inserted`,
+        [slug, p.pid, vendorId, p.formFactor, p.speed, p.speedGbps,
+         p.reachMeters, p.reachLabel, p.fiberType, p.connector,
+         p.wavelengths ?? null, p.standard ?? null, p.notes ?? null]
+      );
+      if (res.rows[0]?.was_inserted) inserted++; else updated++;
+    } catch (err) {
+      console.warn(`  Skip ${p.pid}: ${(err as Error).message.slice(0, 80)}`);
+      errors++;
+    }
+  }
+
+  console.log(`\n=== AudioCodes OEM Seed Complete ===`);
+  console.log(`  Inserted: ${inserted}`);
+  console.log(`  Updated:  ${updated}`);
+  console.log(`  Errors:   ${errors}`);
+  console.log(`  Total PIDs: ${AUDIOCODES_PIDS.length}\n`);
+}
+
+if (require.main === module) {
+  scrapeAudioCodesOem()
+    .then(() => pool.end())
+    .catch((err) => {
+      console.error("Fatal:", err);
+      pool.end();
+      process.exit(1);
+    });
+}

packages/scraper/src/seed-batch35.ts

@@ -0,0 +1,16 @@
+import { pool } from "./utils/db";
+import { scrapeSierraWirelessOem } from "./scrapers/sierra-wireless-oem";
+import { scrapeSenaoOem }          from "./scrapers/senao-oem";
+import { scrapeEmcoreOem }         from "./scrapers/emcore-oem";
+import { scrapeReflexPhotonicsOem }from "./scrapers/reflex-photonics-oem";
+
+async function main() {
+  await scrapeSierraWirelessOem();
+  await scrapeSenaoOem();
+  await scrapeEmcoreOem();
+  await scrapeReflexPhotonicsOem();
+}
+
+main()
+  .then(() => pool.end())
+  .catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); });

packages/scraper/src/seed-batch36.ts

@@ -0,0 +1,16 @@
+import { pool } from "./utils/db";
+import { scrapeEngeniusOem }          from "./scrapers/engenius-oem";
+import { scrapePaloaltoNetworksOem }  from "./scrapers/paloalto-networks-oem";
+import { scrapeBrocadeLegacyOem }     from "./scrapers/brocade-legacy-oem";
+import { scrapeFoundryNetworksOem }   from "./scrapers/foundry-networks-oem";
+
+async function main() {
+  await scrapeEngeniusOem();
+  await scrapePaloaltoNetworksOem();
+  await scrapeBrocadeLegacyOem();
+  await scrapeFoundryNetworksOem();
+}
+
+main()
+  .then(() => pool.end())
+  .catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); });

packages/scraper/src/seed-batch37.ts

@@ -0,0 +1,16 @@
+import { pool } from "./utils/db";
+import { scrapeExtremeLegacyOem }   from "./scrapers/extreme-legacy-oem";
+import { scrapeNortelLegacyOem }    from "./scrapers/nortel-legacy-oem";
+import { scrape3comLegacyOem }      from "./scrapers/3com-legacy-oem";
+import { scrapeAvayaLegacyOem }     from "./scrapers/avaya-legacy-oem";
+
+async function main() {
+  await scrapeExtremeLegacyOem();
+  await scrapeNortelLegacyOem();
+  await scrape3comLegacyOem();
+  await scrapeAvayaLegacyOem();
+}
+
+main()
+  .then(() => pool.end())
+  .catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); });

packages/scraper/src/utils/db.ts

@@ -1,16 +1,25 @@
 import { Pool } from "pg";
+import type { PoolConfig } from "pg";
 import { config } from "dotenv";
 import { join } from "path";
 import { contentHash } from "./hash";
 
 config({ path: join(__dirname, "..", "..", "..", "..", ".env") });
 
+const hasExplicitDbConfig = Boolean(process.env.POSTGRES_HOST || process.env.TIP_DB_HOST);
+const poolConfig: PoolConfig = !hasExplicitDbConfig && process.env.DATABASE_URL
+  ? { connectionString: process.env.DATABASE_URL, ssl: false }
+  : {
+    host: process.env.POSTGRES_HOST || process.env.TIP_DB_HOST || "localhost",
+    port: parseInt(process.env.POSTGRES_PORT || process.env.TIP_DB_PORT || "5433"),
+    database: process.env.POSTGRES_DB || process.env.TIP_DB_NAME || "transceiver_db",
+    user: process.env.POSTGRES_USER || process.env.TIP_DB_USER || "tip",
+    password: process.env.POSTGRES_PASSWORD || process.env.TIP_DB_PASS || "tip_dev_2026",
+    ssl: false,
+  };
+
 export const pool = new Pool({
-  host: process.env.POSTGRES_HOST || "localhost",
+  ...poolConfig,
-  port: parseInt(process.env.POSTGRES_PORT || "5433"),
-  database: process.env.POSTGRES_DB || "transceiver_db",
-  user: process.env.POSTGRES_USER || "tip",
-  password: process.env.POSTGRES_PASSWORD || "tip_dev_2026",
   max: 5,
   idleTimeoutMillis: 10000,
   connectionTimeoutMillis: 5000,

scripts/import-flexoptix-catalog.ts

@@ -0,0 +1,233 @@
+/**
+ * Import normalized Flexoptix shop catalog rows produced by Magatama/Pulso.
+ *
+ * Source file:
+ *   ../magatama/data/pulso/shop-products.normalized.jsonl
+ *
+ * Safe first run:
+ *   npm run flexoptix:catalog:import -- --dry-run
+ *
+ * Live DB import:
+ *   TIP_DB_PASS=... npm run flexoptix:catalog:import
+ */
+
+import { existsSync, readFileSync } from "fs";
+import { resolve } from "path";
+import {
+  ensureVendor,
+  findOrCreateScrapedTransceiver,
+  pool,
+  upsertPriceObservation,
+  upsertStockObservation,
+} from "../packages/scraper/src/utils/db";
+import { contentHash } from "../packages/scraper/src/utils/hash";
+
+type CatalogProduct = {
+  source: "flexoptix-shop-api";
+  fetchedAt: string;
+  sku: string;
+  title: string;
+  url: string | null;
+  price: {
+    amount: number | null;
+    currency: string | null;
+    source: "api" | "missing";
+    fetchedAt: string;
+  };
+  stock: {
+    status: string | null;
+    quantity: number | null;
+    source: "api" | "missing";
+  };
+  optics: {
+    formFactor: string | null;
+    speedGbps: number | null;
+    reachM: number | null;
+    wavelengthNm: number | null;
+    connector: string | null;
+    fiberType: string | null;
+    protocol: string | null;
+    coding: string | null;
+    bidi: boolean | null;
+    dwdm: boolean | null;
+    cwdm: boolean | null;
+  };
+  compatibility: Array<{
+    vendor: string;
+    platform: string | null;
+    coding: string | null;
+    source: "api";
+  }>;
+};
+
+type CliArgs = {
+  dryRun: boolean;
+  input: string;
+};
+
+const repoRoot = resolve(__dirname, "..");
+const defaultInput = resolve(repoRoot, "..", "..", "magatama", "data", "pulso", "shop-products.normalized.jsonl");
+
+function parseArgs(argv: string[]): CliArgs {
+  const args: CliArgs = {
+    dryRun: false,
+    input: process.env.PULSO_SHOP_CATALOG_JSONL || defaultInput,
+  };
+
+  for (let index = 0; index < argv.length; index += 1) {
+    const value = argv[index];
+    if (value === "--") continue;
+    if (value === "--dry-run") {
+      args.dryRun = true;
+      continue;
+    }
+    if (value === "--input") {
+      args.input = resolve(argv[index + 1] || args.input);
+      index += 1;
+      continue;
+    }
+    if (value === "--help" || value === "-h") {
+      console.log(`Usage:
+  npm run flexoptix:catalog:import -- --dry-run
+  npm run flexoptix:catalog:import -- --input /path/shop-products.normalized.jsonl
+
+Env:
+  PULSO_SHOP_CATALOG_JSONL  Optional input path override.
+  TIP_DB_HOST/TIP_DB_PORT/TIP_DB_NAME/TIP_DB_USER/TIP_DB_PASS or PGPASSWORD for live DB import.
+`);
+      process.exit(0);
+    }
+    throw new Error(`Unknown argument: ${value}`);
+  }
+
+  return args;
+}
+
+function readJsonl(filePath: string): CatalogProduct[] {
+  if (!existsSync(filePath)) return [];
+  return readFileSync(filePath, "utf8")
+    .split("\n")
+    .map(line => line.trim())
+    .filter(Boolean)
+    .map(line => JSON.parse(line) as CatalogProduct);
+}
+
+function reachLabel(reachM: number | null): string | undefined {
+  if (reachM === null) return undefined;
+  if (reachM >= 1000 && reachM % 1000 === 0) return `${reachM / 1000}km`;
+  return `${reachM}m`;
+}
+
+function speedLabel(speedGbps: number | null): string | undefined {
+  if (speedGbps === null) return undefined;
+  return `${speedGbps}G`;
+}
+
+function categoryFor(product: CatalogProduct): string {
+  const text = `${product.title} ${product.optics.protocol || ""}`.toLowerCase();
+  if (/\bdac\b|direct attach|copper/.test(text)) return "DAC";
+  if (/\baoc\b|active optical/.test(text)) return "AOC";
+  if (/coherent|zr|dco/.test(text)) return "Coherent";
+  return "DataCenter";
+}
+
+function canImportProduct(product: CatalogProduct): boolean {
+  return Boolean(
+    product.sku
+    && product.title
+    && product.optics.formFactor
+    && product.optics.speedGbps !== null
+    && product.optics.reachM !== null
+  );
+}
+
+async function importProduct(product: CatalogProduct, vendorId: string): Promise<{ priceWritten: boolean; stockWritten: boolean }> {
+  const transceiverId = await findOrCreateScrapedTransceiver({
+    partNumber: product.sku,
+    vendorId,
+    productUrl: product.url || undefined,
+    formFactor: product.optics.formFactor || undefined,
+    speedGbps: product.optics.speedGbps ?? undefined,
+    speed: speedLabel(product.optics.speedGbps),
+    reachMeters: product.optics.reachM ?? undefined,
+    reachLabel: reachLabel(product.optics.reachM),
+    fiberType: product.optics.fiberType || undefined,
+    wavelengths: product.optics.wavelengthNm === null ? undefined : `${product.optics.wavelengthNm}nm`,
+    category: categoryFor(product),
+  });
+
+  let priceWritten = false;
+  if (product.price.amount !== null && product.price.currency) {
+    priceWritten = await upsertPriceObservation({
+      transceiverId,
+      sourceVendorId: vendorId,
+      price: product.price.amount,
+      currency: product.price.currency,
+      stockLevel: product.stock.status || "unknown",
+      quantityAvailable: product.stock.quantity ?? undefined,
+      url: product.url || undefined,
+      contentHash: contentHash({
+        source: product.source,
+        sku: product.sku,
+        price: product.price.amount,
+        currency: product.price.currency,
+        fetchedAt: product.price.fetchedAt,
+      }),
+    });
+  }
+
+  const stockWritten = await upsertStockObservation({
+    transceiverId,
+    sourceVendorId: vendorId,
+    stockLevel: product.stock.status || "unknown",
+    quantityAvailable: product.stock.quantity ?? undefined,
+    priceNet: product.price.amount ?? undefined,
+    productUrl: product.url || undefined,
+    priceCurrency: product.price.currency || undefined,
+    stockConfidence: product.stock.quantity === null ? 1 : 2,
+  });
+
+  return { priceWritten, stockWritten };
+}
+
+async function main(): Promise<void> {
+  const args = parseArgs(process.argv.slice(2));
+  const products = readJsonl(args.input);
+  const importable = products.filter(canImportProduct);
+  const skipped = products.length - importable.length;
+  const priced = importable.filter(product => product.price.amount !== null && product.price.currency).length;
+  const stocked = importable.filter(product => product.stock.status || product.stock.quantity !== null).length;
+
+  console.log("Flexoptix normalized catalog import");
+  console.log(`Input: ${args.input}`);
+  console.log(`Rows: ${products.length} | importable: ${importable.length} | skipped_missing_technical_fields: ${skipped}`);
+  console.log(`With price: ${priced} | with stock signal: ${stocked}`);
+
+  if (args.dryRun) {
+    console.log("Dry-run only: no TIP database writes performed.");
+    await pool.end();
+    return;
+  }
+
+  const vendorId = await ensureVendor("Flexoptix", "compatible", "https://www.flexoptix.net", "https://www.flexoptix.net");
+  let priceWrites = 0;
+  let stockWrites = 0;
+
+  try {
+    for (const product of importable) {
+      const result = await importProduct(product, vendorId);
+      if (result.priceWritten) priceWrites += 1;
+      if (result.stockWritten) stockWrites += 1;
+    }
+  } finally {
+    await pool.end();
+  }
+
+  console.log(`Import complete: ${importable.length} products processed, ${priceWrites} price observations, ${stockWrites} stock observations.`);
+}
+
+main().catch(async error => {
+  console.error(error instanceof Error ? error.message : error);
+  await pool.end().catch(() => undefined);
+  process.exit(1);
+});

sql/102-product-verification-reconcile.sql

@@ -0,0 +1,81 @@
+-- Migration 102: Product photo/details verification reconciliation
+-- Applied after the scraper started storing many image_url/product URLs without
+-- consistently promoting image_verified/details_verified.
+
+BEGIN;
+
+-- Backfill canonical product URLs from recent real price observations.
+UPDATE transceivers t
+SET product_page_url = latest.url,
+    updated_at = NOW()
+FROM (
+  SELECT DISTINCT ON (po.transceiver_id)
+         po.transceiver_id, po.url
+  FROM price_observations po
+  WHERE po.url IS NOT NULL
+    AND po.url != ''
+    AND po.time > NOW() - INTERVAL '180 days'
+  ORDER BY po.transceiver_id, po.time DESC
+) latest
+WHERE t.id = latest.transceiver_id
+  AND (t.product_page_url IS NULL OR t.product_page_url = '');
+
+-- Any non-placeholder product image URL written by a scraper counts as an
+-- image verification source. Older scrapers often set only has_image/image_url.
+UPDATE transceivers
+SET has_image = true,
+    image_verified = true,
+    image_verified_at = COALESCE(image_verified_at, NOW()),
+    image_verified_url = COALESCE(NULLIF(image_verified_url, ''), image_url),
+    updated_at = NOW()
+WHERE image_url IS NOT NULL
+  AND image_url != ''
+  AND image_url !~* '(placeholder|no-image|no_image|keinbild|logo)'
+  AND (image_verified = false OR image_verified IS NULL);
+
+-- Details are verified once a crawled source URL and the core product identity
+-- fields are present. This avoids marking malformed scraper rows as complete.
+UPDATE transceivers
+SET details_verified = true,
+    details_verified_at = COALESCE(details_verified_at, NOW()),
+    details_source_url = COALESCE(NULLIF(details_source_url, ''), product_page_url),
+    data_confidence = CASE
+      WHEN data_confidence IS NULL OR data_confidence IN ('unknown', 'enriched_estimated')
+        THEN 'scraped_unverified'
+      ELSE data_confidence
+    END,
+    updated_at = NOW()
+WHERE product_page_url IS NOT NULL
+  AND product_page_url != ''
+  AND form_factor IS NOT NULL
+  AND speed_gbps IS NOT NULL
+  AND part_number IS NOT NULL
+  AND part_number != ''
+  AND reach_label IS NOT NULL
+  AND reach_label != ''
+  AND fiber_type IS NOT NULL
+  AND fiber_type != ''
+  AND COALESCE(data_confidence, 'unknown') != 'garbage'
+  AND (details_verified = false OR details_verified IS NULL);
+
+-- Refresh full badge after the promotions above.
+UPDATE transceivers
+SET fully_verified = true,
+    fully_verified_at = COALESCE(fully_verified_at, NOW()),
+    updated_at = NOW()
+WHERE competitor_verified = true
+  AND price_verified = true
+  AND image_verified = true
+  AND details_verified = true
+  AND fully_verified = false;
+
+COMMIT;
+
+SELECT
+  COUNT(*) AS total,
+  COUNT(*) FILTER (WHERE image_url IS NOT NULL AND image_url != '') AS has_image_url,
+  COUNT(*) FILTER (WHERE image_verified) AS image_verified,
+  COUNT(*) FILTER (WHERE product_page_url IS NOT NULL AND product_page_url != '') AS has_product_page_url,
+  COUNT(*) FILTER (WHERE details_verified) AS details_verified,
+  COUNT(*) FILTER (WHERE fully_verified) AS fully_verified
+FROM transceivers;

sync/history/2026-05-09-magatama-all-lane-runpod-training-start.md

@@ -28,6 +28,7 @@ Date: 2026-05-09 23:09 UTC
 ## Runner Fix
 
 - Fixed `scripts/trigger_lane_training_once.py` locally and on Erik.
+- MAGATAMA Gitea commit: `76d4054`.
 - The script previously used stale API keys:
   - `iterations`
   - `seedOnly`
@@ -60,4 +61,3 @@ Date: 2026-05-09 23:09 UTC
   - MAGATAMA imports/adopts it locally,
   - smoke checks pass,
   - the active alias/version is updated.
-