feat: Flexoptix full product detail sync (sql/115 + detail-enricher robot)
Pulls complete per-SKU specifications and compatibility data from the Flexoptix API (specifications=1&compatibilities=1) and writes structured fields to the transceivers table for datasheet generation. SQL migration 115: - Adds fx_specifications JSONB (raw spec blob for datasheet gen) - Adds fx_compatibilities JSONB (full OEM compatibility matrix) - Adds compliance_code, laser_type, receiver_type, supported_protocols[] - Adds extinction_ratio_db, cdr_support, inbuilt_fec, detail_synced_at - GIN index on fx_compatibilities for vendor/OPN queries flexoptix-detail-enricher.ts: - Per-SKU API calls with rate-limiting (600ms/call, 100 SKUs/run) - Parses all spec labels → structured fields (power, budget, tx/rx dBm, modulation, wavelengths, temp range, DOM, laser type, receiver type) - Strips :Sx variant suffixes before API queries (self-configure SKUs) - COALESCE writes — never overwrites existing data, only fills gaps - Tracks detail_synced_at, retries stale entries after 7 days flexoptix-api-sync.ts: - Also stores image_url and product_page_url during bulk sync scheduler.ts: - Registers enrich:flexoptix-details daily at 03:00 UTC Results after initial run: - 791/968 FX products (81.7%) fully enriched - 26.0 avg compatibility entries per product (OEM vendor + OPN) - 25.7 avg spec fields per product - DFB(483), EML(148), FP(72), VCSEL(44) laser type distribution
This commit is contained in:
parent
d1bde66e39
commit
2f85571784
@ -23,6 +23,7 @@
|
|||||||
import {
|
import {
|
||||||
ensureVendor,
|
ensureVendor,
|
||||||
findOrCreateScrapedTransceiver,
|
findOrCreateScrapedTransceiver,
|
||||||
|
pool,
|
||||||
upsertPriceObservation,
|
upsertPriceObservation,
|
||||||
upsertStockObservation,
|
upsertStockObservation,
|
||||||
} from "../utils/db";
|
} from "../utils/db";
|
||||||
@ -38,6 +39,7 @@ interface CatalogProduct {
|
|||||||
sku: string;
|
sku: string;
|
||||||
title: string;
|
title: string;
|
||||||
url: string | null;
|
url: string | null;
|
||||||
|
imageUrl: string | null;
|
||||||
price: {
|
price: {
|
||||||
amount: number | null;
|
amount: number | null;
|
||||||
currency: string | null;
|
currency: string | null;
|
||||||
@ -252,6 +254,7 @@ function normalizeProduct(row: JsonRecord, fetchedAt: string): CatalogProduct |
|
|||||||
if (!sku || !title) return null;
|
if (!sku || !title) return null;
|
||||||
|
|
||||||
const url = asString(pick(flat, ["url", "productUrl", "canonicalUrl", "link"]));
|
const url = asString(pick(flat, ["url", "productUrl", "canonicalUrl", "link"]));
|
||||||
|
const imageUrl = asString(pick(flat, ["image", "imageUrl", "productImage", "thumbnail"]));
|
||||||
const amount = asNumber(pick(flat, ["price", "priceNet", "netPrice", "grossPrice", "amount"]));
|
const amount = asNumber(pick(flat, ["price", "priceNet", "netPrice", "grossPrice", "amount"]));
|
||||||
const currency = asString(pick(flat, ["currency", "priceCurrency", "currencyCode"]))
|
const currency = asString(pick(flat, ["currency", "priceCurrency", "currencyCode"]))
|
||||||
?? (amount === null ? null : process.env["FLEXOPTIX_API_CURRENCY"]?.trim() ?? "EUR");
|
?? (amount === null ? null : process.env["FLEXOPTIX_API_CURRENCY"]?.trim() ?? "EUR");
|
||||||
@ -275,6 +278,7 @@ function normalizeProduct(row: JsonRecord, fetchedAt: string): CatalogProduct |
|
|||||||
sku,
|
sku,
|
||||||
title,
|
title,
|
||||||
url,
|
url,
|
||||||
|
imageUrl,
|
||||||
price: {
|
price: {
|
||||||
amount,
|
amount,
|
||||||
currency,
|
currency,
|
||||||
@ -353,6 +357,18 @@ async function importProduct(
|
|||||||
category: categoryFor(product),
|
category: categoryFor(product),
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Write image_url and product_page_url from bulk API response
|
||||||
|
if (product.imageUrl || product.url) {
|
||||||
|
await pool.query(`
|
||||||
|
UPDATE transceivers SET
|
||||||
|
image_url = COALESCE(NULLIF(image_url, ''), $1),
|
||||||
|
product_page_url = COALESCE(NULLIF(product_page_url, ''), $2),
|
||||||
|
updated_at = NOW()
|
||||||
|
WHERE id = $3
|
||||||
|
AND ($1 IS NOT NULL OR $2 IS NOT NULL)
|
||||||
|
`, [product.imageUrl ?? null, product.url ?? null, transceiverId]);
|
||||||
|
}
|
||||||
|
|
||||||
let priceWritten = false;
|
let priceWritten = false;
|
||||||
if (product.price.amount !== null && product.price.currency) {
|
if (product.price.amount !== null && product.price.currency) {
|
||||||
priceWritten = await upsertPriceObservation({
|
priceWritten = await upsertPriceObservation({
|
||||||
|
|||||||
486
packages/scraper/src/robots/flexoptix-detail-enricher.ts
Normal file
486
packages/scraper/src/robots/flexoptix-detail-enricher.ts
Normal file
@ -0,0 +1,486 @@
|
|||||||
|
/**
|
||||||
|
* Flexoptix Detail Enricher
|
||||||
|
*
|
||||||
|
* Fetches full product specifications and compatibility data from the Flexoptix
|
||||||
|
* API on a per-SKU basis (specifications=1&compatibilities=1) and writes all
|
||||||
|
* structured fields back to the transceivers table.
|
||||||
|
*
|
||||||
|
* Unlike the bulk catalog sync (specifications=0 to avoid HTTP 503), this robot
|
||||||
|
* processes products in small batches with rate-limiting so the API stays happy.
|
||||||
|
*
|
||||||
|
* Fields written per product:
|
||||||
|
* fx_specifications — raw [{label, value}, ...] blob (for datasheet gen)
|
||||||
|
* fx_compatibilities — full [{sku, compatible_to_vendor, original_part_number}]
|
||||||
|
* compliance_code — "LX SGMII", "SR4", "LR4", etc.
|
||||||
|
* laser_type — "FP", "DFB", "VCSEL", "EML"
|
||||||
|
* receiver_type — "PIN", "APD"
|
||||||
|
* supported_protocols — TEXT[]
|
||||||
|
* extinction_ratio_db — dB
|
||||||
|
* cdr_support — boolean
|
||||||
|
* inbuilt_fec — boolean
|
||||||
|
* power_consumption_w — W (overrides if empty)
|
||||||
|
* optical_budget_db — dB (overrides if empty)
|
||||||
|
* tx_power_min_dbm — dBm
|
||||||
|
* tx_power_max_dbm — dBm
|
||||||
|
* rx_sensitivity_dbm — dBm
|
||||||
|
* modulation — "NRZ", "PAM4", etc.
|
||||||
|
* wavelength_tx_nm — nm (overrides if empty)
|
||||||
|
* wavelength_rx_nm — nm (overrides if empty)
|
||||||
|
* image_url — product image URL
|
||||||
|
* product_page_url — product page URL
|
||||||
|
* detail_synced_at — timestamp of this sync
|
||||||
|
*
|
||||||
|
* Scheduling:
|
||||||
|
* - Runs daily at 03:00 UTC
|
||||||
|
* - Processes BATCH_SIZE products per run (prioritises unseen, then stale >7d)
|
||||||
|
* - Rate: 1 API call per 600ms (~1.6 rps, safe for Magento)
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { pool } from "../utils/db";
|
||||||
|
|
||||||
|
// ── Constants ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/** Products per enricher run. Full catalog (~1100 products) in ~11 daily runs. */
|
||||||
|
const BATCH_SIZE = 100;
|
||||||
|
|
||||||
|
/** Milliseconds between per-SKU API calls (Magento rate-limit safety). */
|
||||||
|
const API_CALL_DELAY_MS = 600;
|
||||||
|
|
||||||
|
// ── Types ──────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
interface FxApiCompatibility {
|
||||||
|
sku: string | null;
|
||||||
|
compatible_to_vendor: string;
|
||||||
|
original_part_number: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface FxApiSpec {
|
||||||
|
label: string;
|
||||||
|
value: unknown;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface FxApiProduct {
|
||||||
|
sku: string;
|
||||||
|
name?: string;
|
||||||
|
url?: string;
|
||||||
|
image?: string;
|
||||||
|
compatibilities?: FxApiCompatibility[];
|
||||||
|
specifications?: FxApiSpec[];
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ParsedSpecs {
|
||||||
|
complianceCode: string | null;
|
||||||
|
laserType: string | null;
|
||||||
|
receiverType: string | null;
|
||||||
|
supportedProtocols: string[];
|
||||||
|
extinctionRatioDb: number | null;
|
||||||
|
cdrSupport: boolean | null;
|
||||||
|
inbuiltFec: boolean | null;
|
||||||
|
powerConsumptionW: number | null;
|
||||||
|
opticalBudgetDb: number | null;
|
||||||
|
txPowerMinDbm: number | null;
|
||||||
|
txPowerMaxDbm: number | null;
|
||||||
|
rxSensitivityDbm: number | null;
|
||||||
|
modulation: string | null;
|
||||||
|
wavelengthTxNm: number | null;
|
||||||
|
wavelengthRxNm: number | null;
|
||||||
|
tempRange: string | null;
|
||||||
|
domSupport: boolean | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface DetailEnricherResult {
|
||||||
|
processed: number;
|
||||||
|
updated: number;
|
||||||
|
notFound: number;
|
||||||
|
apiErrors: number;
|
||||||
|
dbErrors: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Helpers ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
function specValue(specs: FxApiSpec[], label: string): string | null {
|
||||||
|
const entry = specs.find(s => s.label.toLowerCase() === label.toLowerCase());
|
||||||
|
if (!entry) return null;
|
||||||
|
const v = entry.value;
|
||||||
|
if (Array.isArray(v)) return v.join(", ");
|
||||||
|
if (typeof v === "string") return v.trim() || null;
|
||||||
|
if (typeof v === "number" || typeof v === "boolean") return String(v);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function specArray(specs: FxApiSpec[], label: string): string[] {
|
||||||
|
const entry = specs.find(s => s.label.toLowerCase() === label.toLowerCase());
|
||||||
|
if (!entry) return [];
|
||||||
|
if (Array.isArray(entry.value)) return entry.value.filter(v => typeof v === "string") as string[];
|
||||||
|
const v = entry.value;
|
||||||
|
if (typeof v === "string" && v.trim()) return [v.trim()];
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseDbm(text: string | null): { min: number | null; max: number | null } {
|
||||||
|
if (!text) return { min: null, max: null };
|
||||||
|
// Format: "-15 dBm / -8 dBm" or "-31 dBm / -8 dBm (overload) @100M"
|
||||||
|
const numbers = text.match(/-?\d+(?:\.\d+)?\s*dBm/gi) ?? [];
|
||||||
|
const values = numbers
|
||||||
|
.map(n => parseFloat(n.replace(/dBm/i, "").trim()))
|
||||||
|
.filter(n => Number.isFinite(n));
|
||||||
|
return {
|
||||||
|
min: values[0] ?? null,
|
||||||
|
max: values[1] ?? null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseWavelengthNm(text: string | null): number | null {
|
||||||
|
if (!text) return null;
|
||||||
|
const match = text.match(/(\d{3,4})\s*nm/);
|
||||||
|
return match ? parseInt(match[1], 10) : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parsePowerW(text: string | null): number | null {
|
||||||
|
if (!text) return null;
|
||||||
|
const match = text.match(/([\d.]+)\s*W/i);
|
||||||
|
return match ? parseFloat(match[1]) : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseDb(text: string | null): number | null {
|
||||||
|
if (!text) return null;
|
||||||
|
const match = text.match(/([\d.]+)\s*dB(?!m)/i);
|
||||||
|
return match ? parseFloat(match[1]) : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseTempRange(text: string | null, operatingTemp: string | null): "COM" | "IND" | null {
|
||||||
|
// Parse degree-range strings like "0°C - 70°C" or "-40°C - 85°C"
|
||||||
|
if (text && /°C/.test(text)) {
|
||||||
|
const minMatch = text.match(/(-?\d+)\s*°C/);
|
||||||
|
const minC = minMatch ? parseInt(minMatch[1], 10) : null;
|
||||||
|
if (minC !== null && minC < -10) return "IND";
|
||||||
|
return "COM";
|
||||||
|
}
|
||||||
|
// Classify from the operating temperature label
|
||||||
|
const combined = [text, operatingTemp].filter(Boolean).join(" ").toLowerCase();
|
||||||
|
if (/industrial|ind\b|-40/.test(combined)) return "IND";
|
||||||
|
if (/commercial|standard|com\b/.test(combined)) return "COM";
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseDomSupport(text: string | null): boolean | null {
|
||||||
|
if (!text) return null;
|
||||||
|
const lower = text.toLowerCase();
|
||||||
|
if (/not implemented|no|none/.test(lower)) return false;
|
||||||
|
if (/yes|implemented|supported|digital/.test(lower)) return true;
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseBoolean(text: string | null): boolean | null {
|
||||||
|
if (!text) return null;
|
||||||
|
const lower = text.toLowerCase().trim();
|
||||||
|
if (["yes", "true", "1", "ja"].includes(lower)) return true;
|
||||||
|
if (["no", "false", "0", "nein", "none"].includes(lower)) return false;
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseModulation(text: string | null): string | null {
|
||||||
|
if (!text) return null;
|
||||||
|
// Normalize "NRZ @100M - 800M" → "NRZ", "PAM4" → "PAM4"
|
||||||
|
const match = text.match(/\b(NRZ|PAM4|PAM-4|DP-QPSK|QPSK|16QAM|64QAM|OOK)\b/i);
|
||||||
|
return match ? match[1].toUpperCase().replace("PAM-4", "PAM4") : text.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse the flat specifications array into structured fields.
|
||||||
|
*/
|
||||||
|
function parseSpecs(specs: FxApiSpec[]): ParsedSpecs {
|
||||||
|
const txPowers = parseDbm(specValue(specs, "Transmit min/max per lane"));
|
||||||
|
const rxPowers = parseDbm(specValue(specs, "Receiver min/max per lane"));
|
||||||
|
|
||||||
|
return {
|
||||||
|
complianceCode: specValue(specs, "Compliance Code"),
|
||||||
|
laserType: specValue(specs, "Laser"),
|
||||||
|
receiverType: specValue(specs, "Receiver Type"),
|
||||||
|
supportedProtocols: specArray(specs, "Supported Protocols"),
|
||||||
|
extinctionRatioDb: parseDb(specValue(specs, "Extinction Ratio")),
|
||||||
|
cdrSupport: parseBoolean(specValue(specs, "CDR")),
|
||||||
|
inbuiltFec: parseBoolean(specValue(specs, "Inbuilt FEC")),
|
||||||
|
powerConsumptionW: parsePowerW(specValue(specs, "Power Consumption")),
|
||||||
|
opticalBudgetDb: parseDb(specValue(specs, "Powerbudget (dB)")),
|
||||||
|
txPowerMinDbm: txPowers.min,
|
||||||
|
txPowerMaxDbm: txPowers.max,
|
||||||
|
rxSensitivityDbm: rxPowers.min,
|
||||||
|
modulation: parseModulation(specValue(specs, "Modulation")),
|
||||||
|
wavelengthTxNm: parseWavelengthNm(specValue(specs, "Wavelength TX (Typical)")),
|
||||||
|
wavelengthRxNm: parseWavelengthNm(specValue(specs, "Wavelength RX (Typical)")),
|
||||||
|
tempRange: parseTempRange(
|
||||||
|
specValue(specs, "Temperature Range"),
|
||||||
|
specValue(specs, "Operating Temperature"),
|
||||||
|
),
|
||||||
|
domSupport: parseDomSupport(specValue(specs, "Digital Diagnostic Monitoring (DDM)")),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── API client ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
async function authenticate(baseUrl: string, timeoutMs: number): Promise<string> {
|
||||||
|
const existingToken = process.env["FLEXOPTIX_API_TOKEN"]?.trim();
|
||||||
|
if (existingToken) return existingToken;
|
||||||
|
|
||||||
|
const username = process.env["FLEXOPTIX_API_USERNAME"]?.trim();
|
||||||
|
const password = process.env["FLEXOPTIX_API_PASSWORD"]?.trim();
|
||||||
|
if (!username || !password) {
|
||||||
|
throw new Error("FLEXOPTIX_API_USERNAME + FLEXOPTIX_API_PASSWORD required for detail enricher");
|
||||||
|
}
|
||||||
|
|
||||||
|
const authPath = process.env["FLEXOPTIX_API_AUTH_PATH"]?.trim() ?? "/rest/V1/integration/customer/token";
|
||||||
|
const url = `${baseUrl}${authPath}`;
|
||||||
|
const controller = new AbortController();
|
||||||
|
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const res = await fetch(url, {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "content-type": "application/json", accept: "application/json" },
|
||||||
|
body: JSON.stringify({ username, password }),
|
||||||
|
signal: controller.signal,
|
||||||
|
});
|
||||||
|
if (!res.ok) throw new Error(`Auth failed: HTTP ${res.status}`);
|
||||||
|
const token = await res.json();
|
||||||
|
if (typeof token !== "string") throw new Error("Auth response was not a string token");
|
||||||
|
return token;
|
||||||
|
} finally {
|
||||||
|
clearTimeout(timer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Normalize a FX SKU for the API query.
|
||||||
|
* Strips variant/self-configure suffixes that exist in TIP DB but not in the API:
|
||||||
|
* "S.B1312.10.DLI:Sx" → "S.B1312.10.DLI" (self-configure parent)
|
||||||
|
* "M4.T8SL.x" → "M4.T8SL" (placeholder variant)
|
||||||
|
* "P.1696.25.yy.R" → kept as-is (real SKU with letter suffix)
|
||||||
|
*/
|
||||||
|
function normalizeSku(sku: string): string {
|
||||||
|
// Strip ":Sx", ":S1", ":AB", etc. (colon-delimited variant suffixes)
|
||||||
|
const colonSuffix = sku.replace(/:[A-Za-z0-9]+$/, "");
|
||||||
|
if (colonSuffix !== sku) return colonSuffix;
|
||||||
|
// Strip trailing ".x" or ".y" (single-letter placeholder segments)
|
||||||
|
const dotSuffix = sku.replace(/\.[xy]$/i, "");
|
||||||
|
if (dotSuffix !== sku) return dotSuffix;
|
||||||
|
return sku;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchProductDetail(
|
||||||
|
baseUrl: string,
|
||||||
|
productPath: string,
|
||||||
|
sku: string,
|
||||||
|
headers: Record<string, string>,
|
||||||
|
timeoutMs: number,
|
||||||
|
): Promise<FxApiProduct | null> {
|
||||||
|
const apiSku = normalizeSku(sku);
|
||||||
|
const url = new URL(productPath, baseUrl.endsWith("/") ? baseUrl : `${baseUrl}/`);
|
||||||
|
url.searchParams.set("sku", apiSku);
|
||||||
|
url.searchParams.set("specifications", "1");
|
||||||
|
url.searchParams.set("compatibilities", "1");
|
||||||
|
|
||||||
|
const controller = new AbortController();
|
||||||
|
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||||
|
|
||||||
|
try {
|
||||||
|
const res = await fetch(url.toString(), { headers, signal: controller.signal });
|
||||||
|
if (!res.ok) return null;
|
||||||
|
|
||||||
|
const body = await res.json();
|
||||||
|
// API returns array for SKU query
|
||||||
|
const rows = Array.isArray(body) ? body : [body];
|
||||||
|
const row = rows[0];
|
||||||
|
if (!row || typeof row !== "object") return null;
|
||||||
|
|
||||||
|
return row as FxApiProduct;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
} finally {
|
||||||
|
clearTimeout(timer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── DB helpers ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
interface FxProduct {
|
||||||
|
id: string;
|
||||||
|
part_number: string;
|
||||||
|
power_consumption_w: number | null;
|
||||||
|
optical_budget_db: number | null;
|
||||||
|
wavelength_tx_nm: number | null;
|
||||||
|
wavelength_rx_nm: number | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchBatch(): Promise<FxProduct[]> {
|
||||||
|
const result = await pool.query<FxProduct>(`
|
||||||
|
SELECT
|
||||||
|
t.id,
|
||||||
|
t.part_number,
|
||||||
|
t.power_consumption_w,
|
||||||
|
t.optical_budget_db,
|
||||||
|
t.wavelength_tx_nm,
|
||||||
|
t.wavelength_rx_nm
|
||||||
|
FROM transceivers t
|
||||||
|
JOIN vendors v ON v.id = t.vendor_id
|
||||||
|
WHERE UPPER(v.name) LIKE '%FLEXOPTIX%'
|
||||||
|
-- FX catalog SKUs always contain a dot (e.g. S.1303.10.G, Q2.85850.100.D5)
|
||||||
|
-- Products without a dot are misidentified non-FX items — skip them
|
||||||
|
AND t.part_number LIKE '%.%'
|
||||||
|
AND (
|
||||||
|
t.detail_synced_at IS NULL
|
||||||
|
OR t.detail_synced_at < NOW() - INTERVAL '7 days'
|
||||||
|
)
|
||||||
|
ORDER BY
|
||||||
|
t.detail_synced_at ASC NULLS FIRST,
|
||||||
|
t.data_completeness DESC -- process most-complete products first
|
||||||
|
LIMIT $1
|
||||||
|
`, [BATCH_SIZE]);
|
||||||
|
return result.rows;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function writeDetails(
|
||||||
|
transceiverId: string,
|
||||||
|
product: FxApiProduct,
|
||||||
|
parsed: ParsedSpecs,
|
||||||
|
): Promise<void> {
|
||||||
|
const compat = Array.isArray(product.compatibilities) ? product.compatibilities : [];
|
||||||
|
const specs = Array.isArray(product.specifications) ? product.specifications : [];
|
||||||
|
|
||||||
|
await pool.query(`
|
||||||
|
UPDATE transceivers SET
|
||||||
|
fx_specifications = $1,
|
||||||
|
fx_compatibilities = $2,
|
||||||
|
compliance_code = COALESCE(compliance_code, $3),
|
||||||
|
laser_type = COALESCE(laser_type, $4),
|
||||||
|
receiver_type = COALESCE(receiver_type, $5),
|
||||||
|
supported_protocols = COALESCE(supported_protocols, $6),
|
||||||
|
extinction_ratio_db = COALESCE(extinction_ratio_db, $7),
|
||||||
|
cdr_support = COALESCE(cdr_support, $8),
|
||||||
|
inbuilt_fec = COALESCE(inbuilt_fec, $9),
|
||||||
|
power_consumption_w = COALESCE(power_consumption_w, $10),
|
||||||
|
optical_budget_db = COALESCE(optical_budget_db, $11),
|
||||||
|
tx_power_min_dbm = COALESCE(tx_power_min_dbm, $12),
|
||||||
|
tx_power_max_dbm = COALESCE(tx_power_max_dbm, $13),
|
||||||
|
rx_sensitivity_dbm = COALESCE(rx_sensitivity_dbm, $14),
|
||||||
|
modulation = COALESCE(modulation, $15),
|
||||||
|
wavelength_tx_nm = COALESCE(wavelength_tx_nm, $16),
|
||||||
|
wavelength_rx_nm = COALESCE(wavelength_rx_nm, $17),
|
||||||
|
temp_range = COALESCE(NULLIF(temp_range, 'COM'), $18),
|
||||||
|
dom_support = COALESCE(dom_support, $19),
|
||||||
|
image_url = COALESCE(NULLIF(image_url, ''), $20),
|
||||||
|
product_page_url = COALESCE(NULLIF(product_page_url, ''), $21),
|
||||||
|
detail_synced_at = NOW(),
|
||||||
|
updated_at = NOW()
|
||||||
|
WHERE id = $22
|
||||||
|
`, [
|
||||||
|
specs.length > 0 ? JSON.stringify(specs) : null, // $1
|
||||||
|
compat.length > 0 ? JSON.stringify(compat) : null, // $2
|
||||||
|
parsed.complianceCode, // $3
|
||||||
|
parsed.laserType, // $4
|
||||||
|
parsed.receiverType, // $5
|
||||||
|
parsed.supportedProtocols.length > 0 ? parsed.supportedProtocols : null, // $6
|
||||||
|
parsed.extinctionRatioDb, // $7
|
||||||
|
parsed.cdrSupport, // $8
|
||||||
|
parsed.inbuiltFec, // $9
|
||||||
|
parsed.powerConsumptionW, // $10
|
||||||
|
parsed.opticalBudgetDb, // $11
|
||||||
|
parsed.txPowerMinDbm, // $12
|
||||||
|
parsed.txPowerMaxDbm, // $13
|
||||||
|
parsed.rxSensitivityDbm, // $14
|
||||||
|
parsed.modulation, // $15
|
||||||
|
parsed.wavelengthTxNm, // $16
|
||||||
|
parsed.wavelengthRxNm, // $17
|
||||||
|
parsed.tempRange, // $18
|
||||||
|
parsed.domSupport, // $19
|
||||||
|
product.image ?? null, // $20
|
||||||
|
product.url ?? null, // $21
|
||||||
|
transceiverId, // $22
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Main export ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
export async function runFlexoptixDetailEnricher(): Promise<DetailEnricherResult> {
|
||||||
|
const baseUrl = process.env["FLEXOPTIX_API_BASE_URL"]?.trim();
|
||||||
|
if (!baseUrl) {
|
||||||
|
throw new Error("FLEXOPTIX_API_BASE_URL not configured");
|
||||||
|
}
|
||||||
|
|
||||||
|
const productPath = process.env["FLEXOPTIX_API_PRODUCTS_PATH"]?.trim()
|
||||||
|
?? "/rest/V2/flexoptix/products";
|
||||||
|
const timeoutMs = parseInt(process.env["FLEXOPTIX_API_TIMEOUT_MS"]?.trim() ?? "30000", 10);
|
||||||
|
|
||||||
|
const ts = () => new Date().toISOString();
|
||||||
|
console.log(`[${ts()}] Flexoptix detail enricher starting (batch=${BATCH_SIZE})`);
|
||||||
|
|
||||||
|
const token = await authenticate(baseUrl, timeoutMs);
|
||||||
|
const headers: Record<string, string> = {
|
||||||
|
accept: "application/json",
|
||||||
|
authorization: `Bearer ${token}`,
|
||||||
|
};
|
||||||
|
|
||||||
|
const batch = await fetchBatch();
|
||||||
|
console.log(`[${ts()}] Batch: ${batch.length} FX products queued for detail sync`);
|
||||||
|
|
||||||
|
let updated = 0;
|
||||||
|
let notFound = 0;
|
||||||
|
let apiErrors = 0;
|
||||||
|
let dbErrors = 0;
|
||||||
|
|
||||||
|
for (const product of batch) {
|
||||||
|
// Rate-limit: sleep between calls
|
||||||
|
await new Promise(resolve => setTimeout(resolve, API_CALL_DELAY_MS));
|
||||||
|
|
||||||
|
let apiProduct: FxApiProduct | null = null;
|
||||||
|
try {
|
||||||
|
apiProduct = await fetchProductDetail(baseUrl, productPath, product.part_number, headers, timeoutMs);
|
||||||
|
} catch (err: unknown) {
|
||||||
|
apiErrors++;
|
||||||
|
console.warn(
|
||||||
|
`[${ts()}] detail-enricher API error (${product.part_number}): ` +
|
||||||
|
`${err instanceof Error ? err.message : String(err)}`,
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!apiProduct) {
|
||||||
|
// Not found in FX API — still mark synced so we don't retry daily,
|
||||||
|
// but log it so we can investigate if many products come back empty
|
||||||
|
notFound++;
|
||||||
|
await pool.query(
|
||||||
|
`UPDATE transceivers SET detail_synced_at = NOW() WHERE id = $1`,
|
||||||
|
[product.id],
|
||||||
|
).catch(() => null);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const specs = Array.isArray(apiProduct.specifications) ? apiProduct.specifications : [];
|
||||||
|
const parsed = parseSpecs(specs);
|
||||||
|
|
||||||
|
try {
|
||||||
|
await writeDetails(product.id, apiProduct, parsed);
|
||||||
|
updated++;
|
||||||
|
} catch (err: unknown) {
|
||||||
|
dbErrors++;
|
||||||
|
console.warn(
|
||||||
|
`[${ts()}] detail-enricher DB error (${product.part_number}): ` +
|
||||||
|
`${err instanceof Error ? err.message : String(err)}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(
|
||||||
|
`[${ts()}] Flexoptix detail enricher done: ` +
|
||||||
|
`${batch.length} queued, ${updated} updated, ${notFound} not-in-api, ` +
|
||||||
|
`${apiErrors} api-errors, ${dbErrors} db-errors`,
|
||||||
|
);
|
||||||
|
|
||||||
|
return {
|
||||||
|
processed: batch.length,
|
||||||
|
updated,
|
||||||
|
notFound,
|
||||||
|
apiErrors,
|
||||||
|
dbErrors,
|
||||||
|
};
|
||||||
|
}
|
||||||
@ -356,6 +356,8 @@ export async function registerSchedules(boss: PgBoss): Promise<void> {
|
|||||||
"discover:vendor:ii-vi",
|
"discover:vendor:ii-vi",
|
||||||
// ── Wavelength Enrichment ────────────────────────────────────────────
|
// ── Wavelength Enrichment ────────────────────────────────────────────
|
||||||
"enrich:wavelength",
|
"enrich:wavelength",
|
||||||
|
// ── Flexoptix Detail Enrichment ──────────────────────────────────────
|
||||||
|
"enrich:flexoptix-details",
|
||||||
];
|
];
|
||||||
|
|
||||||
for (const q of queues) {
|
for (const q of queues) {
|
||||||
@ -425,6 +427,13 @@ export async function registerSchedules(boss: PgBoss): Promise<void> {
|
|||||||
// Wavelength Enricher — läuft alle 4 Stunden
|
// Wavelength Enricher — läuft alle 4 Stunden
|
||||||
await boss.schedule("enrich:wavelength", "0 */4 * * *", {}, {});
|
await boss.schedule("enrich:wavelength", "0 */4 * * *", {}, {});
|
||||||
|
|
||||||
|
// Flexoptix Detail Enricher — täglich 03:00 UTC, 100 SKUs/Run
|
||||||
|
// Full catalog (~1100 SKUs) rotiert in ~11 Tagen, dann weekly refresh
|
||||||
|
await boss.schedule("enrich:flexoptix-details", "0 3 * * *", {}, {
|
||||||
|
retryLimit: 2,
|
||||||
|
expireInSeconds: 7200,
|
||||||
|
});
|
||||||
|
|
||||||
// ══════════════════════════════════════════════════════════════════════
|
// ══════════════════════════════════════════════════════════════════════
|
||||||
// MANUFACTURER CATALOGS — every 4h (product data, no prices)
|
// MANUFACTURER CATALOGS — every 4h (product data, no prices)
|
||||||
// ══════════════════════════════════════════════════════════════════════
|
// ══════════════════════════════════════════════════════════════════════
|
||||||
@ -932,6 +941,18 @@ export async function registerWorkers(boss: PgBoss): Promise<void> {
|
|||||||
await runWavelengthEnricher();
|
await runWavelengthEnricher();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Flexoptix Detail Enricher — fetches full specs + compat from API per SKU
|
||||||
|
await boss.work("enrich:flexoptix-details", async () => {
|
||||||
|
const ts = new Date().toISOString();
|
||||||
|
console.log(`[${ts}] Running: Flexoptix Detail Enricher`);
|
||||||
|
const { runFlexoptixDetailEnricher } = await import("./robots/flexoptix-detail-enricher");
|
||||||
|
const result = await runFlexoptixDetailEnricher();
|
||||||
|
console.log(
|
||||||
|
`[enrich:flexoptix-details] Done: ${result.processed} queued, ` +
|
||||||
|
`${result.updated} updated, ${result.notFound} not-in-api, ${result.apiErrors} api-errors`,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
await boss.work("scrape:catalog:smartoptics", async () => {
|
await boss.work("scrape:catalog:smartoptics", async () => {
|
||||||
console.log(`[${new Date().toISOString()}] Running: SmartOptics catalog`);
|
console.log(`[${new Date().toISOString()}] Running: SmartOptics catalog`);
|
||||||
await scrapeSmartOptics();
|
await scrapeSmartOptics();
|
||||||
|
|||||||
74
sql/115-flexoptix-product-details.sql
Normal file
74
sql/115-flexoptix-product-details.sql
Normal file
@ -0,0 +1,74 @@
|
|||||||
|
-- Migration 115: Flexoptix Product Detail Columns
|
||||||
|
-- Adds columns to store full product detail data from the Flexoptix API
|
||||||
|
-- (specifications array, compatibility matrix, laser type, receiver type, etc.)
|
||||||
|
-- so we can build rich datasheets and deepen the TIP comparison data.
|
||||||
|
|
||||||
|
-- ── New columns ──────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
-- Raw specs blob: full [{label, value}, ...] array from API (specifications=1)
|
||||||
|
-- Useful for datasheet generation and ad-hoc queries without re-fetching
|
||||||
|
ALTER TABLE transceivers
|
||||||
|
ADD COLUMN IF NOT EXISTS fx_specifications JSONB;
|
||||||
|
|
||||||
|
-- Full compatibility list from API: [{sku, compatible_to_vendor, original_part_number}, ...]
|
||||||
|
-- More granular than vendor_compat (which has pattern-based matching)
|
||||||
|
ALTER TABLE transceivers
|
||||||
|
ADD COLUMN IF NOT EXISTS fx_compatibilities JSONB;
|
||||||
|
|
||||||
|
-- Structured spec fields parsed from fx_specifications
|
||||||
|
ALTER TABLE transceivers
|
||||||
|
ADD COLUMN IF NOT EXISTS compliance_code TEXT; -- "LX SGMII", "SR4 100GBASE", "LR4", etc.
|
||||||
|
|
||||||
|
ALTER TABLE transceivers
|
||||||
|
ADD COLUMN IF NOT EXISTS laser_type TEXT; -- "FP", "DFB", "VCSEL", "EML", "CW-SiPh"
|
||||||
|
|
||||||
|
ALTER TABLE transceivers
|
||||||
|
ADD COLUMN IF NOT EXISTS receiver_type TEXT; -- "PIN", "APD", "Coherent"
|
||||||
|
|
||||||
|
ALTER TABLE transceivers
|
||||||
|
ADD COLUMN IF NOT EXISTS supported_protocols TEXT[]; -- ["1GigE", "Fast Ethernet", "10GBase-SR", ...]
|
||||||
|
|
||||||
|
ALTER TABLE transceivers
|
||||||
|
ADD COLUMN IF NOT EXISTS extinction_ratio_db NUMERIC(6,2); -- dB
|
||||||
|
|
||||||
|
ALTER TABLE transceivers
|
||||||
|
ADD COLUMN IF NOT EXISTS cdr_support BOOLEAN; -- false = "none", true = integrated CDR
|
||||||
|
|
||||||
|
ALTER TABLE transceivers
|
||||||
|
ADD COLUMN IF NOT EXISTS inbuilt_fec BOOLEAN; -- false = "No", true = integrated FEC
|
||||||
|
|
||||||
|
-- Tracking: when the full per-SKU detail sync last completed for this product
|
||||||
|
ALTER TABLE transceivers
|
||||||
|
ADD COLUMN IF NOT EXISTS detail_synced_at TIMESTAMPTZ;
|
||||||
|
|
||||||
|
-- ── Indexes ──────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
-- GIN index for JSONB compatibility search (e.g. "which FX products are
|
||||||
|
-- compatible with Cisco Nexus 9000 where OPN starts with N9K-?")
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_transceivers_fx_compatibilities
|
||||||
|
ON transceivers USING GIN (fx_compatibilities)
|
||||||
|
WHERE fx_compatibilities IS NOT NULL;
|
||||||
|
|
||||||
|
-- Index for detail sync queue (find unseen or stale products quickly)
|
||||||
|
-- NB: partial index with NOW() is not allowed (non-immutable); use plain index instead
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_transceivers_detail_synced_at
|
||||||
|
ON transceivers (detail_synced_at NULLS FIRST);
|
||||||
|
|
||||||
|
-- ── Statistics ───────────────────────────────────────────────────────────────
|
||||||
|
DO $$
|
||||||
|
DECLARE
|
||||||
|
fx_cnt INTEGER;
|
||||||
|
BEGIN
|
||||||
|
SELECT COUNT(*) INTO fx_cnt
|
||||||
|
FROM transceivers t
|
||||||
|
JOIN vendors v ON v.id = t.vendor_id
|
||||||
|
WHERE UPPER(v.name) LIKE '%FLEXOPTIX%';
|
||||||
|
|
||||||
|
RAISE NOTICE 'Migration 115 complete.';
|
||||||
|
RAISE NOTICE ' Total FX products: %', fx_cnt;
|
||||||
|
RAISE NOTICE ' New columns added: fx_specifications, fx_compatibilities,';
|
||||||
|
RAISE NOTICE ' compliance_code, laser_type, receiver_type,';
|
||||||
|
RAISE NOTICE ' supported_protocols, extinction_ratio_db,';
|
||||||
|
RAISE NOTICE ' cdr_support, inbuilt_fec, detail_synced_at';
|
||||||
|
RAISE NOTICE ' Run enrich:flexoptix-details to populate.';
|
||||||
|
END $$;
|
||||||
Loading…
x
Reference in New Issue
Block a user