Merge remote-tracking branch 'origin/main'
This commit is contained in:
commit
84a54f01e8
@ -50,6 +50,7 @@ hotTopicsRouter.get("/", async (req, res) => {
|
|||||||
JOIN vendors v ON pc.vendor_id = v.id
|
JOIN vendors v ON pc.vendor_id = v.id
|
||||||
JOIN transceivers t ON pc.transceiver_id = t.id
|
JOIN transceivers t ON pc.transceiver_id = t.id
|
||||||
WHERE pc.delta_pct < -10 AND pc.detected_at > NOW() - INTERVAL '14 days'
|
WHERE pc.delta_pct < -10 AND pc.detected_at > NOW() - INTERVAL '14 days'
|
||||||
|
AND v.name != 'Flexoptix'
|
||||||
ORDER BY pc.delta_pct ASC LIMIT 5
|
ORDER BY pc.delta_pct ASC LIMIT 5
|
||||||
`).catch(() => ({ rows: [] }));
|
`).catch(() => ({ rows: [] }));
|
||||||
|
|
||||||
|
|||||||
@ -36,6 +36,7 @@ priceComparisonRouter.get("/summary", async (_req: Request, res: Response) => {
|
|||||||
po.price,
|
po.price,
|
||||||
po.currency
|
po.currency
|
||||||
FROM price_observations po
|
FROM price_observations po
|
||||||
|
WHERE po.marketplace NOT LIKE 'flexoptix%'
|
||||||
ORDER BY po.transceiver_id, po.source_vendor_id, po.time DESC
|
ORDER BY po.transceiver_id, po.source_vendor_id, po.time DESC
|
||||||
)
|
)
|
||||||
SELECT
|
SELECT
|
||||||
@ -55,6 +56,7 @@ priceComparisonRouter.get("/summary", async (_req: Request, res: Response) => {
|
|||||||
po.price,
|
po.price,
|
||||||
po.currency
|
po.currency
|
||||||
FROM price_observations po
|
FROM price_observations po
|
||||||
|
WHERE po.marketplace NOT LIKE 'flexoptix%'
|
||||||
ORDER BY po.transceiver_id, po.source_vendor_id, po.time DESC
|
ORDER BY po.transceiver_id, po.source_vendor_id, po.time DESC
|
||||||
)
|
)
|
||||||
SELECT
|
SELECT
|
||||||
@ -110,6 +112,7 @@ priceComparisonRouter.get("/", async (req: Request, res: Response) => {
|
|||||||
po.price,
|
po.price,
|
||||||
po.currency
|
po.currency
|
||||||
FROM price_observations po
|
FROM price_observations po
|
||||||
|
WHERE po.marketplace NOT LIKE 'flexoptix%'
|
||||||
ORDER BY po.transceiver_id, po.source_vendor_id, po.time DESC
|
ORDER BY po.transceiver_id, po.source_vendor_id, po.time DESC
|
||||||
)
|
)
|
||||||
SELECT
|
SELECT
|
||||||
@ -216,6 +219,7 @@ priceComparisonRouter.get("/:sku", async (req: Request, res: Response) => {
|
|||||||
po.time
|
po.time
|
||||||
FROM price_observations po
|
FROM price_observations po
|
||||||
WHERE po.transceiver_id = $1
|
WHERE po.transceiver_id = $1
|
||||||
|
AND po.marketplace NOT LIKE 'flexoptix%'
|
||||||
ORDER BY po.transceiver_id, po.source_vendor_id, po.time DESC
|
ORDER BY po.transceiver_id, po.source_vendor_id, po.time DESC
|
||||||
) po
|
) po
|
||||||
JOIN vendors v ON v.id = po.source_vendor_id
|
JOIN vendors v ON v.id = po.source_vendor_id
|
||||||
|
|||||||
@ -40,8 +40,11 @@ interface CatalogProduct {
|
|||||||
title: string;
|
title: string;
|
||||||
url: string | null;
|
url: string | null;
|
||||||
imageUrl: string | null;
|
imageUrl: string | null;
|
||||||
|
productType: string | null;
|
||||||
price: {
|
price: {
|
||||||
amount: number | null;
|
amount: number | null;
|
||||||
|
listAmount: number | null;
|
||||||
|
selfConfigureAmount: number | null;
|
||||||
currency: string | null;
|
currency: string | null;
|
||||||
source: "api" | "missing";
|
source: "api" | "missing";
|
||||||
fetchedAt: string;
|
fetchedAt: string;
|
||||||
@ -78,6 +81,7 @@ export interface FlexoptixSyncResult {
|
|||||||
skipped: number;
|
skipped: number;
|
||||||
priceWrites: number;
|
priceWrites: number;
|
||||||
stockWrites: number;
|
stockWrites: number;
|
||||||
|
mapWrites: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Generic helpers ────────────────────────────────────────────────────────
|
// ── Generic helpers ────────────────────────────────────────────────────────
|
||||||
@ -236,7 +240,7 @@ function extractCompatibility(row: JsonRecord): CatalogProduct["compatibility"]
|
|||||||
const rows = Array.isArray(rawCompat) ? rawCompat.filter(isRecord) : [];
|
const rows = Array.isArray(rawCompat) ? rawCompat.filter(isRecord) : [];
|
||||||
return rows.flatMap(entry => {
|
return rows.flatMap(entry => {
|
||||||
const flat = flatLookup(entry);
|
const flat = flatLookup(entry);
|
||||||
const vendor = asString(pick(flat, ["vendor", "manufacturer", "brand", "systemVendor"]));
|
const vendor = asString(pick(flat, ["vendor", "manufacturer", "brand", "systemVendor", "compatibleToVendor"]));
|
||||||
if (!vendor) return [];
|
if (!vendor) return [];
|
||||||
return [{
|
return [{
|
||||||
vendor,
|
vendor,
|
||||||
@ -255,7 +259,10 @@ function normalizeProduct(row: JsonRecord, fetchedAt: string): CatalogProduct |
|
|||||||
|
|
||||||
const url = asString(pick(flat, ["url", "productUrl", "canonicalUrl", "link"]));
|
const url = asString(pick(flat, ["url", "productUrl", "canonicalUrl", "link"]));
|
||||||
const imageUrl = asString(pick(flat, ["image", "imageUrl", "productImage", "thumbnail"]));
|
const imageUrl = asString(pick(flat, ["image", "imageUrl", "productImage", "thumbnail"]));
|
||||||
|
const productType = asString(pick(flat, ["type", "producttype", "moduletype"]));
|
||||||
const amount = asNumber(pick(flat, ["price", "priceNet", "netPrice", "grossPrice", "amount"]));
|
const amount = asNumber(pick(flat, ["price", "priceNet", "netPrice", "grossPrice", "amount"]));
|
||||||
|
const listAmount = asNumber(pick(flat, ["listprice", "msrp", "rrp", "recommendedprice"]));
|
||||||
|
const selfConfigureAmount = asNumber(pick(flat, ["selfconfigureprice", "selfconfigprice"]));
|
||||||
const currency = asString(pick(flat, ["currency", "priceCurrency", "currencyCode"]))
|
const currency = asString(pick(flat, ["currency", "priceCurrency", "currencyCode"]))
|
||||||
?? (amount === null ? null : process.env["FLEXOPTIX_API_CURRENCY"]?.trim() ?? "EUR");
|
?? (amount === null ? null : process.env["FLEXOPTIX_API_CURRENCY"]?.trim() ?? "EUR");
|
||||||
const quantity = asNumber(pick(flat, ["stock", "stockQuantity", "quantity", "availableQuantity"]));
|
const quantity = asNumber(pick(flat, ["stock", "stockQuantity", "quantity", "availableQuantity"]));
|
||||||
@ -277,10 +284,13 @@ function normalizeProduct(row: JsonRecord, fetchedAt: string): CatalogProduct |
|
|||||||
fetchedAt,
|
fetchedAt,
|
||||||
sku,
|
sku,
|
||||||
title,
|
title,
|
||||||
|
productType,
|
||||||
url,
|
url,
|
||||||
imageUrl,
|
imageUrl,
|
||||||
price: {
|
price: {
|
||||||
amount,
|
amount,
|
||||||
|
listAmount: (listAmount !== null && listAmount !== amount) ? listAmount : null,
|
||||||
|
selfConfigureAmount,
|
||||||
currency,
|
currency,
|
||||||
source: amount === null ? "missing" : "api",
|
source: amount === null ? "missing" : "api",
|
||||||
fetchedAt,
|
fetchedAt,
|
||||||
@ -332,6 +342,8 @@ function speedLabel(speedGbps: number | null): string | undefined {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function categoryFor(product: CatalogProduct): string {
|
function categoryFor(product: CatalogProduct): string {
|
||||||
|
const apiType = product.productType?.toLowerCase() ?? "";
|
||||||
|
if (apiType.includes("cable")) return "Cable";
|
||||||
const text = `${product.title} ${product.optics.protocol ?? ""}`.toLowerCase();
|
const text = `${product.title} ${product.optics.protocol ?? ""}`.toLowerCase();
|
||||||
if (/\bdac\b|direct attach|copper/.test(text)) return "DAC";
|
if (/\bdac\b|direct attach|copper/.test(text)) return "DAC";
|
||||||
if (/\baoc\b|active optical/.test(text)) return "AOC";
|
if (/\baoc\b|active optical/.test(text)) return "AOC";
|
||||||
@ -351,7 +363,7 @@ async function importProduct(
|
|||||||
speedGbps: product.optics.speedGbps ?? undefined,
|
speedGbps: product.optics.speedGbps ?? undefined,
|
||||||
speed: speedLabel(product.optics.speedGbps),
|
speed: speedLabel(product.optics.speedGbps),
|
||||||
reachMeters: product.optics.reachM ?? undefined,
|
reachMeters: product.optics.reachM ?? undefined,
|
||||||
reachLabel: reachLabel(product.optics.reachM),
|
reachLabel: reachLabel(product.optics.reachM) ?? undefined,
|
||||||
fiberType: product.optics.fiberType ?? undefined,
|
fiberType: product.optics.fiberType ?? undefined,
|
||||||
wavelengths: product.optics.wavelengthNm === null ? undefined : `${product.optics.wavelengthNm}nm`,
|
wavelengths: product.optics.wavelengthNm === null ? undefined : `${product.optics.wavelengthNm}nm`,
|
||||||
category: categoryFor(product),
|
category: categoryFor(product),
|
||||||
@ -361,11 +373,11 @@ async function importProduct(
|
|||||||
if (product.imageUrl || product.url) {
|
if (product.imageUrl || product.url) {
|
||||||
await pool.query(`
|
await pool.query(`
|
||||||
UPDATE transceivers SET
|
UPDATE transceivers SET
|
||||||
image_url = COALESCE(NULLIF(image_url, ''), $1),
|
image_url = COALESCE(NULLIF(image_url, ''), $1::text),
|
||||||
product_page_url = COALESCE(NULLIF(product_page_url, ''), $2),
|
product_page_url = COALESCE(NULLIF(product_page_url, ''), $2::text),
|
||||||
updated_at = NOW()
|
updated_at = NOW()
|
||||||
WHERE id = $3
|
WHERE id = $3
|
||||||
AND ($1 IS NOT NULL OR $2 IS NOT NULL)
|
AND ($1::text IS NOT NULL OR $2::text IS NOT NULL)
|
||||||
`, [product.imageUrl ?? null, product.url ?? null, transceiverId]);
|
`, [product.imageUrl ?? null, product.url ?? null, transceiverId]);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -389,6 +401,48 @@ async function importProduct(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (product.price.selfConfigureAmount !== null && product.price.currency) {
|
||||||
|
await upsertPriceObservation({
|
||||||
|
transceiverId,
|
||||||
|
sourceVendorId: vendorId,
|
||||||
|
price: product.price.selfConfigureAmount,
|
||||||
|
currency: product.price.currency,
|
||||||
|
marketplace: "flexoptix-selfconfigure",
|
||||||
|
stockLevel: product.stock.status ?? "unknown",
|
||||||
|
quantityAvailable: product.stock.quantity ?? undefined,
|
||||||
|
url: product.url ?? undefined,
|
||||||
|
contentHash: contentHash({
|
||||||
|
source: product.source,
|
||||||
|
sku: product.sku,
|
||||||
|
price: product.price.selfConfigureAmount,
|
||||||
|
currency: product.price.currency,
|
||||||
|
marketplace: "flexoptix-selfconfigure",
|
||||||
|
fetchedAt: product.price.fetchedAt,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (product.price.listAmount !== null && product.price.currency) {
|
||||||
|
await upsertPriceObservation({
|
||||||
|
transceiverId,
|
||||||
|
sourceVendorId: vendorId,
|
||||||
|
price: product.price.listAmount,
|
||||||
|
currency: product.price.currency,
|
||||||
|
marketplace: "flexoptix-msrp",
|
||||||
|
stockLevel: product.stock.status ?? "unknown",
|
||||||
|
quantityAvailable: product.stock.quantity ?? undefined,
|
||||||
|
url: product.url ?? undefined,
|
||||||
|
contentHash: contentHash({
|
||||||
|
source: product.source,
|
||||||
|
sku: product.sku,
|
||||||
|
price: product.price.listAmount,
|
||||||
|
currency: product.price.currency,
|
||||||
|
marketplace: "flexoptix-msrp",
|
||||||
|
fetchedAt: product.price.fetchedAt,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
const stockWritten = await upsertStockObservation({
|
const stockWritten = await upsertStockObservation({
|
||||||
transceiverId,
|
transceiverId,
|
||||||
sourceVendorId: vendorId,
|
sourceVendorId: vendorId,
|
||||||
@ -403,6 +457,22 @@ async function importProduct(
|
|||||||
return { priceWritten, stockWritten };
|
return { priceWritten, stockWritten };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function writeProductMapEntries(product: CatalogProduct): Promise<number> {
|
||||||
|
const entries = product.compatibility
|
||||||
|
.filter(c => c.coding !== null && c.vendor !== null)
|
||||||
|
.map(c => ({
|
||||||
|
oemPartNumber: c.coding!,
|
||||||
|
oemVendor: c.vendor,
|
||||||
|
flexoptixSku: product.sku,
|
||||||
|
priceEur: product.price.amount,
|
||||||
|
formFactor: product.optics.formFactor,
|
||||||
|
speedGbps: product.optics.speedGbps,
|
||||||
|
reachLabel: reachLabel(product.optics.reachM) ?? null,
|
||||||
|
fiberType: product.optics.fiberType,
|
||||||
|
}));
|
||||||
|
return batchUpsertProductMap(entries);
|
||||||
|
}
|
||||||
|
|
||||||
// ── API client ─────────────────────────────────────────────────────────────
|
// ── API client ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
function validateEnv(): { baseUrl: string; username: string | null; password: string | null; token: string | null } {
|
function validateEnv(): { baseUrl: string; username: string | null; password: string | null; token: string | null } {
|
||||||
@ -492,6 +562,94 @@ function extractRows(payload: unknown): JsonRecord[] {
|
|||||||
|
|
||||||
// ── Main export ────────────────────────────────────────────────────────────
|
// ── Main export ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
// ── Self-configure price enrichment ───────────────────────────────────────
|
||||||
|
// The bulk paginated endpoint omits self_configure_price.
|
||||||
|
// A targeted sku[] fetch returns it. Batch 50 SKUs per call (~10 calls total).
|
||||||
|
|
||||||
|
async function fetchSelfConfigurePrices(
|
||||||
|
skus: string[],
|
||||||
|
baseUrl: string,
|
||||||
|
headers: Record<string, string>,
|
||||||
|
currency: string,
|
||||||
|
timeoutMs: number,
|
||||||
|
): Promise<Map<string, number>> {
|
||||||
|
const BATCH_SIZE = 50;
|
||||||
|
const priceMap = new Map<string, number>();
|
||||||
|
const productPath = process.env["FLEXOPTIX_API_PRODUCTS_PATH"]?.trim() ?? "/rest/V2/flexoptix/products";
|
||||||
|
|
||||||
|
for (let i = 0; i < skus.length; i += BATCH_SIZE) {
|
||||||
|
const batch = skus.slice(i, i + BATCH_SIZE);
|
||||||
|
const url = buildUrl(baseUrl, productPath);
|
||||||
|
url.searchParams.set("currency", currency);
|
||||||
|
batch.forEach(sku => url.searchParams.append("sku[]", sku));
|
||||||
|
|
||||||
|
try {
|
||||||
|
const payload = await fetchJson(url, { headers }, timeoutMs);
|
||||||
|
for (const row of extractRows(payload)) {
|
||||||
|
if (!isRecord(row)) continue;
|
||||||
|
const sku = asString(row["sku"] ?? row["SKU"]);
|
||||||
|
const flat = flatLookup(row);
|
||||||
|
const scPrice = asNumber(flat["selfconfigureprice"] ?? flat["selfconfigprice"] ?? null);
|
||||||
|
if (sku && scPrice !== null) priceMap.set(sku, scPrice);
|
||||||
|
}
|
||||||
|
} catch (err: unknown) {
|
||||||
|
const msg = err instanceof Error ? err.message : String(err);
|
||||||
|
console.warn(`[${new Date().toISOString()}] Self-configure enrichment batch failed: ${msg.slice(0, 80)}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return priceMap;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// ── Product map batch upsert ─────────────────────────────────────────────────
|
||||||
|
// Writes OEM vendor coding entries into flexoptix_product_map.
|
||||||
|
// One row per (oem_part_number, oem_vendor) pair — ON CONFLICT updates specs.
|
||||||
|
|
||||||
|
async function batchUpsertProductMap(entries: Array<{
|
||||||
|
oemPartNumber: string;
|
||||||
|
oemVendor: string;
|
||||||
|
flexoptixSku: string;
|
||||||
|
priceEur: number | null;
|
||||||
|
formFactor: string | null;
|
||||||
|
speedGbps: number | null;
|
||||||
|
reachLabel: string | null;
|
||||||
|
fiberType: string | null;
|
||||||
|
}>): Promise<number> {
|
||||||
|
if (entries.length === 0) return 0;
|
||||||
|
const BATCH_SIZE = 500;
|
||||||
|
let written = 0;
|
||||||
|
for (let i = 0; i < entries.length; i += BATCH_SIZE) {
|
||||||
|
const chunk = entries.slice(i, i + BATCH_SIZE);
|
||||||
|
const placeholders: string[] = [];
|
||||||
|
const params: unknown[] = [];
|
||||||
|
let n = 1;
|
||||||
|
for (const e of chunk) {
|
||||||
|
placeholders.push(`($${n},$${n+1},$${n+2},$${n+3},$${n+4},$${n+5},$${n+6},$${n+7},'exact',NOW())`);
|
||||||
|
params.push(e.oemPartNumber, e.oemVendor, e.flexoptixSku, e.priceEur, e.formFactor, e.speedGbps, e.reachLabel, e.fiberType);
|
||||||
|
n += 8;
|
||||||
|
}
|
||||||
|
await pool.query(`
|
||||||
|
INSERT INTO flexoptix_product_map
|
||||||
|
(oem_part_number, oem_vendor, flexoptix_sku, flexoptix_price_eur,
|
||||||
|
form_factor, speed_gbps, reach_label, fiber_type, match_type, last_verified)
|
||||||
|
VALUES ${placeholders.join(',')}
|
||||||
|
ON CONFLICT (oem_part_number, oem_vendor) DO UPDATE SET
|
||||||
|
flexoptix_sku = EXCLUDED.flexoptix_sku,
|
||||||
|
flexoptix_price_eur = EXCLUDED.flexoptix_price_eur,
|
||||||
|
form_factor = EXCLUDED.form_factor,
|
||||||
|
speed_gbps = EXCLUDED.speed_gbps,
|
||||||
|
reach_label = EXCLUDED.reach_label,
|
||||||
|
fiber_type = EXCLUDED.fiber_type,
|
||||||
|
last_verified = EXCLUDED.last_verified,
|
||||||
|
updated_at = NOW()
|
||||||
|
`, params);
|
||||||
|
written += chunk.length;
|
||||||
|
}
|
||||||
|
return written;
|
||||||
|
}
|
||||||
|
|
||||||
export async function syncFlexoptixCatalog(): Promise<FlexoptixSyncResult> {
|
export async function syncFlexoptixCatalog(): Promise<FlexoptixSyncResult> {
|
||||||
const { baseUrl, username, password, token } = validateEnv();
|
const { baseUrl, username, password, token } = validateEnv();
|
||||||
const timeoutMs = parseInt(process.env["FLEXOPTIX_API_TIMEOUT_MS"]?.trim() ?? "30000", 10);
|
const timeoutMs = parseInt(process.env["FLEXOPTIX_API_TIMEOUT_MS"]?.trim() ?? "30000", 10);
|
||||||
@ -517,25 +675,45 @@ export async function syncFlexoptixCatalog(): Promise<FlexoptixSyncResult> {
|
|||||||
const importable = products.filter(canImportProduct);
|
const importable = products.filter(canImportProduct);
|
||||||
const skipped = products.length - importable.length;
|
const skipped = products.length - importable.length;
|
||||||
|
|
||||||
console.log(`[${new Date().toISOString()}] Normalized: ${products.length} | importable: ${importable.length} | skipped: ${skipped}`);
|
// Enrich importable products with self_configure_price (not in bulk response)
|
||||||
|
const currency = process.env["FLEXOPTIX_API_CURRENCY"]?.trim() ?? "EUR";
|
||||||
|
const scPrices = await fetchSelfConfigurePrices(
|
||||||
|
importable.map(p => p.sku),
|
||||||
|
baseUrl,
|
||||||
|
headers,
|
||||||
|
currency,
|
||||||
|
timeoutMs,
|
||||||
|
);
|
||||||
|
let scEnriched = 0;
|
||||||
|
for (const product of importable) {
|
||||||
|
const scPrice = scPrices.get(product.sku);
|
||||||
|
if (scPrice !== undefined) {
|
||||||
|
product.price.selfConfigureAmount = scPrice;
|
||||||
|
scEnriched++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`[${new Date().toISOString()}] Normalized: ${products.length} | importable: ${importable.length} | skipped: ${skipped} | self-configure enriched: ${scEnriched}`);
|
||||||
|
|
||||||
const vendorId = await ensureVendor("Flexoptix", "compatible", "https://www.flexoptix.net", "https://www.flexoptix.net");
|
const vendorId = await ensureVendor("Flexoptix", "compatible", "https://www.flexoptix.net", "https://www.flexoptix.net");
|
||||||
|
|
||||||
let priceWrites = 0;
|
let priceWrites = 0;
|
||||||
let stockWrites = 0;
|
let stockWrites = 0;
|
||||||
|
let mapWrites = 0;
|
||||||
|
|
||||||
for (const product of importable) {
|
for (const product of importable) {
|
||||||
try {
|
try {
|
||||||
const result = await importProduct(product, vendorId);
|
const result = await importProduct(product, vendorId);
|
||||||
if (result.priceWritten) priceWrites++;
|
if (result.priceWritten) priceWrites++;
|
||||||
if (result.stockWritten) stockWrites++;
|
if (result.stockWritten) stockWrites++;
|
||||||
|
mapWrites += await writeProductMapEntries(product);
|
||||||
} catch (err: unknown) {
|
} catch (err: unknown) {
|
||||||
const message = err instanceof Error ? err.message : String(err);
|
const message = err instanceof Error ? err.message : String(err);
|
||||||
console.warn(`[${new Date().toISOString()}] Flexoptix import error (${product.sku}): ${message.slice(0, 100)}`);
|
console.warn(`[${new Date().toISOString()}] Flexoptix import error (${product.sku}): ${message.slice(0, 100)}`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
console.log(`[${new Date().toISOString()}] Flexoptix API sync complete: ${importable.length} products, ${priceWrites} price writes, ${stockWrites} stock writes`);
|
console.log(`[${new Date().toISOString()}] Flexoptix API sync complete: ${importable.length} products, ${priceWrites} price writes, ${stockWrites} stock writes, ${mapWrites} product-map upserts`);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
fetched: rawRows.length,
|
fetched: rawRows.length,
|
||||||
@ -543,5 +721,6 @@ export async function syncFlexoptixCatalog(): Promise<FlexoptixSyncResult> {
|
|||||||
skipped,
|
skipped,
|
||||||
priceWrites,
|
priceWrites,
|
||||||
stockWrites,
|
stockWrites,
|
||||||
|
mapWrites,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@ -417,9 +417,11 @@ export async function upsertStockObservation(params: {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Compare against the last observation to avoid duplicate writes
|
// Compare against the last observation to avoid duplicate writes.
|
||||||
|
// Bypass dedup if last observation is older than 7 days (heartbeat to keep health view alive).
|
||||||
|
const STOCK_REFRESH_DAYS = 7;
|
||||||
const lastObs = await pool.query(
|
const lastObs = await pool.query(
|
||||||
`SELECT warehouse_de_qty, warehouse_global_qty, backorder_qty, units_sold, quantity_available
|
`SELECT warehouse_de_qty, warehouse_global_qty, backorder_qty, units_sold, quantity_available, time
|
||||||
FROM stock_observations
|
FROM stock_observations
|
||||||
WHERE transceiver_id = $1 AND source_vendor_id = $2
|
WHERE transceiver_id = $1 AND source_vendor_id = $2
|
||||||
ORDER BY time DESC LIMIT 1`,
|
ORDER BY time DESC LIMIT 1`,
|
||||||
@ -428,6 +430,9 @@ export async function upsertStockObservation(params: {
|
|||||||
|
|
||||||
if (lastObs.rows.length > 0) {
|
if (lastObs.rows.length > 0) {
|
||||||
const r = lastObs.rows[0];
|
const r = lastObs.rows[0];
|
||||||
|
const ageMs = Date.now() - new Date(r.time).getTime();
|
||||||
|
const ageDays = ageMs / (1000 * 60 * 60 * 24);
|
||||||
|
if (ageDays < STOCK_REFRESH_DAYS) {
|
||||||
const unchanged =
|
const unchanged =
|
||||||
(r.warehouse_de_qty ?? null) === (params.warehouseDeQty ?? null) &&
|
(r.warehouse_de_qty ?? null) === (params.warehouseDeQty ?? null) &&
|
||||||
(r.warehouse_global_qty ?? null) === (params.warehouseGlobalQty ?? null) &&
|
(r.warehouse_global_qty ?? null) === (params.warehouseGlobalQty ?? null) &&
|
||||||
@ -436,6 +441,7 @@ export async function upsertStockObservation(params: {
|
|||||||
(r.quantity_available ?? null) === (params.quantityAvailable ?? null);
|
(r.quantity_available ?? null) === (params.quantityAvailable ?? null);
|
||||||
if (unchanged) return false;
|
if (unchanged) return false;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const inStock =
|
const inStock =
|
||||||
((params.warehouseDeQty ?? 0) + (params.warehouseGlobalQty ?? 0) + (params.quantityAvailable ?? 0)) > 0;
|
((params.warehouseDeQty ?? 0) + (params.warehouseGlobalQty ?? 0) + (params.quantityAvailable ?? 0)) > 0;
|
||||||
@ -455,8 +461,8 @@ export async function upsertStockObservation(params: {
|
|||||||
$5, $6::date,
|
$5, $6::date,
|
||||||
$7, $8::date,
|
$7, $8::date,
|
||||||
$9, $10::date,
|
$9, $10::date,
|
||||||
$11, $12, $13, $14, $15,
|
$11, $12, $13::text[], $14, $15,
|
||||||
$16, $17, $18, $19
|
$16, $17::bpchar, $18, $19
|
||||||
)`,
|
)`,
|
||||||
[
|
[
|
||||||
params.transceiverId,
|
params.transceiverId,
|
||||||
|
|||||||
60
sql/131-wavelength-column-sync.sql
Normal file
60
sql/131-wavelength-column-sync.sql
Normal file
@ -0,0 +1,60 @@
|
|||||||
|
-- sql/131 — Wavelength column sync (root-cause fix for equivalence review backlog)
|
||||||
|
--
|
||||||
|
-- Root cause: the enricher populates the structured integer column
|
||||||
|
-- transceivers.wavelength_tx_nm, but the equivalence spec-matcher reads the TEXT
|
||||||
|
-- column transceivers.wavelengths via tip_extract_wavelength_nm(). 27,908 rows had
|
||||||
|
-- wavelength_tx_nm set while wavelengths was NULL, so the matcher could not confirm
|
||||||
|
-- wavelength on either side and dumped ~3,155 genuine equivalences into manual review.
|
||||||
|
--
|
||||||
|
-- Fix (no fabrication — copies existing structured data into the format the matcher
|
||||||
|
-- expects, and keeps it in sync going forward):
|
||||||
|
-- A) one-off backfill of transceivers.wavelengths from wavelength_tx_nm
|
||||||
|
-- B) durable BEFORE-trigger so future ingests stay in sync
|
||||||
|
-- C) promote pending -> auto_approved ONLY where the FULL spec-matcher criteria now
|
||||||
|
-- agree (form_factor + speed + fiber_type + reach-tier + wavelength +/-10nm).
|
||||||
|
-- Everything with a genuine reach-tier / fiber / wavelength difference stays pending.
|
||||||
|
|
||||||
|
BEGIN;
|
||||||
|
|
||||||
|
-- A) one-off backfill (existing structured data -> text column the matcher reads)
|
||||||
|
UPDATE transceivers
|
||||||
|
SET wavelengths = wavelength_tx_nm || 'nm'
|
||||||
|
WHERE (wavelengths IS NULL OR wavelengths = '')
|
||||||
|
AND wavelength_tx_nm IS NOT NULL;
|
||||||
|
|
||||||
|
-- B) durable sync trigger for future inserts/updates
|
||||||
|
CREATE OR REPLACE FUNCTION tip_sync_wavelengths_from_tx() RETURNS trigger AS $fn$
|
||||||
|
BEGIN
|
||||||
|
IF (NEW.wavelengths IS NULL OR NEW.wavelengths = '')
|
||||||
|
AND NEW.wavelength_tx_nm IS NOT NULL THEN
|
||||||
|
NEW.wavelengths := NEW.wavelength_tx_nm || 'nm';
|
||||||
|
END IF;
|
||||||
|
RETURN NEW;
|
||||||
|
END;
|
||||||
|
$fn$ LANGUAGE plpgsql;
|
||||||
|
|
||||||
|
DROP TRIGGER IF EXISTS trg_sync_wavelengths ON transceivers;
|
||||||
|
CREATE TRIGGER trg_sync_wavelengths
|
||||||
|
BEFORE INSERT OR UPDATE ON transceivers
|
||||||
|
FOR EACH ROW EXECUTE FUNCTION tip_sync_wavelengths_from_tx();
|
||||||
|
|
||||||
|
-- C) valid promotion: pending -> auto_approved where full spec now confirmed
|
||||||
|
UPDATE transceiver_equivalences e
|
||||||
|
SET status = 'auto_approved',
|
||||||
|
confidence = GREATEST(e.confidence, 0.85),
|
||||||
|
match_notes = COALESCE(e.match_notes,'') || ' | sql/131: full spec confirmed after wavelength column sync',
|
||||||
|
updated_at = now()
|
||||||
|
FROM transceivers fx, transceivers cx
|
||||||
|
WHERE e.flexoptix_id = fx.id
|
||||||
|
AND e.competitor_id = cx.id
|
||||||
|
AND e.status = 'pending'
|
||||||
|
AND fx.form_factor = cx.form_factor
|
||||||
|
AND fx.speed_gbps = cx.speed_gbps
|
||||||
|
AND fx.fiber_type IS NOT DISTINCT FROM cx.fiber_type
|
||||||
|
AND fx.reach_meters >= 10 AND cx.reach_meters >= 10
|
||||||
|
AND tip_reach_tier(fx.reach_meters) = tip_reach_tier(cx.reach_meters)
|
||||||
|
AND tip_extract_wavelength_nm(fx.wavelengths) IS NOT NULL
|
||||||
|
AND tip_extract_wavelength_nm(cx.wavelengths) IS NOT NULL
|
||||||
|
AND abs(tip_extract_wavelength_nm(fx.wavelengths) - tip_extract_wavelength_nm(cx.wavelengths)) <= 10;
|
||||||
|
|
||||||
|
COMMIT;
|
||||||
74
sql/132-wavelength-authority-normalize.sql
Normal file
74
sql/132-wavelength-authority-normalize.sql
Normal file
@ -0,0 +1,74 @@
|
|||||||
|
-- sql/132 — Wavelength authority correction + format normalization
|
||||||
|
--
|
||||||
|
-- Follow-up to sql/131. Discovery: transceivers.wavelengths holds the part-derived
|
||||||
|
-- wavelength as a BARE number ("1470") for ~16.8k rows, which tip_extract_wavelength_nm()
|
||||||
|
-- cannot read (it requires the "nm" suffix). Meanwhile wavelength_tx_nm disagrees with
|
||||||
|
-- that authoritative bare number 33% of the time, and 64% of those disagreements are a
|
||||||
|
-- systematic tx_nm=1310 enricher default. So the bare-number column is authoritative;
|
||||||
|
-- tx_nm is not. This migration prefers the authoritative source.
|
||||||
|
--
|
||||||
|
-- 1) normalize authoritative bare-number wavelengths -> "NNNnm" (format only, no value change)
|
||||||
|
-- 2) clean "N/A" text -> NULL
|
||||||
|
-- 3) correct the few CWDM rows sql/131 backfilled with a wrong tx_nm default, using the
|
||||||
|
-- deterministic CWDM wavelength in the part number
|
||||||
|
-- 4) re-promote pending -> auto_approved where the FULL spec gate now agrees on
|
||||||
|
-- authoritative data (gate already protects against tx_nm noise: both sides must
|
||||||
|
-- agree within +/-10nm, so a wrong 1310 default can never auto-approve a real 1470).
|
||||||
|
|
||||||
|
BEGIN;
|
||||||
|
|
||||||
|
-- 1) normalize authoritative bare-number wavelengths (preserve value, add unit)
|
||||||
|
UPDATE transceivers
|
||||||
|
SET wavelengths = wavelengths || 'nm'
|
||||||
|
WHERE wavelengths ~ '^\d{3,4}$';
|
||||||
|
|
||||||
|
-- 2) non-values -> NULL (N/A, -, empty, etc.)
|
||||||
|
UPDATE transceivers
|
||||||
|
SET wavelengths = NULL
|
||||||
|
WHERE wavelengths IS NOT NULL
|
||||||
|
AND tip_extract_wavelength_nm(wavelengths) IS NULL
|
||||||
|
AND wavelengths !~ '\d';
|
||||||
|
|
||||||
|
-- 3) correct CWDM rows that sql/131 filled from a wrong tx_nm default:
|
||||||
|
-- use the deterministic CWDM wavelength encoded in the part number
|
||||||
|
UPDATE transceivers
|
||||||
|
SET wavelengths = (regexp_match(part_number,'(12[6-9]0|13[0-9]0|14[0-9]0|15[0-9]0|16[01]0)'))[1] || 'nm'
|
||||||
|
WHERE wavelengths = wavelength_tx_nm || 'nm'
|
||||||
|
AND part_number ~ '(12[6-9]|13[0-9]|14[0-9]|15[0-9]|16[01])0'
|
||||||
|
AND (regexp_match(part_number,'(12[6-9]0|13[0-9]0|14[0-9]0|15[0-9]0|16[01]0)'))[1]::int <> wavelength_tx_nm;
|
||||||
|
|
||||||
|
-- 3b) upgrade the sync trigger so future ingests self-normalize:
|
||||||
|
-- prefer an authoritative bare-number wavelength (add "nm"), fall back to tx_nm only
|
||||||
|
-- when no textual wavelength is present at all.
|
||||||
|
CREATE OR REPLACE FUNCTION tip_sync_wavelengths_from_tx() RETURNS trigger AS $fn$
|
||||||
|
BEGIN
|
||||||
|
IF NEW.wavelengths ~ '^\d{3,4}$' THEN
|
||||||
|
NEW.wavelengths := NEW.wavelengths || 'nm';
|
||||||
|
ELSIF (NEW.wavelengths IS NULL OR NEW.wavelengths = '' OR tip_extract_wavelength_nm(NEW.wavelengths) IS NULL)
|
||||||
|
AND NEW.wavelength_tx_nm IS NOT NULL THEN
|
||||||
|
NEW.wavelengths := NEW.wavelength_tx_nm || 'nm';
|
||||||
|
END IF;
|
||||||
|
RETURN NEW;
|
||||||
|
END;
|
||||||
|
$fn$ LANGUAGE plpgsql;
|
||||||
|
|
||||||
|
-- 4) valid promotion on corrected/authoritative data (same full spec gate as spec-matcher)
|
||||||
|
UPDATE transceiver_equivalences e
|
||||||
|
SET status = 'auto_approved',
|
||||||
|
confidence = GREATEST(e.confidence, 0.85),
|
||||||
|
match_notes = COALESCE(e.match_notes,'') || ' | sql/132: full spec confirmed on authoritative wavelength',
|
||||||
|
updated_at = now()
|
||||||
|
FROM transceivers fx, transceivers cx
|
||||||
|
WHERE e.flexoptix_id = fx.id
|
||||||
|
AND e.competitor_id = cx.id
|
||||||
|
AND e.status = 'pending'
|
||||||
|
AND fx.form_factor = cx.form_factor
|
||||||
|
AND fx.speed_gbps = cx.speed_gbps
|
||||||
|
AND fx.fiber_type IS NOT DISTINCT FROM cx.fiber_type
|
||||||
|
AND fx.reach_meters >= 10 AND cx.reach_meters >= 10
|
||||||
|
AND tip_reach_tier(fx.reach_meters) = tip_reach_tier(cx.reach_meters)
|
||||||
|
AND tip_extract_wavelength_nm(fx.wavelengths) IS NOT NULL
|
||||||
|
AND tip_extract_wavelength_nm(cx.wavelengths) IS NOT NULL
|
||||||
|
AND abs(tip_extract_wavelength_nm(fx.wavelengths) - tip_extract_wavelength_nm(cx.wavelengths)) <= 10;
|
||||||
|
|
||||||
|
COMMIT;
|
||||||
46
sql/133-reach-substitution-policy.sql
Normal file
46
sql/133-reach-substitution-policy.sql
Normal file
@ -0,0 +1,46 @@
|
|||||||
|
-- sql/133 — One-directional reach-tier substitution policy
|
||||||
|
--
|
||||||
|
-- Decision (Rene, 2026-07-07): a higher-reach optic may satisfy a lower-reach
|
||||||
|
-- requirement (e.g. a 40km ER can serve where a 10km LR is asked for), never the
|
||||||
|
-- reverse — the shorter-reach part cannot close the link budget on a longer run.
|
||||||
|
-- ZR/coherent/amplified tiers are explicitly EXCLUDED from this automatic rule:
|
||||||
|
-- those can overload a short-haul receiver without an attenuator, which is a real
|
||||||
|
-- power-budget calculation, not a simple tier comparison. Those stay in manual review.
|
||||||
|
--
|
||||||
|
-- Result is tagged as its own status (compatible_superset), never merged into
|
||||||
|
-- auto_approved/exact-spec matches, so downstream consumers (Fossy/Beo/Pulso, pricing)
|
||||||
|
-- can distinguish "identical spec" from "safe higher-spec substitute" — they usually
|
||||||
|
-- differ in price and are not a like-for-like swap.
|
||||||
|
|
||||||
|
BEGIN;
|
||||||
|
|
||||||
|
-- widen the status check (additive — existing values still valid)
|
||||||
|
ALTER TABLE transceiver_equivalences DROP CONSTRAINT transceiver_equivalences_status_check;
|
||||||
|
ALTER TABLE transceiver_equivalences ADD CONSTRAINT transceiver_equivalences_status_check
|
||||||
|
CHECK (status::text = ANY (ARRAY['pending','approved','rejected','auto_approved','compatible_superset']::text[]));
|
||||||
|
|
||||||
|
UPDATE transceiver_equivalences e
|
||||||
|
SET status = 'compatible_superset',
|
||||||
|
confidence = GREATEST(e.confidence, 0.75),
|
||||||
|
match_notes = COALESCE(e.match_notes,'') || ' | sql/133: ' ||
|
||||||
|
CASE WHEN fx.reach_meters > cx.reach_meters
|
||||||
|
THEN 'flexoptix ('||tip_reach_tier(fx.reach_meters)||') covers competitor requirement ('||tip_reach_tier(cx.reach_meters)||')'
|
||||||
|
ELSE 'competitor ('||tip_reach_tier(cx.reach_meters)||') covers flexoptix requirement ('||tip_reach_tier(fx.reach_meters)||')'
|
||||||
|
END || ' — reach-tier substitute, not identical spec',
|
||||||
|
updated_at = now()
|
||||||
|
FROM transceivers fx, transceivers cx
|
||||||
|
WHERE e.flexoptix_id = fx.id
|
||||||
|
AND e.competitor_id = cx.id
|
||||||
|
AND e.status = 'pending'
|
||||||
|
AND fx.form_factor = cx.form_factor
|
||||||
|
AND fx.speed_gbps = cx.speed_gbps
|
||||||
|
AND fx.fiber_type IS NOT DISTINCT FROM cx.fiber_type
|
||||||
|
AND fx.reach_meters >= 10 AND cx.reach_meters >= 10
|
||||||
|
AND tip_extract_wavelength_nm(fx.wavelengths) IS NOT NULL
|
||||||
|
AND tip_extract_wavelength_nm(cx.wavelengths) IS NOT NULL
|
||||||
|
AND abs(tip_extract_wavelength_nm(fx.wavelengths) - tip_extract_wavelength_nm(cx.wavelengths)) <= 10
|
||||||
|
AND tip_reach_tier(fx.reach_meters) <> tip_reach_tier(cx.reach_meters)
|
||||||
|
AND tip_reach_tier(fx.reach_meters) <> 'ZR'
|
||||||
|
AND tip_reach_tier(cx.reach_meters) <> 'ZR';
|
||||||
|
|
||||||
|
COMMIT;
|
||||||
Loading…
x
Reference in New Issue
Block a user