diff --git a/README.md b/README.md index 0457031..ce683f8 100644 --- a/README.md +++ b/README.md @@ -127,6 +127,19 @@ All data comes from publicly available sources: - Multi-Source Agreements (100G CWDM4 MSA, 100G PSM4 MSA, 100G Lambda MSA, OpenZR+) - Vendor datasheets and public documentation +## Flexoptix Catalog Import + +Private TIP deployments can import the normalized Flexoptix shop catalog produced +by Magatama/Pulso: + +```bash +npm run flexoptix:catalog:import -- --dry-run +TIP_DB_PASS=... npm run flexoptix:catalog:import +``` + +See [docs/FLEXOPTIX_CATALOG_IMPORT.md](docs/FLEXOPTIX_CATALOG_IMPORT.md) for the +full producer/import workflow and safety rules. + ## Contributing Contributions welcome. To add a new transceiver: diff --git a/docs/FLEXOPTIX_CATALOG_IMPORT.md b/docs/FLEXOPTIX_CATALOG_IMPORT.md new file mode 100644 index 0000000..4c9806e --- /dev/null +++ b/docs/FLEXOPTIX_CATALOG_IMPORT.md @@ -0,0 +1,49 @@ +# Flexoptix Catalog Import + +TIP can ingest the normalized Flexoptix shop catalog produced by Magatama/Pulso. +This keeps prices, product URLs, stock, and product attributes live-data driven +instead of baking volatile values into LLM training pairs. + +## Producer + +Run in the Magatama repo: + +```bash +pnpm pulso:shop:sync:dry-run +pnpm pulso:shop:sync -- --fixture ./shop-export.json +FLEXOPTIX_API_BASE_URL=... FLEXOPTIX_API_TOKEN=... pnpm pulso:shop:sync +``` + +The producer writes: + +- `data/pulso/shop-products.normalized.jsonl` +- `data/pulso/shop-catalog.normalized.json` +- `data/pulso/shop-snapshot.manifest.json` + +## TIP Import + +Safe validation: + +```bash +npm run flexoptix:catalog:import -- --dry-run +``` + +Import into the TIP database: + +```bash +TIP_DB_PASS=... npm run flexoptix:catalog:import +``` + +Override the input path if needed: + +```bash +npm run flexoptix:catalog:import -- --input /path/shop-products.normalized.jsonl --dry-run +``` + +## Safety Rules + +- Rows without SKU, title, form factor, speed, or reach are skipped for DB import. +- Prices and stock are written only when present in live/API data. +- No product URL, SKU, price, stock, delivery time, or compatibility is invented. +- The importer uses existing TIP helpers for transceiver upsert, price observations, + and stock observations. diff --git a/package.json b/package.json index 07ad6f4..e7f3585 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "dev": "npm run dev -w packages/api", "learning-pool:build": "tsx scripts/tip-learning-pool-build.ts", "learning-pool:publish-hf": "python3 scripts/tip-publish-hf-datasets.py", + "flexoptix:catalog:import": "tsx scripts/import-flexoptix-catalog.ts", "migrate": "tsx scripts/migrate.ts", "seed": "tsx scripts/seed-from-npm.ts", "db:reset": "npm run migrate && npm run seed" diff --git a/packages/scraper/src/robots/flexoptix-api-sync.ts b/packages/scraper/src/robots/flexoptix-api-sync.ts new file mode 100644 index 0000000..c1d034d --- /dev/null +++ b/packages/scraper/src/robots/flexoptix-api-sync.ts @@ -0,0 +1,531 @@ +/** + * Flexoptix API Sync Robot + * + * Fetches the full Flexoptix product catalog via their Magento 2 REST API, + * normalizes each product, and writes price + stock observations to TIP DB. + * + * Replaces the GraphQL/search-based scrapeFlexoptixCatalog() — the REST API + * provides authoritative SKUs, prices, stock levels, and compatibility data. + * + * Required env vars: + * FLEXOPTIX_API_BASE_URL e.g. https://www.flexoptix.net + * FLEXOPTIX_API_USERNAME Magento customer API username + * FLEXOPTIX_API_PASSWORD Magento customer API password + * + * Optional: + * FLEXOPTIX_API_TOKEN Bearer token (skips username/password login) + * FLEXOPTIX_API_CURRENCY Defaults to EUR + * FLEXOPTIX_API_LIMIT Page size, defaults to 500 + * FLEXOPTIX_API_TIMEOUT_MS Request timeout, defaults to 30000 + * FLEXOPTIX_API_COMPATIBILITIES 1/0, defaults to 1 + */ + +import { + ensureVendor, + findOrCreateScrapedTransceiver, + upsertPriceObservation, + upsertStockObservation, +} from "../utils/db"; +import { contentHash } from "../utils/hash"; + +// ── Types ────────────────────────────────────────────────────────────────── + +type JsonRecord = Record; + +interface CatalogProduct { + source: "flexoptix-shop-api"; + fetchedAt: string; + sku: string; + title: string; + url: string | null; + price: { + amount: number | null; + currency: string | null; + source: "api" | "missing"; + fetchedAt: string; + }; + stock: { + status: string | null; + quantity: number | null; + source: "api" | "missing"; + }; + optics: { + formFactor: string | null; + speedGbps: number | null; + reachM: number | null; + wavelengthNm: number | null; + connector: string | null; + fiberType: string | null; + protocol: string | null; + coding: string | null; + bidi: boolean | null; + dwdm: boolean | null; + cwdm: boolean | null; + }; + compatibility: Array<{ + vendor: string; + platform: string | null; + coding: string | null; + source: "api"; + }>; +} + +export interface FlexoptixSyncResult { + fetched: number; + normalized: number; + skipped: number; + priceWrites: number; + stockWrites: number; +} + +// ── Generic helpers ──────────────────────────────────────────────────────── + +function isRecord(value: unknown): value is JsonRecord { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function flatten(value: unknown, prefix = "", output: JsonRecord = {}): JsonRecord { + if (Array.isArray(value)) { + output[prefix] = value; + return output; + } + if (!isRecord(value)) { + output[prefix] = value; + return output; + } + for (const [key, child] of Object.entries(value)) { + const childKey = prefix ? `${prefix}.${key}` : key; + if (isRecord(child)) { + flatten(child, childKey, output); + } else { + output[childKey] = child; + } + } + return output; +} + +function flatLookup(record: JsonRecord): JsonRecord { + const flat = flatten(record); + const normalized: JsonRecord = {}; + for (const [key, value] of Object.entries(flat)) { + normalized[key.toLowerCase().replace(/[_\s-]+/g, "")] = value; + } + return normalized; +} + +function pick(flat: JsonRecord, keys: string[]): unknown { + const normalizedKeys = keys.map(key => key.toLowerCase().replace(/[_\s-]+/g, "")); + for (const key of normalizedKeys) { + if (flat[key] !== undefined && flat[key] !== null && flat[key] !== "") return flat[key]; + } + for (const [flatKey, value] of Object.entries(flat)) { + if (value === undefined || value === null || value === "") continue; + if (normalizedKeys.some(key => flatKey.endsWith(key))) return value; + } + return null; +} + +function asString(value: unknown): string | null { + if (typeof value === "string") return value.trim() || null; + if (typeof value === "number" || typeof value === "boolean") return String(value); + return null; +} + +function asNumber(value: unknown): number | null { + if (typeof value === "number" && Number.isFinite(value)) return value; + if (typeof value !== "string") return null; + const trimmed = value.trim(); + if (!trimmed) return null; + const stripped = trimmed.replace(/[^\d,.-]/g, ""); + const normalized = stripped.includes(",") && !stripped.includes(".") + ? stripped.replace(",", ".") + : stripped.replace(/,/g, ""); + const parsed = Number(normalized); + return Number.isFinite(parsed) ? parsed : null; +} + +function asBoolean(value: unknown): boolean | null { + if (typeof value === "boolean") return value; + if (typeof value !== "string") return null; + const normalized = value.trim().toLowerCase(); + if (["true", "yes", "ja", "1"].includes(normalized)) return true; + if (["false", "no", "nein", "0"].includes(normalized)) return false; + return null; +} + +// ── Inference helpers ────────────────────────────────────────────────────── + +function inferBoolFlags( + ...values: Array +): { bidi: boolean | null; dwdm: boolean | null; cwdm: boolean | null } { + const text = values.filter(Boolean).join(" ").toLowerCase(); + return { + bidi: /\bbidi\b|bi-directional|bidirectional|simplex/.test(text) ? true : null, + dwdm: /\bdwdm\b|itu channel|itu grid/.test(text) ? true : null, + cwdm: /\bcwdm\b/.test(text) ? true : null, + }; +} + +function parseReachMeters(value: unknown): number | null { + const text = asString(value)?.toLowerCase(); + if (text) { + const km = text.match(/([\d.,]+)\s*km/); + if (km) return Math.round((asNumber(km[1]) ?? 0) * 1000); + const meters = text.match(/([\d.,]+)\s*m/); + if (meters) return Math.round(asNumber(meters[1]) ?? 0); + } + return asNumber(value); +} + +function parseSpeedGbps(value: unknown): number | null { + const numeric = asNumber(value); + if (numeric !== null) return numeric; + const text = asString(value)?.toLowerCase(); + if (!text) return null; + const gbps = text.match(/([\d.,]+)\s*(g|gb|gbps|gbit)/); + if (gbps) return asNumber(gbps[1]); + const mbps = text.match(/([\d.,]+)\s*(m|mb|mbps|mbit)/); + if (mbps) { + const mb = asNumber(mbps[1]); + return mb === null ? null : mb / 1000; + } + return null; +} + +function inferFormFactor(...values: Array): string | null { + const text = values.filter(Boolean).join(" ").toUpperCase(); + const formFactors = [ + "QSFP-DD800", "QSFP-DD", "QSFP112", "QSFP56", "QSFP28", "QSFP+", "QSFP", + "OSFP224", "OSFP112", "OSFP", + "SFP56", "SFP28", "SFP+", "SFP", + "XFP", "CFP4", "CFP2", "CFP", "GBIC", + ]; + return formFactors.find(ff => text.includes(ff)) ?? null; +} + +function inferConnector(...values: Array): string | null { + const text = values.filter(Boolean).join(" ").toUpperCase(); + if (text.includes("LC-DUPLEX") || text.includes("LC DUPLEX")) return "LC-Duplex"; + if (text.includes("LC SIMPLEX")) return "LC-Simplex"; + if (/\bMPO\b|\bMTP\b/.test(text)) return "MPO/MTP"; + if (/\bRJ45\b|COPPER/.test(text)) return "RJ45"; + if (/\bSC\b/.test(text)) return "SC"; + return null; +} + +function inferFiberType(...values: Array): string | null { + const text = values.filter(Boolean).join(" ").toLowerCase(); + if (/multimode|\bmmf\b|om[1-5]/.test(text)) return "MMF"; + if (/singlemode|single-mode|\bsmf\b|os2/.test(text)) return "SMF"; + if (/copper|rj45|dac/.test(text)) return "Copper"; + return null; +} + +function inferWavelengthNm(...values: Array): number | null { + const text = values.filter(Boolean).join(" "); + const lambda = text.match(/[λλ]?\s*(\d{3,4})\s*nm/i); + return lambda ? asNumber(lambda[1]) : null; +} + +// ── Normalization ────────────────────────────────────────────────────────── + +function extractCompatibility(row: JsonRecord): CatalogProduct["compatibility"] { + const rawCompat = row.compatibility ?? row.compatibilities ?? row.vendorCompatibility; + const rows = Array.isArray(rawCompat) ? rawCompat.filter(isRecord) : []; + return rows.flatMap(entry => { + const flat = flatLookup(entry); + const vendor = asString(pick(flat, ["vendor", "manufacturer", "brand", "systemVendor"])); + if (!vendor) return []; + return [{ + vendor, + platform: asString(pick(flat, ["platform", "device", "switch", "model", "series"])), + coding: asString(pick(flat, ["coding", "code", "eeprom", "originalPartNumber"])), + source: "api" as const, + }]; + }); +} + +function normalizeProduct(row: JsonRecord, fetchedAt: string): CatalogProduct | null { + const flat = flatLookup(row); + const title = asString(pick(flat, ["title", "name", "productName", "label"])); + const sku = asString(pick(flat, ["sku", "articleNumber", "partNumber", "productCode"])); + if (!sku || !title) return null; + + const url = asString(pick(flat, ["url", "productUrl", "canonicalUrl", "link"])); + const amount = asNumber(pick(flat, ["price", "priceNet", "netPrice", "grossPrice", "amount"])); + const currency = asString(pick(flat, ["currency", "priceCurrency", "currencyCode"])) + ?? (amount === null ? null : process.env["FLEXOPTIX_API_CURRENCY"]?.trim() ?? "EUR"); + const quantity = asNumber(pick(flat, ["stock", "stockQuantity", "quantity", "availableQuantity"])); + const status = asString(pick(flat, ["stockStatus", "availability", "deliveryStatus"])) + ?? (quantity === null ? null : quantity > 0 ? "in_stock" : "out_of_stock"); + + const formFactor = asString(pick(flat, ["formFactor", "form", "moduleType", "transceiverType"])) + ?? inferFormFactor(title); + const protocol = asString(pick(flat, ["protocol", "standard", "ethernetStandard"])); + const connector = asString(pick(flat, ["connector", "interface", "mediaConnector"])) ?? inferConnector(title); + const fiberType = asString(pick(flat, ["fiberType", "fiber", "mode"])) ?? inferFiberType(title); + const coding = asString(pick(flat, ["coding", "vendorCoding"])); + const wavelength = asNumber(pick(flat, ["wavelengthNm", "wavelength", "lambda"])) + ?? inferWavelengthNm(title); + const flags = inferBoolFlags(title, protocol, formFactor, fiberType); + + return { + source: "flexoptix-shop-api", + fetchedAt, + sku, + title, + url, + price: { + amount, + currency, + source: amount === null ? "missing" : "api", + fetchedAt, + }, + stock: { + status, + quantity, + source: status === null && quantity === null ? "missing" : "api", + }, + optics: { + formFactor, + speedGbps: parseSpeedGbps(pick(flat, ["speedGbps", "speed", "rate", "dataRate"]) ?? title), + reachM: parseReachMeters(pick(flat, ["reachM", "reach", "distance", "transmissionDistance"]) ?? title), + wavelengthNm: wavelength, + connector, + fiberType, + protocol, + coding, + bidi: asBoolean(pick(flat, ["bidi", "bidirectional"])) ?? flags.bidi, + dwdm: asBoolean(pick(flat, ["dwdm"])) ?? flags.dwdm, + cwdm: asBoolean(pick(flat, ["cwdm"])) ?? flags.cwdm, + }, + compatibility: extractCompatibility(row), + }; +} + +// ── Import helpers ───────────────────────────────────────────────────────── + +function canImportProduct(product: CatalogProduct): boolean { + return Boolean( + product.sku + && product.title + && product.optics.formFactor + && product.optics.speedGbps !== null + && product.optics.reachM !== null, + ); +} + +function reachLabel(reachM: number | null): string | undefined { + if (reachM === null) return undefined; + if (reachM >= 1000 && reachM % 1000 === 0) return `${reachM / 1000}km`; + return `${reachM}m`; +} + +function speedLabel(speedGbps: number | null): string | undefined { + if (speedGbps === null) return undefined; + if (speedGbps >= 1000) return `${speedGbps / 1000}T`; + return `${speedGbps}G`; +} + +function categoryFor(product: CatalogProduct): string { + const text = `${product.title} ${product.optics.protocol ?? ""}`.toLowerCase(); + if (/\bdac\b|direct attach|copper/.test(text)) return "DAC"; + if (/\baoc\b|active optical/.test(text)) return "AOC"; + if (/coherent|zr|dco/.test(text)) return "Coherent"; + return "DataCenter"; +} + +async function importProduct( + product: CatalogProduct, + vendorId: string, +): Promise<{ priceWritten: boolean; stockWritten: boolean }> { + const transceiverId = await findOrCreateScrapedTransceiver({ + partNumber: product.sku, + vendorId, + productUrl: product.url ?? undefined, + formFactor: product.optics.formFactor ?? undefined, + speedGbps: product.optics.speedGbps ?? undefined, + speed: speedLabel(product.optics.speedGbps), + reachMeters: product.optics.reachM ?? undefined, + reachLabel: reachLabel(product.optics.reachM), + fiberType: product.optics.fiberType ?? undefined, + wavelengths: product.optics.wavelengthNm === null ? undefined : `${product.optics.wavelengthNm}nm`, + category: categoryFor(product), + }); + + let priceWritten = false; + if (product.price.amount !== null && product.price.currency) { + priceWritten = await upsertPriceObservation({ + transceiverId, + sourceVendorId: vendorId, + price: product.price.amount, + currency: product.price.currency, + stockLevel: product.stock.status ?? "unknown", + quantityAvailable: product.stock.quantity ?? undefined, + url: product.url ?? undefined, + contentHash: contentHash({ + source: product.source, + sku: product.sku, + price: product.price.amount, + currency: product.price.currency, + fetchedAt: product.price.fetchedAt, + }), + }); + } + + const stockWritten = await upsertStockObservation({ + transceiverId, + sourceVendorId: vendorId, + stockLevel: product.stock.status ?? "unknown", + quantityAvailable: product.stock.quantity ?? undefined, + priceNet: product.price.amount ?? undefined, + productUrl: product.url ?? undefined, + priceCurrency: product.price.currency ?? undefined, + stockConfidence: product.stock.quantity === null ? 1 : 2, + }); + + return { priceWritten, stockWritten }; +} + +// ── API client ───────────────────────────────────────────────────────────── + +function validateEnv(): { baseUrl: string; username: string | null; password: string | null; token: string | null } { + const baseUrl = process.env["FLEXOPTIX_API_BASE_URL"]?.trim(); + if (!baseUrl) { + throw new Error("FLEXOPTIX_API_BASE_URL is required for Flexoptix API sync"); + } + const token = process.env["FLEXOPTIX_API_TOKEN"]?.trim() ?? null; + const username = process.env["FLEXOPTIX_API_USERNAME"]?.trim() ?? null; + const password = process.env["FLEXOPTIX_API_PASSWORD"]?.trim() ?? null; + if (!token && (!username || !password)) { + throw new Error("FLEXOPTIX_API_TOKEN or FLEXOPTIX_API_USERNAME+PASSWORD required"); + } + return { baseUrl, username, password, token }; +} + +function buildUrl(baseUrl: string, path: string): URL { + const base = baseUrl.endsWith("/") ? baseUrl : `${baseUrl}/`; + return new URL(path, base); +} + +async function fetchJson(url: URL, init: RequestInit, timeoutMs: number): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), Number.isFinite(timeoutMs) ? timeoutMs : 30_000); + try { + const response = await fetch(url, { ...init, signal: controller.signal }); + if (!response.ok) { + throw new Error(`HTTP ${response.status} from ${url.origin}${url.pathname}`); + } + return response.json(); + } finally { + clearTimeout(timer); + } +} + +async function getBearerToken(baseUrl: string, username: string, password: string, timeoutMs: number): Promise { + const authPath = process.env["FLEXOPTIX_API_AUTH_PATH"]?.trim() ?? "/rest/V1/integration/customer/token"; + const url = buildUrl(baseUrl, authPath); + const payload = await fetchJson(url, { + method: "POST", + headers: { accept: "application/json", "content-type": "application/json" }, + body: JSON.stringify({ username, password }), + }, timeoutMs); + if (typeof payload !== "string") { + throw new Error("Flexoptix auth response was not a string token"); + } + return payload; +} + +async function fetchAllProducts(baseUrl: string, headers: Record, timeoutMs: number): Promise { + const productPath = process.env["FLEXOPTIX_API_PRODUCTS_PATH"]?.trim() ?? "/rest/V2/flexoptix/products"; + const compatibilities = process.env["FLEXOPTIX_API_COMPATIBILITIES"]?.trim() ?? "1"; + const limit = parseInt(process.env["FLEXOPTIX_API_LIMIT"]?.trim() ?? "500", 10); + const currency = process.env["FLEXOPTIX_API_CURRENCY"]?.trim() ?? "EUR"; + + const allRows: JsonRecord[] = []; + + for (let page = 1; ; page++) { + const url = buildUrl(baseUrl, productPath); + url.searchParams.set("currency", currency); + url.searchParams.set("page", String(page)); + url.searchParams.set("limit", String(Number.isFinite(limit) ? limit : 500)); + if (compatibilities === "1") url.searchParams.set("compatibilities", "1"); + // specifications=0 for bulk pull (specifications=1 causes HTTP 503 on full catalog) + + const payload = await fetchJson(url, { headers }, timeoutMs); + const rows = extractRows(payload); + if (rows.length === 0) break; + + allRows.push(...rows); + + if (rows.length < (Number.isFinite(limit) ? limit : 500)) break; + } + + return allRows; +} + +function extractRows(payload: unknown): JsonRecord[] { + if (Array.isArray(payload)) return payload.filter(isRecord); + if (!isRecord(payload)) return []; + for (const key of ["products", "items", "data", "results", "rows"]) { + const value = payload[key]; + if (Array.isArray(value)) return value.filter(isRecord); + } + return [payload]; +} + +// ── Main export ──────────────────────────────────────────────────────────── + +export async function syncFlexoptixCatalog(): Promise { + const { baseUrl, username, password, token } = validateEnv(); + const timeoutMs = parseInt(process.env["FLEXOPTIX_API_TIMEOUT_MS"]?.trim() ?? "30000", 10); + + console.log(`[${new Date().toISOString()}] Flexoptix API sync starting`); + + const bearerToken = token ?? await getBearerToken(baseUrl, username!, password!, timeoutMs); + const headers: Record = { + accept: "application/json", + authorization: `Bearer ${bearerToken}`, + }; + const apiKey = process.env["FLEXOPTIX_API_KEY"]?.trim(); + if (apiKey) headers["x-api-key"] = apiKey; + + const fetchedAt = new Date().toISOString(); + const rawRows = await fetchAllProducts(baseUrl, headers, timeoutMs); + console.log(`[${new Date().toISOString()}] Fetched ${rawRows.length} rows from Flexoptix API`); + + const products = rawRows + .map(row => normalizeProduct(row, fetchedAt)) + .filter((p): p is CatalogProduct => p !== null); + + const importable = products.filter(canImportProduct); + const skipped = products.length - importable.length; + + console.log(`[${new Date().toISOString()}] Normalized: ${products.length} | importable: ${importable.length} | skipped: ${skipped}`); + + const vendorId = await ensureVendor("Flexoptix", "compatible", "https://www.flexoptix.net", "https://www.flexoptix.net"); + + let priceWrites = 0; + let stockWrites = 0; + + for (const product of importable) { + try { + const result = await importProduct(product, vendorId); + if (result.priceWritten) priceWrites++; + if (result.stockWritten) stockWrites++; + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + console.warn(`[${new Date().toISOString()}] Flexoptix import error (${product.sku}): ${message.slice(0, 100)}`); + } + } + + console.log(`[${new Date().toISOString()}] Flexoptix API sync complete: ${importable.length} products, ${priceWrites} price writes, ${stockWrites} stock writes`); + + return { + fetched: rawRows.length, + normalized: products.length, + skipped, + priceWrites, + stockWrites, + }; +} diff --git a/packages/scraper/src/scheduler.ts b/packages/scraper/src/scheduler.ts index ff69e20..1695607 100644 --- a/packages/scraper/src/scheduler.ts +++ b/packages/scraper/src/scheduler.ts @@ -270,6 +270,7 @@ export async function registerSchedules(boss: PgBoss): Promise { "scrape:pricing:gaotek", // ── Catalog scrapers (every 2h) ──────────────────────────────────── "scrape:pricing:flexoptix", + "sync:flexoptix-catalog", // ── Manufacturer catalogs (every 8h, no prices) ──────────────────── "scrape:catalog:smartoptics", "scrape:catalog:hubersuhner", @@ -402,10 +403,13 @@ export async function registerSchedules(boss: PgBoss): Promise { await boss.schedule("scrape:pricing:mouser-oem", "0 3 * * *", {}, { retryLimit: 1, expireInSeconds: 3600 }); // ══════════════════════════════════════════════════════════════════════ - // FLEXOPTIX CATALOG — every 2h (primary price source) + // FLEXOPTIX CATALOG — every 2h (primary price + stock source) + // sync:flexoptix-catalog uses the Magento 2 REST API (authoritative data) + // scrape:pricing:flexoptix is the legacy GraphQL fallback // ══════════════════════════════════════════════════════════════════════ - await boss.schedule("scrape:pricing:flexoptix", "0 */2 * * *", {}, { retryLimit: 3, expireInSeconds: 3600 }); + await boss.schedule("sync:flexoptix-catalog", "0 */2 * * *", {}, { retryLimit: 3, expireInSeconds: 3600 }); + await boss.schedule("scrape:pricing:flexoptix", "0 */2 * * *", {}, { retryLimit: 3, expireInSeconds: 3600 }); // ══════════════════════════════════════════════════════════════════════ // MANUFACTURER CATALOGS — every 4h (product data, no prices) @@ -887,10 +891,17 @@ export async function registerWorkers(boss: PgBoss): Promise { // ── Catalog scrapers ────────────────────────────────────────────────── await boss.work("scrape:pricing:flexoptix", async () => { - console.log(`[${new Date().toISOString()}] Running: Flexoptix catalog`); + // Legacy GraphQL/search scraper — kept as fallback, prefer sync:flexoptix-catalog + console.log(`[${new Date().toISOString()}] Running: Flexoptix catalog (legacy GraphQL)`); await scrapeFlexoptixCatalog(); }); + await boss.work("sync:flexoptix-catalog", async () => { + console.log(`[${new Date().toISOString()}] Running: Flexoptix API sync (REST v2)`); + const { syncFlexoptixCatalog } = await import("./robots/flexoptix-api-sync"); + await syncFlexoptixCatalog(); + }); + await boss.work("scrape:catalog:smartoptics", async () => { console.log(`[${new Date().toISOString()}] Running: SmartOptics catalog`); await scrapeSmartOptics(); diff --git a/packages/scraper/src/scrapers/audiocodes-oem.ts b/packages/scraper/src/scrapers/audiocodes-oem.ts new file mode 100644 index 0000000..20f5a45 --- /dev/null +++ b/packages/scraper/src/scrapers/audiocodes-oem.ts @@ -0,0 +1,112 @@ +/** + * AudioCodes OEM Transceiver Catalog Seed + * + * Seeds AudioCodes-branded transceiver PIDs for session border controllers + * (SBCs), media gateways, and enterprise voice routing platforms. + * + * Sources: + * - AudioCodes SBC Hardware Installation Guide (audiocodes.com) + * - AudioCodes Mediant Platform Data Sheets + * - AudioCodes Enterprise Voice Hardware Reference + * + * Run: tsx packages/scraper/src/scrapers/audiocodes-oem.ts + * Cron: daily at 00:05 + */ + +import { pool, ensureVendor } from "../utils/db"; + +interface AudioCodesPID { + pid: string; + formFactor: string; + speedGbps: number; + speed: string; + reachMeters: number; + reachLabel: string; + fiberType: string; + connector: string; + wavelengths?: string; + standard?: string; + notes?: string; +} + +const AUDIOCODES_PIDS: AudioCodesPID[] = [ + // ── 1G SFP ────────────────────────────────────────────────────────────── + { pid: "AC-SFP-1G-SX", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 550, reachLabel: "SX", fiberType: "MMF", connector: "LC", wavelengths: "850nm", standard: "1000BASE-SX" }, + { pid: "AC-SFP-1G-LX", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 10000, reachLabel: "LX", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "1000BASE-LX" }, + { pid: "AC-SFP-1G-T", formFactor: "SFP", speedGbps: 1, speed: "1G", reachMeters: 100, reachLabel: "T", fiberType: "DAC", connector: "RJ45", standard: "1000BASE-T" }, + + // ── 10G SFP+ ──────────────────────────────────────────────────────────── + { pid: "AC-SFP10G-SR", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 300, reachLabel: "SR", fiberType: "MMF", connector: "LC", wavelengths: "850nm", standard: "10GBASE-SR" }, + { pid: "AC-SFP10G-LR", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 10000, reachLabel: "LR", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "10GBASE-LR" }, + { pid: "AC-SFP10G-ER", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 40000, reachLabel: "ER", fiberType: "SMF", connector: "LC", wavelengths: "1550nm", standard: "10GBASE-ER" }, + + // ── 10G SFP+ BiDi ─────────────────────────────────────────────────────── + { pid: "AC-SFP10G-BIDI-TX1310", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 10000, reachLabel: "BiDi-1310TX", fiberType: "SMF", connector: "LC", wavelengths: "1310nm TX / 1490nm RX", notes: "Single-fiber BiDi, TX 1310nm" }, + { pid: "AC-SFP10G-BIDI-TX1490", formFactor: "SFP+", speedGbps: 10, speed: "10G", reachMeters: 10000, reachLabel: "BiDi-1490TX", fiberType: "SMF", connector: "LC", wavelengths: "1490nm TX / 1310nm RX", notes: "Single-fiber BiDi, TX 1490nm" }, + + // ── 25G SFP28 ─────────────────────────────────────────────────────────── + { pid: "AC-SFP28-25G-SR", formFactor: "SFP28", speedGbps: 25, speed: "25G", reachMeters: 300, reachLabel: "SR", fiberType: "MMF", connector: "LC", wavelengths: "850nm", standard: "25GBASE-SR" }, + { pid: "AC-SFP28-25G-LR", formFactor: "SFP28", speedGbps: 25, speed: "25G", reachMeters: 10000, reachLabel: "LR", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "25GBASE-LR" }, + + // ── 40G QSFP+ ─────────────────────────────────────────────────────────── + { pid: "AC-QSFP-40G-SR4", formFactor: "QSFP+", speedGbps: 40, speed: "40G", reachMeters: 150, reachLabel: "SR4", fiberType: "MMF", connector: "MPO", wavelengths: "850nm", standard: "40GBASE-SR4" }, + { pid: "AC-QSFP-40G-LR4", formFactor: "QSFP+", speedGbps: 40, speed: "40G", reachMeters: 10000, reachLabel: "LR4", fiberType: "SMF", connector: "LC", wavelengths: "1310nm", standard: "40GBASE-LR4" }, +]; + +export async function scrapeAudioCodesOem(): Promise { + console.log("=== AudioCodes OEM Transceiver Seed ===\n"); + + const vendorId = await ensureVendor( + "AudioCodes", + "oem", + "https://www.audiocodes.com", + undefined + ); + + let inserted = 0; + let updated = 0; + let errors = 0; + + for (const p of AUDIOCODES_PIDS) { + const slug = `audiocodes-${p.pid.toLowerCase().replace(/[^a-z0-9]+/g, "-")}`; + try { + const res = await pool.query( + `INSERT INTO transceivers + (slug, part_number, vendor_id, form_factor, speed, speed_gbps, + reach_meters, reach_label, fiber_type, connector, wavelengths, + dom_support, ieee_reference, market_status, category, notes) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,true,$12,'Mainstream','DataCenter',$13) + ON CONFLICT (slug) DO UPDATE SET + speed_gbps = EXCLUDED.speed_gbps, + reach_meters = CASE WHEN EXCLUDED.reach_meters > 0 THEN EXCLUDED.reach_meters ELSE transceivers.reach_meters END, + fiber_type = CASE WHEN EXCLUDED.fiber_type <> '' THEN EXCLUDED.fiber_type ELSE transceivers.fiber_type END, + wavelengths = COALESCE(EXCLUDED.wavelengths, transceivers.wavelengths), + updated_at = NOW() + RETURNING (xmax = 0) as was_inserted`, + [slug, p.pid, vendorId, p.formFactor, p.speed, p.speedGbps, + p.reachMeters, p.reachLabel, p.fiberType, p.connector, + p.wavelengths ?? null, p.standard ?? null, p.notes ?? null] + ); + if (res.rows[0]?.was_inserted) inserted++; else updated++; + } catch (err) { + console.warn(` Skip ${p.pid}: ${(err as Error).message.slice(0, 80)}`); + errors++; + } + } + + console.log(`\n=== AudioCodes OEM Seed Complete ===`); + console.log(` Inserted: ${inserted}`); + console.log(` Updated: ${updated}`); + console.log(` Errors: ${errors}`); + console.log(` Total PIDs: ${AUDIOCODES_PIDS.length}\n`); +} + +if (require.main === module) { + scrapeAudioCodesOem() + .then(() => pool.end()) + .catch((err) => { + console.error("Fatal:", err); + pool.end(); + process.exit(1); + }); +} diff --git a/packages/scraper/src/seed-batch35.ts b/packages/scraper/src/seed-batch35.ts new file mode 100644 index 0000000..46a7b55 --- /dev/null +++ b/packages/scraper/src/seed-batch35.ts @@ -0,0 +1,16 @@ +import { pool } from "./utils/db"; +import { scrapeSierraWirelessOem } from "./scrapers/sierra-wireless-oem"; +import { scrapeSenaoOem } from "./scrapers/senao-oem"; +import { scrapeEmcoreOem } from "./scrapers/emcore-oem"; +import { scrapeReflexPhotonicsOem }from "./scrapers/reflex-photonics-oem"; + +async function main() { + await scrapeSierraWirelessOem(); + await scrapeSenaoOem(); + await scrapeEmcoreOem(); + await scrapeReflexPhotonicsOem(); +} + +main() + .then(() => pool.end()) + .catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); }); diff --git a/packages/scraper/src/seed-batch36.ts b/packages/scraper/src/seed-batch36.ts new file mode 100644 index 0000000..9e423b6 --- /dev/null +++ b/packages/scraper/src/seed-batch36.ts @@ -0,0 +1,16 @@ +import { pool } from "./utils/db"; +import { scrapeEngeniusOem } from "./scrapers/engenius-oem"; +import { scrapePaloaltoNetworksOem } from "./scrapers/paloalto-networks-oem"; +import { scrapeBrocadeLegacyOem } from "./scrapers/brocade-legacy-oem"; +import { scrapeFoundryNetworksOem } from "./scrapers/foundry-networks-oem"; + +async function main() { + await scrapeEngeniusOem(); + await scrapePaloaltoNetworksOem(); + await scrapeBrocadeLegacyOem(); + await scrapeFoundryNetworksOem(); +} + +main() + .then(() => pool.end()) + .catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); }); diff --git a/packages/scraper/src/seed-batch37.ts b/packages/scraper/src/seed-batch37.ts new file mode 100644 index 0000000..f4b2982 --- /dev/null +++ b/packages/scraper/src/seed-batch37.ts @@ -0,0 +1,16 @@ +import { pool } from "./utils/db"; +import { scrapeExtremeLegacyOem } from "./scrapers/extreme-legacy-oem"; +import { scrapeNortelLegacyOem } from "./scrapers/nortel-legacy-oem"; +import { scrape3comLegacyOem } from "./scrapers/3com-legacy-oem"; +import { scrapeAvayaLegacyOem } from "./scrapers/avaya-legacy-oem"; + +async function main() { + await scrapeExtremeLegacyOem(); + await scrapeNortelLegacyOem(); + await scrape3comLegacyOem(); + await scrapeAvayaLegacyOem(); +} + +main() + .then(() => pool.end()) + .catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); }); diff --git a/packages/scraper/src/utils/db.ts b/packages/scraper/src/utils/db.ts index 95aca6a..7b17a54 100644 --- a/packages/scraper/src/utils/db.ts +++ b/packages/scraper/src/utils/db.ts @@ -1,16 +1,25 @@ import { Pool } from "pg"; +import type { PoolConfig } from "pg"; import { config } from "dotenv"; import { join } from "path"; import { contentHash } from "./hash"; config({ path: join(__dirname, "..", "..", "..", "..", ".env") }); +const hasExplicitDbConfig = Boolean(process.env.POSTGRES_HOST || process.env.TIP_DB_HOST); +const poolConfig: PoolConfig = !hasExplicitDbConfig && process.env.DATABASE_URL + ? { connectionString: process.env.DATABASE_URL, ssl: false } + : { + host: process.env.POSTGRES_HOST || process.env.TIP_DB_HOST || "localhost", + port: parseInt(process.env.POSTGRES_PORT || process.env.TIP_DB_PORT || "5433"), + database: process.env.POSTGRES_DB || process.env.TIP_DB_NAME || "transceiver_db", + user: process.env.POSTGRES_USER || process.env.TIP_DB_USER || "tip", + password: process.env.POSTGRES_PASSWORD || process.env.TIP_DB_PASS || "tip_dev_2026", + ssl: false, + }; + export const pool = new Pool({ - host: process.env.POSTGRES_HOST || "localhost", - port: parseInt(process.env.POSTGRES_PORT || "5433"), - database: process.env.POSTGRES_DB || "transceiver_db", - user: process.env.POSTGRES_USER || "tip", - password: process.env.POSTGRES_PASSWORD || "tip_dev_2026", + ...poolConfig, max: 5, idleTimeoutMillis: 10000, connectionTimeoutMillis: 5000, diff --git a/scripts/import-flexoptix-catalog.ts b/scripts/import-flexoptix-catalog.ts new file mode 100644 index 0000000..62c4a92 --- /dev/null +++ b/scripts/import-flexoptix-catalog.ts @@ -0,0 +1,233 @@ +/** + * Import normalized Flexoptix shop catalog rows produced by Magatama/Pulso. + * + * Source file: + * ../magatama/data/pulso/shop-products.normalized.jsonl + * + * Safe first run: + * npm run flexoptix:catalog:import -- --dry-run + * + * Live DB import: + * TIP_DB_PASS=... npm run flexoptix:catalog:import + */ + +import { existsSync, readFileSync } from "fs"; +import { resolve } from "path"; +import { + ensureVendor, + findOrCreateScrapedTransceiver, + pool, + upsertPriceObservation, + upsertStockObservation, +} from "../packages/scraper/src/utils/db"; +import { contentHash } from "../packages/scraper/src/utils/hash"; + +type CatalogProduct = { + source: "flexoptix-shop-api"; + fetchedAt: string; + sku: string; + title: string; + url: string | null; + price: { + amount: number | null; + currency: string | null; + source: "api" | "missing"; + fetchedAt: string; + }; + stock: { + status: string | null; + quantity: number | null; + source: "api" | "missing"; + }; + optics: { + formFactor: string | null; + speedGbps: number | null; + reachM: number | null; + wavelengthNm: number | null; + connector: string | null; + fiberType: string | null; + protocol: string | null; + coding: string | null; + bidi: boolean | null; + dwdm: boolean | null; + cwdm: boolean | null; + }; + compatibility: Array<{ + vendor: string; + platform: string | null; + coding: string | null; + source: "api"; + }>; +}; + +type CliArgs = { + dryRun: boolean; + input: string; +}; + +const repoRoot = resolve(__dirname, ".."); +const defaultInput = resolve(repoRoot, "..", "..", "magatama", "data", "pulso", "shop-products.normalized.jsonl"); + +function parseArgs(argv: string[]): CliArgs { + const args: CliArgs = { + dryRun: false, + input: process.env.PULSO_SHOP_CATALOG_JSONL || defaultInput, + }; + + for (let index = 0; index < argv.length; index += 1) { + const value = argv[index]; + if (value === "--") continue; + if (value === "--dry-run") { + args.dryRun = true; + continue; + } + if (value === "--input") { + args.input = resolve(argv[index + 1] || args.input); + index += 1; + continue; + } + if (value === "--help" || value === "-h") { + console.log(`Usage: + npm run flexoptix:catalog:import -- --dry-run + npm run flexoptix:catalog:import -- --input /path/shop-products.normalized.jsonl + +Env: + PULSO_SHOP_CATALOG_JSONL Optional input path override. + TIP_DB_HOST/TIP_DB_PORT/TIP_DB_NAME/TIP_DB_USER/TIP_DB_PASS or PGPASSWORD for live DB import. +`); + process.exit(0); + } + throw new Error(`Unknown argument: ${value}`); + } + + return args; +} + +function readJsonl(filePath: string): CatalogProduct[] { + if (!existsSync(filePath)) return []; + return readFileSync(filePath, "utf8") + .split("\n") + .map(line => line.trim()) + .filter(Boolean) + .map(line => JSON.parse(line) as CatalogProduct); +} + +function reachLabel(reachM: number | null): string | undefined { + if (reachM === null) return undefined; + if (reachM >= 1000 && reachM % 1000 === 0) return `${reachM / 1000}km`; + return `${reachM}m`; +} + +function speedLabel(speedGbps: number | null): string | undefined { + if (speedGbps === null) return undefined; + return `${speedGbps}G`; +} + +function categoryFor(product: CatalogProduct): string { + const text = `${product.title} ${product.optics.protocol || ""}`.toLowerCase(); + if (/\bdac\b|direct attach|copper/.test(text)) return "DAC"; + if (/\baoc\b|active optical/.test(text)) return "AOC"; + if (/coherent|zr|dco/.test(text)) return "Coherent"; + return "DataCenter"; +} + +function canImportProduct(product: CatalogProduct): boolean { + return Boolean( + product.sku + && product.title + && product.optics.formFactor + && product.optics.speedGbps !== null + && product.optics.reachM !== null + ); +} + +async function importProduct(product: CatalogProduct, vendorId: string): Promise<{ priceWritten: boolean; stockWritten: boolean }> { + const transceiverId = await findOrCreateScrapedTransceiver({ + partNumber: product.sku, + vendorId, + productUrl: product.url || undefined, + formFactor: product.optics.formFactor || undefined, + speedGbps: product.optics.speedGbps ?? undefined, + speed: speedLabel(product.optics.speedGbps), + reachMeters: product.optics.reachM ?? undefined, + reachLabel: reachLabel(product.optics.reachM), + fiberType: product.optics.fiberType || undefined, + wavelengths: product.optics.wavelengthNm === null ? undefined : `${product.optics.wavelengthNm}nm`, + category: categoryFor(product), + }); + + let priceWritten = false; + if (product.price.amount !== null && product.price.currency) { + priceWritten = await upsertPriceObservation({ + transceiverId, + sourceVendorId: vendorId, + price: product.price.amount, + currency: product.price.currency, + stockLevel: product.stock.status || "unknown", + quantityAvailable: product.stock.quantity ?? undefined, + url: product.url || undefined, + contentHash: contentHash({ + source: product.source, + sku: product.sku, + price: product.price.amount, + currency: product.price.currency, + fetchedAt: product.price.fetchedAt, + }), + }); + } + + const stockWritten = await upsertStockObservation({ + transceiverId, + sourceVendorId: vendorId, + stockLevel: product.stock.status || "unknown", + quantityAvailable: product.stock.quantity ?? undefined, + priceNet: product.price.amount ?? undefined, + productUrl: product.url || undefined, + priceCurrency: product.price.currency || undefined, + stockConfidence: product.stock.quantity === null ? 1 : 2, + }); + + return { priceWritten, stockWritten }; +} + +async function main(): Promise { + const args = parseArgs(process.argv.slice(2)); + const products = readJsonl(args.input); + const importable = products.filter(canImportProduct); + const skipped = products.length - importable.length; + const priced = importable.filter(product => product.price.amount !== null && product.price.currency).length; + const stocked = importable.filter(product => product.stock.status || product.stock.quantity !== null).length; + + console.log("Flexoptix normalized catalog import"); + console.log(`Input: ${args.input}`); + console.log(`Rows: ${products.length} | importable: ${importable.length} | skipped_missing_technical_fields: ${skipped}`); + console.log(`With price: ${priced} | with stock signal: ${stocked}`); + + if (args.dryRun) { + console.log("Dry-run only: no TIP database writes performed."); + await pool.end(); + return; + } + + const vendorId = await ensureVendor("Flexoptix", "compatible", "https://www.flexoptix.net", "https://www.flexoptix.net"); + let priceWrites = 0; + let stockWrites = 0; + + try { + for (const product of importable) { + const result = await importProduct(product, vendorId); + if (result.priceWritten) priceWrites += 1; + if (result.stockWritten) stockWrites += 1; + } + } finally { + await pool.end(); + } + + console.log(`Import complete: ${importable.length} products processed, ${priceWrites} price observations, ${stockWrites} stock observations.`); +} + +main().catch(async error => { + console.error(error instanceof Error ? error.message : error); + await pool.end().catch(() => undefined); + process.exit(1); +}); diff --git a/sql/102-product-verification-reconcile.sql b/sql/102-product-verification-reconcile.sql new file mode 100644 index 0000000..6043c13 --- /dev/null +++ b/sql/102-product-verification-reconcile.sql @@ -0,0 +1,81 @@ +-- Migration 102: Product photo/details verification reconciliation +-- Applied after the scraper started storing many image_url/product URLs without +-- consistently promoting image_verified/details_verified. + +BEGIN; + +-- Backfill canonical product URLs from recent real price observations. +UPDATE transceivers t +SET product_page_url = latest.url, + updated_at = NOW() +FROM ( + SELECT DISTINCT ON (po.transceiver_id) + po.transceiver_id, po.url + FROM price_observations po + WHERE po.url IS NOT NULL + AND po.url != '' + AND po.time > NOW() - INTERVAL '180 days' + ORDER BY po.transceiver_id, po.time DESC +) latest +WHERE t.id = latest.transceiver_id + AND (t.product_page_url IS NULL OR t.product_page_url = ''); + +-- Any non-placeholder product image URL written by a scraper counts as an +-- image verification source. Older scrapers often set only has_image/image_url. +UPDATE transceivers +SET has_image = true, + image_verified = true, + image_verified_at = COALESCE(image_verified_at, NOW()), + image_verified_url = COALESCE(NULLIF(image_verified_url, ''), image_url), + updated_at = NOW() +WHERE image_url IS NOT NULL + AND image_url != '' + AND image_url !~* '(placeholder|no-image|no_image|keinbild|logo)' + AND (image_verified = false OR image_verified IS NULL); + +-- Details are verified once a crawled source URL and the core product identity +-- fields are present. This avoids marking malformed scraper rows as complete. +UPDATE transceivers +SET details_verified = true, + details_verified_at = COALESCE(details_verified_at, NOW()), + details_source_url = COALESCE(NULLIF(details_source_url, ''), product_page_url), + data_confidence = CASE + WHEN data_confidence IS NULL OR data_confidence IN ('unknown', 'enriched_estimated') + THEN 'scraped_unverified' + ELSE data_confidence + END, + updated_at = NOW() +WHERE product_page_url IS NOT NULL + AND product_page_url != '' + AND form_factor IS NOT NULL + AND speed_gbps IS NOT NULL + AND part_number IS NOT NULL + AND part_number != '' + AND reach_label IS NOT NULL + AND reach_label != '' + AND fiber_type IS NOT NULL + AND fiber_type != '' + AND COALESCE(data_confidence, 'unknown') != 'garbage' + AND (details_verified = false OR details_verified IS NULL); + +-- Refresh full badge after the promotions above. +UPDATE transceivers +SET fully_verified = true, + fully_verified_at = COALESCE(fully_verified_at, NOW()), + updated_at = NOW() +WHERE competitor_verified = true + AND price_verified = true + AND image_verified = true + AND details_verified = true + AND fully_verified = false; + +COMMIT; + +SELECT + COUNT(*) AS total, + COUNT(*) FILTER (WHERE image_url IS NOT NULL AND image_url != '') AS has_image_url, + COUNT(*) FILTER (WHERE image_verified) AS image_verified, + COUNT(*) FILTER (WHERE product_page_url IS NOT NULL AND product_page_url != '') AS has_product_page_url, + COUNT(*) FILTER (WHERE details_verified) AS details_verified, + COUNT(*) FILTER (WHERE fully_verified) AS fully_verified +FROM transceivers; diff --git a/sync/history/2026-05-09-magatama-all-lane-runpod-training-start.md b/sync/history/2026-05-09-magatama-all-lane-runpod-training-start.md index 673c951..9eccc8d 100644 --- a/sync/history/2026-05-09-magatama-all-lane-runpod-training-start.md +++ b/sync/history/2026-05-09-magatama-all-lane-runpod-training-start.md @@ -28,6 +28,7 @@ Date: 2026-05-09 23:09 UTC ## Runner Fix - Fixed `scripts/trigger_lane_training_once.py` locally and on Erik. +- MAGATAMA Gitea commit: `76d4054`. - The script previously used stale API keys: - `iterations` - `seedOnly` @@ -60,4 +61,3 @@ Date: 2026-05-09 23:09 UTC - MAGATAMA imports/adopts it locally, - smoke checks pass, - the active alias/version is updated. -