cc85d3d0f8
- cisco-tmg.ts: upsert Cisco OEM transceivers from TMG API instead of
SELECT-only. Parsers for formFactor/speed/reach/fiberType/tempRange.
Fixes market_status ('EOL') + temp_range ('COM'/'IND') check constraints.
- arista-oem.ts: seed scraper for 69 Arista OEM PIDs (1G→800G,
SFP/SFP28/QSFP+/QSFP28/QSFP-DD/OSFP/QSFP-DD800) with full specs.
- scheduler.ts: daily arista-oem seed at 04:00 UTC
389 lines
15 KiB
TypeScript
389 lines
15 KiB
TypeScript
/**
|
|
* Cisco TMG Matrix Scraper — Transceiver Compatibility + OEM Catalog
|
|
*
|
|
* Source: tmgmatrix.cisco.com (JSON API — no auth required)
|
|
* Extracts: Switch model ↔ Transceiver compatibility data
|
|
* + upserts Cisco OEM transceivers into transceivers table
|
|
* Stores: transceivers (Cisco OEM), switches, compatibility
|
|
*
|
|
* Uses POST /public/api/networkdevice/search endpoint directly.
|
|
*/
|
|
import { pool, ensureVendor } from "../utils/db";
|
|
|
|
const TMG_API = "https://tmgmatrix.cisco.com/public/api/networkdevice/search";
|
|
|
|
interface TmgTransceiver {
|
|
tmgId: number;
|
|
productId: string;
|
|
productFamily: string;
|
|
formFactor: string;
|
|
reach: string;
|
|
temperatureRange: string;
|
|
cableType: string;
|
|
media: string;
|
|
connectorType: string;
|
|
transmissionStandard: string;
|
|
dataRate: string;
|
|
endOfSale: string;
|
|
softReleaseMinVer: string;
|
|
breakoutMode: string;
|
|
osType: string;
|
|
domSupport: string;
|
|
type: string;
|
|
}
|
|
|
|
interface TmgCompatEntry {
|
|
productId: string; // switch PID
|
|
transceivers: TmgTransceiver[];
|
|
}
|
|
|
|
interface TmgDevice {
|
|
productFamily: string;
|
|
networkAndTransceiverCompatibility: TmgCompatEntry[];
|
|
}
|
|
|
|
interface TmgSearchResponse {
|
|
totalCount: number;
|
|
filters: Array<{ name: string; values: Array<{ id: number; name: string; count: number }> }>;
|
|
networkDevices: TmgDevice[];
|
|
}
|
|
|
|
/** Key Nexus/Catalyst platform family IDs from the TMG API */
|
|
const PLATFORM_FAMILIES = [
|
|
// ── Nexus Data Center ───────────────────────────────────────────────────
|
|
{ id: 74, name: "N9300" }, // Nexus 9300 — 8,515 entries
|
|
{ id: 77, name: "N9500" }, // Nexus 9500 — 2,266 entries
|
|
{ id: 78, name: "N9200" }, // Nexus 9200 — 708 entries
|
|
{ id: 661, name: "N9800" }, // Nexus 9800 — 238 entries
|
|
// ── Catalyst Campus ─────────────────────────────────────────────────────
|
|
{ id: 76, name: "C9300" }, // Catalyst 9300 — 260 entries
|
|
{ id: 601, name: "C9300L" }, // Catalyst 9300L — 720 entries
|
|
{ id: 1181, name: "C9300X" }, // Catalyst 9300X — 413 entries
|
|
{ id: 8, name: "C9500" }, // Catalyst 9500 — 1,141 entries
|
|
{ id: 521, name: "C9600" }, // Catalyst 9600 — 771 entries
|
|
{ id: 7, name: "C9400" }, // Catalyst 9400 — 561 entries
|
|
{ id: 341, name: "C9200" }, // Catalyst 9200 — 222 entries
|
|
// ── Service Provider / High-Capacity ────────────────────────────────────
|
|
{ id: 83, name: "ASR9000" }, // ASR 9000 — 3,644 entries
|
|
{ id: 1021, name: "8000" }, // Cisco 8000 Series — 1,954 entries
|
|
{ id: 20, name: "NCS5500" }, // NCS 5500 — 3,843 entries
|
|
{ id: 121, name: "NCS540" }, // NCS 540 — 2,684 entries
|
|
{ id: 141, name: "NCS560" }, // NCS 560 — 229 entries
|
|
{ id: 17, name: "NCS1000" }, // NCS 1000 (optical) — 325 entries
|
|
];
|
|
|
|
function buildTmgBody(
|
|
familyFilter?: { id: number; name: string },
|
|
deviceIdFilter?: { id: number; name: string }
|
|
): object {
|
|
return {
|
|
cableType: [],
|
|
dataRate: [],
|
|
formFactor: [],
|
|
reach: [],
|
|
searchInput: [""],
|
|
osType: [],
|
|
transceiverProductFamily: [],
|
|
transceiverProductID: [],
|
|
networkDeviceProductFamily: familyFilter ? [familyFilter] : [],
|
|
networkDeviceProductID: deviceIdFilter ? [deviceIdFilter] : [],
|
|
media: [],
|
|
connectorType: [],
|
|
caseTemperature: [],
|
|
performanceMonitoring: [],
|
|
};
|
|
}
|
|
|
|
async function searchTmg(familyFilter: { id: number; name: string }): Promise<TmgSearchResponse> {
|
|
const res = await fetch(TMG_API, {
|
|
method: "POST",
|
|
headers: {
|
|
"Content-Type": "application/json",
|
|
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
|
|
"Accept": "application/json",
|
|
},
|
|
body: JSON.stringify(buildTmgBody(familyFilter)),
|
|
signal: AbortSignal.timeout(30000),
|
|
});
|
|
|
|
if (!res.ok) {
|
|
throw new Error(`TMG API ${res.status}: ${res.statusText}`);
|
|
}
|
|
|
|
return res.json() as Promise<TmgSearchResponse>;
|
|
}
|
|
|
|
/** Search for a specific switch by its TMG Product ID to get full compat data */
|
|
async function searchTmgByDeviceId(deviceId: { id: number; name: string }): Promise<TmgSearchResponse> {
|
|
const res = await fetch(TMG_API, {
|
|
method: "POST",
|
|
headers: {
|
|
"Content-Type": "application/json",
|
|
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
|
|
"Accept": "application/json",
|
|
},
|
|
body: JSON.stringify(buildTmgBody(undefined, deviceId)),
|
|
signal: AbortSignal.timeout(30000),
|
|
});
|
|
|
|
if (!res.ok) {
|
|
throw new Error(`TMG API ${res.status}: ${res.statusText}`);
|
|
}
|
|
|
|
return res.json() as Promise<TmgSearchResponse>;
|
|
}
|
|
|
|
// ── TMG field parsers ────────────────────────────────────────────────────────
|
|
|
|
function parseSpeedGbps(dataRate: string): number {
|
|
const s = (dataRate || "").toUpperCase();
|
|
if (s.includes("1.6T")) return 1600;
|
|
if (s.includes("800G")) return 800;
|
|
if (s.includes("400G")) return 400;
|
|
if (s.includes("200G")) return 200;
|
|
if (s.includes("100G")) return 100;
|
|
if (s.includes("40G") || s.includes("40GBASE")) return 40;
|
|
if (s.includes("25G") || s.includes("25GBASE")) return 25;
|
|
if (s.includes("10G") || s.includes("10GBASE")) return 10;
|
|
if (s.includes("1G") || s.includes("1000BASE") || s.includes("GIGE")) return 1;
|
|
if (s.includes("100M") || s.includes("100BASE")) return 0.1;
|
|
return 0;
|
|
}
|
|
|
|
function parseSpeedLabel(dataRate: string, gbps: number): string {
|
|
if (gbps >= 1600) return "1.6T";
|
|
if (gbps >= 800) return "800G";
|
|
if (gbps >= 400) return "400G";
|
|
if (gbps >= 200) return "200G";
|
|
if (gbps >= 100) return "100G";
|
|
if (gbps >= 40) return "40G";
|
|
if (gbps >= 25) return "25G";
|
|
if (gbps >= 10) return "10G";
|
|
if (gbps >= 1) return "1G";
|
|
return dataRate || "Unknown";
|
|
}
|
|
|
|
function parseFiberType(cableType: string, media: string): string {
|
|
const c = (cableType + " " + media).toUpperCase();
|
|
if (c.includes("TWINAX") || c.includes("COPPER") || c.includes("DAC")) return "DAC";
|
|
if (c.includes("AOC")) return "MMF";
|
|
if (c.includes("SMF") || c.includes("SINGLE")) return "SMF";
|
|
if (c.includes("MMF") || c.includes("MULTI")) return "MMF";
|
|
return "";
|
|
}
|
|
|
|
function parseReachMeters(reach: string, cableType: string): number {
|
|
const r = (reach || "").toUpperCase();
|
|
const c = (cableType || "").toUpperCase();
|
|
// Explicit distance strings
|
|
const mMatch = r.match(/(\d+)\s*M\b/);
|
|
const kmMatch = r.match(/(\d+)\s*KM\b/);
|
|
if (kmMatch) return parseInt(kmMatch[1]) * 1000;
|
|
if (mMatch) return parseInt(mMatch[1]);
|
|
// Standard reach codes
|
|
if (r.includes("ZR+") || r.includes("ZRP")) return 120000;
|
|
if (r.includes("ZR")) return 80000;
|
|
if (r.includes("ER4") || r.includes("ER")) return 40000;
|
|
if (r.includes("LR4") || r.includes("LR")) return 10000;
|
|
if (r.includes("FR4") || r.includes("FR")) return 2000;
|
|
if (r.includes("DR4") || r.includes("DR")) return 500;
|
|
if (r.includes("SR4") || r.includes("SR")) return 100;
|
|
if (c.includes("TWINAX") || c.includes("DAC")) return 5;
|
|
return 0;
|
|
}
|
|
|
|
function parseFormFactor(ff: string): string {
|
|
const f = (ff || "").toUpperCase().replace(/[-\s]/g, "");
|
|
if (f === "QSFPDD" || f === "QSFP-DD") return "QSFP-DD";
|
|
if (f === "QSFPDD800") return "QSFP-DD800";
|
|
if (f === "QSFP112") return "QSFP112";
|
|
if (f === "QSFP56") return "QSFP56";
|
|
if (f === "QSFP28") return "QSFP28";
|
|
if (f.startsWith("QSFP")) return "QSFP+";
|
|
if (f === "SFP56DD") return "SFP56-DD";
|
|
if (f === "SFP56") return "SFP56";
|
|
if (f === "SFP28") return "SFP28";
|
|
if (f.startsWith("SFP") && (f.includes("+") || f.includes("PLUS"))) return "SFP+";
|
|
if (f.startsWith("SFP")) return "SFP+";
|
|
if (f.startsWith("XFP")) return "XFP";
|
|
if (f.startsWith("CFP")) return ff.toUpperCase().replace(/\s/g, "");
|
|
if (f.startsWith("OSFP")) return "OSFP";
|
|
return ff || "SFP+";
|
|
}
|
|
|
|
/** Upsert a Cisco OEM transceiver from TMG data — returns its DB id */
|
|
async function upsertCiscoTransceiver(vendorId: string, t: TmgTransceiver): Promise<string> {
|
|
const pid = t.productId.trim();
|
|
const ff = parseFormFactor(t.formFactor);
|
|
const gbps = parseSpeedGbps(t.dataRate);
|
|
const speed = parseSpeedLabel(t.dataRate, gbps);
|
|
const reach = parseReachMeters(t.reach, t.cableType);
|
|
const fiber = parseFiberType(t.cableType, t.media);
|
|
// temp_range CHECK constraint: only 'COM' or 'IND' allowed
|
|
const tr = (t.temperatureRange || "COM").toUpperCase();
|
|
const tempRng = (tr === "IND" || tr === "INDUSTRIAL" || tr === "EXT" || tr === "EXTENDED") ? "IND" : "COM";
|
|
const dom = t.domSupport?.toLowerCase() === "yes";
|
|
// market_status CHECK: 'Mainstream' | 'Growth' | 'Emerging' | 'Legacy' | 'EOL'
|
|
const marketStatus = (t.endOfSale && t.endOfSale !== "N/A" && t.endOfSale !== "") ? "EOL" : "Mainstream";
|
|
const slug = `cisco-${pid.toLowerCase().replace(/[^a-z0-9]+/g, "-")}`;
|
|
|
|
const result = await pool.query(
|
|
`INSERT INTO transceivers
|
|
(slug, part_number, vendor_id, form_factor, speed, speed_gbps,
|
|
reach_meters, reach_label, fiber_type, connector, temp_range,
|
|
dom_support, ieee_reference, market_status, category)
|
|
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,'DataCenter')
|
|
ON CONFLICT (slug) DO UPDATE SET
|
|
speed_gbps = EXCLUDED.speed_gbps,
|
|
reach_meters = CASE WHEN EXCLUDED.reach_meters > 0 THEN EXCLUDED.reach_meters ELSE transceivers.reach_meters END,
|
|
fiber_type = CASE WHEN EXCLUDED.fiber_type <> '' THEN EXCLUDED.fiber_type ELSE transceivers.fiber_type END,
|
|
temp_range = EXCLUDED.temp_range,
|
|
dom_support = EXCLUDED.dom_support,
|
|
ieee_reference = CASE WHEN EXCLUDED.ieee_reference <> '' THEN EXCLUDED.ieee_reference ELSE transceivers.ieee_reference END,
|
|
market_status = CASE WHEN EXCLUDED.market_status = 'EOL' THEN 'EOL' ELSE transceivers.market_status END,
|
|
updated_at = NOW()
|
|
RETURNING id`,
|
|
[slug, pid, vendorId, ff, speed, gbps,
|
|
reach, t.reach || "", fiber, t.connectorType || "", tempRng,
|
|
dom, t.transmissionStandard || "", marketStatus]
|
|
);
|
|
return result.rows[0].id;
|
|
}
|
|
|
|
async function upsertCiscoSwitch(vendorId: string, model: string, series: string): Promise<string> {
|
|
const result = await pool.query(
|
|
`INSERT INTO switches (vendor_id, model, series, category, layer, managed)
|
|
VALUES ($1, $2, $3, 'DataCenter', 'L3', true)
|
|
ON CONFLICT (vendor_id, model) DO UPDATE SET series = EXCLUDED.series
|
|
RETURNING id`,
|
|
[vendorId, model, series]
|
|
);
|
|
return result.rows[0].id;
|
|
}
|
|
|
|
async function upsertCompatibility(
|
|
switchId: string,
|
|
transceiverId: string,
|
|
firmwareMin: string,
|
|
formFactor: string,
|
|
reach: string,
|
|
cableType: string,
|
|
media: string,
|
|
dataRate: string
|
|
): Promise<void> {
|
|
await pool.query(
|
|
`INSERT INTO compatibility (switch_id, transceiver_id, verified_by, verification_method, status, firmware_min, source_url, notes)
|
|
VALUES ($1, $2, 'Cisco TMG Matrix', 'vendor_matrix', 'compatible', $3, $4, $5)
|
|
ON CONFLICT (switch_id, transceiver_id) DO UPDATE SET
|
|
firmware_min = EXCLUDED.firmware_min,
|
|
notes = EXCLUDED.notes`,
|
|
[
|
|
switchId,
|
|
transceiverId,
|
|
firmwareMin || null,
|
|
"https://tmgmatrix.cisco.com",
|
|
`${formFactor} ${dataRate} ${reach} ${media} ${cableType}`.trim(),
|
|
]
|
|
);
|
|
}
|
|
|
|
export async function scrapeCiscoTmg(): Promise<void> {
|
|
console.log("=== Cisco TMG Matrix Scraper Starting (API mode) ===\n");
|
|
|
|
const ciscoVendorId = await ensureVendor(
|
|
"Cisco",
|
|
"oem",
|
|
"https://www.cisco.com",
|
|
undefined
|
|
);
|
|
|
|
let totalSwitches = 0;
|
|
let totalCompat = 0;
|
|
let totalTransceivers = 0;
|
|
|
|
/** Process one networkDevice compat response — writes switches+compat to DB */
|
|
async function processDevices(
|
|
devices: TmgDevice[],
|
|
familyName: string
|
|
): Promise<{ switches: number; transceivers: number; compat: number }> {
|
|
let sw = 0; let tx = 0; let cp = 0;
|
|
for (const device of devices) {
|
|
for (const compat of device.networkAndTransceiverCompatibility) {
|
|
if (!compat.productId) continue;
|
|
const switchId = await upsertCiscoSwitch(ciscoVendorId, compat.productId, familyName);
|
|
sw++;
|
|
for (const t of compat.transceivers) {
|
|
if (!t.productId) continue;
|
|
tx++;
|
|
try {
|
|
// Always upsert the Cisco OEM transceiver — creates if missing, updates if stale
|
|
const txId = await upsertCiscoTransceiver(ciscoVendorId, t);
|
|
await upsertCompatibility(switchId, txId, t.softReleaseMinVer, t.formFactor, t.reach, t.cableType, t.media, t.dataRate);
|
|
cp++;
|
|
} catch (txErr) {
|
|
console.warn(` Skip transceiver ${t.productId}: ${(txErr as Error).message.slice(0, 80)}`);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return { switches: sw, transceivers: tx, compat: cp };
|
|
}
|
|
|
|
for (const family of PLATFORM_FAMILIES) {
|
|
console.log(`\nFetching ${family.name}...`);
|
|
|
|
try {
|
|
// Step 1: Fetch family-level response to get all device IDs in the filter
|
|
const familyData = await searchTmg(family);
|
|
const deviceIdFilter = familyData.filters
|
|
?.find((f) => f.name === "Network Device Product ID")
|
|
?.values ?? [];
|
|
|
|
console.log(` ${family.name}: ${deviceIdFilter.length} switch models`);
|
|
|
|
if (deviceIdFilter.length === 0) {
|
|
// Fallback: process whatever family-level search returned
|
|
const r = await processDevices(familyData.networkDevices, family.name);
|
|
totalSwitches += r.switches; totalTransceivers += r.transceivers; totalCompat += r.compat;
|
|
continue;
|
|
}
|
|
|
|
// Step 2: Iterate every switch model by its specific TMG Product ID
|
|
// (family search only returns 1 switch; per-device search returns full compat list)
|
|
for (const dev of deviceIdFilter) {
|
|
try {
|
|
await new Promise((r) => setTimeout(r, 1000)); // 1s between requests
|
|
const devData = await searchTmgByDeviceId({ id: dev.id, name: dev.name });
|
|
const r = await processDevices(devData.networkDevices, family.name);
|
|
totalSwitches += r.switches; totalTransceivers += r.transceivers; totalCompat += r.compat;
|
|
if (totalSwitches % 20 === 0) {
|
|
console.log(` ... ${totalSwitches} switches processed, ${totalCompat} compat matches`);
|
|
}
|
|
} catch (devErr) {
|
|
console.warn(` Skip ${dev.name}: ${(devErr as Error).message.slice(0, 60)}`);
|
|
}
|
|
}
|
|
|
|
// Rate limit: 2 seconds between platform families
|
|
await new Promise((r) => setTimeout(r, 2000));
|
|
} catch (err) {
|
|
console.error(` Error fetching ${family.name}:`, err);
|
|
}
|
|
}
|
|
|
|
console.log(`\n=== Cisco TMG Scraper Complete ===`);
|
|
console.log(` Switches upserted: ${totalSwitches}`);
|
|
console.log(` Transceivers upserted: ${totalTransceivers}`);
|
|
console.log(` Compatibility entries: ${totalCompat}\n`);
|
|
}
|
|
|
|
if (require.main === module) {
|
|
scrapeCiscoTmg()
|
|
.then(() => pool.end())
|
|
.catch((err) => {
|
|
console.error("Fatal:", err);
|
|
pool.end();
|
|
process.exit(1);
|
|
});
|
|
}
|