transceiver-db/packages/scraper/src/scrapers/cisco-tmg.ts

/**
 * Cisco TMG Matrix Scraper — Transceiver Compatibility
 *
 * Source: tmgmatrix.cisco.com (JSON API — no auth required)
 * Extracts: Switch model ↔ Transceiver compatibility data
 * Stores: switches, compatibility table
 *
 * Uses POST /public/api/networkdevice/search endpoint directly.
 */
import { pool, ensureVendor } from "../utils/db";

const TMG_API = "https://tmgmatrix.cisco.com/public/api/networkdevice/search";

interface TmgTransceiver {
  tmgId: number;
  productId: string;
  productFamily: string;
  formFactor: string;
  reach: string;
  temperatureRange: string;
  cableType: string;
  media: string;
  connectorType: string;
  transmissionStandard: string;
  dataRate: string;
  endOfSale: string;
  softReleaseMinVer: string;
  breakoutMode: string;
  osType: string;
  domSupport: string;
  type: string;
}

interface TmgCompatEntry {
  productId: string; // switch PID
  transceivers: TmgTransceiver[];
}

interface TmgDevice {
  productFamily: string;
  networkAndTransceiverCompatibility: TmgCompatEntry[];
}

interface TmgSearchResponse {
  totalCount: number;
  filters: Array<{ name: string; values: Array<{ id: number; name: string; count: number }> }>;
  networkDevices: TmgDevice[];
}

/** Key Nexus/Catalyst platform family IDs from the TMG API */
const PLATFORM_FAMILIES = [
  { id: 74, name: "N9300" },   // Nexus 9300 — 8,515 entries
  { id: 77, name: "N9500" },   // Nexus 9500 — 2,266 entries
  { id: 78, name: "N9200" },   // Nexus 9200 — 708 entries
  { id: 661, name: "N9800" },  // Nexus 9800 — 238 entries
  { id: 76, name: "C9300" },   // Catalyst 9300 — 260 entries
  { id: 601, name: "C9300L" }, // Catalyst 9300L — 720 entries
  { id: 1181, name: "C9300X" }, // Catalyst 9300X — 413 entries
  { id: 8, name: "C9500" },   // Catalyst 9500 — 1,141 entries
  { id: 521, name: "C9600" }, // Catalyst 9600 — 771 entries
  { id: 7, name: "C9400" },   // Catalyst 9400 — 561 entries
  { id: 341, name: "C9200" }, // Catalyst 9200 — 222 entries
  { id: 83, name: "ASR9000" }, // ASR 9000 — 3,644 entries
];

async function searchTmg(familyFilter: { id: number; name: string }): Promise<TmgSearchResponse> {
  const body = {
    cableType: [],
    dataRate: [],
    formFactor: [],
    reach: [],
    searchInput: [""],
    osType: [],
    transceiverProductFamily: [],
    transceiverProductID: [],
    networkDeviceProductFamily: [familyFilter],
    networkDeviceProductID: [],
    media: [],
    connectorType: [],
    caseTemperature: [],
    performanceMonitoring: [],
  };

  const res = await fetch(TMG_API, {
    method: "POST",
    headers: {
      "Content-Type": "application/json",
      "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
      "Accept": "application/json",
    },
    body: JSON.stringify(body),
  });

  if (!res.ok) {
    throw new Error(`TMG API ${res.status}: ${res.statusText}`);
  }

  return res.json() as Promise<TmgSearchResponse>;
}

async function upsertCiscoSwitch(vendorId: string, model: string, series: string): Promise<string> {
  const result = await pool.query(
    `INSERT INTO switches (vendor_id, model, series, category, layer, managed)
     VALUES ($1, $2, $3, 'DataCenter', 'L3', true)
     ON CONFLICT (vendor_id, model) DO UPDATE SET series = EXCLUDED.series
     RETURNING id`,
    [vendorId, model, series]
  );
  return result.rows[0].id;
}

async function upsertCompatibility(
  switchId: string,
  transceiverId: string,
  firmwareMin: string,
  formFactor: string,
  reach: string,
  cableType: string,
  media: string,
  dataRate: string
): Promise<void> {
  await pool.query(
    `INSERT INTO compatibility (switch_id, transceiver_id, verified_by, verification_method, status, firmware_min, source_url, notes)
     VALUES ($1, $2, 'Cisco TMG Matrix', 'vendor_matrix', 'compatible', $3, $4, $5)
     ON CONFLICT (switch_id, transceiver_id) DO UPDATE SET
       firmware_min = EXCLUDED.firmware_min,
       notes = EXCLUDED.notes`,
    [
      switchId,
      transceiverId,
      firmwareMin || null,
      "https://tmgmatrix.cisco.com",
      `${formFactor} ${dataRate} ${reach} ${media} ${cableType}`.trim(),
    ]
  );
}

export async function scrapeCiscoTmg(): Promise<void> {
  console.log("=== Cisco TMG Matrix Scraper Starting (API mode) ===\n");

  const ciscoVendorId = await ensureVendor(
    "Cisco",
    "oem",
    "https://www.cisco.com",
    undefined
  );

  let totalSwitches = 0;
  let totalCompat = 0;
  let totalTransceivers = 0;

  for (const family of PLATFORM_FAMILIES) {
    console.log(`\nFetching ${family.name}...`);

    try {
      const data = await searchTmg(family);
      console.log(`  ${family.name}: ${data.totalCount} total entries, ${data.networkDevices.length} device groups`);

      for (const device of data.networkDevices) {
        for (const compat of device.networkAndTransceiverCompatibility) {
          if (!compat.productId) continue;

          const switchId = await upsertCiscoSwitch(
            ciscoVendorId,
            compat.productId,
            device.productFamily
          );
          totalSwitches++;

          for (const tx of compat.transceivers) {
            if (!tx.productId) continue;
            totalTransceivers++;

            // Try to match transceiver in our DB by Cisco PID
            const txResult = await pool.query(
              `SELECT id FROM transceivers
               WHERE part_number = $1
                  OR part_number = $2
               LIMIT 1`,
              [tx.productId, tx.productId.replace(/-S$/, "")]
            );

            if (txResult.rows.length > 0) {
              await upsertCompatibility(
                switchId,
                txResult.rows[0].id,
                tx.softReleaseMinVer,
                tx.formFactor,
                tx.reach,
                tx.cableType,
                tx.media,
                tx.dataRate
              );
              totalCompat++;
            }
          }
        }
      }

      // Rate limit: 2 seconds between platform families
      await new Promise((r) => setTimeout(r, 2000));
    } catch (err) {
      console.error(`  Error fetching ${family.name}:`, err);
    }
  }

  console.log(`\n=== Cisco TMG Scraper Complete ===`);
  console.log(`  Switches upserted: ${totalSwitches}`);
  console.log(`  Transceiver entries scanned: ${totalTransceivers}`);
  console.log(`  Compatibility matches: ${totalCompat}\n`);
}

if (require.main === module) {
  scrapeCiscoTmg()
    .then(() => pool.end())
    .catch((err) => {
      console.error("Fatal:", err);
      pool.end();
      process.exit(1);
    });
}