transceiver-db/packages/scraper/src/scrapers/cisco-tmg.ts

/**
 * Cisco TMG Matrix Scraper — Transceiver Compatibility
 *
 * Source: tmgmatrix.cisco.com (JSON API — no auth required)
 * Extracts: Switch model ↔ Transceiver compatibility data
 * Stores: switches, compatibility table
 *
 * Uses POST /public/api/networkdevice/search endpoint directly.
 */
import { pool, ensureVendor } from "../utils/db";

const TMG_API = "https://tmgmatrix.cisco.com/public/api/networkdevice/search";

interface TmgTransceiver {
  tmgId: number;
  productId: string;
  productFamily: string;
  formFactor: string;
  reach: string;
  temperatureRange: string;
  cableType: string;
  media: string;
  connectorType: string;
  transmissionStandard: string;
  dataRate: string;
  endOfSale: string;
  softReleaseMinVer: string;
  breakoutMode: string;
  osType: string;
  domSupport: string;
  type: string;
}

interface TmgCompatEntry {
  productId: string; // switch PID
  transceivers: TmgTransceiver[];
}

interface TmgDevice {
  productFamily: string;
  networkAndTransceiverCompatibility: TmgCompatEntry[];
}

interface TmgSearchResponse {
  totalCount: number;
  filters: Array<{ name: string; values: Array<{ id: number; name: string; count: number }> }>;
  networkDevices: TmgDevice[];
}

/** Key Nexus/Catalyst platform family IDs from the TMG API */
const PLATFORM_FAMILIES = [
  // ── Nexus Data Center ───────────────────────────────────────────────────
  { id: 74,   name: "N9300" },    // Nexus 9300 — 8,515 entries
  { id: 77,   name: "N9500" },    // Nexus 9500 — 2,266 entries
  { id: 78,   name: "N9200" },    // Nexus 9200 — 708 entries
  { id: 661,  name: "N9800" },    // Nexus 9800 — 238 entries
  // ── Catalyst Campus ─────────────────────────────────────────────────────
  { id: 76,   name: "C9300" },    // Catalyst 9300 — 260 entries
  { id: 601,  name: "C9300L" },   // Catalyst 9300L — 720 entries
  { id: 1181, name: "C9300X" },   // Catalyst 9300X — 413 entries
  { id: 8,    name: "C9500" },    // Catalyst 9500 — 1,141 entries
  { id: 521,  name: "C9600" },    // Catalyst 9600 — 771 entries
  { id: 7,    name: "C9400" },    // Catalyst 9400 — 561 entries
  { id: 341,  name: "C9200" },    // Catalyst 9200 — 222 entries
  // ── Service Provider / High-Capacity ────────────────────────────────────
  { id: 83,   name: "ASR9000" },  // ASR 9000 — 3,644 entries
  { id: 1021, name: "8000" },     // Cisco 8000 Series — 1,954 entries
  { id: 20,   name: "NCS5500" },  // NCS 5500 — 3,843 entries
  { id: 121,  name: "NCS540" },   // NCS 540 — 2,684 entries
  { id: 141,  name: "NCS560" },   // NCS 560 — 229 entries
  { id: 17,   name: "NCS1000" },  // NCS 1000 (optical) — 325 entries
];

function buildTmgBody(
  familyFilter?: { id: number; name: string },
  deviceIdFilter?: { id: number; name: string }
): object {
  return {
    cableType: [],
    dataRate: [],
    formFactor: [],
    reach: [],
    searchInput: [""],
    osType: [],
    transceiverProductFamily: [],
    transceiverProductID: [],
    networkDeviceProductFamily: familyFilter ? [familyFilter] : [],
    networkDeviceProductID: deviceIdFilter ? [deviceIdFilter] : [],
    media: [],
    connectorType: [],
    caseTemperature: [],
    performanceMonitoring: [],
  };
}

async function searchTmg(familyFilter: { id: number; name: string }): Promise<TmgSearchResponse> {
  const res = await fetch(TMG_API, {
    method: "POST",
    headers: {
      "Content-Type": "application/json",
      "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
      "Accept": "application/json",
    },
    body: JSON.stringify(buildTmgBody(familyFilter)),
    signal: AbortSignal.timeout(30000),
  });

  if (!res.ok) {
    throw new Error(`TMG API ${res.status}: ${res.statusText}`);
  }

  return res.json() as Promise<TmgSearchResponse>;
}

/** Search for a specific switch by its TMG Product ID to get full compat data */
async function searchTmgByDeviceId(deviceId: { id: number; name: string }): Promise<TmgSearchResponse> {
  const res = await fetch(TMG_API, {
    method: "POST",
    headers: {
      "Content-Type": "application/json",
      "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
      "Accept": "application/json",
    },
    body: JSON.stringify(buildTmgBody(undefined, deviceId)),
    signal: AbortSignal.timeout(30000),
  });

  if (!res.ok) {
    throw new Error(`TMG API ${res.status}: ${res.statusText}`);
  }

  return res.json() as Promise<TmgSearchResponse>;
}

async function upsertCiscoSwitch(vendorId: string, model: string, series: string): Promise<string> {
  const result = await pool.query(
    `INSERT INTO switches (vendor_id, model, series, category, layer, managed)
     VALUES ($1, $2, $3, 'DataCenter', 'L3', true)
     ON CONFLICT (vendor_id, model) DO UPDATE SET series = EXCLUDED.series
     RETURNING id`,
    [vendorId, model, series]
  );
  return result.rows[0].id;
}

async function upsertCompatibility(
  switchId: string,
  transceiverId: string,
  firmwareMin: string,
  formFactor: string,
  reach: string,
  cableType: string,
  media: string,
  dataRate: string
): Promise<void> {
  await pool.query(
    `INSERT INTO compatibility (switch_id, transceiver_id, verified_by, verification_method, status, firmware_min, source_url, notes)
     VALUES ($1, $2, 'Cisco TMG Matrix', 'vendor_matrix', 'compatible', $3, $4, $5)
     ON CONFLICT (switch_id, transceiver_id) DO UPDATE SET
       firmware_min = EXCLUDED.firmware_min,
       notes = EXCLUDED.notes`,
    [
      switchId,
      transceiverId,
      firmwareMin || null,
      "https://tmgmatrix.cisco.com",
      `${formFactor} ${dataRate} ${reach} ${media} ${cableType}`.trim(),
    ]
  );
}

export async function scrapeCiscoTmg(): Promise<void> {
  console.log("=== Cisco TMG Matrix Scraper Starting (API mode) ===\n");

  const ciscoVendorId = await ensureVendor(
    "Cisco",
    "oem",
    "https://www.cisco.com",
    undefined
  );

  let totalSwitches = 0;
  let totalCompat = 0;
  let totalTransceivers = 0;

  /** Process one networkDevice compat response — writes switches+compat to DB */
  async function processDevices(
    devices: TmgDevice[],
    familyName: string
  ): Promise<{ switches: number; transceivers: number; compat: number }> {
    let sw = 0; let tx = 0; let cp = 0;
    for (const device of devices) {
      for (const compat of device.networkAndTransceiverCompatibility) {
        if (!compat.productId) continue;
        const switchId = await upsertCiscoSwitch(ciscoVendorId, compat.productId, familyName);
        sw++;
        for (const t of compat.transceivers) {
          if (!t.productId) continue;
          tx++;
          const txResult = await pool.query(
            `SELECT id FROM transceivers WHERE part_number = $1 OR part_number = $2 LIMIT 1`,
            [t.productId, t.productId.replace(/-S$/, "")]
          );
          if (txResult.rows.length > 0) {
            await upsertCompatibility(switchId, txResult.rows[0].id, t.softReleaseMinVer, t.formFactor, t.reach, t.cableType, t.media, t.dataRate);
            cp++;
          }
        }
      }
    }
    return { switches: sw, transceivers: tx, compat: cp };
  }

  for (const family of PLATFORM_FAMILIES) {
    console.log(`\nFetching ${family.name}...`);

    try {
      // Step 1: Fetch family-level response to get all device IDs in the filter
      const familyData = await searchTmg(family);
      const deviceIdFilter = familyData.filters
        ?.find((f) => f.name === "Network Device Product ID")
        ?.values ?? [];

      console.log(`  ${family.name}: ${deviceIdFilter.length} switch models`);

      if (deviceIdFilter.length === 0) {
        // Fallback: process whatever family-level search returned
        const r = await processDevices(familyData.networkDevices, family.name);
        totalSwitches += r.switches; totalTransceivers += r.transceivers; totalCompat += r.compat;
        continue;
      }

      // Step 2: Iterate every switch model by its specific TMG Product ID
      // (family search only returns 1 switch; per-device search returns full compat list)
      for (const dev of deviceIdFilter) {
        try {
          await new Promise((r) => setTimeout(r, 1000)); // 1s between requests
          const devData = await searchTmgByDeviceId({ id: dev.id, name: dev.name });
          const r = await processDevices(devData.networkDevices, family.name);
          totalSwitches += r.switches; totalTransceivers += r.transceivers; totalCompat += r.compat;
          if (totalSwitches % 20 === 0) {
            console.log(`  ... ${totalSwitches} switches processed, ${totalCompat} compat matches`);
          }
        } catch (devErr) {
          console.warn(`    Skip ${dev.name}: ${(devErr as Error).message.slice(0, 60)}`);
        }
      }

      // Rate limit: 2 seconds between platform families
      await new Promise((r) => setTimeout(r, 2000));
    } catch (err) {
      console.error(`  Error fetching ${family.name}:`, err);
    }
  }

  console.log(`\n=== Cisco TMG Scraper Complete ===`);
  console.log(`  Switches upserted: ${totalSwitches}`);
  console.log(`  Transceiver entries scanned: ${totalTransceivers}`);
  console.log(`  Compatibility matches: ${totalCompat}\n`);
}

if (require.main === module) {
  scrapeCiscoTmg()
    .then(() => pool.end())
    .catch((err) => {
      console.error("Fatal:", err);
      pool.end();
      process.exit(1);
    });
}