transceiver-db/packages/scraper/src/scrapers/cisco-tmg.ts

/**
 * Cisco TMG Matrix Scraper — Transceiver Compatibility
 *
 * Source: tmgmatrix.cisco.com
 * Extracts: Switch model ↔ Transceiver compatibility data
 * Stores: switches, compatibility table
 *
 * The TMG Matrix has a JSON API behind the scenes.
 */
import { CheerioCrawler } from "crawlee";
import { pool, ensureVendor } from "../utils/db";

const TMG_BASE = "https://tmgmatrix.cisco.com";

interface TmgEntry {
  switchModel: string;
  switchSeries: string;
  transceiverPid: string;
  transceiverDescription: string;
  speed: string;
  reach: string;
  cableType: string;
  connector: string;
  minSoftware: string;
}

async function upsertCiscoSwitch(vendorId: string, model: string, series: string): Promise<string> {
  const result = await pool.query(
    `INSERT INTO switches (vendor_id, model, series, category, layer, managed)
     VALUES ($1, $2, $3, 'DataCenter', 'L3', true)
     ON CONFLICT (vendor_id, model) DO UPDATE SET series = EXCLUDED.series
     RETURNING id`,
    [vendorId, model, series]
  );
  return result.rows[0].id;
}

async function upsertCompatibility(
  switchId: string,
  transceiverId: string,
  firmwareMin: string
): Promise<void> {
  await pool.query(
    `INSERT INTO compatibility (switch_id, transceiver_id, verified_by, verification_method, status, firmware_min, source_url)
     VALUES ($1, $2, 'Cisco TMG Matrix', 'vendor_matrix', 'compatible', $3, $4)
     ON CONFLICT (switch_id, transceiver_id) DO UPDATE SET firmware_min = EXCLUDED.firmware_min`,
    [switchId, transceiverId, firmwareMin || null, TMG_BASE]
  );
}

export async function scrapeCiscoTmg(): Promise<void> {
  console.log("=== Cisco TMG Matrix Scraper Starting ===\n");

  const ciscoVendorId = await ensureVendor(
    "Cisco",
    "oem",
    "https://www.cisco.com",
    undefined
  );

  const entries: TmgEntry[] = [];

  // TMG Matrix uses a search API
  // First, try the public HTML interface
  const crawler = new CheerioCrawler({
    maxConcurrency: 1,
    maxRequestsPerMinute: 10, // Very respectful — Cisco rate limits aggressively

    async requestHandler({ request, $, log }) {
      log.info(`Scraping: ${request.url}`);

      // The TMG Matrix renders a table with compatibility data
      $("table tbody tr, .matrix-row, [class*='result-row']").each((_i, el) => {
        const $row = $(el);
        const cells = $row.find("td").map((_j, td) => $(td).text().trim()).get();

        if (cells.length >= 4) {
          entries.push({
            switchModel: cells[0] || "",
            switchSeries: cells[0]?.split(" ")[0] || "Nexus",
            transceiverPid: cells[1] || "",
            transceiverDescription: cells[2] || "",
            speed: cells[3] || "",
            reach: cells[4] || "",
            cableType: cells[5] || "",
            connector: cells[6] || "",
            minSoftware: cells[7] || "",
          });
        }
      });
    },
  });

  // Start with Nexus switches (most relevant for Flexoptix)
  await crawler.run([
    `${TMG_BASE}/public/tmg?searchValue=Nexus+9000`,
    `${TMG_BASE}/public/tmg?searchValue=Nexus+3000`,
    `${TMG_BASE}/public/tmg?searchValue=Nexus+7000`,
    `${TMG_BASE}/public/tmg?searchValue=Catalyst+9000`,
  ]);

  console.log(`\nEntries found: ${entries.length}`);

  // Write to database
  let switches = 0;
  let compat = 0;

  for (const entry of entries) {
    if (!entry.switchModel || !entry.transceiverPid) continue;

    try {
      const switchId = await upsertCiscoSwitch(
        ciscoVendorId,
        entry.switchModel,
        entry.switchSeries
      );
      switches++;

      // Try to match transceiver in our DB
      const txResult = await pool.query(
        `SELECT id FROM transceivers
         WHERE part_number = $1
            OR slug LIKE $2
            OR standard_name ILIKE $3
         LIMIT 1`,
        [
          entry.transceiverPid,
          `%${entry.transceiverPid.toLowerCase().replace(/[^a-z0-9]/g, "")}%`,
          `%${entry.speed}%${entry.reach}%`,
        ]
      );

      if (txResult.rows.length > 0) {
        await upsertCompatibility(switchId, txResult.rows[0].id, entry.minSoftware);
        compat++;
      }
    } catch (err) {
      // Skip duplicates silently
    }
  }

  console.log(`Switches upserted: ${switches}`);
  console.log(`Compatibility entries: ${compat}`);
  console.log("=== Cisco TMG Scraper Complete ===\n");
}

if (require.main === module) {
  scrapeCiscoTmg()
    .then(() => pool.end())
    .catch((err) => {
      console.error("Fatal:", err);
      pool.end();
      process.exit(1);
    });
}