transceiver-db/packages/scraper/src/scrapers/flexoptix-compat.ts

/**
 * Flexoptix Switch Compatibility Scraper
 *
 * Populates the `compatibility` table by querying Flexoptix search/product API
 * with each switch model number. Flexoptix names compatible transceivers like
 * "Cisco N9K-C9364C compatible QSFP28 100GBASE-LR4" — so searching by switch
 * model returns all transceivers that are explicitly marked compatible.
 *
 * Strategy:
 *   1. For each switch in DB, query Flexoptix AJAX search
 *   2. Parse returned transceiver names for form-factor and vendor hints
 *   3. Match against transceivers table (Flexoptix vendor)
 *   4. Insert into compatibility with status = 'compatible'
 *
 * Fallback (when search returns nothing):
 *   Form-factor matching — for each port type in switch.ports_config,
 *   find Flexoptix transceivers matching that form factor.
 *   Used as spec_match rather than vendor_compat — shown in dashboard
 *   as "by form factor" rather than "explicitly verified".
 *
 * Rate limit: 1 req/1.5sec.
 */
import { pool } from "../utils/db";

const BASE = "https://www.flexoptix.net";
const HEADERS = {
  "User-Agent": "Mozilla/5.0 (compatible; TIP-Bot/1.0; internal-flexoptix)",
  Accept: "application/json, text/html",
};

function sleep(ms: number): Promise<void> {
  return new Promise((r) => setTimeout(r, ms));
}

// ── Ports-config → form factor map ─────────────────────────────────────────

const PORT_KEY_TO_FORM_FACTOR: Array<[RegExp, string]> = [
  [/QSFP-DD/i,  "QSFP-DD"],
  [/QSFP-DD800/i, "QSFP-DD800"],
  [/OSFP224/i,  "OSFP224"],
  [/OSFP/i,     "OSFP"],
  [/QSFP28/i,   "QSFP28"],
  [/QSFP\+/i,   "QSFP+"],
  [/QSFP/i,     "QSFP+"],
  [/SFP28/i,    "SFP28"],
  [/SFP\+/i,    "SFP+"],
  [/SFP/i,      "SFP"],
  [/CFP2/i,     "CFP2"],
  [/CFP4/i,     "CFP4"],
  [/CFP/i,      "CFP"],
];

function portsConfigToFormFactors(portsConfig: Record<string, number>): string[] {
  const factors = new Set<string>();
  for (const key of Object.keys(portsConfig)) {
    for (const [pattern, ff] of PORT_KEY_TO_FORM_FACTOR) {
      if (pattern.test(key)) {
        factors.add(ff);
        break;
      }
    }
  }
  return [...factors];
}

// ── Flexoptix search API ────────────────────────────────────────────────────

interface FlexoptixSuggestion {
  name: string;
  url: string;
  sku?: string;
  image?: string;
  price?: string;
}

async function searchFlexoptix(query: string): Promise<FlexoptixSuggestion[]> {
  const url = `${BASE}/en/search/ajax/suggest/?q=${encodeURIComponent(query)}`;
  try {
    const resp = await fetch(url, {
      headers: HEADERS,
      signal: AbortSignal.timeout(15000),
    });
    if (!resp.ok) return [];
    const data = await resp.json() as unknown;
    // Magento suggest returns array or object with products key
    if (Array.isArray(data)) return data as FlexoptixSuggestion[];
    const obj = data as Record<string, unknown>;
    if (obj["products"] && Array.isArray(obj["products"])) {
      return obj["products"] as FlexoptixSuggestion[];
    }
    return [];
  } catch {
    return [];
  }
}

// ── Main scraper ────────────────────────────────────────────────────────────

export async function scrapeFlexoptixCompatibility(): Promise<void> {
  console.log("=== Flexoptix Compatibility Scraper ===\n");

  // Get all switches
  const { rows: switches } = await pool.query<{
    id: string;
    model: string;
    vendor_name: string;
    ports_config: Record<string, number> | null;
  }>(
    `SELECT sw.id, sw.model, v.name AS vendor_name, sw.ports_config
     FROM switches sw
     JOIN vendors v ON v.id = sw.vendor_id
     ORDER BY sw.max_speed_gbps DESC`,
  );

  console.log(`  ${switches.length} switches to process\n`);

  // Get Flexoptix vendor ID
  const { rows: [flexoptixVendor] } = await pool.query<{ id: string }>(
    `SELECT id FROM vendors WHERE UPPER(name) LIKE '%FLEXOPTIX%' LIMIT 1`,
  );

  if (!flexoptixVendor) {
    console.error("  [FAIL] Flexoptix vendor not found in DB — run flexoptix-catalog scraper first");
    return;
  }

  const flexoptixVendorId = flexoptixVendor.id;
  let totalCompatAdded = 0;
  let totalSwitchesProcessed = 0;

  for (const sw of switches) {
    console.log(`\n[${sw.vendor_name}] ${sw.model}`);
    const addedForSwitch: string[] = [];

    // ── Strategy 1: Search Flexoptix by switch model ──────────────────────
    await sleep(1500);
    const suggestions = await searchFlexoptix(sw.model);

    const matchedBySku = new Set<string>();

    if (suggestions.length > 0) {
      console.log(`  Search: ${suggestions.length} results`);

      for (const suggestion of suggestions) {
        if (!suggestion.sku && !suggestion.name) continue;
        const sku = suggestion.sku?.trim();
        const name = suggestion.name?.trim();
        if (!sku && !name) continue;

        // Find matching transceiver in DB
        let txRow: { id: string } | undefined;
        if (sku) {
          const r = await pool.query<{ id: string }>(
            `SELECT id FROM transceivers WHERE vendor_id = $1 AND (part_number = $2 OR slug = $3) LIMIT 1`,
            [flexoptixVendorId, sku, sku.toLowerCase().replace(/[^a-z0-9]/g, "-")],
          );
          txRow = r.rows[0];
        }

        if (!txRow && name) {
          // Try matching by name similarity to part_number
          const r = await pool.query<{ id: string }>(
            `SELECT id FROM transceivers
             WHERE vendor_id = $1
               AND (part_number ILIKE $2 OR standard_name ILIKE $3)
             LIMIT 1`,
            [flexoptixVendorId, `%${name.slice(0, 20)}%`, `%${name.slice(0, 20)}%`],
          );
          txRow = r.rows[0];
        }

        if (txRow) {
          matchedBySku.add(txRow.id);
          await upsertCompat(sw.id, txRow.id, "Flexoptix", "vendor_compat");
          addedForSwitch.push(txRow.id);
        }
      }

      console.log(`  Vendor-compat matches: ${addedForSwitch.length}`);
    }

    // ── Strategy 2: Form-factor fallback ─────────────────────────────────
    if (sw.ports_config) {
      const formFactors = portsConfigToFormFactors(sw.ports_config);
      console.log(`  Form factors: ${formFactors.join(", ")}`);

      for (const ff of formFactors) {
        const { rows: txRows } = await pool.query<{ id: string }>(
          `SELECT t.id
           FROM transceivers t
           WHERE t.vendor_id = $1
             AND t.form_factor = $2
             AND NOT EXISTS (
               SELECT 1 FROM compatibility c
               WHERE c.switch_id = $3 AND c.transceiver_id = t.id
             )`,
          [flexoptixVendorId, ff, sw.id],
        );

        for (const tx of txRows) {
          if (matchedBySku.has(tx.id)) continue; // already added via search
          await upsertCompat(sw.id, tx.id, "Form Factor Match", "spec_match");
          addedForSwitch.push(tx.id);
        }

        if (txRows.length > 0) {
          console.log(`  Form-factor ${ff}: +${txRows.length} transceivers`);
        }
      }
    }

    totalCompatAdded += addedForSwitch.length;
    totalSwitchesProcessed++;

    console.log(`  → Added ${addedForSwitch.length} compat entries`);
  }

  console.log(`\n=== Flexoptix Compatibility Scraper Complete ===`);
  console.log(`  Switches processed: ${totalSwitchesProcessed}`);
  console.log(`  Compatibility entries added: ${totalCompatAdded}`);
}

async function upsertCompat(
  switchId: string,
  transceiverId: string,
  verifiedBy: string,
  verificationMethod: string,
): Promise<void> {
  await pool.query(
    `INSERT INTO compatibility (switch_id, transceiver_id, verified_by, verification_method, status, source_url)
     VALUES ($1, $2, $3, $4, 'compatible', 'https://www.flexoptix.net')
     ON CONFLICT (switch_id, transceiver_id) DO UPDATE SET
       verified_by = CASE
         WHEN EXCLUDED.verification_method = 'vendor_compat' THEN EXCLUDED.verified_by
         ELSE compatibility.verified_by
       END,
       verification_method = CASE
         WHEN EXCLUDED.verification_method = 'vendor_compat' THEN EXCLUDED.verification_method
         ELSE compatibility.verification_method
       END`,
    [switchId, transceiverId, verifiedBy, verificationMethod],
  );
}

if (require.main === module) {
  scrapeFlexoptixCompatibility()
    .then(() => pool.end())
    .catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); });
}