transceiver-db/packages/scraper/src/scrapers/gbics.ts

/**
 * GBICS.com Scraper — UK-based compatible transceiver vendor
 *
 * gbics.com — BigCommerce store, server-rendered HTML, GBP prices.
 * Products in <li> cards with <h3> product names, "Now: £XX.XX" pricing.
 * Pagination via ?page=N. Rate limited: 1 req/2sec.
 */
import { pool, findOrCreateScrapedTransceiver, ensureVendor, upsertPriceObservation } from "../utils/db";
import { contentHash } from "../utils/hash";
import { SocksProxyAgent } from "socks-proxy-agent";

const BASE = "https://www.gbics.com";
const HEADERS = {
  "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
  Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
  "Accept-Language": "en-US,en;q=0.9",
};

// SOCKS5 proxy pool — rotate across CT130/131/132 to avoid IP blocks
const PROXY_URLS = (process.env["PROXY_URLS"] ?? "")
  .split(",")
  .map((u) => u.trim())
  .filter(Boolean);

let proxyIndex = 0;
function getNextProxyAgent(): SocksProxyAgent | undefined {
  if (PROXY_URLS.length === 0) return undefined;
  const url = PROXY_URLS[proxyIndex % PROXY_URLS.length];
  proxyIndex++;
  return new SocksProxyAgent(url);
}

const CATEGORIES = [
  { path: "/800g-osfp/", formFactor: "OSFP", speed: "800G", speedGbps: 800 },
  { path: "/400g-qsfp112/", formFactor: "QSFP112", speed: "400G", speedGbps: 400 },
  { path: "/400g-qsfp-dd/", formFactor: "QSFP-DD", speed: "400G", speedGbps: 400 },
  { path: "/400g-osfp/", formFactor: "OSFP", speed: "400G", speedGbps: 400 },
  { path: "/200g-qsfp56/", formFactor: "QSFP56", speed: "200G", speedGbps: 200 },
  { path: "/100g-qsfp28/", formFactor: "QSFP28", speed: "100G", speedGbps: 100 },
  { path: "/40g-qsfp/", formFactor: "QSFP+", speed: "40G", speedGbps: 40 },
  { path: "/25g-sfp28/", formFactor: "SFP28", speed: "25G", speedGbps: 25 },
  { path: "/10g-sfp/", formFactor: "SFP+", speed: "10G", speedGbps: 10 },
  { path: "/1g-sfp/", formFactor: "SFP", speed: "1G", speedGbps: 1 },
];

interface Product {
  partNumber: string;
  name: string;
  url: string;
  price?: number;
  formFactor: string;
  speed: string;
  speedGbps: number;
  reachLabel?: string;
  reachMeters?: number;
  fiberType?: string;
  wavelength?: string;
  compatibleWith?: string;
}

function sleep(ms: number): Promise<void> {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

function detectReach(text: string): { label: string; meters: number } | undefined {
  const patterns: [RegExp, string, number][] = [
    [/\b120\s*km\b/i, "120km", 120000],
    [/\b80\s*km\b/i, "80km", 80000],
    [/\b40\s*km\b/i, "40km", 40000],
    [/\b20\s*km\b/i, "20km", 20000],
    [/\b10\s*km\b/i, "10km", 10000],
    [/\b2\s*km\b/i, "2km", 2000],
    [/\b550\s*m\b/i, "550m", 550],
    [/\b500\s*m\b/i, "500m", 500],
    [/\b400\s*m\b/i, "400m", 400],
    [/\b300\s*m\b/i, "300m", 300],
    [/\b150\s*m\b/i, "150m", 150],
    [/\b100\s*m\b/i, "100m", 100],
    [/\bLR4\b/, "10km", 10000],
    [/\bLR\b/, "10km", 10000],
    [/\bER4?\b/, "40km", 40000],
    [/\bZR4?\b/, "80km", 80000],
    [/\bSR4?\b/, "300m", 300],
    [/\bDR4?\b/, "500m", 500],
    [/\bFR4?\b/, "2km", 2000],
  ];
  for (const [regex, label, meters] of patterns) {
    if (regex.test(text)) return { label, meters };
  }
  return undefined;
}

function detectFiber(text: string): string {
  if (/single.?mode|smf|[^a-z]lx[^a-z]|[^a-z]lr[^a-z]|[^a-z]er[^a-z]|[^a-z]zr[^a-z]|bidi|cwdm|dwdm/i.test(text)) return "SMF";
  if (/multi.?mode|mmf|[^a-z]sx[^a-z]|[^a-z]sr[^a-z]/i.test(text)) return "MMF";
  if (/copper|dac|twinax|rj.?45|base-t|cat[56x]/i.test(text)) return "Copper";
  return "";
}

function detectWavelength(text: string): string {
  const match = text.match(/(\d{3,4})\s*nm/i);
  if (match) return match[1];
  return "";
}

function extractCompatibleVendor(name: string): string {
  const match = name.match(/^(\w+(?:\s+\w+)?)\s+Compatible\b/i);
  return match ? match[1] : "";
}

function parseProductList(html: string, cat: typeof CATEGORIES[number]): Product[] {
  const products: Product[] = [];

  // Collapse whitespace for easier regex matching
  const collapsed = html.replace(/\s+/g, " ");

  // BigCommerce card pattern (attribute order varies by theme version):
  // Old: <a aria-label="Name, £XX.XX" href="URL" data-event-type="product-click">
  // New: <a href="URL" class="card-figure__link..." aria-label="Name, £XX.XX">
  // Two-pass approach: find all product <a> tags regardless of attribute order
  const productRegex = /href="(https?:\/\/(?:www\.)?gbics\.com\/[^"]+)"[^>]*aria-label="([^"]+)"/gi;
  const productRegex2 = /aria-label="([^"]+)"[^>]*href="(https?:\/\/(?:www\.)?gbics\.com\/[^"]+)"/gi;
  let match;
  const rawMatches: { url: string; label: string; index: number }[] = [];
  while ((match = productRegex.exec(collapsed)) !== null) {
    rawMatches.push({ url: match[1].trim(), label: match[2].trim(), index: match.index });
  }
  if (rawMatches.length === 0) {
    while ((match = productRegex2.exec(collapsed)) !== null) {
      rawMatches.push({ url: match[2].trim(), label: match[1].trim(), index: match.index });
    }
  }
  for (const { url, label: rawLabel, index: matchIndex } of rawMatches) {
    const label = rawLabel.replace(/\s+/g, " ").trim();

    // aria-label contains "Product Name, £XX.XX"
    // Split on last comma to separate name and price
    const priceInLabel = label.match(/,\s*£\s*([\d,.]+)\s*$/);
    const name = priceInLabel ? label.slice(0, label.lastIndexOf(",")).trim() : label;
    let price = priceInLabel ? parseFloat(priceInLabel[1].replace(",", "")) : undefined;

    // Fallback: extract price from data-price-asc attribute on parent <li>
    if (!price) {
      const priceContext = collapsed.slice(Math.max(0, matchIndex - 500), matchIndex);
      const dataPriceMatch = priceContext.match(/data-price-asc="(\d+)"/);
      if (dataPriceMatch) price = parseFloat(dataPriceMatch[1]);
    }

    if (name.length < 10) continue;

    const reach = detectReach(name);
    const partParts = name.split(/\s+-\s+/);
    const partNumber = partParts[0]?.trim().slice(0, 80) || url.split("/").filter(Boolean).pop() || "";

    products.push({
      partNumber, name, url,
      price: price && price > 0 && price < 50000 ? price : undefined,
      formFactor: cat.formFactor, speed: cat.speed, speedGbps: cat.speedGbps,
      reachLabel: reach?.label, reachMeters: reach?.meters,
      fiberType: detectFiber(name), wavelength: detectWavelength(name),
      compatibleWith: extractCompatibleVendor(name),
    });
  }

  // Fallback: aria-label pattern
  if (products.length === 0) {
    const ariaRegex = /aria-label="([^"]+£[^"]+)"\s+href="(https?:\/\/(?:www\.)?gbics\.com\/[^"]+)"/gi;
    while ((match = ariaRegex.exec(collapsed)) !== null) {
      const label = match[1].trim();
      const url = match[2];
      const priceMatch = label.match(/£\s*([\d,.]+)/);
      const name = label.split(",")[0]?.trim() || label;
      const price = priceMatch ? parseFloat(priceMatch[1].replace(",", "")) : undefined;
      if (name.length < 10) continue;
      const reach = detectReach(name);
      products.push({
        partNumber: name.split(/\s+-\s+/)[0]?.trim().slice(0, 80) || "",
        name, url,
        price: price && price > 0 && price < 50000 ? price : undefined,
        formFactor: cat.formFactor, speed: cat.speed, speedGbps: cat.speedGbps,
        reachLabel: reach?.label, reachMeters: reach?.meters,
        fiberType: detectFiber(name), wavelength: detectWavelength(name),
        compatibleWith: extractCompatibleVendor(name),
      });
    }
  }

  // Fallback: try "Now: £XX.XX" pattern near product links
  if (products.length === 0) {
    const altRegex = /href="(https?:\/\/(?:www\.)?gbics\.com\/[^"]+)"[^>]*>\s*([^<]{15,})<\/a>/gi;
    while ((match = altRegex.exec(collapsed)) !== null) {
      const url = match[1];
      const name = match[2].trim();
      if (name.length < 10 || products.find((p) => p.url === url)) continue;
      if (!/transceiver|sfp|qsfp|xfp|osfp|base/i.test(name)) continue;

      const context = collapsed.slice(Math.max(0, match.index - 300), match.index + 600);
      const priceMatch = context.match(/Now:\s*£\s*([\d,.]+)/) || context.match(/£\s*([\d,.]+)/);
      const price = priceMatch ? parseFloat(priceMatch[1].replace(",", "")) : undefined;
      const reach = detectReach(name);

      products.push({
        partNumber: name.split(/\s+-\s+/)[0]?.trim().slice(0, 80) || "",
        name, url,
        price: price && price > 0 && price < 50000 ? price : undefined,
        formFactor: cat.formFactor, speed: cat.speed, speedGbps: cat.speedGbps,
        reachLabel: reach?.label, reachMeters: reach?.meters,
        fiberType: detectFiber(name), wavelength: detectWavelength(name),
        compatibleWith: extractCompatibleVendor(name),
      });
    }
  }

  const seen = new Set<string>();
  return products.filter((p) => {
    if (seen.has(p.url)) return false;
    seen.add(p.url);
    return true;
  });
}

async function fetchPage(url: string): Promise<string> {
  const agent = getNextProxyAgent();
  const opts: RequestInit = { headers: HEADERS, signal: AbortSignal.timeout(30000) };
  if (agent) (opts as Record<string, unknown>)["dispatcher"] = agent;
  const resp = await fetch(url, opts);
  if (!resp.ok) throw new Error(`HTTP ${resp.status} for ${url}`);
  return resp.text();
}

export async function scrapeGbics(): Promise<void> {
  console.log("=== GBICS.com Scraper Starting ===\n");

  const vendorId = await ensureVendor("GBICS", "compatible", "https://www.gbics.com", "https://www.gbics.com/optical-transceivers/");

  let totalProducts = 0;
  let priceUpdates = 0;

  for (const cat of CATEGORIES) {
    console.log(`\n--- ${cat.formFactor} (${cat.speed}) [${cat.path}] ---`);

    try {
      const html = await fetchPage(BASE + cat.path);
      const catProducts = parseProductList(html, cat);
      console.log(`  Found ${catProducts.length} products`);

      for (const product of catProducts) {
        try {
          const txId = await findOrCreateScrapedTransceiver({
            partNumber: product.partNumber, vendorId,
            formFactor: product.formFactor, speedGbps: product.speedGbps,
            speed: product.speed, reachMeters: product.reachMeters,
            reachLabel: product.reachLabel, fiberType: product.fiberType,
            wavelengths: product.wavelength, category: "DataCenter",
          });

          if (product.price && product.price > 0) {
            const hash = contentHash(JSON.stringify({ price: product.price, part: product.partNumber }));
            const updated = await upsertPriceObservation({
              transceiverId: txId, sourceVendorId: vendorId,
              price: product.price, currency: "GBP",
              stockLevel: "in_stock", url: product.url, contentHash: hash,
            });
            if (updated) priceUpdates++;
          }
          totalProducts++;
        } catch (err) {
          console.warn(`    Error: ${(err as Error).message.slice(0, 80)}`);
        }
      }
    } catch (err) {
      console.error(`  Category failed: ${(err as Error).message}`);
    }

    await sleep(2000);
  }

  console.log(`\n=== GBICS Complete: ${totalProducts} products, ${priceUpdates} prices ===`);
}

if (require.main === module) {
  scrapeGbics()
    .then(() => pool.end())
    .catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); });
}