transceiver-db/packages/scraper/src/scrapers/fluxlight.ts

/**
 * Fluxlight Scraper — US-based compatible transceiver vendor
 *
 * www.fluxlight.com — BigCommerce, server-rendered HTML with real prices.
 * ~144+ products across 6 pages. Uses pagination via ?page=N.
 *
 * Rate limited: 1 req/2sec.
 */
import { pool, findOrCreateScrapedTransceiver, ensureVendor, upsertPriceObservation } from "../utils/db";
import { contentHash } from "../utils/hash";

const BASE = "https://fluxlight.com";
const CATALOG_PATH = "/transceivers/";
const HEADERS = {
  "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
  Accept: "text/html,application/xhtml+xml",
};

interface Product {
  partNumber: string;
  name: string;
  url: string;
  price?: number;
  formFactor: string;
  speed: string;
  speedGbps: number;
  reachLabel?: string;
  reachMeters?: number;
  fiberType?: string;
  wavelength?: string;
}

function sleep(ms: number): Promise<void> {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

function detectFormFactor(text: string): { formFactor: string; speed: string; speedGbps: number } {
  const lower = text.toLowerCase();
  if (lower.includes("osfp") && !lower.includes("qsfp")) return { formFactor: "OSFP", speed: "400G", speedGbps: 400 };
  if (lower.includes("qsfp-dd")) return { formFactor: "QSFP-DD", speed: "400G", speedGbps: 400 };
  if (lower.includes("qsfp28")) return { formFactor: "QSFP28", speed: "100G", speedGbps: 100 };
  if (lower.includes("qsfp+") || lower.includes("qsfp plus")) return { formFactor: "QSFP+", speed: "40G", speedGbps: 40 };
  if (lower.includes("sfp56")) return { formFactor: "SFP56", speed: "50G", speedGbps: 50 };
  if (lower.includes("sfp28") || lower.includes("25g")) return { formFactor: "SFP28", speed: "25G", speedGbps: 25 };
  if (lower.includes("sfp+") || lower.includes("10gbase") || lower.includes("10g")) return { formFactor: "SFP+", speed: "10G", speedGbps: 10 };
  if (lower.includes("xfp")) return { formFactor: "XFP", speed: "10G", speedGbps: 10 };
  if (lower.includes("1000base") || lower.includes("1g")) return { formFactor: "SFP", speed: "1G", speedGbps: 1 };
  if (lower.includes("sfp") && !lower.includes("qsfp")) return { formFactor: "SFP", speed: "1G", speedGbps: 1 };
  return { formFactor: "SFP+", speed: "10G", speedGbps: 10 };
}

function detectReach(text: string): { label: string; meters: number } | undefined {
  const patterns: [RegExp, string, number][] = [
    [/\b80\s*km\b/i, "80km", 80000],
    [/\b40\s*km\b/i, "40km", 40000],
    [/\b20\s*km\b/i, "20km", 20000],
    [/\b10\s*km\b/i, "10km", 10000],
    [/\b2\s*km\b/i, "2km", 2000],
    [/\b550\s*m\b/i, "550m", 550],
    [/\b500\s*m\b/i, "500m", 500],
    [/\b300\s*m\b/i, "300m", 300],
    [/\b100\s*m\b/i, "100m", 100],
    [/\bLR4\b/, "10km", 10000],
    [/\bLR\b/, "10km", 10000],
    [/\bER4?\b/, "40km", 40000],
    [/\bZR4?\b/, "80km", 80000],
    [/\bSR4?\b/, "300m", 300],
    [/\bDR4?\b/, "500m", 500],
    [/\bFR4?\b/, "2km", 2000],
  ];
  for (const [regex, label, meters] of patterns) {
    if (regex.test(text)) return { label, meters };
  }
  return undefined;
}

function detectFiber(text: string): string {
  if (/single.?mode|smf|[^a-z]lx[^a-z]|[^a-z]lr[^a-z]|[^a-z]er[^a-z]|[^a-z]zr[^a-z]|bidi|cwdm|dwdm/i.test(text)) return "SMF";
  if (/multi.?mode|mmf|[^a-z]sx[^a-z]|[^a-z]sr[^a-z]/i.test(text)) return "MMF";
  if (/copper|dac|twinax|rj45|base-t|catx/i.test(text)) return "Copper";
  return "";
}

function detectWavelength(text: string): string {
  const match = text.match(/(\d{3,4})\s*nm/i);
  if (match) return match[1];
  return "";
}

function parseProductList(html: string): Product[] {
  const products: Product[] = [];

  // BigCommerce product card pattern: product link + price
  // Pattern: <a href="https://www.fluxlight.com/PARTNUM-FL/">Product Name</a> ... $29.99
  const productRegex = /href="(https?:\/\/(?:www\.)?fluxlight\.com\/[^"]*-FL\/)"[^>]*>\s*([^<]{10,})<\/a>/gi;
  let match;
  while ((match = productRegex.exec(html)) !== null) {
    const url = match[1];
    const name = match[2].trim();
    if (name.length < 10 || name.length > 200) continue;

    // Look for price in surrounding context — BigCommerce uses data-product-price-without-tax
    const context = html.slice(Math.max(0, match.index - 300), match.index + 1200);
    const priceMatch = context.match(/data-product-price-without-tax[^>]*>\s*\$([\d,]+\.?\d{0,2})/)
      || context.match(/class="price price--withoutTax">\s*\$([\d,]+\.?\d{0,2})/)
      || context.match(/\$\s*([\d,]+\.?\d{0,2})/);

    const price = priceMatch ? parseFloat(priceMatch[1].replace(",", "")) : undefined;

    const ff = detectFormFactor(name);
    const reach = detectReach(name);
    const partNum = url.split("/").filter(Boolean).pop() || name.replace(/\s+/g, "-").slice(0, 80);

    products.push({
      partNumber: partNum,
      name,
      url,
      price: price && price > 0 && price < 50000 ? price : undefined,
      ...ff,
      reachLabel: reach?.label,
      reachMeters: reach?.meters,
      fiberType: detectFiber(name),
      wavelength: detectWavelength(name),
    });
  }

  // Fallback: broader link pattern
  if (products.length === 0) {
    const simpleRegex = /href="(https?:\/\/(?:www\.)?fluxlight\.com\/[^"]+)"[^>]*>([^<]{10,}(?:SFP|QSFP|XFP|Base)[^<]*)<\/a>/gi;
    while ((match = simpleRegex.exec(html)) !== null) {
      const url = match[1];
      const name = match[2].trim();
      if (products.find((p) => p.url === url)) continue;

      const context = html.slice(match.index, match.index + 500);
      const priceMatch = context.match(/\$\s*([\d,]+\.?\d{0,2})/);
      const price = priceMatch ? parseFloat(priceMatch[1].replace(",", "")) : undefined;
      const ff = detectFormFactor(name);
      const reach = detectReach(name);

      products.push({
        partNumber: url.split("/").filter(Boolean).pop() || name.replace(/\s+/g, "-").slice(0, 80),
        name, url,
        price: price && price > 0 && price < 50000 ? price : undefined,
        ...ff,
        reachLabel: reach?.label, reachMeters: reach?.meters,
        fiberType: detectFiber(name), wavelength: detectWavelength(name),
      });
    }
  }

  const seen = new Set<string>();
  return products.filter((p) => {
    if (seen.has(p.url)) return false;
    seen.add(p.url);
    return true;
  });
}

/** Detect max page by probing — page 1 may not have pagination links */
const MAX_PAGES = 6;

async function fetchPage(url: string): Promise<string> {
  const resp = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(30000) });
  if (!resp.ok) throw new Error(`HTTP ${resp.status} for ${url}`);
  return resp.text();
}

export async function scrapeFluxlight(): Promise<void> {
  console.log("=== Fluxlight Scraper Starting ===\n");

  const vendorId = await ensureVendor("Fluxlight", "compatible", "https://fluxlight.com", "https://www.fluxlight.com/transceivers/");

  let allProducts: Product[] = [];

  for (let page = 1; page <= MAX_PAGES; page++) {
    try {
      const url = page === 1 ? BASE + CATALOG_PATH : `${BASE}${CATALOG_PATH}?page=${page}`;
      const html = await fetchPage(url);
      const pageProducts = parseProductList(html);
      allProducts.push(...pageProducts);
      console.log(`  Page ${page}: ${pageProducts.length} products`);
      if (pageProducts.length === 0) {
        console.log(`  Empty page ${page}, continuing...`);
      }
      if (page < MAX_PAGES) await sleep(2000);
    } catch (err) {
      console.warn(`  Page ${page} failed: ${(err as Error).message}`);
    }
  }

  // Dedupe
  const seen = new Set<string>();
  allProducts = allProducts.filter((p) => {
    if (seen.has(p.url)) return false;
    seen.add(p.url);
    return true;
  });

  console.log(`\nTotal unique products: ${allProducts.length}`);

  let totalProducts = 0;
  let priceUpdates = 0;

  for (const product of allProducts) {
    try {
      const txId = await findOrCreateScrapedTransceiver({
        partNumber: product.partNumber, vendorId,
        formFactor: product.formFactor, speedGbps: product.speedGbps,
        speed: product.speed, reachMeters: product.reachMeters,
        reachLabel: product.reachLabel, fiberType: product.fiberType,
        wavelengths: product.wavelength, category: "DataCenter",
      });

      if (product.price && product.price > 0) {
        const hash = contentHash(JSON.stringify({ price: product.price, part: product.partNumber }));
        const updated = await upsertPriceObservation({
          transceiverId: txId, sourceVendorId: vendorId,
          price: product.price, currency: "USD",
          stockLevel: "in_stock", url: product.url, contentHash: hash,
        });
        if (updated) priceUpdates++;
      }
      totalProducts++;
    } catch (err) {
      console.warn(`  Error: ${(err as Error).message.slice(0, 80)}`);
    }
  }

  console.log(`\n=== Fluxlight Complete: ${totalProducts} products, ${priceUpdates} prices ===`);
}

if (require.main === module) {
  scrapeFluxlight()
    .then(() => pool.end())
    .catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); });
}