transceiver-db/packages/scraper/src/scrapers/hubersuhner.ts

/**
 * HUBER+SUHNER Scraper — Swiss transceiver manufacturer
 *
 * www.hubersuhner.com — no direct prices, catalog-only scrape.
 * Tries DE and EN product listing pages; may expose product API.
 *
 * Rate limited: 1 req/2sec.
 */
import { pool, findOrCreateScrapedTransceiver, ensureVendor } from "../utils/db";
import * as cheerio from "cheerio";

const BASE = "https://www.hubersuhner.com";

// Try multiple entry points — HS has EN and DE variants
const CATALOG_URLS = [
  "/en/transceiver",
  "/de/transceiver",
  "/en/products/fiber-optics/fiber-optic-transceivers",
  "/de/products/fiber-optics/fiber-optic-transceivers",
  "/en/products/fiber-optics",
];

// Known HS product API endpoint pattern (JSON-based catalog)
const API_URLS = [
  "/api/products?category=transceiver&format=json",
  "/api/catalog/transceiver",
];

const HEADERS = {
  "User-Agent": "Mozilla/5.0 (compatible; TIP-Bot/1.0; research)",
  Accept: "text/html,application/xhtml+xml,application/json",
  "Accept-Language": "en-US,en;q=0.9,de;q=0.8",
};

interface Product {
  partNumber: string;
  name: string;
  url: string;
  formFactor: string;
  speed: string;
  speedGbps: number;
  reachLabel?: string;
  reachMeters?: number;
  fiberType?: string;
  wavelength?: string;
}

function sleep(ms: number): Promise<void> {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

function detectFormFactor(text: string): { formFactor: string; speed: string; speedGbps: number } {
  const lower = text.toLowerCase();
  if (lower.includes("osfp") && !lower.includes("qsfp")) return { formFactor: "OSFP", speed: "400G", speedGbps: 400 };
  if (lower.includes("qsfp-dd")) return { formFactor: "QSFP-DD", speed: "400G", speedGbps: 400 };
  if (lower.includes("qsfp28")) return { formFactor: "QSFP28", speed: "100G", speedGbps: 100 };
  if (lower.includes("qsfp+") || lower.includes("qsfp plus")) return { formFactor: "QSFP+", speed: "40G", speedGbps: 40 };
  if (lower.includes("sfp56")) return { formFactor: "SFP56", speed: "50G", speedGbps: 50 };
  if (lower.includes("sfp28") || lower.includes("25g")) return { formFactor: "SFP28", speed: "25G", speedGbps: 25 };
  if (lower.includes("sfp+") || lower.includes("10gbase") || lower.includes("10g")) return { formFactor: "SFP+", speed: "10G", speedGbps: 10 };
  if (lower.includes("xfp")) return { formFactor: "XFP", speed: "10G", speedGbps: 10 };
  if (lower.includes("1000base") || lower.includes("1g")) return { formFactor: "SFP", speed: "1G", speedGbps: 1 };
  if (lower.includes("sfp") && !lower.includes("qsfp")) return { formFactor: "SFP", speed: "1G", speedGbps: 1 };
  return { formFactor: "SFP+", speed: "10G", speedGbps: 10 };
}

function detectReach(text: string): { label: string; meters: number } | undefined {
  const patterns: [RegExp, string, number][] = [
    [/\b80\s*km\b/i, "80km", 80000],
    [/\b40\s*km\b/i, "40km", 40000],
    [/\b20\s*km\b/i, "20km", 20000],
    [/\b10\s*km\b/i, "10km", 10000],
    [/\b2\s*km\b/i, "2km", 2000],
    [/\b550\s*m\b/i, "550m", 550],
    [/\b300\s*m\b/i, "300m", 300],
    [/\b100\s*m\b/i, "100m", 100],
    [/\bLR4\b/, "10km", 10000], [/\bLR\b/, "10km", 10000],
    [/\bER4?\b/, "40km", 40000], [/\bZR4?\b/, "80km", 80000],
    [/\bSR4?\b/, "300m", 300], [/\bDR4?\b/, "500m", 500], [/\bFR4?\b/, "2km", 2000],
  ];
  for (const [regex, label, meters] of patterns) {
    if (regex.test(text)) return { label, meters };
  }
  return undefined;
}

function detectFiber(text: string): string {
  if (/single.?mode|smf|[^a-z]lx[^a-z]|[^a-z]lr[^a-z]|[^a-z]er[^a-z]|[^a-z]zr[^a-z]|bidi|cwdm|dwdm/i.test(text)) return "SMF";
  if (/multi.?mode|mmf|[^a-z]sx[^a-z]|[^a-z]sr[^a-z]/i.test(text)) return "MMF";
  if (/copper|dac|twinax|rj45|base-t/i.test(text)) return "Copper";
  return "SMF";
}

function detectWavelength(text: string): string {
  const match = text.match(/(\d{3,4})\s*nm/i);
  return match ? match[1] : "";
}

function parseHtmlProducts(html: string, sourceUrl: string): Product[] {
  const $ = cheerio.load(html);
  const products: Product[] = [];

  // HS uses a custom product portal — try multiple selectors
  const cardSelectors = [
    ".product-item", ".product-tile", ".product-card", ".product",
    "tr[data-part]", "tr", ".item", ".result-item", ".catalog-item",
    ".product-list-item", "li[data-id]",
  ];

  for (const sel of cardSelectors) {
    if ($(sel).length >= 2) {
      $(sel).each((_i, el) => {
        const text = $(el).text().trim();
        if (!/sfp|qsfp|xfp|transceiver|optic|fiber/i.test(text)) return;

        const nameEl = $(el).find("h2, h3, h4, .name, .title, .product-name, a").first();
        const name = nameEl.text().trim() || text.slice(0, 100);
        if (!name || name.length < 5) return;

        const linkEl = $(el).find("a[href]").first();
        const href = linkEl.attr("href") || sourceUrl;
        const url = href.startsWith("http") ? href : BASE + href;

        // HUBER+SUHNER part numbers often follow: 85XXXXXXX or similar numeric patterns
        const partNumMatch = text.match(/\b(85\d{7,})\b/) ||
          text.match(/\b([A-Z]{2,}[-_][A-Z0-9]+[-_][A-Z0-9]+)\b/) ||
          name.match(/[A-Z0-9][-A-Z0-9]{5,}/);
        const partNumber = partNumMatch?.[0] || name.replace(/\s+/g, "-").slice(0, 60);

        const ff = detectFormFactor(name + " " + text);
        const reach = detectReach(name + " " + text);

        products.push({
          partNumber,
          name,
          url,
          ...ff,
          reachLabel: reach?.label,
          reachMeters: reach?.meters,
          fiberType: detectFiber(name + " " + text),
          wavelength: detectWavelength(name + " " + text),
        });
      });
      if (products.length > 0) break;
    }
  }

  // Fallback: all product-relevant links
  if (products.length === 0) {
    $("a[href]").each((_i, el) => {
      const name = $(el).text().trim();
      const href = $(el).attr("href") || "";
      if (name.length < 5 || name.length > 200 || !/sfp|qsfp|transceiver|optic/i.test(name)) return;
      const url = href.startsWith("http") ? href : BASE + href;
      const ff = detectFormFactor(name);
      const reach = detectReach(name);
      products.push({
        partNumber: name.match(/[A-Z0-9][-A-Z0-9]{5,}/)?.[0] || name.replace(/\s+/g, "-").slice(0, 60),
        name, url, ...ff,
        reachLabel: reach?.label, reachMeters: reach?.meters,
        fiberType: detectFiber(name), wavelength: detectWavelength(name),
      });
    });
  }

  const seen = new Set<string>();
  return products.filter((p) => {
    if (!p.url || seen.has(p.url)) return false;
    seen.add(p.url);
    return true;
  });
}

// Try JSON API endpoint — some HS portals expose catalog APIs
function parseJsonProducts(json: unknown): Product[] {
  const products: Product[] = [];
  const items = Array.isArray(json)
    ? json
    : (json as Record<string, unknown>)?.products || (json as Record<string, unknown>)?.items || [];
  if (!Array.isArray(items)) return products;

  for (const item of items) {
    const name = String((item as Record<string, unknown>).name || (item as Record<string, unknown>).title || "");
    if (!name || !/sfp|qsfp|transceiver/i.test(name)) continue;
    const partNumber = String(
      (item as Record<string, unknown>).partNumber ||
      (item as Record<string, unknown>).sku ||
      (item as Record<string, unknown>).id ||
      name.replace(/\s+/g, "-").slice(0, 60)
    );
    const url = String((item as Record<string, unknown>).url || BASE);
    const ff = detectFormFactor(name);
    const reach = detectReach(name);
    products.push({
      partNumber, name,
      url: url.startsWith("http") ? url : BASE + url,
      ...ff,
      reachLabel: reach?.label, reachMeters: reach?.meters,
      fiberType: detectFiber(name), wavelength: detectWavelength(name),
    });
  }
  return products;
}

async function fetchPage(url: string): Promise<string> {
  const resp = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(30000) });
  if (!resp.ok) throw new Error(`HTTP ${resp.status} for ${url}`);
  return resp.text();
}

export async function scrapeHuberSuhner(): Promise<void> {
  console.log("=== HUBER+SUHNER Scraper Starting (catalog-only, no prices) ===\n");

  const vendorId = await ensureVendor(
    "HUBER+SUHNER",
    "manufacturer",
    "https://www.hubersuhner.com",
    BASE + CATALOG_URLS[0],
  );

  const allProducts: Product[] = [];
  const seenUrls = new Set<string>();

  // Try JSON API endpoints first
  for (const apiPath of API_URLS) {
    const apiUrl = BASE + apiPath;
    console.log(`  Trying API: ${apiUrl}`);
    try {
      const resp = await fetch(apiUrl, {
        headers: { ...HEADERS, Accept: "application/json" },
        signal: AbortSignal.timeout(15000),
      });
      if (resp.ok) {
        const json: unknown = await resp.json();
        const parsed = parseJsonProducts(json);
        console.log(`    API returned ${parsed.length} products`);
        for (const p of parsed) {
          if (!seenUrls.has(p.url)) { seenUrls.add(p.url); allProducts.push(p); }
        }
      }
      await sleep(2000);
    } catch {
      // API endpoint not available — continue to HTML scraping
    }
  }

  // HTML catalog fallback
  for (const catalogPath of CATALOG_URLS) {
    const catalogUrl = BASE + catalogPath;
    console.log(`  Fetching catalog: ${catalogUrl}`);
    try {
      const html = await fetchPage(catalogUrl);
      const pageProducts = parseHtmlProducts(html, catalogUrl);
      for (const p of pageProducts) {
        if (!seenUrls.has(p.url)) { seenUrls.add(p.url); allProducts.push(p); }
      }
      console.log(`    Found ${pageProducts.length} products`);
      await sleep(2000);
    } catch (err) {
      console.warn(`    Failed ${catalogPath}: ${(err as Error).message}`);
    }
  }

  console.log(`\nTotal unique products: ${allProducts.length}`);

  let totalProducts = 0;

  for (const product of allProducts) {
    try {
      await findOrCreateScrapedTransceiver({
        partNumber: product.partNumber,
        vendorId,
        formFactor: product.formFactor,
        speedGbps: product.speedGbps,
        speed: product.speed,
        reachMeters: product.reachMeters,
        reachLabel: product.reachLabel,
        fiberType: product.fiberType,
        wavelengths: product.wavelength,
        category: "DataCenter",
      });
      totalProducts++;
    } catch (err) {
      console.warn(`  Error saving ${product.partNumber}: ${(err as Error).message.slice(0, 80)}`);
    }
  }

  console.log(`\n=== HUBER+SUHNER Complete: ${totalProducts} products catalogued (no prices — contact sales) ===`);
}

if (require.main === module) {
  scrapeHuberSuhner()
    .then(() => pool.end())
    .catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); });
}