/**
 * NADDOD Scraper v2 — Chinese compatible transceiver vendor
 *
 * naddod.com — Migrated from WooCommerce → Astro/Shopify-style in 2025.
 * Product URLs: /products/XXXXX.html (numeric IDs, not category slugs)
 * Sitemap: /sitemaps/products.xml
 *
 * Phase 1: Parse sitemap to collect all product URLs (plain HTTP)
 * Phase 2: Fetch product detail pages — extract name, price, stock count
 *   Stock format: "In Stock: 543" (exact) | "In Stock: 2.1k+" (rounded) | "In Stock: Available" (boolean)
 *   Per-warehouse JSON (warehouse_stock: {us, nl, sg, cn}) is in a JS hydration
 *   payload that requires JS execution — only the display count is in plain HTML.
 *   → stock_confidence=2 (aggregated global count) for exact/rounded counts
 *   → stock_confidence=1 (boolean) for "Available" only
 *
 * Rate limited: 1 req/2sec.
 */
import {
  pool,
  findOrCreateScrapedTransceiver,
  ensureVendor,
  upsertPriceObservation,
  upsertStockObservation,
  markImageVerified,
  markDetailsVerified,
} from "../utils/db";
import { contentHash } from "../utils/hash";
import { readFileSync, writeFileSync, existsSync } from "node:fs";
import { join } from "node:path";

const BASE = "https://www.naddod.com";
const SITEMAP_URL = `${BASE}/sitemaps/products.xml`;
const HEADERS = {
  "User-Agent": "Mozilla/5.0 (compatible; TIP-Bot/1.0; research)",
  Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
  "Accept-Language": "en-US,en;q=0.9",
};

// Limit detail-page fetches per run to stay reasonable
const MAX_DETAIL_PAGES = Math.max(1, Math.min(1000, parseInt(process.env["NADDOD_MAX_DETAIL_PAGES"] || "600", 10)));
const DB_DETAIL_ONLY = process.env["NADDOD_DB_DETAIL_ONLY"] === "1";

// Cursor file: persists across runs so each run covers the next 600 URLs
const CURSOR_FILE = join(process.env["TIP_STORAGE_DIR"] ?? "/opt/tip", "naddod-cursor.json");

function readCursor(): number {
  try {
    if (existsSync(CURSOR_FILE)) {
      const data = JSON.parse(readFileSync(CURSOR_FILE, "utf-8")) as { offset: number };
      return typeof data.offset === "number" && data.offset >= 0 ? data.offset : 0;
    }
  } catch { /* fall through */ }
  return 0;
}

function writeCursor(offset: number): void {
  try {
    writeFileSync(CURSOR_FILE, JSON.stringify({ offset, updated: new Date().toISOString() }));
  } catch (err) {
    console.warn(`  [cursor] Failed to write cursor: ${(err as Error).message}`);
  }
}

function sleep(ms: number): Promise<void> {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

// ── Classification helpers ──────────────────────────────────────────────────

function detectFormFactor(text: string): string {
  const t = text.toLowerCase();
  if (/\bosfp\b/.test(t))                                return "OSFP";
  if (/\bqsfp.?dd800\b|\bqsfp.?dd\s+800\b/.test(t))    return "QSFP-DD800";
  if (/\bqsfp.?dd\b/.test(t))                            return "QSFP-DD";
  if (/\bqsfp56\b/.test(t))                              return "QSFP56";
  if (/\bqsfp112\b/.test(t))                             return "QSFP112";
  if (/\bqsfp28\b/.test(t))                              return "QSFP28";
  if (/\bqsfp\+|\bqsfp\s*plus\b/.test(t))               return "QSFP+";
  if (/\bsfp28\b/.test(t))                               return "SFP28";
  if (/\bsfp.?\+|10g.*sfp|sfp.*10g/.test(t))            return "SFP+";
  if (/\bsfp\b/.test(t))                                 return "SFP";
  if (/\bxfp\b/.test(t))                                 return "XFP";
  return "SFP+"; // default
}

function detectSpeedGbps(text: string): { speed: string; speedGbps: number } {
  const t = text.toUpperCase();
  if (/\b800G\b|\b800GBE\b/.test(t))  return { speed: "800G", speedGbps: 800 };
  if (/\b400G\b|\b400GBE\b/.test(t))  return { speed: "400G", speedGbps: 400 };
  if (/\b200G\b|\b200GBE\b/.test(t))  return { speed: "200G", speedGbps: 200 };
  if (/\b100G\b|\b100GBE\b/.test(t))  return { speed: "100G", speedGbps: 100 };
  if (/\b40G\b|\b40GBE\b/.test(t))    return { speed: "40G",  speedGbps: 40  };
  if (/\b25G\b|\b25GBE\b/.test(t))    return { speed: "25G",  speedGbps: 25  };
  if (/\b10G\b|\b10GBE\b/.test(t))    return { speed: "10G",  speedGbps: 10  };
  if (/\b1G\b|\b1GBE\b|\bGIGABIT\b/.test(t)) return { speed: "1G", speedGbps: 1 };
  return { speed: "Unknown", speedGbps: 0 };
}

function detectReach(text: string): { label: string; meters: number } | undefined {
  const cableCode = text.match(/\b(?:AOC|DAC|CU|COPPER|MCP|MFS)[A-Z0-9._-]*?(\d+(?:\.\d+)?)M\b/i);
  if (cableCode) {
    const value = parseFloat(cableCode[1]);
    if (Number.isFinite(value) && value > 0 && value <= 500) {
      return { label: `${String(value).replace(/\.0$/, "")}m`, meters: Math.round(value) };
    }
  }

  const generic = text.match(/\b(\d+(?:\.\d+)?)\s*(km|m)\b/i);
  if (generic) {
    const value = parseFloat(generic[1]);
    const unit = generic[2].toLowerCase();
    if (Number.isFinite(value) && value > 0) {
      const meters = unit === "km" ? Math.round(value * 1000) : Math.round(value);
      const labelValue = String(value).replace(/\.0$/, "");
      const label = unit === "km" ? `${labelValue}km` : `${labelValue}m`;
      return { label, meters };
    }
  }

  const patterns: [RegExp, string, number][] = [
    [/\b120\s*km\b/i, "120km", 120000],
    [/\b80\s*km\b/i, "80km", 80000],
    [/\b40\s*km\b/i, "40km", 40000],
    [/\b20\s*km\b/i, "20km", 20000],
    [/\b10\s*km\b/i, "10km", 10000],
    [/\b2\s*km\b/i, "2km", 2000],
    [/\b550\s*m\b/i, "550m", 550],
    [/\b500\s*m\b/i, "500m", 500],
    [/\b400\s*m\b/i, "400m", 400],
    [/\b300\s*m\b/i, "300m", 300],
    [/\b150\s*m\b/i, "150m", 150],
    [/\b100\s*m\b/i, "100m", 100],
    [/\bLR4\b/, "10km", 10000], [/\bLR\b/, "10km", 10000],
    [/\bER4?\b/, "40km", 40000], [/\bZRP?\b|\bZR4?\b/, "80km", 80000],
    [/\bSR\d*\b|\bVR\d*\b/, "100m", 100],
    [/\bDR4?\b|\bXDR\d*\b/, "500m", 500],
    [/\bFR4?\b/, "2km", 2000],
  ];
  for (const [re, label, meters] of patterns) {
    if (re.test(text)) return { label, meters };
  }
  return undefined;
}

function detectFiber(text: string): string {
  if (/copper|dac|twinax|rj.?45|base-t/i.test(text)) return "Copper";
  if (/aoc|active.?optical|active.?fiber|[^a-z]sr\d*[^a-z]?|[^a-z]vr\d*[^a-z]?|850\s*nm/i.test(text)) return "MMF";
  if (/single.?mode|smf|[^a-z]lx[^a-z]|[^a-z]lr[^a-z]|[^a-z]er[^a-z]|[^a-z]zr[^a-z]?|[^a-z]dr\d*[^a-z]?|[^a-z]fr\d*[^a-z]?|xdr\d*|psm|bidi|cwdm|dwdm|1310\s*nm|1550\s*nm/i.test(text)) return "SMF";
  if (/multi.?mode|mmf|[^a-z]sx[^a-z]/i.test(text)) return "MMF";
  return "";
}

function detectWavelength(text: string): string {
  const m = text.match(/(\d{3,4})\s*nm/i);
  return m ? m[1] : "";
}

function isTransceiver(name: string): boolean {
  const lower = name.toLowerCase();
  // Include: SFP, QSFP, OSFP, XFP, DAC/AOC cables count as transceivers for stock tracking
  return /sfp|qsfp|osfp|xfp|transceiver|dac|aoc|cwdm|dwdm/i.test(lower);
}

// ── Stock parsing ───────────────────────────────────────────────────────────

/**
 * Parse NADDOD stock display text.
 * "In Stock: 543"   → { qty: 543, confidence: 2 }
 * "In Stock: 2.1k+" → { qty: 2100, confidence: 2 } (approximate, rounded)
 * "In Stock: Available" → { qty: undefined, confidence: 1 }
 * Returns null if no stock text found.
 */
function parseStockText(html: string): { qty?: number; confidence: 1 | 2 } | null {
  // Look for "In Stock: X" pattern in page text
  const m = html.match(/In\s+Stock[:\s]+([^\s<"]+)/i);
  if (!m) return null;

  const raw = m[1].trim().toLowerCase();

  // "Available" = boolean only
  if (/^avail/i.test(raw)) return { confidence: 1 };

  // Numeric: "543" or "2.1k+" or "1.5k+"
  const kMatch = raw.match(/^([\d.]+)k\+?$/);
  if (kMatch) {
    const qty = Math.round(parseFloat(kMatch[1]) * 1000);
    return { qty: isNaN(qty) ? undefined : qty, confidence: 2 };
  }

  const exact = parseInt(raw.replace(/[^0-9]/g, ""), 10);
  if (!isNaN(exact) && exact >= 0) return { qty: exact, confidence: 2 };

  return { confidence: 1 }; // fallback: boolean
}


/**
 * Parse per-warehouse stock breakdown from NADDOD HTML.
 * The data lives as HTML-entity-encoded JSON in a hydration payload:
 *   &quot;warehouse_stock&quot;:[0,{&quot;us&quot;:[0,543],&quot;nl&quot;:[0,211],&quot;sg&quot;:[0,0],&quot;cn&quot;:[0,0]}]
 * Format per region: [signalBit, quantity]
 * Mapping: us → US, nl → NL/EU (closest to DE), sg → APAC, cn → CN
 * We use nl as DE-equivalent (EU warehouse) and sum all for global.
 */
function parseWarehouseStock(html: string): {
  eu: number | null;   // nl warehouse → DE-equivalent
  global: number | null; // us+nl+sg+cn total
} | null {
  // Strip HTML entities so we can JSON.parse
  const decoded = html
    .replace(/&quot;/g, '"')
    .replace(/&amp;/g, '&')
    .replace(/&#39;/g, "'");

  const m = decoded.match(/"warehouse_stock":\[0,(\{[^}]+\})\]/);
  if (!m) return null;

  try {
    const ws = JSON.parse(m[1]) as Record<string, unknown>;
    function qty(key: string): number {
      const v = ws[key];
      if (Array.isArray(v) && v.length >= 2 && typeof v[1] === 'number') return Math.max(0, v[1]);
      return 0;
    }
    const us = qty('us'), nl = qty('nl'), sg = qty('sg'), cn = qty('cn');
    const total = us + nl + sg + cn;
    // Only return data if at least one warehouse has stock
    if (total === 0 && nl === 0) return null; // no warehouse breakdown available
    return { eu: nl, global: total };
  } catch {
    return null;
  }
}

// ── HTTP helpers ────────────────────────────────────────────────────────────

async function fetchText(url: string): Promise<string> {
  const resp = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(30000) });
  if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
  return resp.text();
}

/**
 * Parse product detail page HTML → extract name, price, stock.
 */
function parseDetailPage(html: string, url: string): {
  name: string;
  imageUrl?: string;
  price?: number;
  stock: { qty?: number; confidence: 1 | 2 } | null;
} | null {
  // Product name: og:title or <title> or <h1>
  const ogTitle = html.match(/<meta\s+property="og:title"\s+content="([^"]+)"/i)?.[1];
  const h1 = html.match(/<h1[^>]*>([^<]{15,})<\/h1>/i)?.[1]?.trim();
  const titleTag = html.match(/<title[^>]*>([^<]+)<\/title>/i)?.[1]?.trim();
  const name = (ogTitle || h1 || titleTag || "").replace(/\s+/g, " ").slice(0, 200);

  if (!name || name.length < 10) return null;
  if (!isTransceiver(name)) return null;

  const imageUrl = (() => {
    const candidates = [
      html.match(/<meta\s+property="og:image"\s+content="([^"]+)"/i)?.[1],
      html.match(/<meta\s+name="twitter:image"\s+content="([^"]+)"/i)?.[1],
      html.match(/"image"\s*:\s*"([^"]+)"/i)?.[1],
    ].filter(Boolean) as string[];

    const img = candidates.find((candidate) =>
      !/(logo|placeholder|default|no-image|icon|sprite)/i.test(candidate)
    );
    if (!img) return undefined;
    return img.startsWith("http") ? img : `${BASE}${img.startsWith("/") ? "" : "/"}${img}`;
  })();

  // Price: prefer LD+JSON offers.price (Astro/Shopify structure, e.g. "price":"731.00")
  // Fall back to "US$ 10.90" or "$10.90" visible text patterns
  let price: number | undefined;
  const ldJsonMatch = html.match(/<script[^>]+type="application\/ld\+json"[^>]*>([\s\S]*?)<\/script>/gi);
  if (ldJsonMatch) {
    for (const block of ldJsonMatch) {
      try {
        const jsonStr = block.replace(/<script[^>]*>/, "").replace(/<\/script>/, "");
        const ld = JSON.parse(jsonStr) as Record<string, unknown>;
        const offers = ld["offers"] as Record<string, unknown> | undefined;
        if (offers && typeof offers["price"] === "string") {
          const p = parseFloat(offers["price"].replace(/,/g, ""));
          if (p > 0 && p < 500000) { price = p; break; }
        }
        if (offers && typeof offers["price"] === "number" && offers["price"] > 0 && (offers["price"] as number) < 500000) {
          price = offers["price"] as number;
          break;
        }
      } catch { /* skip malformed blocks */ }
    }
  }
  if (!price) {
    const priceMatch = html.match(/US\$\s*([\d,]+\.?\d{0,2})/i) ||
                       html.match(/\$\s*([\d,]+\.\d{2})\b/);
    if (priceMatch) {
      const p = parseFloat(priceMatch[1].replace(/,/g, ""));
      if (p > 0 && p < 500000) price = p;
    }
  }

  // Stock count
  const stock = parseStockText(html);

  return { name, imageUrl, price, stock };
}

// ── Sitemap parsing ─────────────────────────────────────────────────────────

async function fetchProductUrlsFromSitemap(): Promise<string[]> {
  console.log(`  Fetching sitemap: ${SITEMAP_URL}`);
  const xml = await fetchText(SITEMAP_URL);

  // Extract all <loc> product URLs — supports both:
  //   /products/12345.html  (most products)
  //   /products/brand-slug/12345  (new URL style without .html)
  const urls: string[] = [];
  const locRegex = /<loc>([^<]+\/products\/[^<\s]+)<\/loc>/gi;
  let m: RegExpExecArray | null;
  while ((m = locRegex.exec(xml)) !== null) {
    const url = m[1].trim();
    // Keep only canonical English URLs (no language prefix like /en/, /de/, /fr/, /ja/ etc.)
    const hasLangPrefix = /\/[a-z]{2}\/products\//.test(url);
    if (!hasLangPrefix) {
      urls.push(url);
    }
  }

  return [...new Set(urls)]; // deduplicate
}

async function fetchDbTargets(limit: number): Promise<Array<{ url: string; targetTransceiverId: string }>> {
  const result = await pool.query<{ id: string; product_page_url: string }>(`
    SELECT t.id, t.product_page_url
    FROM transceivers t
    JOIN vendors v ON v.id = t.vendor_id
    WHERE v.name = 'NADDOD'
      AND t.product_page_url IS NOT NULL
      AND t.product_page_url != ''
      AND (t.details_verified = false OR t.image_verified = false)
    ORDER BY t.details_verified ASC, t.image_verified ASC, t.updated_at ASC
    LIMIT $1
  `, [limit]);

  return result.rows.map((row) => ({
    url: row.product_page_url,
    targetTransceiverId: row.id,
  }));
}

// ── Main scraper ────────────────────────────────────────────────────────────

export async function scrapeNaddod(): Promise<void> {
  console.log("=== NADDOD Scraper v2 Starting (sitemap + detail mode) ===\n");

  const vendorId = await ensureVendor(
    "NADDOD",
    "compatible",
    "https://www.naddod.com",
    "https://www.naddod.com/collection/optical-transceivers",
  );

  // ── Phase 1: Discover product URLs via sitemap ────────────────────────────
  console.log("[Phase 1] Discovering products from sitemap...");
  let productUrls: string[] = [];
  let targets: Array<{ url: string; targetTransceiverId?: string }> = [];
  try {
    if (DB_DETAIL_ONLY) {
      targets = await fetchDbTargets(MAX_DETAIL_PAGES);
      productUrls = targets.map((target) => target.url);
      console.log(`  DB detail targets: ${productUrls.length}`);
    } else {
      productUrls = await fetchProductUrlsFromSitemap();
      console.log(`  Found ${productUrls.length} product URLs in sitemap`);
    }
  } catch (err) {
    console.error(`  Target discovery failed: ${(err as Error).message}`);
    return;
  }

  if (productUrls.length === 0) {
    console.warn("  No product URLs found — aborting");
    return;
  }

  // Cursor-based rotation: each run advances by MAX_DETAIL_PAGES so over ~12 runs
  // (24 hours) we cover all ~7300 products. Wraps around when exhausted.
  const totalUrls = productUrls.length;
  const offset = DB_DETAIL_ONLY ? 0 : readCursor() % totalUrls;
  const endIdx = Math.min(offset + MAX_DETAIL_PAGES, totalUrls);
  let batchTargets: Array<{ url: string; targetTransceiverId?: string }> =
    DB_DETAIL_ONLY ? targets : productUrls.slice(offset, endIdx).map((url) => ({ url }));
  if (!DB_DETAIL_ONLY && batchTargets.length < MAX_DETAIL_PAGES && totalUrls > MAX_DETAIL_PAGES) {
    const wrap = MAX_DETAIL_PAGES - batchTargets.length;
    batchTargets = batchTargets.concat(productUrls.slice(0, wrap).map((url) => ({ url })));
  }
  const nextOffset = DB_DETAIL_ONLY ? offset : (offset + MAX_DETAIL_PAGES) % totalUrls;
  console.log(`  Offset: ${offset}/${totalUrls} → processing ${batchTargets.length} products (next run starts at ${nextOffset})`);

  // ── Phase 2: Fetch detail pages + write to DB ─────────────────────────────
  console.log("\n[Phase 2] Fetching product detail pages...");

  let processed = 0;
  let priceUpdates = 0;
  let stockWritten = 0;
  let stockSkipped = 0;
  let skippedNonTx = 0;
  let errors = 0;

  for (const target of batchTargets) {
    const url = target.url;
    await sleep(2000);
    try {
      const html = await fetchText(url);
      const detail = parseDetailPage(html, url);

      if (!detail) {
        skippedNonTx++;
        continue;
      }

      const { name, imageUrl, price, stock } = detail;
      const evidenceText = `${name} ${html.replace(/<[^>]+>/g, " ").slice(0, 20000)}`;
      const { speed, speedGbps } = detectSpeedGbps(evidenceText);
      const formFactor = detectFormFactor(evidenceText);
      const reach = detectReach(evidenceText);
      const fiberType = detectFiber(evidenceText);
      const wavelength = detectWavelength(evidenceText);

      // Extract part number from name (first word-group before "Compatible" or vendor name)
      const partNumber = name.split(/\s+(?:compatible|for\s+[A-Z]|sfp|qsfp)/i)[0]?.trim().slice(0, 80) || name.slice(0, 60);

      let txId = target.targetTransceiverId;

      if (txId) {
        await pool.query(`
          UPDATE transceivers
          SET product_page_url = COALESCE(NULLIF(product_page_url, ''), $2),
              form_factor = COALESCE(NULLIF($3::text, ''), form_factor),
              speed_gbps = CASE WHEN $4::numeric > 0 THEN $4::numeric ELSE speed_gbps END,
              speed = CASE WHEN $4::numeric > 0 THEN $5 ELSE speed END,
              reach_meters = CASE WHEN $6::int IS NOT NULL AND $6::int > 0 THEN $6::int ELSE reach_meters END,
              reach_label = COALESCE(NULLIF($7::text, ''), reach_label),
              fiber_type = COALESCE(NULLIF($8::text, ''), fiber_type),
              wavelengths = COALESCE(NULLIF($9::text, ''), wavelengths),
              category = COALESCE(NULLIF(category, ''), 'DataCenter'),
              updated_at = NOW()
          WHERE id = $1
            AND vendor_id = $10
        `, [
          txId,
          url,
          formFactor,
          speedGbps,
          speed,
          reach?.meters ?? null,
          reach?.label ?? null,
          fiberType || null,
          wavelength || null,
          vendorId,
        ]);
        if (imageUrl) await markImageVerified(txId, imageUrl);
        await markDetailsVerified({ transceiverId: txId, sourceUrl: url });
      } else {
        txId = await findOrCreateScrapedTransceiver({
          partNumber,
          vendorId,
          productUrl: url,
          formFactor,
          speedGbps,
          speed,
          reachMeters: reach?.meters,
          reachLabel: reach?.label,
          fiberType,
          wavelengths: wavelength,
          category: "DataCenter",
          imageUrl,
        });
      }

      // Price observation
      if (price && price > 0) {
        const hash = contentHash({ price, part: partNumber });
        const isNew = await upsertPriceObservation({
          transceiverId: txId,
          sourceVendorId: vendorId,
          price,
          currency: "USD",
          stockLevel: stock?.qty !== undefined && stock.qty > 0 ? "in_stock" :
                      stock?.confidence === 1 ? "in_stock" : "on_request",
          url,
          contentHash: hash,
        });
        if (isNew) priceUpdates++;
      }

      // Stock observation — enhanced with per-warehouse breakdown
      if (stock !== null) {
        const stockLevel = stock.qty !== undefined ? (stock.qty > 0 ? "in_stock" : "out_of_stock") : "in_stock";
        const warehouseData = parseWarehouseStock(html);
        const isNew = await upsertStockObservation({
          transceiverId: txId,
          sourceVendorId: vendorId,
          stockLevel,
          quantityAvailable: stock.qty !== undefined && stock.qty > 0 ? stock.qty : undefined,
          // NL warehouse ≈ EU/DE equivalent; sum of all warehouses = global
          warehouseDeQty: warehouseData?.eu ?? undefined,
          warehouseGlobalQty: warehouseData?.global ?? (stock.qty !== undefined && stock.qty > 0 ? stock.qty : undefined),
          productUrl: url,
          stockConfidence: warehouseData ? 3 : stock.confidence, // L3 when per-warehouse available
          priceCurrency: "USD",
          priceIncludesTax: false,
        });
        if (isNew) stockWritten++;
        else stockSkipped++;
      }

      processed++;
      if (processed % 50 === 0) {
        console.log(`  Progress: ${processed}/${batchTargets.length} | prices: ${priceUpdates} | stock: ${stockWritten} new / ${stockSkipped} unchanged`);
      }
    } catch (err) {
      errors++;
      if (errors <= 10) console.warn(`  Error for ${url}: ${(err as Error).message.slice(0, 120)}`);
    }
  }

  // Advance cursor for next run
  if (!DB_DETAIL_ONLY) writeCursor(nextOffset);

  console.log("\n=== NADDOD Scraper v2 Complete ===");
  console.log(`  URL range processed:   ${offset}–${offset + batchTargets.length - 1} of ${totalUrls}`);
  console.log(`  Products processed:    ${processed}`);
  console.log(`  Non-transceivers skip: ${skippedNonTx}`);
  console.log(`  Price observations:    ${priceUpdates} new`);
  console.log(`  Stock observations:    ${stockWritten} new, ${stockSkipped} unchanged`);
  if (errors > 0) console.warn(`  Errors:                ${errors}`);
  console.log(`  Next run starts at:    offset ${nextOffset}`);
}

if (require.main === module) {
  scrapeNaddod()
    .then(() => pool.end())
    .catch((err) => { console.error("Fatal:", err); pool.end(); process.exit(1); });
}
