transceiver-db/packages/scraper/src/scrapers/ebay-velocity.ts

/**
 * eBay Marketplace Velocity Scraper
 *
 * Tracks SOLD listing counts and price distributions to measure
 * actual market demand velocity — a 1-3 month leading indicator.
 *
 * Scrapes eBay's public completed/sold listings page (no API needed).
 * Looks at: sold count last 30 days, active listing count, price spread.
 *
 * High sold velocity + rising prices → buy signal
 * Falling velocity + dropping prices → commodity transition in progress
 */

import * as cheerio from "cheerio";
import { pool } from "../utils/db";
import { logger } from "../utils/logger";

const HEADERS = {
  "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
  "Accept-Language": "en-US,en;q=0.9",
  "Accept": "text/html,application/xhtml+xml,application/xhtml,application/xml;q=0.9,*/*;q=0.8",
};

const SEARCH_TERMS: Array<{ keyword: string; formFactor: string; speedLabel: string }> = [
  { keyword: "QSFP28 100G transceiver",     formFactor: "QSFP28",   speedLabel: "100G"  },
  { keyword: "QSFP+ 40G transceiver",       formFactor: "QSFP+",    speedLabel: "40G"   },
  { keyword: "QSFP28 400G transceiver",     formFactor: "QSFP28",   speedLabel: "400G"  },
  { keyword: "QSFP-DD 400G transceiver",    formFactor: "QSFP-DD",  speedLabel: "400G"  },
  { keyword: "QSFP-DD 800G transceiver",    formFactor: "QSFP-DD",  speedLabel: "800G"  },
  { keyword: "SFP+ 10G transceiver",        formFactor: "SFP+",     speedLabel: "10G"   },
  { keyword: "OSFP 400G transceiver",       formFactor: "OSFP",     speedLabel: "400G"  },
  { keyword: "CFP2 100G coherent DCO",      formFactor: "CFP2",     speedLabel: "100G"  },
  { keyword: "400ZR coherent transceiver",  formFactor: "QSFP-DD",  speedLabel: "400G-ZR"},
];

function parsePrice(text: string): number | null {
  const m = text.replace(/,/g, "").match(/[\$£€]?\s*([\d.]+)/);
  return m ? parseFloat(m[1]) : null;
}

async function scrapeEbaySoldListings(
  keyword: string
): Promise<{ soldCount: number; activeCount: number; avgSoldPrice: number | null; minPrice: number | null; maxPrice: number | null }> {

  const enc = encodeURIComponent(keyword);
  // Completed (sold) listings
  const soldUrl = `https://www.ebay.com/sch/i.html?_nkw=${enc}&LH_Complete=1&LH_Sold=1&_sop=12&_ipg=200`;

  const res = await fetch(soldUrl, { headers: HEADERS, signal: AbortSignal.timeout(20000) });
  if (!res.ok) throw new Error(`eBay returned ${res.status}`);
  const html = await res.text();
  const $ = cheerio.load(html);

  const prices: number[] = [];
  let soldCount = 0;

  // Count sold items and extract prices
  $(".s-item").each((_, el) => {
    const priceText = $(el).find(".s-item__price").first().text().trim();
    const price = parsePrice(priceText);
    const isSold = $(el).find(".POSITIVE").length > 0 || $(el).find(".s-item__caption--signal").text().includes("Sold");
    if (price && price > 0 && price < 50000) {
      prices.push(price);
      if (isSold) soldCount++;
    }
  });

  // Total count from results summary
  const resultText = $(".srp-controls__count-heading").text();
  const totalMatch = resultText.replace(/,/g, "").match(/([\d,]+)/);
  if (totalMatch) soldCount = Math.max(soldCount, parseInt(totalMatch[1]));

  // Active listings (separate search)
  await new Promise(r => setTimeout(r, 1500));
  const activeUrl = `https://www.ebay.com/sch/i.html?_nkw=${enc}&_sop=12&_ipg=1&LH_BIN=1`;
  let activeCount = 0;
  try {
    const activeRes = await fetch(activeUrl, { headers: HEADERS, signal: AbortSignal.timeout(15000) });
    if (activeRes.ok) {
      const activeHtml = await activeRes.text();
      const $a = cheerio.load(activeHtml);
      const activeText = $a(".srp-controls__count-heading").text().replace(/,/g, "");
      const am = activeText.match(/([\d]+)/);
      if (am) activeCount = parseInt(am[1]);
    }
  } catch { /* ignore active count failure */ }

  const avgSoldPrice = prices.length > 0
    ? Math.round(prices.reduce((a, b) => a + b, 0) / prices.length * 100) / 100
    : null;

  return {
    soldCount,
    activeCount,
    avgSoldPrice,
    minPrice: prices.length > 0 ? Math.min(...prices) : null,
    maxPrice: prices.length > 0 ? Math.max(...prices) : null,
  };
}

export async function scrapeEbayVelocity(): Promise<void> {
  logger.info("eBay velocity scraper starting");
  let recorded = 0;

  for (const term of SEARCH_TERMS) {
    try {
      logger.info(`Checking eBay velocity: ${term.keyword}`);
      await new Promise(r => setTimeout(r, 3000)); // be gentle

      const result = await scrapeEbaySoldListings(term.keyword);

      await pool.query(`
        INSERT INTO marketplace_velocity
          (marketplace, keyword, form_factor, speed_label, sold_count_30d,
           active_listings, avg_sold_price, min_price, max_price, currency)
        VALUES ('ebay', $1, $2, $3, $4, $5, $6, $7, $8, 'USD')
      `, [
        term.keyword, term.formFactor, term.speedLabel,
        result.soldCount, result.activeCount, result.avgSoldPrice,
        result.minPrice, result.maxPrice,
      ]);
      recorded++;

      logger.info(
        `${term.speedLabel} ${term.formFactor}: ${result.soldCount} sold, ` +
        `${result.activeCount} active, avg $${result.avgSoldPrice ?? "?"}`
      );
    } catch (err) {
      logger.warn(`eBay velocity failed: ${term.keyword}`, { err });
    }
  }

  logger.info(`eBay velocity scraper done — ${recorded} records`);
}