diff --git a/packages/scraper/src/scrapers/ascentoptics.ts b/packages/scraper/src/scrapers/ascentoptics.ts index d23b7c1..3fe4e90 100644 --- a/packages/scraper/src/scrapers/ascentoptics.ts +++ b/packages/scraper/src/scrapers/ascentoptics.ts @@ -19,7 +19,7 @@ const CATALOG_URLS = [ ]; const MAX_PAGES = 15; const HEADERS = { - "User-Agent": "Mozilla/5.0 (compatible; TIP-Bot/1.0; research)", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", Accept: "text/html,application/xhtml+xml", "Accept-Language": "en-US,en;q=0.9", }; diff --git a/packages/scraper/src/scrapers/blueoptics.ts b/packages/scraper/src/scrapers/blueoptics.ts index f651c3b..5e605a1 100644 --- a/packages/scraper/src/scrapers/blueoptics.ts +++ b/packages/scraper/src/scrapers/blueoptics.ts @@ -14,7 +14,7 @@ const BASE = "https://www.blueoptics.de"; const CATALOG_PATH = "/transceivers/"; const MAX_PAGES = 20; const HEADERS = { - "User-Agent": "Mozilla/5.0 (compatible; TIP-Bot/1.0; research)", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", Accept: "text/html,application/xhtml+xml", "Accept-Language": "de-DE,de;q=0.9,en;q=0.8", }; diff --git a/packages/scraper/src/scrapers/champion-one.ts b/packages/scraper/src/scrapers/champion-one.ts index e8a1c6b..b426cd2 100644 --- a/packages/scraper/src/scrapers/champion-one.ts +++ b/packages/scraper/src/scrapers/champion-one.ts @@ -11,7 +11,7 @@ import { contentHash } from "../utils/hash"; const BASE = "https://www.championone.com"; const HEADERS = { - "User-Agent": "Mozilla/5.0 (compatible; TIP-Bot/1.0; research)", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", Accept: "text/html,application/xhtml+xml", }; diff --git a/packages/scraper/src/scrapers/comms-express.ts b/packages/scraper/src/scrapers/comms-express.ts index b8aed74..a41afb3 100644 --- a/packages/scraper/src/scrapers/comms-express.ts +++ b/packages/scraper/src/scrapers/comms-express.ts @@ -53,7 +53,7 @@ interface Product { async function fetchCategory(cat: typeof CATEGORIES[0], vendorId: string, page = 1): Promise { const url = `${BASE}${cat.url}?page=${page}`; const resp = await fetch(url, { - headers: { "User-Agent": "Mozilla/5.0 (compatible; TIPBot/1.0; +https://tip.context-x.org)" }, + headers: { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" }, signal: AbortSignal.timeout(30_000), }); if (!resp.ok) return []; diff --git a/packages/scraper/src/scrapers/fiber24.ts b/packages/scraper/src/scrapers/fiber24.ts index 8d93009..de2eb97 100644 --- a/packages/scraper/src/scrapers/fiber24.ts +++ b/packages/scraper/src/scrapers/fiber24.ts @@ -14,7 +14,7 @@ const BASE = "https://shop.fiber24.net"; const CATALOG_PATH = "/FO-TRANSCEIVER/de"; const MAX_PAGES = 20; const HEADERS = { - "User-Agent": "Mozilla/5.0 (compatible; TIP-Bot/1.0; research)", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", Accept: "text/html,application/xhtml+xml", "Accept-Language": "de-DE,de;q=0.9,en;q=0.8", }; diff --git a/packages/scraper/src/scrapers/fluxlight.ts b/packages/scraper/src/scrapers/fluxlight.ts index 3f4fdae..5278a13 100644 --- a/packages/scraper/src/scrapers/fluxlight.ts +++ b/packages/scraper/src/scrapers/fluxlight.ts @@ -12,7 +12,7 @@ import { contentHash } from "../utils/hash"; const BASE = "https://fluxlight.com"; const CATALOG_PATH = "/transceivers/"; const HEADERS = { - "User-Agent": "Mozilla/5.0 (compatible; TIP-Bot/1.0; research)", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", Accept: "text/html,application/xhtml+xml", }; diff --git a/packages/scraper/src/scrapers/gaotek.ts b/packages/scraper/src/scrapers/gaotek.ts index 6775a59..e02c83a 100644 --- a/packages/scraper/src/scrapers/gaotek.ts +++ b/packages/scraper/src/scrapers/gaotek.ts @@ -14,7 +14,7 @@ const BASE = "https://gaotek.com"; const CATALOG_PATH = "/category/fiber-optics/transceivers/"; const MAX_PAGES = 20; const HEADERS = { - "User-Agent": "Mozilla/5.0 (compatible; TIP-Bot/1.0; research)", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", Accept: "text/html,application/xhtml+xml", "Accept-Language": "en-US,en;q=0.9", }; diff --git a/packages/scraper/src/scrapers/gbics.ts b/packages/scraper/src/scrapers/gbics.ts index 92aafe0..c231f7f 100644 --- a/packages/scraper/src/scrapers/gbics.ts +++ b/packages/scraper/src/scrapers/gbics.ts @@ -10,8 +10,9 @@ import { contentHash } from "../utils/hash"; const BASE = "https://www.gbics.com"; const HEADERS = { - "User-Agent": "Mozilla/5.0 (compatible; TIP-Bot/1.0; research)", - Accept: "text/html,application/xhtml+xml", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", }; const CATEGORIES = [ diff --git a/packages/scraper/src/scrapers/hubersuhner.ts b/packages/scraper/src/scrapers/hubersuhner.ts index b16448a..f1d95ea 100644 --- a/packages/scraper/src/scrapers/hubersuhner.ts +++ b/packages/scraper/src/scrapers/hubersuhner.ts @@ -27,7 +27,7 @@ const API_URLS = [ ]; const HEADERS = { - "User-Agent": "Mozilla/5.0 (compatible; TIP-Bot/1.0; research)", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", Accept: "text/html,application/xhtml+xml,application/json", "Accept-Language": "en-US,en;q=0.9,de;q=0.8", }; diff --git a/packages/scraper/src/scrapers/multimode-inc.ts b/packages/scraper/src/scrapers/multimode-inc.ts index 6115028..9bebc4c 100644 --- a/packages/scraper/src/scrapers/multimode-inc.ts +++ b/packages/scraper/src/scrapers/multimode-inc.ts @@ -38,7 +38,7 @@ export async function scrapeMultimodeInc(): Promise { for (const cat of CATEGORIES) { try { const resp = await fetch(`${BASE}${cat.path}`, { - headers: { "User-Agent": "Mozilla/5.0 (compatible; TIPBot/1.0)" }, + headers: { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" }, signal: AbortSignal.timeout(25_000), }); if (!resp.ok) continue; diff --git a/packages/scraper/src/scrapers/optcore.ts b/packages/scraper/src/scrapers/optcore.ts index 2a37f25..979ff56 100644 --- a/packages/scraper/src/scrapers/optcore.ts +++ b/packages/scraper/src/scrapers/optcore.ts @@ -110,7 +110,7 @@ async function fetchTransceiverUrls(): Promise { - logger.info("OpticTransceiver.com scraper starting"); + // Domain repurposed as plant shop (2026-04-06) — skip entirely + logger.warn("OpticTransceiver.com is no longer an optics vendor (domain repurposed). Scraper disabled."); + return; const vendorId = await ensureVendor("OpticTransceiver", BASE); let total = 0; diff --git a/packages/scraper/src/scrapers/router-switch.ts b/packages/scraper/src/scrapers/router-switch.ts index 25e0d76..a3ac564 100644 --- a/packages/scraper/src/scrapers/router-switch.ts +++ b/packages/scraper/src/scrapers/router-switch.ts @@ -35,7 +35,7 @@ async function fetchPage(catUrl: string, form_factor: string, vendorId: string, const sep = catUrl.includes("?") ? "&" : "?"; const url = `${BASE}${catUrl}${page > 1 ? `${sep}p=${page}` : ""}`; const resp = await fetch(url, { - headers: { "User-Agent": "Mozilla/5.0 (compatible; TIPBot/1.0)" }, + headers: { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" }, signal: AbortSignal.timeout(30_000), }); if (!resp.ok) return 0; diff --git a/packages/scraper/src/scrapers/sfpcables.ts b/packages/scraper/src/scrapers/sfpcables.ts index 46493d2..5ec6f2b 100644 --- a/packages/scraper/src/scrapers/sfpcables.ts +++ b/packages/scraper/src/scrapers/sfpcables.ts @@ -10,7 +10,7 @@ import { contentHash } from "../utils/hash"; const BASE = "https://www.sfpcables.com"; const HEADERS = { - "User-Agent": "Mozilla/5.0 (compatible; TIP-Bot/1.0; research)", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", Accept: "text/html,application/xhtml+xml", }; diff --git a/packages/scraper/src/scrapers/skylane.ts b/packages/scraper/src/scrapers/skylane.ts index ca0d90f..2576880 100644 --- a/packages/scraper/src/scrapers/skylane.ts +++ b/packages/scraper/src/scrapers/skylane.ts @@ -21,7 +21,7 @@ const CATALOG_URLS = [ ]; const MAX_PAGES = 10; const HEADERS = { - "User-Agent": "Mozilla/5.0 (compatible; TIP-Bot/1.0; research)", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", Accept: "text/html,application/xhtml+xml", "Accept-Language": "en-US,en;q=0.9", }; diff --git a/packages/scraper/src/scrapers/tenGtek.ts b/packages/scraper/src/scrapers/tenGtek.ts index e30fdcb..91bed9e 100644 --- a/packages/scraper/src/scrapers/tenGtek.ts +++ b/packages/scraper/src/scrapers/tenGtek.ts @@ -12,7 +12,7 @@ import { contentHash, parsePrice } from "../utils/hash"; const BASE = "https://www.10gtek.com"; const HEADERS = { - "User-Agent": "Mozilla/5.0 (compatible; TIP-Bot/1.0; research)", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", Accept: "text/html,application/xhtml+xml", }; diff --git a/packages/scraper/src/scrapers/tscom.ts b/packages/scraper/src/scrapers/tscom.ts index 8a1c240..79e59d8 100644 --- a/packages/scraper/src/scrapers/tscom.ts +++ b/packages/scraper/src/scrapers/tscom.ts @@ -14,7 +14,7 @@ const BASE = "https://www.china-tscom.com"; const CATALOG_PATH = "/products/fiber-optic-transceivers/"; const MAX_PAGES = 15; const HEADERS = { - "User-Agent": "Mozilla/5.0 (compatible; TIP-Bot/1.0; research)", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", Accept: "text/html,application/xhtml+xml", "Accept-Language": "en-US,en;q=0.9", }; diff --git a/packages/scraper/src/scrapers/wiitek.ts b/packages/scraper/src/scrapers/wiitek.ts index 5bc32c4..82edec3 100644 --- a/packages/scraper/src/scrapers/wiitek.ts +++ b/packages/scraper/src/scrapers/wiitek.ts @@ -47,7 +47,7 @@ export async function scrapeWiitek(): Promise { for (const cat of CATEGORIES) { try { const resp = await fetch(`${BASE}${cat.path}`, { - headers: { "User-Agent": "Mozilla/5.0 (compatible; TIPBot/1.0)" }, + headers: { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" }, signal: AbortSignal.timeout(20_000), }); if (!resp.ok) continue;