rene/transceiver-db
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: flexoptix catalog scraper — 1G SFP coverage + SKU suffix + pagination

Browse Source 
- Add 1G SFP search queries ("1G SFP", "SFP LX", "SFP SX", "SFP ZX") — were completely missing
- Strip vendor-compat suffix from SKU (S.1303.10.DG:Sx → S.1303.10.DG) to match existing records
- Remove 200-product cap, use full API pagination (page >= 50 limit only)
- Result: FLEXOPTIX 1G SFP coverage 50% → 97%, overall price coverage 62% → 88%
This commit is contained in:
Rene Fichtmueller 2026-04-04 07:26:13 +02:00
parent 71d6c20b5e
commit 441d9721c9
1 changed files with 11 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
packages/scraper/src/scrapers/flexoptix-catalog.ts
View File

@ -397,6 +397,10 @@ export async function scrapeFlexoptixCatalog(): Promise<void> {
const GRAPHQL_URL = `${BASE}/graphql`; const GRAPHQL_URL = `${BASE}/graphql`;
const GRAPHQL_QUERIES = [ const GRAPHQL_QUERIES = [
{ search: "1G SFP", defaultFF: "SFP", defaultGbps: 1 },
{ search: "SFP LX", defaultFF: "SFP", defaultGbps: 1 },
{ search: "SFP SX", defaultFF: "SFP", defaultGbps: 1 },
{ search: "SFP ZX", defaultFF: "SFP", defaultGbps: 1 },
{ search: "SFP+", defaultFF: "SFP+", defaultGbps: 10 }, { search: "SFP+", defaultFF: "SFP+", defaultGbps: 10 },
{ search: "SFP28", defaultFF: "SFP28", defaultGbps: 25 }, { search: "SFP28", defaultFF: "SFP28", defaultGbps: 25 },
{ search: "QSFP+", defaultFF: "QSFP+", defaultGbps: 40 }, { search: "QSFP+", defaultFF: "QSFP+", defaultGbps: 40 },
@ -494,9 +498,13 @@ export async function scrapeFlexoptixCatalog(): Promise<void> {
const rawImg = item.small_image?.url; const rawImg = item.small_image?.url;
const imageUrl = rawImg && !rawImg.includes("placeholder") ? rawImg : undefined; const imageUrl = rawImg && !rawImg.includes("placeholder") ? rawImg : undefined;
// Strip the vendor-compatibility suffix (e.g. ":Sx", ":Ci", ":Ju") from SKU
// The base SKU (before ":") is the canonical FLEXOPTIX part number
const baseSku = item.sku.includes(":") ? item.sku.split(":")[0] : item.sku;
allProducts.set(url, { allProducts.set(url, {
name: item.name, name: item.name,
partNumber: item.sku, partNumber: baseSku,
url, url,
price: price && price > 0 && price < 100000 ? price : undefined, price: price && price > 0 && price < 100000 ? price : undefined,
currency: item.price_range?.minimum_price?.final_price?.currency || "EUR", currency: item.price_range?.minimum_price?.final_price?.currency || "EUR",
@ -515,8 +523,8 @@ export async function scrapeFlexoptixCatalog(): Promise<void> {
totalFetched += products.items.length; totalFetched += products.items.length;
if (newCount > 0) console.log(` GraphQL "${gq.search}" p${page}: +${newCount} new (${products.items.length} items, ${products.total_count} total)`); if (newCount > 0) console.log(` GraphQL "${gq.search}" p${page}: +${newCount} new (${products.items.length} items, ${products.total_count} total)`);
// Stop if we've fetched all pages or reached a reasonable limit // Stop if we've fetched all pages (no artificial cap — let the API determine the limit)
if (totalFetched >= products.total_count || totalFetched >= 200 || page >= 10) break; if (totalFetched >= products.total_count || page >= 50) break;
page++; page++;
await sleep(800); await sleep(800);
} catch (err) { } catch (err) {