Rene Fichtmueller b43bdd3060 feat: TIP Phase 0+1 — monorepo, DB schema, API, scraper engine
Phase 0 - Foundation:
- Restructure into npm workspace monorepo (packages/core, api, scraper)
- PostgreSQL 17 + TimescaleDB schema (15 tables incl. hypertables)
- Docker Compose for local dev (PostgreSQL on 5433 + Qdrant)
- Express 5 API on port 3200 with 6 routes
- Seed script to migrate 159 transceivers + 42 standards from npm package
- Erik server setup script + PM2 ecosystem config

Phase 1 - Scraper Engine:
- Crawlee + Playwright framework with pg-boss scheduler
- FS.com scraper (PlaywrightCrawler, anti-bot workaround)
- Optcore.net scraper (WP REST API enumeration + PlaywrightCrawler)
  - Uses /wp-json/wp/v2/product to get 2000+ product URLs
  - Playwright renders individual product pages for price extraction
- Cisco TMG Matrix scraper (compatibility data)
- News RSS aggregator (optics.org, SPIE, Network World, Nature Photonics)
  - Keyword relevance scoring for transceiver/fiber topics
  - xml2js with malformed XML sanitization
- SHA-256 content hashing for change detection (skip unchanged records)
- pg-boss v10 with explicit queue creation before scheduling
2026-03-27 16:27:31 +13:00

212 lines
5.7 KiB
TypeScript

import { pool } from "./client";
export interface SearchParams {
q?: string;
form_factor?: string;
speed?: string;
speed_gbps?: number;
category?: string;
fiber_type?: string;
reach_min?: number;
reach_max?: number;
wdm_type?: string;
coherent?: boolean;
market_status?: string;
limit?: number;
offset?: number;
}
export async function searchTransceivers(params: SearchParams) {
const conditions: string[] = [];
const values: any[] = [];
let idx = 1;
if (params.q) {
conditions.push(`search_vector @@ plainto_tsquery('english', $${idx})`);
values.push(params.q);
idx++;
}
if (params.form_factor) {
conditions.push(`form_factor = $${idx}`);
values.push(params.form_factor);
idx++;
}
if (params.speed) {
conditions.push(`speed = $${idx}`);
values.push(params.speed);
idx++;
}
if (params.speed_gbps) {
conditions.push(`speed_gbps = $${idx}`);
values.push(params.speed_gbps);
idx++;
}
if (params.category) {
conditions.push(`category = $${idx}`);
values.push(params.category);
idx++;
}
if (params.fiber_type) {
conditions.push(`fiber_type = $${idx}`);
values.push(params.fiber_type);
idx++;
}
if (params.reach_min) {
conditions.push(`reach_meters >= $${idx}`);
values.push(params.reach_min);
idx++;
}
if (params.reach_max) {
conditions.push(`reach_meters <= $${idx}`);
values.push(params.reach_max);
idx++;
}
if (params.wdm_type) {
conditions.push(`wdm_type = $${idx}`);
values.push(params.wdm_type);
idx++;
}
if (params.coherent !== undefined) {
conditions.push(`coherent = $${idx}`);
values.push(params.coherent);
idx++;
}
if (params.market_status) {
conditions.push(`market_status = $${idx}`);
values.push(params.market_status);
idx++;
}
const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
const limit = params.limit || 50;
const offset = params.offset || 0;
// Add relevance ranking when full-text search is used
const orderBy = params.q
? `ORDER BY ts_rank(search_vector, plainto_tsquery('english', $1)) DESC`
: `ORDER BY speed_gbps DESC, reach_meters ASC`;
const query = `
SELECT t.*, v.name as vendor_name
FROM transceivers t
LEFT JOIN vendors v ON t.vendor_id = v.id
${where}
${orderBy}
LIMIT ${limit} OFFSET ${offset}
`;
const countQuery = `SELECT COUNT(*) FROM transceivers ${where}`;
const [dataResult, countResult] = await Promise.all([
pool.query(query, values),
pool.query(countQuery, values),
]);
return {
data: dataResult.rows,
total: parseInt(countResult.rows[0].count),
limit,
offset,
};
}
export async function getTransceiverById(id: string) {
const result = await pool.query(
`SELECT t.*, v.name as vendor_name, s.name as standard_full_name
FROM transceivers t
LEFT JOIN vendors v ON t.vendor_id = v.id
LEFT JOIN standards s ON t.standard_id = s.id
WHERE t.id = $1 OR t.slug = $1`,
[id]
);
return result.rows[0] || null;
}
export async function searchSwitches(params: SearchParams) {
const conditions: string[] = [];
const values: any[] = [];
let idx = 1;
if (params.q) {
conditions.push(`sw.search_vector @@ plainto_tsquery('english', $${idx})`);
values.push(params.q);
idx++;
}
if (params.category) {
conditions.push(`sw.category = $${idx}`);
values.push(params.category);
idx++;
}
const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
const limit = params.limit || 50;
const offset = params.offset || 0;
const query = `
SELECT sw.*, v.name as vendor_name
FROM switches sw
LEFT JOIN vendors v ON sw.vendor_id = v.id
${where}
ORDER BY sw.max_speed_gbps DESC NULLS LAST
LIMIT ${limit} OFFSET ${offset}
`;
const result = await pool.query(query, values);
return { data: result.rows, limit, offset };
}
export async function getSwitchById(id: string) {
const result = await pool.query(
`SELECT sw.*, v.name as vendor_name
FROM switches sw
LEFT JOIN vendors v ON sw.vendor_id = v.id
WHERE sw.id = $1`,
[id]
);
return result.rows[0] || null;
}
export async function getCompatibleTransceivers(switchId: string) {
const result = await pool.query(
`SELECT t.*, c.status, c.verified_by, c.notes as compat_notes
FROM compatibility c
JOIN transceivers t ON c.transceiver_id = t.id
WHERE c.switch_id = $1 AND c.status = 'compatible'
ORDER BY t.speed_gbps DESC`,
[switchId]
);
return result.rows;
}
export async function listVendors(type?: string) {
const query = type
? `SELECT * FROM vendors WHERE type = $1 ORDER BY name`
: `SELECT * FROM vendors ORDER BY name`;
const result = await pool.query(query, type ? [type] : []);
return result.rows;
}
export async function listStandards(speed?: string) {
const query = speed
? `SELECT * FROM standards WHERE speed = $1 ORDER BY year_ratified DESC`
: `SELECT * FROM standards ORDER BY year_ratified DESC`;
const result = await pool.query(query, speed ? [speed] : []);
return result.rows;
}
export async function getDbStats() {
const result = await pool.query(`
SELECT
(SELECT COUNT(*) FROM vendors) as vendor_count,
(SELECT COUNT(*) FROM standards) as standard_count,
(SELECT COUNT(*) FROM transceivers) as transceiver_count,
(SELECT COUNT(*) FROM switches) as switch_count,
(SELECT COUNT(*) FROM compatibility) as compatibility_count,
(SELECT COUNT(*) FROM breakouts) as breakout_count,
(SELECT COUNT(*) FROM knowledge_base) as kb_count,
(SELECT COUNT(*) FROM documents) as document_count,
(SELECT COUNT(*) FROM news_articles) as news_count
`);
return result.rows[0];
}