Phase 0 - Foundation: - Restructure into npm workspace monorepo (packages/core, api, scraper) - PostgreSQL 17 + TimescaleDB schema (15 tables incl. hypertables) - Docker Compose for local dev (PostgreSQL on 5433 + Qdrant) - Express 5 API on port 3200 with 6 routes - Seed script to migrate 159 transceivers + 42 standards from npm package - Erik server setup script + PM2 ecosystem config Phase 1 - Scraper Engine: - Crawlee + Playwright framework with pg-boss scheduler - FS.com scraper (PlaywrightCrawler, anti-bot workaround) - Optcore.net scraper (WP REST API enumeration + PlaywrightCrawler) - Uses /wp-json/wp/v2/product to get 2000+ product URLs - Playwright renders individual product pages for price extraction - Cisco TMG Matrix scraper (compatibility data) - News RSS aggregator (optics.org, SPIE, Network World, Nature Photonics) - Keyword relevance scoring for transceiver/fiber topics - xml2js with malformed XML sanitization - SHA-256 content hashing for change detection (skip unchanged records) - pg-boss v10 with explicit queue creation before scheduling
212 lines
5.7 KiB
TypeScript
212 lines
5.7 KiB
TypeScript
import { pool } from "./client";
|
|
|
|
export interface SearchParams {
|
|
q?: string;
|
|
form_factor?: string;
|
|
speed?: string;
|
|
speed_gbps?: number;
|
|
category?: string;
|
|
fiber_type?: string;
|
|
reach_min?: number;
|
|
reach_max?: number;
|
|
wdm_type?: string;
|
|
coherent?: boolean;
|
|
market_status?: string;
|
|
limit?: number;
|
|
offset?: number;
|
|
}
|
|
|
|
export async function searchTransceivers(params: SearchParams) {
|
|
const conditions: string[] = [];
|
|
const values: any[] = [];
|
|
let idx = 1;
|
|
|
|
if (params.q) {
|
|
conditions.push(`search_vector @@ plainto_tsquery('english', $${idx})`);
|
|
values.push(params.q);
|
|
idx++;
|
|
}
|
|
if (params.form_factor) {
|
|
conditions.push(`form_factor = $${idx}`);
|
|
values.push(params.form_factor);
|
|
idx++;
|
|
}
|
|
if (params.speed) {
|
|
conditions.push(`speed = $${idx}`);
|
|
values.push(params.speed);
|
|
idx++;
|
|
}
|
|
if (params.speed_gbps) {
|
|
conditions.push(`speed_gbps = $${idx}`);
|
|
values.push(params.speed_gbps);
|
|
idx++;
|
|
}
|
|
if (params.category) {
|
|
conditions.push(`category = $${idx}`);
|
|
values.push(params.category);
|
|
idx++;
|
|
}
|
|
if (params.fiber_type) {
|
|
conditions.push(`fiber_type = $${idx}`);
|
|
values.push(params.fiber_type);
|
|
idx++;
|
|
}
|
|
if (params.reach_min) {
|
|
conditions.push(`reach_meters >= $${idx}`);
|
|
values.push(params.reach_min);
|
|
idx++;
|
|
}
|
|
if (params.reach_max) {
|
|
conditions.push(`reach_meters <= $${idx}`);
|
|
values.push(params.reach_max);
|
|
idx++;
|
|
}
|
|
if (params.wdm_type) {
|
|
conditions.push(`wdm_type = $${idx}`);
|
|
values.push(params.wdm_type);
|
|
idx++;
|
|
}
|
|
if (params.coherent !== undefined) {
|
|
conditions.push(`coherent = $${idx}`);
|
|
values.push(params.coherent);
|
|
idx++;
|
|
}
|
|
if (params.market_status) {
|
|
conditions.push(`market_status = $${idx}`);
|
|
values.push(params.market_status);
|
|
idx++;
|
|
}
|
|
|
|
const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
const limit = params.limit || 50;
|
|
const offset = params.offset || 0;
|
|
|
|
// Add relevance ranking when full-text search is used
|
|
const orderBy = params.q
|
|
? `ORDER BY ts_rank(search_vector, plainto_tsquery('english', $1)) DESC`
|
|
: `ORDER BY speed_gbps DESC, reach_meters ASC`;
|
|
|
|
const query = `
|
|
SELECT t.*, v.name as vendor_name
|
|
FROM transceivers t
|
|
LEFT JOIN vendors v ON t.vendor_id = v.id
|
|
${where}
|
|
${orderBy}
|
|
LIMIT ${limit} OFFSET ${offset}
|
|
`;
|
|
|
|
const countQuery = `SELECT COUNT(*) FROM transceivers ${where}`;
|
|
|
|
const [dataResult, countResult] = await Promise.all([
|
|
pool.query(query, values),
|
|
pool.query(countQuery, values),
|
|
]);
|
|
|
|
return {
|
|
data: dataResult.rows,
|
|
total: parseInt(countResult.rows[0].count),
|
|
limit,
|
|
offset,
|
|
};
|
|
}
|
|
|
|
export async function getTransceiverById(id: string) {
|
|
const result = await pool.query(
|
|
`SELECT t.*, v.name as vendor_name, s.name as standard_full_name
|
|
FROM transceivers t
|
|
LEFT JOIN vendors v ON t.vendor_id = v.id
|
|
LEFT JOIN standards s ON t.standard_id = s.id
|
|
WHERE t.id = $1 OR t.slug = $1`,
|
|
[id]
|
|
);
|
|
return result.rows[0] || null;
|
|
}
|
|
|
|
export async function searchSwitches(params: SearchParams) {
|
|
const conditions: string[] = [];
|
|
const values: any[] = [];
|
|
let idx = 1;
|
|
|
|
if (params.q) {
|
|
conditions.push(`sw.search_vector @@ plainto_tsquery('english', $${idx})`);
|
|
values.push(params.q);
|
|
idx++;
|
|
}
|
|
if (params.category) {
|
|
conditions.push(`sw.category = $${idx}`);
|
|
values.push(params.category);
|
|
idx++;
|
|
}
|
|
|
|
const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
const limit = params.limit || 50;
|
|
const offset = params.offset || 0;
|
|
|
|
const query = `
|
|
SELECT sw.*, v.name as vendor_name
|
|
FROM switches sw
|
|
LEFT JOIN vendors v ON sw.vendor_id = v.id
|
|
${where}
|
|
ORDER BY sw.max_speed_gbps DESC NULLS LAST
|
|
LIMIT ${limit} OFFSET ${offset}
|
|
`;
|
|
|
|
const result = await pool.query(query, values);
|
|
return { data: result.rows, limit, offset };
|
|
}
|
|
|
|
export async function getSwitchById(id: string) {
|
|
const result = await pool.query(
|
|
`SELECT sw.*, v.name as vendor_name
|
|
FROM switches sw
|
|
LEFT JOIN vendors v ON sw.vendor_id = v.id
|
|
WHERE sw.id = $1`,
|
|
[id]
|
|
);
|
|
return result.rows[0] || null;
|
|
}
|
|
|
|
export async function getCompatibleTransceivers(switchId: string) {
|
|
const result = await pool.query(
|
|
`SELECT t.*, c.status, c.verified_by, c.notes as compat_notes
|
|
FROM compatibility c
|
|
JOIN transceivers t ON c.transceiver_id = t.id
|
|
WHERE c.switch_id = $1 AND c.status = 'compatible'
|
|
ORDER BY t.speed_gbps DESC`,
|
|
[switchId]
|
|
);
|
|
return result.rows;
|
|
}
|
|
|
|
export async function listVendors(type?: string) {
|
|
const query = type
|
|
? `SELECT * FROM vendors WHERE type = $1 ORDER BY name`
|
|
: `SELECT * FROM vendors ORDER BY name`;
|
|
const result = await pool.query(query, type ? [type] : []);
|
|
return result.rows;
|
|
}
|
|
|
|
export async function listStandards(speed?: string) {
|
|
const query = speed
|
|
? `SELECT * FROM standards WHERE speed = $1 ORDER BY year_ratified DESC`
|
|
: `SELECT * FROM standards ORDER BY year_ratified DESC`;
|
|
const result = await pool.query(query, speed ? [speed] : []);
|
|
return result.rows;
|
|
}
|
|
|
|
export async function getDbStats() {
|
|
const result = await pool.query(`
|
|
SELECT
|
|
(SELECT COUNT(*) FROM vendors) as vendor_count,
|
|
(SELECT COUNT(*) FROM standards) as standard_count,
|
|
(SELECT COUNT(*) FROM transceivers) as transceiver_count,
|
|
(SELECT COUNT(*) FROM switches) as switch_count,
|
|
(SELECT COUNT(*) FROM compatibility) as compatibility_count,
|
|
(SELECT COUNT(*) FROM breakouts) as breakout_count,
|
|
(SELECT COUNT(*) FROM knowledge_base) as kb_count,
|
|
(SELECT COUNT(*) FROM documents) as document_count,
|
|
(SELECT COUNT(*) FROM news_articles) as news_count
|
|
`);
|
|
return result.rows[0];
|
|
}
|