de5bdb24ca
- PostgreSQL 17 + TimescaleDB schema with 12 tables - 48 standards (IEEE, SFF, ITU-T, OIF, MSA) - 33 form factors (SFP through OSFP-XD/CPO) - 85+ vendors (OEM, compatible, manufacturers, marketplaces) - 80+ seed transceivers (1G-1.6T, CWDM, BiDi, DAC, AOC, FC, PON) - 60+ network devices (Cisco, Juniper, Arista, HPE, Dell, etc.) - Crawler framework with fs.com and eBay crawlers - REST API (15 endpoints) on port 3200 - MCP server (12 tools) on port 3201 - PM2 ecosystem for production deployment on Erik (.82)
41 lines
1.1 KiB
TypeScript
41 lines
1.1 KiB
TypeScript
import 'dotenv/config';
|
|
import { pino } from 'pino';
|
|
import { FsComCrawler } from '../src/crawlers/fscom.js';
|
|
import { EbayCrawler } from '../src/crawlers/ebay.js';
|
|
|
|
const log = pino({ name: 'crawl-orchestrator' });
|
|
|
|
const CRAWLERS: Record<string, () => { run: () => Promise<void> }> = {
|
|
fscom: () => new FsComCrawler(),
|
|
ebay: () => new EbayCrawler(),
|
|
};
|
|
|
|
async function main() {
|
|
const target = process.argv[2];
|
|
|
|
if (target && CRAWLERS[target]) {
|
|
log.info({ crawler: target }, 'Running single crawler');
|
|
const crawler = CRAWLERS[target]!();
|
|
await crawler.run();
|
|
} else if (target === 'all' || !target) {
|
|
log.info('Running all crawlers sequentially');
|
|
for (const [name, factory] of Object.entries(CRAWLERS)) {
|
|
log.info({ crawler: name }, 'Starting crawler');
|
|
try {
|
|
const crawler = factory();
|
|
await crawler.run();
|
|
} catch (err) {
|
|
log.error({ err, crawler: name }, 'Crawler failed');
|
|
}
|
|
}
|
|
} else {
|
|
log.error(`Unknown crawler: ${target}. Available: ${Object.keys(CRAWLERS).join(', ')}`);
|
|
process.exit(1);
|
|
}
|
|
|
|
log.info('Crawl orchestration complete');
|
|
process.exit(0);
|
|
}
|
|
|
|
main();
|