b43bdd3060
Phase 0 - Foundation: - Restructure into npm workspace monorepo (packages/core, api, scraper) - PostgreSQL 17 + TimescaleDB schema (15 tables incl. hypertables) - Docker Compose for local dev (PostgreSQL on 5433 + Qdrant) - Express 5 API on port 3200 with 6 routes - Seed script to migrate 159 transceivers + 42 standards from npm package - Erik server setup script + PM2 ecosystem config Phase 1 - Scraper Engine: - Crawlee + Playwright framework with pg-boss scheduler - FS.com scraper (PlaywrightCrawler, anti-bot workaround) - Optcore.net scraper (WP REST API enumeration + PlaywrightCrawler) - Uses /wp-json/wp/v2/product to get 2000+ product URLs - Playwright renders individual product pages for price extraction - Cisco TMG Matrix scraper (compatibility data) - News RSS aggregator (optics.org, SPIE, Network World, Nature Photonics) - Keyword relevance scoring for transceiver/fiber topics - xml2js with malformed XML sanitization - SHA-256 content hashing for change detection (skip unchanged records) - pg-boss v10 with explicit queue creation before scheduling
43 lines
1.1 KiB
TypeScript
43 lines
1.1 KiB
TypeScript
import { readFileSync, readdirSync } from "fs";
|
|
import { join } from "path";
|
|
import { Pool } from "pg";
|
|
import { config } from "dotenv";
|
|
|
|
config();
|
|
|
|
const pool = new Pool({
|
|
host: process.env.POSTGRES_HOST || "localhost",
|
|
port: parseInt(process.env.POSTGRES_PORT || "5432"),
|
|
database: process.env.POSTGRES_DB || "transceiver_db",
|
|
user: process.env.POSTGRES_USER || "tip",
|
|
password: process.env.POSTGRES_PASSWORD || "tip_dev_2026",
|
|
});
|
|
|
|
async function migrate(): Promise<void> {
|
|
const sqlDir = join(__dirname, "..", "sql");
|
|
const files = readdirSync(sqlDir)
|
|
.filter((f) => f.endsWith(".sql"))
|
|
.sort();
|
|
|
|
console.log(`Found ${files.length} migration files`);
|
|
|
|
const client = await pool.connect();
|
|
try {
|
|
for (const file of files) {
|
|
const sql = readFileSync(join(sqlDir, file), "utf-8");
|
|
console.log(`Running: ${file}...`);
|
|
await client.query(sql);
|
|
console.log(` Done: ${file}`);
|
|
}
|
|
console.log("\nAll migrations completed successfully.");
|
|
} catch (err) {
|
|
console.error("Migration failed:", err);
|
|
process.exit(1);
|
|
} finally {
|
|
client.release();
|
|
await pool.end();
|
|
}
|
|
}
|
|
|
|
migrate();
|