b43bdd3060
Phase 0 - Foundation: - Restructure into npm workspace monorepo (packages/core, api, scraper) - PostgreSQL 17 + TimescaleDB schema (15 tables incl. hypertables) - Docker Compose for local dev (PostgreSQL on 5433 + Qdrant) - Express 5 API on port 3200 with 6 routes - Seed script to migrate 159 transceivers + 42 standards from npm package - Erik server setup script + PM2 ecosystem config Phase 1 - Scraper Engine: - Crawlee + Playwright framework with pg-boss scheduler - FS.com scraper (PlaywrightCrawler, anti-bot workaround) - Optcore.net scraper (WP REST API enumeration + PlaywrightCrawler) - Uses /wp-json/wp/v2/product to get 2000+ product URLs - Playwright renders individual product pages for price extraction - Cisco TMG Matrix scraper (compatibility data) - News RSS aggregator (optics.org, SPIE, Network World, Nature Photonics) - Keyword relevance scoring for transceiver/fiber topics - xml2js with malformed XML sanitization - SHA-256 content hashing for change detection (skip unchanged records) - pg-boss v10 with explicit queue creation before scheduling
32 lines
861 B
JSON
32 lines
861 B
JSON
{
|
|
"name": "@tip/scraper",
|
|
"version": "0.1.0",
|
|
"private": true,
|
|
"description": "TIP scraper engine — Crawlee + Playwright for competitor pricing, stock, datasheets, FAQs",
|
|
"main": "dist/index.js",
|
|
"scripts": {
|
|
"build": "tsc",
|
|
"dev": "tsx src/index.ts",
|
|
"scrape:fs": "tsx src/scrapers/fs-com.ts",
|
|
"scrape:cisco": "tsx src/scrapers/cisco-tmg.ts",
|
|
"scrape:optcore": "tsx src/scrapers/optcore.ts",
|
|
"scrape:news": "tsx src/scrapers/news.ts",
|
|
"scrape:all": "tsx src/index.ts --all"
|
|
},
|
|
"dependencies": {
|
|
"crawlee": "^3.12.0",
|
|
"playwright": "^1.50.0",
|
|
"pg": "^8.13.1",
|
|
"pg-boss": "^10.1.5",
|
|
"dotenv": "^16.4.7",
|
|
"cheerio": "^1.0.0",
|
|
"xml2js": "^0.6.2"
|
|
},
|
|
"devDependencies": {
|
|
"@types/pg": "^8.11.11",
|
|
"@types/xml2js": "^0.4.14",
|
|
"typescript": "^5.9.3",
|
|
"tsx": "^4.19.0"
|
|
}
|
|
}
|