60cffb3585
Close the missing feedback loop. Detection (v_scraper_health, v_data_quality_alerts, check_switch_quality) was accurate but inert; dispatch was blind fixed-cadence (boss.send() is never called from any health path). tip-dqo is a separate, read-mostly control plane that closes DETECT -> PRIORITIZE -> DISPATCH -> VERIFY -> RECHECK against observation-table freshness (never pg-boss job state), with DB-enforced guardrails. - sql/129: source_registry (signal->queue map + per-source SLA + guardrail contract + global kill-switch), self_heal_dispatch append-only ledger, scraper_run_log, acknowledged_exceptions, v_dqo_targets ranked view. - orchestrator/: control-loop, self-heal-guards (pure, unit-tested), escalation-sink (ntfy + no-op fallback), index entrypoint. Reuses the verification-robots dispatch idiom (extracted shared dispatchQueues) and the quality views as-is. - Guardrails: kill-switch, plan-mode default, per-source cooldown/budget/backoff/ circuit-breaker, load-guard, erik-safe profile cap, delta-gated escalation. Root-cause routing escalates code_fix/blocked (e.g. ATGBICS silent stock-write break) to a human instead of looping retries; only staleness is auto-dispatched. - security: remove hardcoded 'tip_dev_2026' DB password fallback across 6 sites -> fail-closed requireDbPassword() / dbConnectionString(). SELF_HEAL_PLAN_ONLY defaults true (dispatches nothing; writes plan rows for review). Verified against the live DB: detects ATGBICS-stock dead/54d -> escalate, Flexoptix-stock stale/30d -> plan-dispatch; guards + ledger correct; 8/8 guard unit tests pass.
58 lines
2.7 KiB
JSON
58 lines
2.7 KiB
JSON
{
|
|
"name": "@tip/scraper",
|
|
"version": "0.1.0",
|
|
"private": true,
|
|
"description": "TIP scraper engine — Crawlee + Playwright for competitor pricing, stock, datasheets, FAQs",
|
|
"main": "dist/index.js",
|
|
"scripts": {
|
|
"build": "tsc",
|
|
"dev": "tsx src/index.ts",
|
|
"scrape:fs": "tsx src/scrapers/fs-com.ts",
|
|
"scrape:fs:db-detail": "FS_DB_DETAIL_ONLY=1 TIP_FORCE_REVALIDATE=1 tsx src/scrapers/fs-com.ts",
|
|
"scrape:fs:url-discovery": "FS_URL_DISCOVERY_ONLY=1 TIP_FORCE_REVALIDATE=1 tsx src/scrapers/fs-com.ts",
|
|
"scrape:atgbics:details": "tsx src/scrapers/atgbics-detail-pages.ts",
|
|
"scrape:vendors:details": "tsx src/scrapers/shopfiber24-fibermall-detail-pages.ts",
|
|
"scrape:gaotek:details": "tsx src/scrapers/gaotek-detail-pages.ts",
|
|
"scrape:10gtek": "tsx src/scrapers/tenGtek.ts",
|
|
"scrape:gbics": "tsx src/scrapers/gbics.ts",
|
|
"scrape:tscom": "tsx src/scrapers/tscom.ts",
|
|
"scrape:sfpcables": "tsx src/scrapers/sfpcables.ts",
|
|
"verify:catalog:details": "tsx src/utils/verify-catalog-details.ts",
|
|
"verify:quarantine:non-transceivers": "tsx src/utils/quarantine-non-transceivers.ts",
|
|
"verify:product-page-assets": "tsx src/utils/verify-product-page-assets.ts",
|
|
"verify:product-page-prices": "tsx src/utils/verify-product-page-prices.ts",
|
|
"verify:10gtek:datasheets": "tsx src/utils/verify-10gtek-datasheets.ts",
|
|
"verify:part-number-details": "tsx src/utils/verify-part-number-details.ts",
|
|
"verify:normalize:product-urls": "tsx src/utils/normalize-product-urls.ts",
|
|
"verify:fs:sku-aliases": "tsx src/utils/quarantine-fs-sku-aliases.ts",
|
|
"verify:price-availability": "tsx src/utils/resolve-price-availability.ts",
|
|
"verify:research-status": "tsx src/utils/resolve-research-status.ts",
|
|
"verify:no-valid-competitor": "tsx src/utils/resolve-no-valid-competitor.ts",
|
|
"verify:open-competitor-status": "tsx src/utils/resolve-open-competitor-status.ts",
|
|
"scrape:cisco": "tsx src/scrapers/cisco-tmg.ts",
|
|
"scrape:optcore": "tsx src/scrapers/optcore.ts",
|
|
"scrape:news": "tsx src/scrapers/news.ts",
|
|
"scrape:all": "tsx src/index.ts --all",
|
|
"robots:verification": "tsx src/robots/verification-robots.ts",
|
|
"dqo": "tsx src/orchestrator/index.ts",
|
|
"dqo:once": "tsx src/orchestrator/index.ts --once",
|
|
"dqo:test": "node --import tsx --test src/orchestrator/self-heal-guards.test.ts"
|
|
},
|
|
"dependencies": {
|
|
"crawlee": "^3.12.0",
|
|
"playwright": "^1.50.0",
|
|
"pg": "^8.13.1",
|
|
"pg-boss": "^10.1.5",
|
|
"dotenv": "^16.4.7",
|
|
"cheerio": "^1.0.0",
|
|
"xml2js": "^0.6.2",
|
|
"socks-proxy-agent": "^8.0.5"
|
|
},
|
|
"devDependencies": {
|
|
"@types/pg": "^8.11.11",
|
|
"@types/xml2js": "^0.4.14",
|
|
"typescript": "^5.9.3",
|
|
"tsx": "^4.19.0"
|
|
}
|
|
}
|