rene/transceiver-db
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: sync-to-erik now includes scraper rebuild + daemon restart; changelog

Browse Source
This commit is contained in:
Rene Fichtmueller 2026-04-18 01:37:12 +02:00
parent 9965d8e43c
commit c51af82a6b
2 changed files with 15 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG_PENDING.md
View File

@ -3,6 +3,8 @@
Format: `{"d":"YYYY-MM-DD","t":"TYPE","m":"Description"}` Format: `{"d":"YYYY-MM-DD","t":"TYPE","m":"Description"}`
Types: FEAT · FIX · UI · DATA · AI · INFRA Types: FEAT · FIX · UI · DATA · AI · INFRA
{"d":"2026-04-18","t":"FIX","m":"Crawlee concurrent-worker storage isolation: replaced global CRAWLEE_STORAGE_DIR env-var mutation (race condition) with instance-level Configuration(storageClientOptions.localDataDirectory) per scraper. Fixed 6 scrapers (optcore, atgbics, community-issues, edgecore, ufispace, market-intel). Added missing withIsolatedStorage to optcore+market-intel scheduler workers. eBay enricher vendor type 'marketplace'->'reseller' (constraint fix). Playwright chromium installed on Erik. sync-to-erik.sh now syncs scraper src + rebuilds + restarts tip-scraper-daemon."}
{"d":"2026-04-18","t":"FIX","m":"NADDOD stockLevel 'unknown' -> 'on_request': invalid value for price_observations_stock_level_check constraint — was causing all NADDOD price insertions to fail."}
{"d":"2026-04-18","t":"FEAT","m":"Price Comparison Dashboard: public /api/price-comparison (summary, list top-50 SKUs by vendor coverage, per-SKU detail). Express Router, no auth required. New '💲 Price Comparison' dashboard tab with stat cards, form-factor breakdown table, top-50 SKU table (clickable rows), and SKU detail lookup with per-vendor prices + stock + spread %."} {"d":"2026-04-18","t":"FEAT","m":"Price Comparison Dashboard: public /api/price-comparison (summary, list top-50 SKUs by vendor coverage, per-SKU detail). Express Router, no auth required. New '💲 Price Comparison' dashboard tab with stat cards, form-factor breakdown table, top-50 SKU table (clickable rows), and SKU detail lookup with per-vendor prices + stock + spread %."}
{"d":"2026-04-18","t":"DATA","m":"Eoptolink OEM catalog scraper: harvests 93 product-solution pages from eoptolink.com, extracts part numbers (EOLO-*/EOLQ-* format), seeds transceivers table as manufacturer=Eoptolink entries with form_factor/speed/fiber/category. No prices (B2B OEM). Scheduled every 4h (40 */4 * * *)."} {"d":"2026-04-18","t":"DATA","m":"Eoptolink OEM catalog scraper: harvests 93 product-solution pages from eoptolink.com, extracts part numbers (EOLO-*/EOLQ-* format), seeds transceivers table as manufacturer=Eoptolink entries with form_factor/speed/fiber/category. No prices (B2B OEM). Scheduled every 4h (40 */4 * * *)."}
{"d":"2026-04-18","t":"FIX","m":"stock_observations repopulated after TRUNCATE: storage-fs/request_queues/default/ directory re-created on Erik; NADDOD scraper manual-triggered; 4+ prices confirmed written within 20s."} {"d":"2026-04-18","t":"FIX","m":"stock_observations repopulated after TRUNCATE: storage-fs/request_queues/default/ directory re-created on Erik; NADDOD scraper manual-triggered; 4+ prices confirmed written within 20s."}

16
scripts/sync-to-erik.sh
View File

@ -31,17 +31,27 @@ rsync -avz \
"$LOCAL_PATH/packages/dashboard/index.html" \ "$LOCAL_PATH/packages/dashboard/index.html" \
"$ERIK:$REMOTE_PATH/packages/dashboard/index.html" "$ERIK:$REMOTE_PATH/packages/dashboard/index.html"
# Sync MCP Server source # Sync MCP Server + Scraper source
echo "" echo ""
echo "[4/5] Syncing MCP Server source..." echo "[4/5] Syncing MCP Server + Scraper source..."
rsync -avz --delete \ rsync -avz --delete \
"$LOCAL_PATH/packages/mcp-server/src/" \ "$LOCAL_PATH/packages/mcp-server/src/" \
"$ERIK:$REMOTE_PATH/packages/mcp-server/src/" "$ERIK:$REMOTE_PATH/packages/mcp-server/src/"
rsync -avz --delete \
"$LOCAL_PATH/packages/scraper/src/" \
"$ERIK:$REMOTE_PATH/packages/scraper/src/"
# Rebuild and restart # Rebuild and restart
echo "" echo ""
echo "[5/5] Rebuilding and restarting on Erik..." echo "[5/5] Rebuilding and restarting on Erik..."
ssh "$ERIK" "cd $REMOTE_PATH/packages/api && npx tsc 2>&1 | tail -5; pm2 restart tip-api 2>/dev/null; pm2 restart tip-mcp 2>/dev/null; echo 'Done. PM2 status:'; pm2 list 2>/dev/null | head -15" ssh "$ERIK" "\
cd $REMOTE_PATH/packages/api && npx tsc 2>&1 | tail -3; \
cd $REMOTE_PATH/packages/scraper && npx tsc 2>&1 | tail -3; \
pm2 restart tip-api 2>/dev/null; \
pm2 restart tip-mcp 2>/dev/null; \
pm2 restart tip-scraper-daemon 2>/dev/null; \
echo 'Done. PM2 status:'; pm2 list 2>/dev/null | head -15"
echo "" echo ""
echo "=== Sync complete ===" echo "=== Sync complete ==="