017ed78d2b
- Replaces Playwright with pure fetch() — static HTML has prices - Correct collection handles (compatible-transceivers-sfpp-10g etc.) - Cookie: cart_currency=GBP forces GBP pricing from any geo-IP - Handles 35+ pages per category × 24 products = 840+ SFP+ products - No IP-blocking with static HTML (Playwright was the trigger) - Adds scripts/run-atgbics-mac.sh for Mac-side runner if needed
56 lines
1.5 KiB
Bash
Executable File
56 lines
1.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# run-atgbics-mac.sh — Run ATGBICS scraper from Mac (bypasses Erik datacenter IP block)
|
|
#
|
|
# Uses Shopify JSON API (no Playwright needed).
|
|
# Connects to Erik's PostgreSQL via SSH tunnel.
|
|
#
|
|
# Usage: ./scripts/run-atgbics-mac.sh
|
|
# Requirements: SSH access to Erik (root@82.165.222.127)
|
|
|
|
set -euo pipefail
|
|
|
|
REPO_DIR="$(cd "$(dirname "$0")/.." && pwd)"
|
|
TUNNEL_PORT=5434
|
|
TUNNEL_PID=""
|
|
LOG="/tmp/atgbics-mac-run.log"
|
|
|
|
cleanup() {
|
|
if [[ -n "$TUNNEL_PID" ]]; then
|
|
echo "Closing SSH tunnel (PID $TUNNEL_PID)..."
|
|
kill "$TUNNEL_PID" 2>/dev/null || true
|
|
fi
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
echo "=== ATGBICS Mac Runner ==="
|
|
echo "Repo: $REPO_DIR"
|
|
|
|
# Open SSH tunnel to Erik's PostgreSQL
|
|
echo "Opening SSH tunnel to Erik DB (port 5433 → local 5434)..."
|
|
ssh -fNL "${TUNNEL_PORT}:127.0.0.1:5433" root@82.165.222.127
|
|
TUNNEL_PID=$(lsof -ti "TCP:${TUNNEL_PORT}" -sTCP:LISTEN 2>/dev/null | head -1)
|
|
echo "Tunnel open (listener PID: ${TUNNEL_PID:-unknown})"
|
|
|
|
# Wait briefly for tunnel to stabilize
|
|
sleep 1
|
|
|
|
# Build scraper if needed
|
|
if [[ ! -f "$REPO_DIR/packages/scraper/dist/scrapers/atgbics.js" ]]; then
|
|
echo "Building scraper package..."
|
|
cd "$REPO_DIR/packages/scraper" && npm run build
|
|
fi
|
|
|
|
# Run scraper
|
|
echo "Running ATGBICS scraper..."
|
|
cd "$REPO_DIR"
|
|
POSTGRES_HOST=127.0.0.1 \
|
|
POSTGRES_PORT="${TUNNEL_PORT}" \
|
|
POSTGRES_USER=tip \
|
|
POSTGRES_PASSWORD=tip_prod_2026 \
|
|
POSTGRES_DB=transceiver_db \
|
|
node packages/scraper/dist/scrapers/atgbics.js 2>&1 | tee "$LOG"
|
|
|
|
echo ""
|
|
echo "Log saved to: $LOG"
|
|
echo "=== Done ==="
|