74 lines
2.5 KiB
Bash
Executable File
74 lines
2.5 KiB
Bash
Executable File
#!/bin/bash
|
|
# FS.com Scraper — Mac-side runner
|
|
# Runs from this Mac (residential IP) so FS.com isn't blocked.
|
|
# Opens SSH tunnel to Erik's DB → runs scraper → closes tunnel.
|
|
#
|
|
# Schedule: launchd at 02:00, 10:00, 18:00 daily
|
|
# Log: ~/Library/Logs/tip-fs-scraper.log
|
|
|
|
set -euo pipefail
|
|
|
|
LOG="$HOME/Library/Logs/tip-fs-scraper.log"
|
|
REPO="/Users/renefichtmueller/Desktop/Claude Code/github-repos/transceiver-db"
|
|
NODE="/opt/homebrew/bin/node"
|
|
NPX="/opt/homebrew/bin/npx"
|
|
TUNNEL_PID_FILE="/tmp/tip-db-tunnel.pid"
|
|
DB_LOCAL_PORT=5433
|
|
|
|
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG"; }
|
|
|
|
# ── Open SSH tunnel if not already running ────────────────────────────────────
|
|
open_tunnel() {
|
|
if [ -f "$TUNNEL_PID_FILE" ]; then
|
|
PID=$(cat "$TUNNEL_PID_FILE")
|
|
if kill -0 "$PID" 2>/dev/null; then
|
|
log "Tunnel already running (PID $PID)"
|
|
return 0
|
|
fi
|
|
fi
|
|
log "Opening SSH tunnel → Erik PostgreSQL on port $DB_LOCAL_PORT…"
|
|
ssh -N -f -L "${DB_LOCAL_PORT}:localhost:${DB_LOCAL_PORT}" erik
|
|
# -f forks to background, no PID tracking needed — use pkill to close
|
|
log "Tunnel opened"
|
|
sleep 2 # Give the tunnel a moment to establish
|
|
}
|
|
|
|
close_tunnel() {
|
|
log "Closing SSH tunnel…"
|
|
pkill -f "ssh -N -f -L ${DB_LOCAL_PORT}:localhost:${DB_LOCAL_PORT}" 2>/dev/null || true
|
|
rm -f "$TUNNEL_PID_FILE"
|
|
}
|
|
|
|
# ── Main ──────────────────────────────────────────────────────────────────────
|
|
mkdir -p "$(dirname "$LOG")"
|
|
log "=== FS.com Mac Scraper starting ==="
|
|
|
|
# Only close tunnel if we opened it (not if one was already running)
|
|
OPENED_TUNNEL=0
|
|
if ! pgrep -f "ssh -N.*${DB_LOCAL_PORT}:localhost" >/dev/null 2>&1; then
|
|
open_tunnel
|
|
OPENED_TUNNEL=1
|
|
trap close_tunnel EXIT
|
|
fi
|
|
|
|
cd "$REPO"
|
|
|
|
export POSTGRES_HOST=localhost
|
|
export POSTGRES_PORT=$DB_LOCAL_PORT
|
|
export POSTGRES_DB=transceiver_db
|
|
export POSTGRES_USER=tip
|
|
# POSTGRES_PASSWORD must be set in your environment or ~/.tip/.env
|
|
export POSTGRES_PASSWORD="${POSTGRES_PASSWORD:-}"
|
|
export NODE_ENV=production
|
|
|
|
# Load local credentials file if present (not committed to git)
|
|
if [ -f "$HOME/.tip/.env" ]; then
|
|
# shellcheck disable=SC1091
|
|
source "$HOME/.tip/.env"
|
|
fi
|
|
|
|
log "Running fs-com scraper via tsx…"
|
|
"$NPX" tsx packages/scraper/src/scrapers/fs-com.ts 2>&1 | tee -a "$LOG"
|
|
|
|
log "=== FS.com Mac Scraper complete ==="
|