25 lines
632 B
Bash
Executable File
25 lines
632 B
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
VENV_DIR="${TIP_CRAWLEE_PY_VENV:-/opt/tip-crawlee-python-venv}"
|
|
EXTRA="${TIP_CRAWLEE_PY_EXTRA:-beautifulsoup}"
|
|
|
|
python3 -m venv "$VENV_DIR"
|
|
"$VENV_DIR/bin/python" -m pip install -U pip
|
|
"$VENV_DIR/bin/python" -m pip install -e "$ROOT_DIR/packages/crawlee-python[$EXTRA]"
|
|
|
|
cat <<EOF
|
|
TIP Crawlee Python worker installed.
|
|
|
|
Venv:
|
|
$VENV_DIR
|
|
|
|
Smoke test:
|
|
$VENV_DIR/bin/python -m tip_crawlee_worker \\
|
|
--mode beautifulsoup \\
|
|
--url https://crawlee.dev \\
|
|
--out /tmp/tip-crawlee-python-smoke.jsonl \\
|
|
--max-requests 1
|
|
EOF
|