chore(security): remove credential-like defaults
This commit is contained in:
Rene Fichtmueller
parent
92a2d41fb1
commit
421926768a
1
.gitignore
vendored
1
.gitignore
vendored
@ -17,3 +17,4 @@ ecosystem.config.js.bak-*
|
||||
|
||||
# Codex/editor backup directories
|
||||
.codex-backup-*/
|
||||
.serena/
|
||||
|
||||
@ -2,21 +2,24 @@
|
||||
"""Load blog-training-alpaca.jsonl into PostgreSQL learning_corpus table via psql."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
JSONL_FILE = Path(__file__).parent.parent / "data" / "blog-training-alpaca.jsonl"
|
||||
DB_URL = "postgresql://llm:llm_secure_2026@127.0.0.1:15432/llm_gateway"
|
||||
DB_URL = os.environ.get("FT_DB_URL", "postgresql://llm@localhost:15432/llm_gateway")
|
||||
PSQL_BIN = os.environ.get("PSQL_BIN", "psql")
|
||||
TASK_TYPE = "tip_blog"
|
||||
|
||||
# Parse connection details
|
||||
parts = DB_URL.replace("postgresql://", "").split("@")
|
||||
creds = parts[0].split(":")
|
||||
db_user, db_pass = creds[0], creds[1]
|
||||
host_port = parts[1].split("/")
|
||||
host, port = host_port[0].split(":")
|
||||
db_name = host_port[1]
|
||||
parsed = urlparse(DB_URL)
|
||||
db_user = parsed.username or "llm"
|
||||
db_pass = parsed.password or os.environ.get("PGPASSWORD")
|
||||
host = parsed.hostname or "localhost"
|
||||
port = str(parsed.port or 5432)
|
||||
db_name = parsed.path.lstrip("/") or "llm_gateway"
|
||||
|
||||
print("🔄 Loading blog training data into PostgreSQL...")
|
||||
print(f" File: {JSONL_FILE}")
|
||||
@ -66,10 +69,12 @@ with open(tmpfile, "w") as f:
|
||||
|
||||
# Execute via psql
|
||||
try:
|
||||
env = {"PGPASSWORD": db_pass}
|
||||
env = os.environ.copy()
|
||||
if db_pass:
|
||||
env["PGPASSWORD"] = db_pass
|
||||
result = subprocess.run(
|
||||
[
|
||||
"/opt/homebrew/bin/psql",
|
||||
PSQL_BIN,
|
||||
"-h", host,
|
||||
"-p", port,
|
||||
"-U", db_user,
|
||||
|
||||
@ -1,16 +1,13 @@
|
||||
#!/bin/bash
|
||||
# Load blog-training-alpaca.jsonl into PostgreSQL learning_corpus table
|
||||
|
||||
JSONL_FILE="/Users/renefichtmueller/Desktop/Claude Code/llm-gateway/packages/fine-tuner/data/blog-training-alpaca.jsonl"
|
||||
DB_URL="postgresql://llm:llm_secure_2026@127.0.0.1:15432/llm_gateway"
|
||||
JSONL_FILE="${JSONL_FILE:-$(cd "$(dirname "$0")/.." && pwd)/data/blog-training-alpaca.jsonl}"
|
||||
DB_HOST="${DB_HOST:-localhost}"
|
||||
DB_PORT="${DB_PORT:-15432}"
|
||||
DB_NAME="${DB_NAME:-llm_gateway}"
|
||||
DB_USER="${DB_USER:-llm}"
|
||||
TASK_TYPE="tip_blog"
|
||||
|
||||
# Parse connection string
|
||||
DB_HOST=$(echo "$DB_URL" | sed -n 's/.*@\([^:]*\).*/\1/p')
|
||||
DB_PORT=$(echo "$DB_URL" | sed -n 's/.*:\([0-9]*\)\/.*/\1/p')
|
||||
DB_NAME=$(echo "$DB_URL" | sed -n 's/.*\/\(.*\)$/\1/p')
|
||||
DB_USER="llm"
|
||||
|
||||
echo "🔄 Loading blog training data into PostgreSQL..."
|
||||
echo " Host: $DB_HOST:$DB_PORT"
|
||||
echo " Database: $DB_NAME"
|
||||
@ -29,7 +26,7 @@ SELECT COUNT(*) as "Rows before" FROM learning_corpus WHERE task_type = :task_ty
|
||||
EOF
|
||||
|
||||
# Use Python to parse JSONL and generate SQL
|
||||
python3 << PYEOF
|
||||
python3 << PYEOF >> "$TMPFILE"
|
||||
import json
|
||||
import sys
|
||||
|
||||
@ -63,10 +60,12 @@ print("\n-- Count after")
|
||||
print(f"SELECT COUNT(*) as \"Rows after\" FROM learning_corpus WHERE task_type = '{task_type}';")
|
||||
print("COMMIT;")
|
||||
|
||||
PYEOF >> "$TMPFILE"
|
||||
PYEOF
|
||||
|
||||
# Execute SQL
|
||||
PGPASSWORD="llm_secure_2026" psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -f "$TMPFILE"
|
||||
: "${PGPASSWORD:?Set PGPASSWORD before running this script}"
|
||||
export PGPASSWORD
|
||||
psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -f "$TMPFILE"
|
||||
|
||||
# Cleanup
|
||||
rm "$TMPFILE"
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
#!/bin/bash
|
||||
# start.sh — Launch fine-tuner with SSH tunnel to Erik's PostgreSQL
|
||||
# start.sh — Launch fine-tuner with SSH tunnel to PostgreSQL
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/start.sh # run fine-tuner
|
||||
@ -9,11 +9,11 @@
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
TUNNEL_PORT=5434
|
||||
SSH_HOST="erik"
|
||||
REMOTE_PG_PORT=5432
|
||||
TUNNEL_PORT="${TUNNEL_PORT:-5434}"
|
||||
SSH_HOST="${FT_SSH_HOST:?Set FT_SSH_HOST before running this script}"
|
||||
REMOTE_PG_PORT="${REMOTE_PG_PORT:-5432}"
|
||||
|
||||
echo "[start.sh] Opening SSH tunnel to Erik:${REMOTE_PG_PORT} → localhost:${TUNNEL_PORT}"
|
||||
echo "[start.sh] Opening SSH tunnel to ${SSH_HOST}:${REMOTE_PG_PORT} -> localhost:${TUNNEL_PORT}"
|
||||
|
||||
# Kill any existing tunnel on that port
|
||||
pkill -f "ssh.*${TUNNEL_PORT}:localhost:${REMOTE_PG_PORT}" 2>/dev/null || true
|
||||
@ -36,9 +36,9 @@ trap cleanup EXIT
|
||||
|
||||
# Run fine-tuner with tunnel DB URL
|
||||
# Fine-tuner reads FT_DB_URL (see src/main.py load_config)
|
||||
export FT_DB_URL="postgresql://llm:llm_secure_2026@localhost:${TUNNEL_PORT}/llm_gateway"
|
||||
export FT_GATEWAY_URL="https://llm-gateway.context-x.org"
|
||||
export FT_OLLAMA_URL="http://localhost:11434"
|
||||
export FT_DB_URL="${FT_DB_URL:-postgresql://llm@localhost:${TUNNEL_PORT}/llm_gateway}"
|
||||
export FT_GATEWAY_URL="${FT_GATEWAY_URL:-http://localhost:3100}"
|
||||
export FT_OLLAMA_URL="${FT_OLLAMA_URL:-http://localhost:11434}"
|
||||
|
||||
echo "[start.sh] Starting fine-tuner..."
|
||||
# Route --status / --dry-run / --task-type / --general / --dpo to manual_trigger.py
|
||||
|
||||
@ -54,9 +54,9 @@ logger = logging.getLogger(__name__)
|
||||
_BASE_DIR = Path(__file__).parent.parent
|
||||
_DEFAULT_CONFIG = _BASE_DIR / "config" / "fine_tuner.yaml"
|
||||
|
||||
DEFAULT_DB_URL = "postgresql://llm:llm_secure_password@localhost:5432/llm_gateway"
|
||||
DEFAULT_DB_URL = "postgresql://llm@localhost:5432/llm_gateway"
|
||||
DEFAULT_GATEWAY_URL = "http://localhost:3100"
|
||||
DEFAULT_OLLAMA_URL = "http://192.168.178.169:11434"
|
||||
DEFAULT_OLLAMA_URL = "http://localhost:11434"
|
||||
|
||||
|
||||
def load_config(path: Optional[str] = None) -> dict:
|
||||
|
||||
@ -203,7 +203,7 @@ LIGHTRAG_PORT=3140
|
||||
ENVIRONMENT=production
|
||||
|
||||
# LLM Backend
|
||||
OLLAMA_URL=http://192.168.178.213:11434
|
||||
OLLAMA_URL=http://localhost:11434
|
||||
OLLAMA_MODEL=qwen2.5:14b
|
||||
|
||||
# Vector Database
|
||||
@ -211,7 +211,7 @@ QDRANT_URL=http://localhost:6333
|
||||
EMBEDDING_MODEL=bge-m3
|
||||
|
||||
# PostgreSQL
|
||||
DATABASE_URL=postgresql://tip_kg:password@localhost:5432/tip_lightrag
|
||||
DATABASE_URL=postgresql://tip_kg@localhost:5432/tip_lightrag
|
||||
DB_POOL_SIZE=10
|
||||
|
||||
# Hybrid Retrieval
|
||||
@ -230,17 +230,17 @@ pip install -r requirements.txt
|
||||
python scripts/init_db.py
|
||||
|
||||
# Run sidecar
|
||||
uvicorn app.main:app --host 0.0.0.0 --port 3140 --reload
|
||||
uvicorn app.main:app --host localhost --port 3140 --reload
|
||||
```
|
||||
|
||||
### Erik Deployment
|
||||
### Remote Deployment
|
||||
|
||||
```bash
|
||||
# Copy to Erik
|
||||
scp -r packages/lightrag-sidecar/ erik:/opt/llm-gateway/packages/
|
||||
scp -r packages/lightrag-sidecar/ "$DEPLOY_HOST:$APP_DIR/packages/"
|
||||
|
||||
# Install on Erik
|
||||
cd /opt/llm-gateway/packages/lightrag-sidecar
|
||||
cd "$APP_DIR/packages/lightrag-sidecar"
|
||||
python -m venv venv
|
||||
source venv/bin/activate
|
||||
pip install -r requirements.txt
|
||||
|
||||
@ -17,7 +17,7 @@ class Settings(BaseSettings):
|
||||
|
||||
# LLM Backend
|
||||
LLM_BACKEND: Literal["ollama", "claude"] = "ollama"
|
||||
OLLAMA_URL: str = "http://192.168.178.213:11434"
|
||||
OLLAMA_URL: str = "http://localhost:11434"
|
||||
OLLAMA_MODEL: str = "qwen2.5:14b" # For entity extraction
|
||||
|
||||
# Vector Search
|
||||
@ -27,7 +27,7 @@ class Settings(BaseSettings):
|
||||
VECTOR_SIMILARITY_THRESHOLD: float = 0.7
|
||||
|
||||
# Database
|
||||
DATABASE_URL: str = "postgresql://tip_kg:password@localhost/tip_lightrag"
|
||||
DATABASE_URL: str = "postgresql://tip_kg@localhost/tip_lightrag"
|
||||
DB_POOL_SIZE: int = 10
|
||||
DB_ECHO: bool = False # SQL logging
|
||||
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
/**
|
||||
* PM2 Ecosystem Config — LightRAG Sidecar on Erik (217.154.82.179)
|
||||
* PM2 Ecosystem Config - LightRAG Sidecar
|
||||
*
|
||||
* Deploy: pm2 start packages/lightrag-sidecar/ecosystem.config.cjs
|
||||
* Reload: pm2 reload lightrag-sidecar
|
||||
@ -12,10 +12,10 @@ module.exports = {
|
||||
{
|
||||
name: 'lightrag-sidecar',
|
||||
script: 'app/main.py',
|
||||
cwd: '/opt/llm-gateway/packages/lightrag-sidecar',
|
||||
cwd: process.env.LIGHTRAG_CWD || process.cwd(),
|
||||
interpreter: '/usr/bin/python3',
|
||||
interpreter_args: '-m uvicorn',
|
||||
args: 'app.main:app --host 0.0.0.0 --port 3140 --workers 2',
|
||||
args: `app.main:app --host ${process.env.HOST || 'localhost'} --port ${process.env.PORT || '3140'} --workers 2`,
|
||||
instances: 1,
|
||||
exec_mode: 'fork',
|
||||
env: {
|
||||
@ -24,11 +24,11 @@ module.exports = {
|
||||
ENVIRONMENT: 'production',
|
||||
LIGHTRAG_DOMAIN: 'transceiver',
|
||||
LLM_BACKEND: 'ollama',
|
||||
OLLAMA_URL: 'https://ollama.fichtmueller.org',
|
||||
OLLAMA_URL: process.env.OLLAMA_URL || 'http://localhost:11434',
|
||||
OLLAMA_MODEL: 'qwen2.5:14b',
|
||||
QDRANT_URL: 'http://localhost:6333',
|
||||
EMBEDDING_MODEL: 'bge-m3',
|
||||
DATABASE_URL: 'postgresql://tip_kg:tip_secure_2026@localhost:5432/tip_lightrag',
|
||||
DATABASE_URL: process.env.DATABASE_URL || 'postgresql://tip_kg@localhost:5432/tip_lightrag',
|
||||
DB_POOL_SIZE: '10',
|
||||
MAX_WORKERS: '4',
|
||||
LOG_LEVEL: 'info',
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user