#!/bin/bash # Load blog-training-alpaca.jsonl into PostgreSQL learning_corpus table JSONL_FILE="${JSONL_FILE:-$(cd "$(dirname "$0")/.." && pwd)/data/blog-training-alpaca.jsonl}" DB_HOST="${DB_HOST:-localhost}" DB_PORT="${DB_PORT:-15432}" DB_NAME="${DB_NAME:-llm_gateway}" DB_USER="${DB_USER:-llm}" TASK_TYPE="tip_blog" echo "🔄 Loading blog training data into PostgreSQL..." echo " Host: $DB_HOST:$DB_PORT" echo " Database: $DB_NAME" echo " JSONL: $JSONL_FILE" echo "" # Create temporary SQL file TMPFILE=$(mktemp) cat > "$TMPFILE" << 'EOF' BEGIN; -- Count before SELECT COUNT(*) as "Rows before" FROM learning_corpus WHERE task_type = :task_type; -- Load from JSONL (using Python/jq approach) EOF # Use Python to parse JSONL and generate SQL python3 << PYEOF >> "$TMPFILE" import json import sys jsonl_path = "$JSONL_FILE" task_type = "$TASK_TYPE" print("-- Insert blog training samples") print("INSERT INTO learning_corpus (task_type, prompt_text, completion_text, quality_score, tags) VALUES") with open(jsonl_path) as f: samples = [json.loads(line) for line in f if line.strip()] for i, sample in enumerate(samples): prompt = sample.get('instruction', '').replace("'", "''") if sample.get('input'): prompt += f"\n{sample['input'].replace(\"'\", \"''\")}".replace('\n', '\\n') output = sample.get('output', '').replace("'", "''").replace('\n', '\\n') quality = sample.get('quality_score', 8.0) source = sample.get('source', 'unknown').replace("'", "''") # SQL VALUES clause values = f"('{task_type}', '{prompt}', '{output}', {quality}, ARRAY['{source}', '{task_type}'])" if i < len(samples) - 1: print(f" {values},") else: print(f" {values};") print("\n-- Count after") print(f"SELECT COUNT(*) as \"Rows after\" FROM learning_corpus WHERE task_type = '{task_type}';") print("COMMIT;") PYEOF # Execute SQL : "${PGPASSWORD:?Set PGPASSWORD before running this script}" export PGPASSWORD psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$DB_NAME" -f "$TMPFILE" # Cleanup rm "$TMPFILE" echo "" echo "✅ Load complete!"