86d1531d52
- Replace plaintext tip (transceiver_db) and llm_gateway DB passwords with env-var references guarded by fail-fast checks, across 15 scripts + sql runbook. Covers quoted exports, unquoted psql calls, python env dicts and env-or-hardcoded-default fallbacks. - Untrack training-data/*.jsonl (~48MB, kept on disk) and gitignore *.jsonl. - Add .security-scan-allowlist for legit false positives: vendor name flexoptix, own homelab infra, sibling-project codenames in the sync journal, env-resolved DB URIs, example placeholders, ML tokenizer tokens. - Pre-push leak scanner goes from 397 to 0 findings, green without --no-verify. Follow-up (not in this commit): rotate the leaked values (llm_gateway first, tip fleet-wide coordinated); git history still holds the old passwords.
56 lines
2.5 KiB
Bash
56 lines
2.5 KiB
Bash
#!/bin/bash
|
|
# Run ON Erik: fix temp_range in SQL, remove transaction, re-apply
|
|
SQL="/tmp/011-flexoptix-enrichment.sql"
|
|
LOG="/tmp/fix-enrich.log"
|
|
|
|
echo "$(date): Fixing SQL..." > "$LOG"
|
|
|
|
# Remove BEGIN/COMMIT
|
|
sed -i 's/^BEGIN;$//' "$SQL"
|
|
sed -i 's/^COMMIT;$//' "$SQL"
|
|
|
|
# Map verbose temp_range strings to DB enum values
|
|
sed -i "s/temp_range = 'Commercial (0°C to 70°C)'/temp_range = 'COM'/g" "$SQL"
|
|
sed -i "s/temp_range = 'Industrial (-40°C to 85°C)'/temp_range = 'IND'/g" "$SQL"
|
|
sed -i "s/temp_range = 'Extended (-5°C to 85°C)'/temp_range = 'EXT'/g" "$SQL"
|
|
sed -i "s/temp_range = 'Extended (-40°C to 85°C)'/temp_range = 'IND'/g" "$SQL"
|
|
# Remove any remaining non-standard temp_range values (keep only COM/IND/EXT)
|
|
# Just remove the temp_range SET clause if it doesn't match
|
|
python3 -c "
|
|
import re, sys
|
|
with open('$SQL', 'r') as f:
|
|
content = f.read()
|
|
# Remove temp_range = 'anything that is not COM/IND/EXT'
|
|
def fix_temp(m):
|
|
val = m.group(1)
|
|
if val in ('COM', 'IND', 'EXT'):
|
|
return m.group(0)
|
|
if 'commercial' in val.lower() or '0' in val:
|
|
return \"temp_range = 'COM'\"
|
|
if 'industrial' in val.lower() or '-40' in val:
|
|
return \"temp_range = 'IND'\"
|
|
return '' # remove invalid temp_range
|
|
content = re.sub(r\"temp_range = '([^']*)'\", fix_temp, content)
|
|
# Clean up dangling commas from removed fields
|
|
content = re.sub(r',\s*,', ',', content)
|
|
content = re.sub(r'SET\s*,', 'SET ', content)
|
|
content = re.sub(r',\s*WHERE', ' WHERE', content)
|
|
with open('$SQL', 'w') as f:
|
|
f.write(content)
|
|
print('Fixed temp_range values')
|
|
" >> "$LOG" 2>&1
|
|
|
|
echo "Re-applying SQL..." >> "$LOG"
|
|
PGPASSWORD="${PGPASSWORD:?set PGPASSWORD}" psql -h localhost -p 5433 -U tip -d transceiver_db -f "$SQL" >> "$LOG" 2>&1
|
|
|
|
echo "" >> "$LOG"
|
|
echo "=== RESULTS ===" >> "$LOG"
|
|
PGPASSWORD="${PGPASSWORD:?set PGPASSWORD}" psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -c "SELECT count(*) as img FROM transceivers WHERE image_url IS NOT NULL" >> "$LOG" 2>&1
|
|
echo " transceivers have images" >> "$LOG"
|
|
PGPASSWORD="${PGPASSWORD:?set PGPASSWORD}" psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -c "SELECT count(*) FROM transceivers WHERE notes IS NOT NULL AND notes != ''" >> "$LOG" 2>&1
|
|
echo " transceivers have enriched notes" >> "$LOG"
|
|
PGPASSWORD="${PGPASSWORD:?set PGPASSWORD}" psql -h localhost -p 5433 -U tip -d transceiver_db -t -A -c "SELECT count(*) FROM transceivers WHERE connector IS NOT NULL" >> "$LOG" 2>&1
|
|
echo " transceivers have connector" >> "$LOG"
|
|
|
|
echo "$(date): DONE" >> "$LOG"
|