#!/usr/bin/env bash # # Daily WSP incremental sync — runs on satra via systemd timer or cron. # # Loads DATABASE_URL from /opt/architools/.env and SEAP secrets from # Infisical. Runs incremental for all configured public + Beletage feeds. # Logs go to /var/log/wsp-sync/ (rotated by logrotate). set -euo pipefail LOG_DIR=/var/log/wsp-sync LOG_FILE="$LOG_DIR/wsp-incremental-$(date +%Y%m%d).log" mkdir -p "$LOG_DIR" cd /opt/vreaudigital/services/seap-scraper # Load DB URL (architools_user — same DB as TED imports) set -a . /opt/architools/.env set +a # Strip ?schema= param (psycopg2 doesn't accept it) export DATABASE_URL="${DATABASE_URL%%\?*}" # Load SEAP secrets from Infisical (using the same loader as claude-start.sh) if [ -f /opt/wsp/.env.seap ]; then # Production: secrets pre-loaded into a static file readable by service account set -a . /opt/wsp/.env.seap set +a else echo "ERROR: /opt/wsp/.env.seap not found — run wsp-secrets-fetch.sh first" >&2 exit 1 fi # Run incremental for each feed in sequence (low-volume daily = ~5 min total) for op in SU_CaNotices SU_CNotices SU_PiNotices SU_RfqNotices SU_RfqInvitations \ SU_DCNotices SU_PCNotices SU_RdcNotices SU_ENotices \ SuContracts SuInvoices SuDirectAcquisitions; do echo "=== $(date -Iseconds) — $op ===" | tee -a "$LOG_FILE" ./.venv/bin/python -m wsp.runner incremental "$op" --lookback-hours 36 \ >> "$LOG_FILE" 2>&1 || echo " (failed, continuing)" | tee -a "$LOG_FILE" done echo "=== $(date -Iseconds) — Done ===" | tee -a "$LOG_FILE" # Show status summary at the end (also captured by journalctl if systemd-run) ./.venv/bin/python -m wsp.runner status 2>&1 | tail -30 | tee -a "$LOG_FILE"