#!/bin/bash # SEAP Achiziții Directe (DA) — daily/weekly backfill of e-licitatie.ro DA notices. # # The DA endpoint is rate-limited and large (~500K rows already + ~8M historical # 2017-2024 pending). The scraper itself is idempotent and resumable via # `seap.sync_state[source='da']`: # - reads last_date, requests notices > last_date # - upserts on natural key, updates sync_state to latest fetched # # Mirrors scrape-anre.sh / scrape-bugetar.sh pattern. Reads DATABASE_URL via # Infisical MI, writes envfile, docker-run with --env-file, deletes file. # # Env knobs: # MODE=da | backfill (default: da; backfill = last 6 months ignoring sync_state) # # Run: # sudo /opt/vreaudigital/services/seap-scraper/cron/scrape-da.sh # sudo MODE=backfill /opt/vreaudigital/services/seap-scraper/cron/scrape-da.sh set -euo pipefail MODE="${MODE:-da}" LOG=/var/log/vreaudigital-da.log log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG"; } log "=== SEAP DA scrape started (mode=$MODE) ===" if docker ps --filter name=vreaudigital-da --format '{{.Names}}' | grep -q '^vreaudigital-da$'; then log "WARN: vreaudigital-da already running, skipping this tick" exit 0 fi docker rm -f vreaudigital-da 2>/dev/null || true # ── Fetch DATABASE_URL via Infisical Machine Identity ── source /opt/vreaudigital/.infisical-mi TOKEN=$(infisical login --method=universal-auth \ --domain="$INFISICAL_API_URL" \ --client-id="$INFISICAL_CLIENT_ID" \ --client-secret="$INFISICAL_CLIENT_SECRET" \ --silent --plain) umask 077 ENVF=$(mktemp /tmp/.vreaudigital-da-env.XXXXXX) DBURL=$(infisical secrets get DATABASE_URL \ --domain="$INFISICAL_API_URL" \ --projectId="$INFISICAL_PROJECT_ID" \ --env="$INFISICAL_ENV" --path="$INFISICAL_PATH" \ --token="$TOKEN" --plain --silent) echo "DATABASE_URL=$DBURL" > "$ENVF" unset DBURL TOKEN cd /opt/vreaudigital/services/seap-scraper if [ ! -d node_modules/tsx ]; then log "Installing seap-scraper deps..." docker run --rm -v "$(pwd):/work" -w /work --user "$(id -u):$(id -g)" \ node:22-alpine npm install --omit=optional 2>&1 | tee -a "$LOG" >/dev/null fi CID=$(docker run -d \ --name vreaudigital-da \ --network host \ --env-file "$ENVF" \ -v "$(pwd):/work" \ -w /work \ --user "$(id -u):$(id -g)" \ --restart no \ node:22-alpine \ npx tsx src/index.ts --mode=$MODE) log "container started: $CID" sleep 3 rm -f "$ENVF" log "envfile cleaned" docker wait vreaudigital-da >/dev/null EXIT_CODE=$(docker inspect -f '{{.State.ExitCode}}' vreaudigital-da 2>/dev/null || echo "?") docker logs vreaudigital-da 2>&1 | tail -40 | tee -a "$LOG" log "=== SEAP DA scrape done (exit=$EXIT_CODE) ===" exit "$EXIT_CODE"