Claude VM
a6c03a091e
Moved from gov-agreg/src/pages/achizitii/* to root (drop prefix). - 22 pages migrated, 127 files total - All internal links: /achizitii/X → /X (176 occurrences fixed) - AchizitiiLayout subnav rewritten: /X paths, top-right link to vreaudigital.ro hub - BaseLayout new (vreau.digital branding, OG tags, site URL) - astro.config.mjs: site https://vreau.digital, server output (was static) - docker-compose: port 5096 (vreaudigital is 5095), container vreau-digital - deploy.sh: paths /opt/vreau-digital, log /var/log/vreau-digital-deploy.log Backend shared with gov-agreg: - PostgreSQL satra (same schemas: seap, firms, anaf, anre, ...) - Photon, Martin tiles - Infisical /vreaudigital path (DATABASE_URL etc. shared) build: PASS (npx astro check 0 errors, npm run build 5s vite + 10s server)
94 lines
3.3 KiB
Bash
Executable File
94 lines
3.3 KiB
Bash
Executable File
#!/bin/bash
|
|
# Curtea de Conturi — Stage 1: listing-page metadata harvest.
|
|
#
|
|
# Mirrors scrape-anre.sh / scrape-bugetar.sh pattern: Infisical Machine
|
|
# Identity → env-file → docker run --env-file (NEVER -e $VAR), file deleted
|
|
# post-launch.
|
|
#
|
|
# Idempotent (UPSERT on slug_id PK = sha1(category|slug)).
|
|
# Safe to run from cron — recommend weekly (new audits drip in slowly).
|
|
#
|
|
# Stage 2 (PDF parse + CUI fuzzy match) is a separate scraper, see
|
|
# services/seap-scraper/CURTEACONT-PLAN.md.
|
|
#
|
|
# Env knobs:
|
|
# SOURCE=all|financiar|conformitate|performanta (default: all)
|
|
# LIMIT=0 (default: 0 = full)
|
|
# START_PAGE=1 (default: 1)
|
|
#
|
|
# Run:
|
|
# sudo SOURCE=financiar LIMIT=500 /opt/vreaudigital/services/seap-scraper/cron/scrape-curteacont.sh
|
|
# sudo /opt/vreaudigital/services/seap-scraper/cron/scrape-curteacont.sh # full all sources
|
|
set -euo pipefail
|
|
|
|
SOURCE="${SOURCE:-all}"
|
|
LIMIT="${LIMIT:-0}"
|
|
START_PAGE="${START_PAGE:-1}"
|
|
LOG=/var/log/vreaudigital-curteacont.log
|
|
|
|
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG"; }
|
|
|
|
log "=== curteacont scrape started (source=$SOURCE limit=$LIMIT start=$START_PAGE) ==="
|
|
|
|
if docker ps --filter name=vreaudigital-curteacont --format '{{.Names}}' | grep -q '^vreaudigital-curteacont$'; then
|
|
log "WARN: vreaudigital-curteacont already running, skipping this tick"
|
|
exit 0
|
|
fi
|
|
docker rm -f vreaudigital-curteacont 2>/dev/null || true
|
|
|
|
# ── Fetch DATABASE_URL via Infisical Machine Identity ──
|
|
source /opt/vreaudigital/.infisical-mi
|
|
TOKEN=$(infisical login --method=universal-auth \
|
|
--domain="$INFISICAL_API_URL" \
|
|
--client-id="$INFISICAL_CLIENT_ID" \
|
|
--client-secret="$INFISICAL_CLIENT_SECRET" \
|
|
--silent --plain)
|
|
|
|
umask 077
|
|
ENVF=$(mktemp /tmp/.vreaudigital-curteacont-env.XXXXXX)
|
|
DBURL=$(infisical secrets get DATABASE_URL \
|
|
--domain="$INFISICAL_API_URL" \
|
|
--projectId="$INFISICAL_PROJECT_ID" \
|
|
--env="$INFISICAL_ENV" --path="$INFISICAL_PATH" \
|
|
--token="$TOKEN" --plain --silent)
|
|
echo "DATABASE_URL=$DBURL" > "$ENVF"
|
|
# curteadeconturi.ro serves an intermediate CA chain that node's bundle doesn't
|
|
# trust by default. Cert is valid OOB; bypass for this scraper. (Same workaround
|
|
# we use for ANRE.)
|
|
echo "NODE_TLS_REJECT_UNAUTHORIZED=0" >> "$ENVF"
|
|
unset DBURL TOKEN
|
|
|
|
cd /opt/vreaudigital/services/seap-scraper
|
|
|
|
if [ ! -d node_modules/tsx ]; then
|
|
log "Installing seap-scraper deps..."
|
|
docker run --rm -v "$(pwd):/work" -w /work --user "$(id -u):$(id -g)" \
|
|
node:22-alpine npm install --omit=optional 2>&1 | tee -a "$LOG" >/dev/null
|
|
fi
|
|
|
|
EXTRA_ARGS="--source=$SOURCE --start-page=$START_PAGE"
|
|
[ "$LIMIT" -gt 0 ] 2>/dev/null && EXTRA_ARGS="$EXTRA_ARGS --limit=$LIMIT"
|
|
|
|
CID=$(docker run -d \
|
|
--name vreaudigital-curteacont \
|
|
--network host \
|
|
--env-file "$ENVF" \
|
|
-v "$(pwd):/work" \
|
|
-w /work \
|
|
--user "$(id -u):$(id -g)" \
|
|
--restart no \
|
|
node:22-alpine \
|
|
npx tsx src/scrape-curteacont.ts $EXTRA_ARGS)
|
|
log "container started: $CID"
|
|
|
|
sleep 3
|
|
rm -f "$ENVF"
|
|
log "envfile cleaned"
|
|
|
|
docker wait vreaudigital-curteacont >/dev/null
|
|
EXIT_CODE=$(docker inspect -f '{{.State.ExitCode}}' vreaudigital-curteacont 2>/dev/null || echo "?")
|
|
docker logs vreaudigital-curteacont 2>&1 | tail -50 | tee -a "$LOG"
|
|
log "=== curteacont scrape done (exit=$EXIT_CODE) ==="
|
|
|
|
exit "$EXIT_CODE"
|