#!/bin/bash # Daily data-freshness heartbeat for vreaudigital.ro # - Queries max(fetched_at) per primary table across 17 schemas # - Compares against per-source expected cadence (days) # - Posts a webhook payload if any source is stale beyond threshold # - Always exits 0 (alerts are signal, not error — cron noise budget = 1 alert/day) # # Run from satra cron at 07:00 daily. # Designed to be paranoid-safe: never echoes the DB password, never fails # loud on transient DB blips (only fails when the heartbeat itself can't run). set -uo pipefail LOG=/var/log/vreaudigital-heartbeat.log log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG"; } WEBHOOK_URL="https://n8n.beletage.ro/webhook/satra-backup-alert" HOSTNAME_TAG="vreaudigital" log "=== Heartbeat started ===" if [ ! -f /opt/vreaudigital/.infisical-mi ]; then log "FATAL: /opt/vreaudigital/.infisical-mi missing" exit 1 fi # shellcheck disable=SC1091 source /opt/vreaudigital/.infisical-mi TOKEN=$(infisical login \ --method=universal-auth \ --domain="$INFISICAL_API_URL" \ --client-id="$INFISICAL_CLIENT_ID" \ --client-secret="$INFISICAL_CLIENT_SECRET" \ --silent --plain) DATABASE_URL=$(infisical run \ --domain="$INFISICAL_API_URL" \ --projectId="$INFISICAL_PROJECT_ID" \ --env="$INFISICAL_ENV" \ --path="$INFISICAL_PATH" \ --silent --token="$TOKEN" \ -- sh -c 'echo "$DATABASE_URL"') DB=$(echo "$DATABASE_URL" | sed -E 's/[?&]schema=[^&]*//; s/\?$//') export PGUSER=$(echo "$DB" | sed -E 's|^postgresql://([^:]+):.*|\1|') export PGPASSWORD=$(echo "$DB" | sed -E 's|^postgresql://[^:]+:([^@]+)@.*|\1|') export PGHOST=$(echo "$DB" | sed -E 's|^postgresql://[^@]+@([^:/]+).*|\1|') export PGPORT=$(echo "$DB" | sed -E 's|^postgresql://[^@]+@[^:]+:([0-9]+)/.*|\1|') export PGDATABASE=$(echo "$DB" | sed -E 's|^postgresql://[^@]+@[^/]+/([^?]+).*|\1|') unset DATABASE_URL TOKEN DB # Per-source cadence query. Each row: source_label, expected_max_days, actual_gap_days, # last_seen_date. Sources stuck at known long staleness (anaf datornici Q1 2016) are # excluded — heartbeat noise budget is for fixable freshness, not known constants. QUERY=$(cat <<'SQL' WITH probes AS ( SELECT 'seap.announcements' AS label, 2 AS expected_days, max(publication_date)::date AS last_seen FROM seap.announcements UNION ALL SELECT 'seap.wsp_sync_state', 1, max(last_run_at)::date FROM seap.wsp_sync_state UNION ALL SELECT 'seap.sync_state(da)', 30, max(updated_at)::date FROM seap.sync_state WHERE source='da' UNION ALL SELECT 'firms.entities', 100, max(updated_at)::date FROM firms.entities UNION ALL SELECT 'firms.financials', 400, max(fetched_at)::date FROM firms.financials UNION ALL SELECT 'fonduri.beneficiar_anunt', 7, max(data_publicare)::date FROM fonduri.beneficiar_anunt UNION ALL SELECT 'fonduri.afir_plati', 365, max(fetched_at)::date FROM fonduri.afir_plati UNION ALL SELECT 'regas.ajutoare', 45, max(fetched_at)::date FROM regas.ajutoare UNION ALL SELECT 'aep.donatii_pj', 60, max(fetched_at)::date FROM aep.donatii_pj UNION ALL SELECT 'ani.declaratii', 400, max(fetched_at)::date FROM ani.declaratii UNION ALL SELECT 'bugetar.entitate', 60, max(updated_at)::date FROM bugetar.entitate UNION ALL SELECT 'anre.licente', 14, max(fetched_at)::date FROM anre.licente UNION ALL SELECT 'ancom.operatori', 14, max(fetched_at)::date FROM ancom.operatori UNION ALL SELECT 'cnsc.decizii', 14, max(fetched_at)::date FROM cnsc.decizii UNION ALL SELECT 'cnas.furnizori', 60, max(fetched_at)::date FROM cnas.furnizori UNION ALL SELECT 'asf.entitati', 14, max(fetched_at)::date FROM asf.entitati UNION ALL SELECT 'aaas.firme', 30, max(fetched_at)::date FROM aaas.firme UNION ALL SELECT 'curteacont.rapoarte', 14, max(fetched_at)::date FROM curteacont.rapoarte UNION ALL SELECT 'apia.fermieri', 60, max(fetched_at)::date FROM apia.fermieri UNION ALL SELECT 'gnm.comunicate', 14, max(fetched_at)::date FROM gnm.comunicate ) SELECT label, expected_days, -- clamp future dates (TED publication-date can be in the future) and -- treat NULL last_seen as ancient (empty table → alert). -- NB: LEAST(NULL, x) = x in PG (returns NULL only if all args NULL), -- so explicit CASE for NULL handling. CASE WHEN last_seen IS NULL THEN 9999 ELSE (now()::date - LEAST(last_seen, now()::date)) END AS gap_days, COALESCE(last_seen::text, 'NEVER') AS last_seen, CASE WHEN last_seen IS NULL THEN 'STALE' WHEN (now()::date - LEAST(last_seen, now()::date)) > expected_days THEN 'STALE' ELSE 'OK' END AS status FROM probes ORDER BY CASE WHEN last_seen IS NULL THEN 9999 ELSE (now()::date - LEAST(last_seen, now()::date)) END DESC; SQL ) OUT=$(psql -v ON_ERROR_STOP=1 -A -F$'\t' -t -c "$QUERY" 2>&1) || { log "ERROR: psql failed — heartbeat skipped this run" log "$OUT" exit 0 } unset PGPASSWORD STALE_LIST=$(echo "$OUT" | awk -F'\t' '$5=="STALE" { printf "%s (gap=%sd, expected≤%sd, last=%s)\n", $1, $3, $2, $4 }') STALE_COUNT=$(echo -n "$STALE_LIST" | grep -c . || true) TOTAL=$(echo -n "$OUT" | grep -c . || true) log "Probed $TOTAL sources, $STALE_COUNT stale" echo "$OUT" | awk -F'\t' '{ printf " %-30s %s gap=%sd last=%s\n", $1, $5, $3, $4 }' | tee -a "$LOG" if [ "$STALE_COUNT" -gt 0 ]; then log "ALERT — posting to webhook" PAYLOAD=$(jq -nc \ --arg s "STALE" \ --arg h "$HOSTNAME_TAG" \ --argjson c "$STALE_COUNT" \ --argjson t "$TOTAL" \ --arg d "$STALE_LIST" \ '{status:$s, host:$h, service:"data-heartbeat", stale_count:$c, total:$t, details:$d}') curl -sS -X POST -H "Content-Type: application/json" --max-time 30 \ -d "$PAYLOAD" "$WEBHOOK_URL" >/dev/null 2>&1 || log "webhook POST failed (non-fatal)" fi log "=== Done ===" exit 0