/**
 * ANRE — Autoritatea Națională de Reglementare în domeniul Energiei.
 *
 * Scrapes 4 public registries from portal.anre.ro/PublicLists:
 *   1. LicenteAutorizatii      — ~4.9K licenses (electricitate, flat columns)
 *   2. LicenteAutorizatiiGN    — ~353 licenses (gaze, parent + Detaliu HTML)
 *   3. Atestate                — ~9.7K atestate (parent + Detaliu HTML)
 *   4. AutorizatiiElectricieniAutorizati — ~101K electricieni (people, flat)
 *
 * All endpoints are Kendo Grid AJAX. POST with `page=1&pageSize=99999` returns
 * the full dataset in a single JSON payload — no VIEWSTATE, no captcha,
 * no pagination needed. ~2-15 MB per source, fast.
 *
 * Idempotent via sha1(license_source|license_no|titular_name|data_emitere|tip_al)
 * UNIQUE PK on anre.licente, plus UNIQUE(nr_autorizare, nume_prenume) on electricieni.
 *
 * Run:
 *   npx tsx src/scrape-anre.ts --source=electricitate
 *   npx tsx src/scrape-anre.ts --source=gaze
 *   npx tsx src/scrape-anre.ts --source=atestat
 *   npx tsx src/scrape-anre.ts --source=electricieni
 *   npx tsx src/scrape-anre.ts --source=all
 *   npx tsx src/scrape-anre.ts --source=electricitate --limit=100   # smoke test
 */
import { Pool } from 'pg';
import { createHash } from 'crypto';

const PORTAL = 'https://portal.anre.ro';
const USER_AGENT = 'gov-agreg/1.0 vreaudigital.ro (contact: m.tarau@beletage.ro)';

type Source = 'electricitate' | 'gaze' | 'atestat' | 'electricieni';

const ENDPOINTS: Record<Source, { listUrl: string; dataUrl: string }> = {
  electricitate: {
    listUrl: `${PORTAL}/PublicLists/LicenteAutorizatii`,
    dataUrl: `${PORTAL}/PublicLists/LicenteAutorizatii/GetLicenteAutorizatii`,
  },
  gaze: {
    listUrl: `${PORTAL}/PublicLists/LicenteAutorizatiiGN`,
    dataUrl: `${PORTAL}/PublicLists/LicenteAutorizatiiGN/GetLicenteAutorizatiiGN`,
  },
  atestat: {
    listUrl: `${PORTAL}/PublicLists/Atestate`,
    dataUrl: `${PORTAL}/PublicLists/Atestate/GetAtestate`,
  },
  electricieni: {
    listUrl: `${PORTAL}/PublicLists/AutorizatiiElectricieniAutorizati`,
    dataUrl: `${PORTAL}/PublicLists/AutorizatiiElectricieniAutorizati/GetAutorizatiiElectricieniAutorizati`,
  },
};

// ── CLI args ────────────────────────────────────────────────────────────────
const args = process.argv.slice(2);
const argSource = (args.find((a) => a.startsWith('--source='))?.split('=')[1] || 'all') as Source | 'all';
const argLimit = parseInt(args.find((a) => a.startsWith('--limit='))?.split('=')[1] || '0');

const pool = new Pool({ connectionString: process.env.DATABASE_URL, max: 4 });

// ── Helpers ─────────────────────────────────────────────────────────────────
function sha1Hex(s: string): string {
  return createHash('sha1').update(s).digest('hex');
}

function trimOrNull(v: any): string | null {
  if (v == null) return null;
  const s = String(v).trim();
  return s === '' ? null : s;
}

/** Parses ISO-ish "2025-09-09T00:00:00" or "DD/MM/YYYY" → "YYYY-MM-DD". */
function parseDate(v: any): string | null {
  if (v == null) return null;
  const s = String(v).trim();
  if (s === '') return null;
  // ISO with time
  const iso = s.match(/^(\d{4})-(\d{2})-(\d{2})/);
  if (iso) {
    const y = parseInt(iso[1], 10);
    if (y < 1900 || y > 2100) return null;
    return `${iso[1]}-${iso[2]}-${iso[3]}`;
  }
  // DD/MM/YYYY
  const dmy = s.match(/^(\d{1,2})\/(\d{1,2})\/(\d{4})$/);
  if (dmy) {
    const dd = dmy[1].padStart(2, '0');
    const mm = dmy[2].padStart(2, '0');
    const yyyy = dmy[3];
    const m = parseInt(mm, 10);
    const d = parseInt(dd, 10);
    if (m < 1 || m > 12 || d < 1 || d > 31) return null;
    return `${yyyy}-${mm}-${dd}`;
  }
  return null;
}

// ── HTTP fetch (Kendo Grid AJAX) ────────────────────────────────────────────
let cookieJar = '';

async function getCookies(): Promise<void> {
  const res = await fetch(ENDPOINTS.electricitate.listUrl, {
    headers: { 'User-Agent': USER_AGENT },
    signal: AbortSignal.timeout(30000),
  });
  const setCookie = res.headers.getSetCookie ? res.headers.getSetCookie() : [];
  const cookies = setCookie.map((c: string) => c.split(';')[0]).join('; ');
  cookieJar = cookies;
}

/**
 * Fetches one Kendo Grid page with retries. The ANRE portal is flaky —
 * occasionally times out on certain pages but recovers on retry, even within
 * a few seconds. Returns { Data, Total }.
 */
async function fetchPage(
  source: Source,
  page: number,
  pageSize: number,
  attempt: number = 1,
  extraFilter: Record<string, string | number> = {}
): Promise<{ Data: any[]; Total: number }> {
  const ep = ENDPOINTS[source];
  const params = new URLSearchParams();
  params.set('page', String(page));
  params.set('pageSize', String(pageSize));
  for (const [k, v] of Object.entries(extraFilter)) params.set(k, String(v));
  const body = params.toString();
  // Per-request timeout: shorter so we can retry faster.
  const timeoutMs = 90000;
  try {
    const res = await fetch(ep.dataUrl, {
      method: 'POST',
      headers: {
        'User-Agent': USER_AGENT,
        'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
        'X-Requested-With': 'XMLHttpRequest',
        Accept: 'application/json,text/plain,*/*',
        Referer: ep.listUrl,
        Cookie: cookieJar,
      },
      body,
      signal: AbortSignal.timeout(timeoutMs),
    });
    if (!res.ok) {
      const txt = await res.text().catch(() => '');
      throw new Error(`fetch ${source} p${page} ${res.status}: ${txt.slice(0, 200)}`);
    }
    const json = (await res.json()) as { Data: any[]; Total: number };
    if (!Array.isArray(json.Data)) throw new Error(`bad payload for ${source} p${page}`);
    return json;
  } catch (e) {
    const maxAttempts = 4;
    if (attempt >= maxAttempts) throw e;
    const backoffMs = 2000 * attempt;
    console.warn(`[anre:${source}] p${page} attempt ${attempt} failed: ${e instanceof Error ? e.message : e}; retry in ${backoffMs}ms`);
    await new Promise((resolve) => setTimeout(resolve, backoffMs));
    return fetchPage(source, page, pageSize, attempt + 1, extraFilter);
  }
}

/**
 * Fetch the judet dictionary so we can drill down by county. The
 * electricieni grid has a server-side OFFSET bug above ~20K rows — every
 * page request times out. The workaround is to filter by `Judet[0].IdJudet=N`
 * so each per-judet result set is small (max ~9K for Bucuresti) and
 * paginatable from offset 0.
 */
async function fetchJudete(): Promise<Array<{ id: number; nume: string }>> {
  const res = await fetch(`${PORTAL}/Dictionary/GetJudete`, {
    method: 'POST',
    headers: {
      'User-Agent': USER_AGENT,
      'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
      'X-Requested-With': 'XMLHttpRequest',
      Accept: 'application/json,text/plain,*/*',
      Cookie: cookieJar,
    },
    body: '',
    signal: AbortSignal.timeout(30000),
  });
  if (!res.ok) throw new Error(`GetJudete ${res.status}`);
  const arr = (await res.json()) as Array<{ IdJudet: number; Nume: string }>;
  return arr.map((j) => ({ id: j.IdJudet, nume: j.Nume }));
}

/**
 * Per-source page size limit. Sources with HTML Detaliu (gaze + atestat) MUST
 * paginate — server times out at pageSize > 100. Flat sources can pull
 * everything in one shot.
 */
// ANRE portal is flaky — certain pages randomly time out regardless of pageSize.
// Keep pages SMALL so the failure blast radius is minimal (skip 25 rows, not 1000).
const PAGE_SIZE_LIMIT: Record<Source, number> = {
  electricitate: 99999,   // ~5K rows flat — full in one shot OK (~13s)
  gaze: 25,               // 353 rows w/ huge HTML Detaliu, 15 pages
  atestat: 50,            // 9.7K rows w/ Detaliu — ~195 pages, ~3 min (fast — 0.5s/page)
  electricieni: 2000,     // 101K rows flat. Server times out at offset >20K with pageSize=5000; 2000/page = ~51 pages, ~10 min
};

async function fetchAll(source: Source): Promise<{ rows: any[]; missedPages: number[] }> {
  // Electricieni has a server-side OFFSET-bug at ~20K rows.
  // Drill down by judet — each county is small (max ~9K) and paginatable.
  if (source === 'electricieni' && argLimit === 0) {
    return fetchAllElectricieniByJudet();
  }

  const missedPages: number[] = [];
  if (argLimit > 0) {
    const ps = Math.min(argLimit, PAGE_SIZE_LIMIT[source]);
    const j = await fetchPage(source, 1, ps);
    return { rows: j.Data.slice(0, argLimit), missedPages };
  }
  const ps = PAGE_SIZE_LIMIT[source];
  const first = await fetchPage(source, 1, ps);
  let rows = first.Data;
  const total = first.Total;
  if (rows.length < total) {
    const totalPages = Math.ceil(total / ps);
    console.log(`[anre:${source}] paginating: total=${total} pages=${totalPages} pageSize=${ps}`);
    for (let p = 2; p <= totalPages; p++) {
      try {
        const j = await fetchPage(source, p, ps);
        rows = rows.concat(j.Data);
      } catch (e) {
        // Hard skip after 4 retries — log + continue. Operator can re-run later.
        console.warn(`[anre:${source}] HARD SKIP page ${p}: ${e instanceof Error ? e.message : e}`);
        missedPages.push(p);
      }
      if (p % 10 === 0 || p === totalPages) {
        console.log(`[anre:${source}]   fetched p${p}/${totalPages} (${rows.length}/${total} rows, ${missedPages.length} pages missed)`);
      }
    }
  }
  return { rows, missedPages };
}

/**
 * Per-judet drilldown for electricieni. Iterates GetJudete, for each judet
 * does a paginated fetch with filter `Judet[0].IdJudet=<id>`. Dedups by
 * NrAutorizare across counties (entities can belong to multiple counties
 * in raw data, the schema's UNIQUE(nr_autorizare, nume_prenume) handles
 * collisions on ingest).
 */
async function fetchAllElectricieniByJudet(): Promise<{ rows: any[]; missedPages: number[] }> {
  const missedPages: number[] = [];  // not used in this mode but kept for compat
  const ps = PAGE_SIZE_LIMIT.electricieni;  // 2000
  const judete = await fetchJudete();
  console.log(`[anre:electricieni] drilldown: ${judete.length} judete to scan`);

  // Also include the "no judet" bucket (IdJudet=0 sometimes works for orphans).
  // Easier: just enumerate the 42 RO judete + foreign codes we discovered.
  const allRows: any[] = [];
  let judetIdx = 0;
  for (const j of judete) {
    judetIdx++;
    const filter = { 'Judet[0].IdJudet': j.id };
    try {
      const first = await fetchPage('electricieni', 1, ps, 1, filter);
      let collected = first.Data;
      const total = first.Total;
      if (collected.length < total) {
        const totalPages = Math.ceil(total / ps);
        for (let p = 2; p <= totalPages; p++) {
          try {
            const r = await fetchPage('electricieni', p, ps, 1, filter);
            collected = collected.concat(r.Data);
          } catch (e) {
            console.warn(`[anre:electricieni] judet=${j.nume} HARD SKIP p${p}: ${e instanceof Error ? e.message : e}`);
            missedPages.push(judetIdx * 1000 + p);
          }
        }
      }
      allRows.push(...collected);
      console.log(`[anre:electricieni]   judet ${judetIdx}/${judete.length} ${j.nume} (id=${j.id}): ${collected.length}/${total} rows (total cumulated: ${allRows.length})`);
    } catch (e) {
      console.warn(`[anre:electricieni] judet=${j.nume} FAILED: ${e instanceof Error ? e.message : e}`);
    }
  }
  return { rows: allRows, missedPages };
}

// ── Detaliu HTML parser (for gaze + atestat) ────────────────────────────────
/**
 * Parses the Detaliu HTML <table> into an array of row-objects.
 * Returns an array of records keyed by column header (Romanian text).
 *
 * Example input for `gaze`:
 *   <table>
 *     <tr><th>Nr. Document</th><th>Tip document</th>...</tr>
 *     <tr><td>2033</td><td>Licenta</td>...</tr>
 *     <tr><td>1234</td><td>Autorizatie</td>...</tr>
 *   </table>
 */
function parseDetaliu(html: string): Record<string, string>[] {
  if (!html) return [];
  // Strip whitespace, remove style attributes for cleaner parsing
  const cleaned = html.replace(/\s+/g, ' ').replace(/style="[^"]*"/g, '');

  // Extract rows
  const rowRe = /<tr>\s*(.*?)\s*<\/tr>/gis;
  const rows: string[] = [];
  let m: RegExpExecArray | null;
  while ((m = rowRe.exec(cleaned)) !== null) rows.push(m[1]);
  if (rows.length < 2) return [];

  // Header from first row (th cells)
  const headerRe = /<th[^>]*>\s*(.*?)\s*<\/th>/gis;
  const headers: string[] = [];
  let h: RegExpExecArray | null;
  while ((h = headerRe.exec(rows[0])) !== null) {
    headers.push(decodeHtml(h[1]).replace(/<[^>]+>/g, '').trim());
  }
  if (headers.length === 0) return [];

  // Data rows
  const out: Record<string, string>[] = [];
  for (let i = 1; i < rows.length; i++) {
    const cellRe = /<td[^>]*>\s*(.*?)\s*<\/td>/gis;
    const cells: string[] = [];
    let c: RegExpExecArray | null;
    while ((c = cellRe.exec(rows[i])) !== null) {
      cells.push(decodeHtml(c[1]).replace(/<[^>]+>/g, '').trim());
    }
    if (cells.length === 0) continue;
    const rec: Record<string, string> = {};
    for (let j = 0; j < headers.length && j < cells.length; j++) {
      rec[headers[j]] = cells[j];
    }
    out.push(rec);
  }
  return out;
}

function decodeHtml(s: string): string {
  return s
    .replace(/&nbsp;/g, ' ')
    .replace(/&amp;/g, '&')
    .replace(/&lt;/g, '<')
    .replace(/&gt;/g, '>')
    .replace(/&quot;/g, '"')
    .replace(/&#39;/g, "'")
    .replace(/&#(\d+);/g, (_, n) => String.fromCharCode(parseInt(n, 10)));
}

// ── Ingest ──────────────────────────────────────────────────────────────────
interface IngestStats { seen: number; inserted: number; updated: number; skipped: number; }

async function ingestElectricitate(rows: any[]): Promise<IngestStats> {
  const stats: IngestStats = { seen: rows.length, inserted: 0, updated: 0, skipped: 0 };
  const client = await pool.connect();
  try {
    for (const r of rows) {
      const titular = trimOrNull(r.Societate);
      const licenseNo = r.NrLicenta != null ? String(r.NrLicenta) : null;
      if (!titular || !licenseNo) { stats.skipped++; continue; }
      const dataEm = parseDate(r.DataEmitere);
      const dataEx = parseDate(r.DataExpirare);
      const tipAl = trimOrNull(r.TipAL);
      const id = sha1Hex(['electricitate', licenseNo, titular.toLowerCase(), dataEm || '', tipAl || ''].join('|'));

      const result = await client.query(
        `INSERT INTO anre.licente (
           id, license_source, license_no, license_type, license_subtype,
           titular_name, sediu, localitate, judet, telefon_fax,
           data_emitere, data_expirare, stare, comentariu, raw_json
         ) VALUES ($1,'electricitate',$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14)
         ON CONFLICT (id) DO UPDATE SET
           license_type = EXCLUDED.license_type,
           license_subtype = EXCLUDED.license_subtype,
           sediu = EXCLUDED.sediu,
           localitate = EXCLUDED.localitate,
           judet = EXCLUDED.judet,
           telefon_fax = EXCLUDED.telefon_fax,
           data_emitere = EXCLUDED.data_emitere,
           data_expirare = EXCLUDED.data_expirare,
           stare = EXCLUDED.stare,
           comentariu = EXCLUDED.comentariu,
           raw_json = EXCLUDED.raw_json,
           fetched_at = now()
         RETURNING (xmax = 0) AS inserted`,
        [
          id,
          licenseNo,
          tipAl,
          trimOrNull(r.TipActivitate),
          titular,
          trimOrNull(r.Sediu),
          trimOrNull(r.Localitate),
          trimOrNull(r.Judet),
          trimOrNull(r.TelefonFax),
          dataEm,
          dataEx,
          trimOrNull(r.Stare),
          trimOrNull(r.Comentariu),
          JSON.stringify(r),
        ]
      );
      if (result.rows[0]?.inserted) stats.inserted++; else stats.updated++;
    }
  } finally {
    client.release();
  }
  return stats;
}

async function ingestGazeOrAtestat(source: 'gaze' | 'atestat', rows: any[]): Promise<IngestStats> {
  const stats: IngestStats = { seen: 0, inserted: 0, updated: 0, skipped: 0 };
  const client = await pool.connect();
  try {
    for (const r of rows) {
      const titular = trimOrNull(r.Societate);
      if (!titular) continue;
      const subRows = parseDetaliu(r.Detaliu || '');
      if (subRows.length === 0) { stats.skipped++; continue; }

      for (const sr of subRows) {
        stats.seen++;
        // GN columns: Nr. Document, Tip document, Tip activitate, Localitate unde isi desfasoara activitatea, Data emitere, Data expirare, Stare, Decizie
        // Atestate columns: Nr. atestat, Tip tarif, Data emitere, Data expirare, Stare
        const licenseNo =
          sr['Nr. Document'] || sr['Nr. atestat'] || sr['Nr. document'] || sr['Nr.Document'] || '';
        if (!licenseNo) { stats.skipped++; continue; }
        const tipAl = source === 'atestat' ? 'Atestat' : (sr['Tip document'] || sr['Tip Document'] || null);
        const subtype = sr['Tip activitate'] || sr['Tip tarif'] || sr['Tip Tarif'] || null;
        const dataEm = parseDate(sr['Data emitere']);
        const dataEx = parseDate(sr['Data expirare']);
        const stare = sr['Stare'] || null;
        const decizie = sr['Decizie'] || null;
        const localitateAct = sr['Localitate unde isi desfasoara activitatea'] || null;

        const id = sha1Hex([source, licenseNo, titular.toLowerCase(), dataEm || '', tipAl || ''].join('|'));

        const result = await client.query(
          `INSERT INTO anre.licente (
             id, license_source, license_no, license_type, license_subtype,
             titular_name, sediu, localitate, judet, telefon_fax,
             data_emitere, data_expirare, stare, decizie, raw_json
           ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15)
           ON CONFLICT (id) DO UPDATE SET
             license_type = EXCLUDED.license_type,
             license_subtype = EXCLUDED.license_subtype,
             sediu = EXCLUDED.sediu,
             localitate = EXCLUDED.localitate,
             judet = EXCLUDED.judet,
             telefon_fax = EXCLUDED.telefon_fax,
             data_emitere = EXCLUDED.data_emitere,
             data_expirare = EXCLUDED.data_expirare,
             stare = EXCLUDED.stare,
             decizie = EXCLUDED.decizie,
             raw_json = EXCLUDED.raw_json,
             fetched_at = now()
           RETURNING (xmax = 0) AS inserted`,
          [
            id,
            source,
            String(licenseNo),
            tipAl,
            subtype,
            titular,
            trimOrNull(r.Sediu),
            trimOrNull(localitateAct || r.Localitate),
            trimOrNull(r.Judet),
            trimOrNull(r.TelefonFax),
            dataEm,
            dataEx,
            trimOrNull(stare),
            trimOrNull(decizie),
            JSON.stringify({ parent: r, sub: sr }),
          ]
        );
        if (result.rows[0]?.inserted) stats.inserted++; else stats.updated++;
      }
    }
  } finally {
    client.release();
  }
  return stats;
}

async function ingestElectricieni(rows: any[]): Promise<IngestStats> {
  const stats: IngestStats = { seen: rows.length, inserted: 0, updated: 0, skipped: 0 };
  // Dedup by (nr_autorizare, nume_prenume) PRIOR to chunking. The drilldown
  // returns a person under each judet they're registered in, and the bulk
  // INSERT...ON CONFLICT errors out if the same conflict key appears twice
  // in a single statement ("cannot affect row a second time"). Last wins.
  const deduped = new Map<string, any>();
  for (const r of rows) {
    const nume = trimOrNull(r.NumePrenume);
    const nrAut = r.NrAutorizare != null ? parseInt(String(r.NrAutorizare), 10) : null;
    if (!nume || nrAut == null || !Number.isFinite(nrAut)) continue;
    const key = `${nrAut}|${nume}`;
    deduped.set(key, r);  // last write wins
  }
  const dedupedRows = Array.from(deduped.values());
  console.log(`[anre:electricieni] dedup: ${rows.length} → ${dedupedRows.length} unique (nr_autorizare, nume_prenume)`);

  const client = await pool.connect();
  try {
    // Bulk approach: batch in chunks of 500 with multi-row insert.
    const chunkSize = 500;
    for (let i = 0; i < dedupedRows.length; i += chunkSize) {
      const chunk = dedupedRows.slice(i, i + chunkSize);
      const params: any[] = [];
      const rowsSql: string[] = [];
      let p = 1;
      for (const r of chunk) {
        const nume = trimOrNull(r.NumePrenume);
        const nrAut = r.NrAutorizare != null ? parseInt(String(r.NrAutorizare), 10) : null;
        if (!nume || nrAut == null || !Number.isFinite(nrAut)) { stats.skipped++; continue; }
        rowsSql.push(`($${p++},$${p++},$${p++},$${p++},$${p++},$${p++},$${p++},$${p++},$${p++},$${p++},$${p++})`);
        params.push(
          nume,
          r.NrRegistru != null ? parseInt(String(r.NrRegistru), 10) || null : null,
          nrAut,
          trimOrNull(r.TipAutorizare),
          trimOrNull(r.TarifAutorizare),
          trimOrNull(r.Localitate),
          trimOrNull(r.Judet),
          trimOrNull(r.TelefonFax),
          parseDate(r.DataExpirare),
          trimOrNull(r.Stare),
          JSON.stringify(r)
        );
      }
      if (rowsSql.length === 0) continue;
      const sql = `
        INSERT INTO anre.electricieni
          (nume_prenume, nr_registru, nr_autorizare, tip_autorizare, tarif,
           localitate, judet, telefon_fax, data_expirare, stare, raw_json)
        VALUES ${rowsSql.join(',')}
        ON CONFLICT (nr_autorizare, nume_prenume) DO UPDATE SET
          nr_registru = EXCLUDED.nr_registru,
          tip_autorizare = EXCLUDED.tip_autorizare,
          tarif = EXCLUDED.tarif,
          localitate = EXCLUDED.localitate,
          judet = EXCLUDED.judet,
          telefon_fax = EXCLUDED.telefon_fax,
          data_expirare = EXCLUDED.data_expirare,
          stare = EXCLUDED.stare,
          raw_json = EXCLUDED.raw_json,
          fetched_at = now()
        RETURNING (xmax = 0) AS inserted`;
      const result = await client.query(sql, params);
      const ins = result.rows.filter((r) => r.inserted).length;
      stats.inserted += ins;
      stats.updated += result.rows.length - ins;
    }
  } finally {
    client.release();
  }
  return stats;
}

// ── Orchestration ──────────────────────────────────────────────────────────
async function logScrape(scraper: string, sourceUrl: string, stats: IngestStats, durationMs: number, startedAt: Date, error?: string) {
  await pool.query(
    `INSERT INTO anre.scrape_log (scraper, source_url, rows_seen, rows_inserted, rows_updated, rows_skipped, duration_ms, started_at, error)
     VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)`,
    [scraper, sourceUrl, stats.seen, stats.inserted, stats.updated, stats.skipped, durationMs, startedAt, error || null]
  );
}

async function runOne(source: Source): Promise<IngestStats> {
  const ep = ENDPOINTS[source];
  const startedAt = new Date();
  const t0 = Date.now();
  console.log(`[anre:${source}] fetching ${ep.dataUrl} (${argLimit > 0 ? `limit=${argLimit}` : 'full'})...`);
  let stats: IngestStats = { seen: 0, inserted: 0, updated: 0, skipped: 0 };
  let error: string | undefined;
  let isHardError = false;
  try {
    const { rows, missedPages } = await fetchAll(source);
    console.log(`[anre:${source}] fetched ${rows.length} rows (missed pages: ${missedPages.length}); upserting...`);
    if (source === 'electricitate') stats = await ingestElectricitate(rows);
    else if (source === 'gaze') stats = await ingestGazeOrAtestat('gaze', rows);
    else if (source === 'atestat') stats = await ingestGazeOrAtestat('atestat', rows);
    else if (source === 'electricieni') stats = await ingestElectricieni(rows);
    if (missedPages.length > 0) {
      // Soft warning — log + continue. Operator can re-run for the missed pages.
      error = `missed_pages=${missedPages.length}: ${missedPages.slice(0, 20).join(',')}`;
    }
  } catch (e) {
    error = e instanceof Error ? e.message : String(e);
    isHardError = true;
    console.error(`[anre:${source}] ERROR: ${error}`);
  }
  const dur = Date.now() - t0;
  await logScrape(source, ep.dataUrl, stats, dur, startedAt, error);
  console.log(
    `[anre:${source}] done in ${(dur / 1000).toFixed(1)}s seen=${stats.seen} ins=${stats.inserted} upd=${stats.updated} skip=${stats.skipped}${error ? (isHardError ? ' ERROR' : ' WARN') : ''}`
  );
  if (isHardError) throw new Error(error);
  return stats;
}

async function main(): Promise<void> {
  console.log(`[anre] source=${argSource} limit=${argLimit || 'none'}`);
  await getCookies();
  const sources: Source[] = ['electricitate', 'gaze', 'atestat', 'electricieni'];
  if (argSource === 'all') {
    for (const s of sources) await runOne(s);
  } else if (sources.includes(argSource as Source)) {
    await runOne(argSource as Source);
  } else {
    console.error(`Unknown --source=${argSource}. Use ${sources.join('|')}|all`);
    process.exit(2);
  }
  // Refresh MV (cheap)
  console.log(`[anre] refreshing materialized views...`);
  try {
    await pool.query('REFRESH MATERIALIZED VIEW CONCURRENTLY anre.mv_licente_per_cui');
  } catch (e) {
    // Falls back to non-concurrent if first refresh
    await pool.query('REFRESH MATERIALIZED VIEW anre.mv_licente_per_cui');
  }
  console.log(`[anre] done.`);
}

main()
  .catch((e) => {
    console.error(e);
    process.exit(1);
  })
  .finally(() => pool.end());
