/**
 * Scraper pentru RegAS — Registrul Ajutoarelor de Stat (Consiliul Concurenței).
 * Source: https://regas.consiliulconcurentei.ro/transparenta/index.html
 *
 * API: POST /apitransparenta/cautareTransparenta
 *   - Authorize via XSRF-TOKEN cookie (set by GET /transparenta/index.html)
 *   - Body: {"model":{}, "page":N, "size":5000, "sort":null}
 *   - Returns array of ajutor records (no envelope)
 *   - Companion: POST /apitransparenta/countTransparenta with same body
 *
 * Volume @ 2026-05-09: 132,363 records, ~30s per 5000-row page → ~14 min full.
 *
 * Run: npx tsx src/scrape-regas.ts [--page-size=5000] [--start-page=0]
 */
import { Pool } from 'pg';
import { createHash } from 'crypto';

// RegAS cert chain not in default node CA bundle (intermediate signed by
// public-but-unbundled CA). We've verified the cert OOB. Disable verify on
// the global undici dispatcher used by fetch().
//
// Setting NODE_TLS_REJECT_UNAUTHORIZED=0 at the env level would also work
// but globalThis-dispatcher is cleaner — and we already have undici bundled
// inside node 22 (just not exposed as a top-level public module from npm).
// Using the dynamic import path through node:internal is unsafe; simplest
// portable fix: set NODE_TLS_REJECT_UNAUTHORIZED in bash wrapper.

const BASE = 'https://regas.consiliulconcurentei.ro';
const REFERER = `${BASE}/transparenta/index.html`;
const USER_AGENT = 'gov-agreg/1.0 vreaudigital.ro (contact: m.tarau@beletage.ro)';

const args = process.argv.slice(2);
const argPageSize = Math.max(100, Math.min(5000, parseInt(args.find((a) => a.startsWith('--page-size='))?.split('=')[1] || '5000')));
const argStartPage = Math.max(0, parseInt(args.find((a) => a.startsWith('--start-page='))?.split('=')[1] || '0'));
const argMaxPages = parseInt(args.find((a) => a.startsWith('--max-pages='))?.split('=')[1] || '0');

const pool = new Pool({
  connectionString: process.env.DATABASE_URL,
  max: 4,
});

interface RegasRecord {
  denumireBeneficiar: string;
  cui: string | number | null;
  raf?: string | null;
  altCod?: string | null;
  dimensiuneIntreprindere?: string | null;
  regiune?: string[];
  domeniuActivitate?: string | null;
  dataAcordare?: string | null; // "DD/MM/YYYY"
  denumireMasura?: string | null;
  referintaMasura?: string | null;
  activitatiFinantate?: string[];
  categorie?: string | null;
  obiectiv?: string | null;
  ajutorAcordatCategorie?: number | null;
  intermediariFinanciari?: string[];
  executanti?: string[];
  subcategorie?: string | null;
  ajutorAcordatSubcategorie?: number | null;
  instrumentAcordare?: string | null;
  intensitate?: number | null;
  finantator?: string | null;
  pdfMasura?: string | null;
  idMasura: number;
}

// ── Session management ─────────────────────────────────────────────────────
let xsrfToken: string | null = null;
let cookieJar: string = '';

async function bootstrapSession(): Promise<void> {
  const res = await fetch(REFERER, {
    headers: { 'User-Agent': USER_AGENT, Accept: 'text/html' },
    signal: AbortSignal.timeout(30000),
  });
  const setCookies = res.headers.getSetCookie?.() || [];
  // Build cookie jar string + extract XSRF-TOKEN value
  const parts: string[] = [];
  for (const sc of setCookies) {
    const m = sc.match(/^([^=]+)=([^;]+)/);
    if (!m) continue;
    parts.push(`${m[1]}=${m[2]}`);
    if (m[1] === 'XSRF-TOKEN') xsrfToken = m[2];
  }
  cookieJar = parts.join('; ');
  if (!xsrfToken) throw new Error('Could not bootstrap XSRF-TOKEN from RegAS landing page');
  console.log(`[session] XSRF-TOKEN obtained, ${parts.length} cookies in jar`);
}

async function apiPost<T>(path: string, body: any): Promise<T> {
  if (!xsrfToken) await bootstrapSession();
  const res = await fetch(`${BASE}${path}`, {
    method: 'POST',
    headers: {
      'User-Agent': USER_AGENT,
      Accept: 'application/json, text/plain, */*',
      'Content-Type': 'application/json;charset=UTF-8',
      Origin: BASE,
      Referer: REFERER,
      'X-XSRF-TOKEN': xsrfToken!,
      Cookie: cookieJar,
    },
    body: JSON.stringify(body),
    signal: AbortSignal.timeout(120000),
  });
  if (res.status === 403) {
    // Session expired? Reset and retry once.
    xsrfToken = null;
    await bootstrapSession();
    return apiPost(path, body);
  }
  if (!res.ok) {
    const txt = await res.text().catch(() => '');
    throw new Error(`POST ${path} → ${res.status}: ${txt.slice(0, 200)}`);
  }
  return (await res.json()) as T;
}

// ── Helpers ────────────────────────────────────────────────────────────────
function parseRoDate(s: string | null | undefined): string | null {
  if (!s) return null;
  const m = s.trim().match(/^(\d{2})\/(\d{2})\/(\d{4})$/);
  return m ? `${m[3]}-${m[2]}-${m[1]}` : null;
}

function normCui(c: string | number | null | undefined): string | null {
  if (c == null) return null;
  const s = String(c).trim().replace(/^RO/i, '').replace(/\s+/g, '');
  if (!/^\d+$/.test(s)) return null;
  return s;
}

function recordId(r: RegasRecord): string {
  const cui = normCui(r.cui) || 'unknown';
  const key = [
    cui,
    r.idMasura,
    r.dataAcordare || '',
    r.subcategorie || '',
    r.ajutorAcordatSubcategorie ?? '',
  ].join('|');
  return createHash('sha1').update(key).digest('hex');
}

// ── DB upsert ──────────────────────────────────────────────────────────────
async function upsertBatch(rows: RegasRecord[]): Promise<{ inserted: number; updated: number; skipped: number }> {
  if (rows.length === 0) return { inserted: 0, updated: 0, skipped: 0 };
  const client = await pool.connect();
  let inserted = 0, updated = 0, skipped = 0;
  try {
    await client.query('BEGIN');
    for (const r of rows) {
      const cui = normCui(r.cui);
      if (!cui) { skipped++; continue; }
      const id = recordId(r);
      const dateRO = parseRoDate(r.dataAcordare);
      const result = await client.query(
        `INSERT INTO regas.ajutoare (
            id, cui, denumire_beneficiar, raf, alt_cod, dimensiune_intreprindere,
            regiune, domeniu_activitate, data_acordare, id_masura, denumire_masura,
            referinta_masura, activitati_finantate, categorie, subcategorie,
            obiectiv, ajutor_acordat_categorie, ajutor_acordat_subcategorie,
            intermediari_financiari, executanti, instrument_acordare, intensitate,
            finantator, pdf_masura, raw_json
          ) VALUES (
            $1, $2, $3, $4, $5, $6,
            $7, $8, $9, $10, $11,
            $12, $13, $14, $15,
            $16, $17, $18,
            $19, $20, $21, $22,
            $23, $24, $25
          )
          ON CONFLICT (id) DO UPDATE SET
            denumire_beneficiar = EXCLUDED.denumire_beneficiar,
            raf = EXCLUDED.raf,
            alt_cod = EXCLUDED.alt_cod,
            dimensiune_intreprindere = EXCLUDED.dimensiune_intreprindere,
            regiune = EXCLUDED.regiune,
            domeniu_activitate = EXCLUDED.domeniu_activitate,
            denumire_masura = EXCLUDED.denumire_masura,
            referinta_masura = EXCLUDED.referinta_masura,
            activitati_finantate = EXCLUDED.activitati_finantate,
            categorie = EXCLUDED.categorie,
            subcategorie = EXCLUDED.subcategorie,
            obiectiv = EXCLUDED.obiectiv,
            ajutor_acordat_categorie = EXCLUDED.ajutor_acordat_categorie,
            ajutor_acordat_subcategorie = EXCLUDED.ajutor_acordat_subcategorie,
            intermediari_financiari = EXCLUDED.intermediari_financiari,
            executanti = EXCLUDED.executanti,
            instrument_acordare = EXCLUDED.instrument_acordare,
            intensitate = EXCLUDED.intensitate,
            finantator = EXCLUDED.finantator,
            pdf_masura = EXCLUDED.pdf_masura,
            raw_json = EXCLUDED.raw_json,
            fetched_at = now()
          RETURNING (xmax = 0) AS inserted`,
        [
          id, cui, r.denumireBeneficiar, r.raf || null, r.altCod || null, r.dimensiuneIntreprindere || null,
          r.regiune && r.regiune.length ? r.regiune : null, r.domeniuActivitate || null, dateRO,
          r.idMasura, r.denumireMasura || '',
          r.referintaMasura || null, r.activitatiFinantate && r.activitatiFinantate.length ? r.activitatiFinantate : null,
          r.categorie || null, r.subcategorie || null,
          r.obiectiv || null, r.ajutorAcordatCategorie ?? null, r.ajutorAcordatSubcategorie ?? null,
          r.intermediariFinanciari && r.intermediariFinanciari.length ? r.intermediariFinanciari : null,
          r.executanti && r.executanti.length ? r.executanti : null,
          r.instrumentAcordare || null, r.intensitate ?? null,
          r.finantator || null, r.pdfMasura || null,
          JSON.stringify(r),
        ]
      );
      if (result.rows[0]?.inserted) inserted++; else updated++;
    }
    await client.query('COMMIT');
  } catch (e) {
    await client.query('ROLLBACK');
    throw e;
  } finally {
    client.release();
  }
  return { inserted, updated, skipped };
}

// ── Main ───────────────────────────────────────────────────────────────────
async function main(): Promise<void> {
  console.log(`[start] page-size=${argPageSize} start-page=${argStartPage} max-pages=${argMaxPages || 'all'}`);

  await bootstrapSession();

  const total = await apiPost<number>('/apitransparenta/countTransparenta', {});
  const totalPages = Math.ceil(total / argPageSize);
  console.log(`[count] total=${total} pages=${totalPages}`);

  let totalInserted = 0, totalUpdated = 0, totalSkipped = 0;
  const startedAt = Date.now();
  const lastPage = argMaxPages > 0 ? Math.min(totalPages - 1, argStartPage + argMaxPages - 1) : totalPages - 1;

  // Stable sort REQUIRED — without it, the API shuffles rows across pages
  // and we'd see ~50K unique IDs instead of all 132K. Sorting by idMasura+cui
  // gives a deterministic ordering with low cardinality at the high end
  // (idMasura asc clusters by program — fine).
  const sortBody = [
    { field: 'idMasura', direction: 'asc', priority: 0 },
    { field: 'cui', direction: 'asc', priority: 1 },
  ];

  for (let page = argStartPage; page <= lastPage; page++) {
    const t0 = Date.now();
    const rows = await apiPost<RegasRecord[]>('/apitransparenta/cautareTransparenta', {
      model: {},
      page,
      size: argPageSize,
      sort: sortBody,
    });
    const t1 = Date.now();
    const stats = await upsertBatch(rows);
    const t2 = Date.now();
    totalInserted += stats.inserted; totalUpdated += stats.updated; totalSkipped += stats.skipped;
    const elapsedSec = ((Date.now() - startedAt) / 1000).toFixed(0);
    console.log(
      `[page ${page + 1}/${totalPages}] got=${rows.length} ins=${stats.inserted} upd=${stats.updated} skip=${stats.skipped} ` +
      `fetch=${t1 - t0}ms db=${t2 - t1}ms total=${totalInserted}+${totalUpdated} elapsed=${elapsedSec}s`
    );
    if (rows.length < argPageSize) {
      console.log('[done] reached last page (short response)');
      break;
    }
  }

  console.log(`[summary] inserted=${totalInserted} updated=${totalUpdated} skipped=${totalSkipped} elapsed=${((Date.now() - startedAt) / 1000).toFixed(0)}s`);

  // Refresh MV at end
  console.log('[mv] refreshing regas.mv_ajutoare_per_cui ...');
  await pool.query('REFRESH MATERIALIZED VIEW regas.mv_ajutoare_per_cui');
  console.log('[mv] done');

  await pool.end();
}

main().catch((e) => {
  console.error('[fatal]', e);
  process.exit(1);
});
