/**
 * CNSC — Consiliul Național de Soluționare a Contestațiilor.
 *
 * Stage 1 ingest: walks the public listing at portal.cnsc.ro/decizii.html.
 * The page is ASP.NET WebForms but the actual data is loaded async via
 * an AJAX call to /Default.aspx/CallWebMethod. We replay that POST.
 *
 * Endpoint discovery (2026-05-10):
 *   1. GET  http://portal.cnsc.ro/decizii.html              → sets ASP.NET_SessionId
 *   2. POST http://portal.cnsc.ro/Default.aspx/CallWebMethod
 *           Content-Type: application/json
 *           Cookie:       (ASP.NET_SessionId from step 1)
 *           Referer:      http://portal.cnsc.ro/decizii.html?...page=N
 *                         (server reads page from Referer query — NOT body)
 *           Body:         {"sender":"67fb8141-f456-4276-b51a-baca731f92ca",
 *                          "methodName":"get",
 *                          "senderParams":"a=search&reg:registrationDate=-&page=N",
 *                          "isBuletin":"0"}
 *           Response:     {"d":"<table>…</table>"}
 *
 * Listing layout: <table><tbody><tr><td>×9</td></tr>×~50</tbody></table>
 *   Cols: decision_no | contestator_names | reg_no | authority_name |
 *         contestator_cuis | authority_cuis | year | reg_date | <a> PDF
 *
 * 50 rows × ~617 pages ≈ 30,850 decisions. Pages 1–N descend chronologically
 * by decision_no (page 1 = current year, last page = ~2 years older).
 *
 * Idempotent: ON CONFLICT (decision_no, decision_year) DO UPDATE.
 *
 * Run:
 *   npx tsx src/scrape-cnsc.ts                         # full walk (all pages)
 *   npx tsx src/scrape-cnsc.ts --max-pages=2           # smoke test (~100 rows)
 *   npx tsx src/scrape-cnsc.ts --start-page=200        # resume mid-walk
 *   npx tsx src/scrape-cnsc.ts --start-page=1 --max-pages=20
 */
import { Pool } from 'pg';

const PORTAL = 'http://portal.cnsc.ro';
const LISTING_URL = `${PORTAL}/decizii.html`;
const WEBMETHOD_URL = `${PORTAL}/Default.aspx/CallWebMethod`;
const DECIZII_SENDER = '67fb8141-f456-4276-b51a-baca731f92ca';
const USER_AGENT = 'gov-agreg/1.0 vreaudigital.ro (contact: m.tarau@beletage.ro)';

// ── CLI args ────────────────────────────────────────────────────────────────
const args = process.argv.slice(2);
const argMaxPages = parseInt(
  args.find((a) => a.startsWith('--max-pages='))?.split('=')[1] || '0',
  10
);
const argStartPage = parseInt(
  args.find((a) => a.startsWith('--start-page='))?.split('=')[1] || '1',
  10
);

const pool = new Pool({ connectionString: process.env.DATABASE_URL, max: 4 });

// ── Helpers ─────────────────────────────────────────────────────────────────
function trimOrNull(v: any): string | null {
  if (v == null) return null;
  const s = String(v).trim();
  return s === '' ? null : s;
}

/** Decodes HTML entities (&amp; &quot; &lt; &gt; &nbsp; numeric refs). */
function htmlDecode(s: string): string {
  return s
    .replace(/&amp;/g, '&')
    .replace(/&quot;/g, '"')
    .replace(/&#39;/g, "'")
    .replace(/&apos;/g, "'")
    .replace(/&lt;/g, '<')
    .replace(/&gt;/g, '>')
    .replace(/&nbsp;/g, ' ')
    .replace(/&#(\d+);/g, (_, n) => String.fromCharCode(parseInt(n, 10)));
}

/** Strips inner HTML tags; collapses whitespace. */
function textify(html: string): string {
  return htmlDecode(html.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ')).trim();
}

/** "17.02.2026" → "2026-02-17". Returns null on bad input. */
function parseDmyDate(v: any): string | null {
  if (v == null) return null;
  const s = String(v).trim();
  const m = s.match(/^(\d{1,2})\.(\d{1,2})\.(\d{4})$/);
  if (!m) return null;
  const d = m[1].padStart(2, '0');
  const mo = m[2].padStart(2, '0');
  const y = m[3];
  if (parseInt(mo, 10) < 1 || parseInt(mo, 10) > 12) return null;
  if (parseInt(d, 10) < 1 || parseInt(d, 10) > 31) return null;
  if (parseInt(y, 10) < 2000 || parseInt(y, 10) > 2100) return null;
  return `${y}-${mo}-${d}`;
}

/** Splits a listing-cell name list (comma-separated, sometimes ", ") into a
 *  trimmed, non-empty array. Preserves order (LSP can be liderul asocierii). */
function splitNames(raw: string | null): string[] {
  if (!raw) return [];
  return raw
    .split(/\s*,\s*/g)
    .map((s) => s.trim())
    .filter((s) => s.length > 0);
}

/** Splits a CUI cell. CNSC uses both ',' and ';' separators inconsistently.
 *  Strips RO prefix, keeps digits-only, dedupes preserving order. */
function splitCuis(raw: string | null): string[] {
  if (!raw) return [];
  const tokens = raw.split(/[,;]+/g);
  const out: string[] = [];
  for (let t of tokens) {
    t = t.trim().replace(/^RO\s*/i, '');
    const m = t.match(/^\d+$/);
    if (!m) continue;
    if (!out.includes(t)) out.push(t);
  }
  return out;
}

// ── HTTP session ────────────────────────────────────────────────────────────
//
// CNSC's pagination is server-side stateful: the AJAX POST to CallWebMethod
// does NOT accept the page in its body (the server ignores `senderParams`
// for the page selector — confirmed by experiment 2026-05-10). Instead,
// the server reads the current page from session state, which is set by
// a GET to /decizii.html?...page=N preceding the POST.
//
// Per-page flow (one HTTP session reused):
//   1. GET  /decizii.html?a=search&reg:registrationDate=-&page=N   (sets state)
//   2. POST /Default.aspx/CallWebMethod                            (returns rows)
//
// First-time session bootstrap also goes through GET /decizii.html (no params).
let cookieJar = '';

async function ensureCookies(): Promise<void> {
  if (cookieJar) return;
  const res = await fetch(LISTING_URL, {
    headers: { 'User-Agent': USER_AGENT },
    signal: AbortSignal.timeout(30000),
  });
  if (!res.ok) throw new Error(`GET ${LISTING_URL} → ${res.status}`);
  const setCookie = res.headers.getSetCookie ? res.headers.getSetCookie() : [];
  cookieJar = setCookie.map((c: string) => c.split(';')[0]).join('; ');
  if (!cookieJar.includes('ASP.NET_SessionId')) {
    throw new Error(`no ASP.NET_SessionId in Set-Cookie: ${setCookie.join(' | ')}`);
  }
}

/** GET /decizii.html?...page=N to set the server-side page state. */
async function setPage(page: number): Promise<void> {
  const url = `${LISTING_URL}?a=search&reg:registrationDate=-&page=${page}`;
  const res = await fetch(url, {
    headers: {
      'User-Agent': USER_AGENT,
      Cookie: cookieJar,
    },
    signal: AbortSignal.timeout(30000),
  });
  if (!res.ok) throw new Error(`GET page=${page} → ${res.status}`);
  // Drain body so the connection can be reused
  await res.text().catch(() => '');
}

/**
 * Fetches one listing page. Performs:
 *   1. GET /decizii.html?...page=N  (sets server-side current page)
 *   2. POST /Default.aspx/CallWebMethod  (returns the rows for that page)
 * Returns the inner HTML payload (json.d).
 */
async function fetchPage(page: number, attempt: number = 1): Promise<string> {
  await setPage(page);
  // senderParams is logged server-side but does NOT drive page selection.
  // We still send a sensible value for forward compatibility.
  const senderParams = `a=search&reg:registrationDate=-&page=${page}`;
  const referer = `${LISTING_URL}?a=search&reg:registrationDate=-&page=${page}`;
  const body = JSON.stringify({
    sender: DECIZII_SENDER,
    methodName: 'get',
    senderParams,
    isBuletin: '0',
  });
  try {
    const res = await fetch(WEBMETHOD_URL, {
      method: 'POST',
      headers: {
        'User-Agent': USER_AGENT,
        'Content-Type': 'application/json; charset=utf-8',
        Accept: 'application/json,text/plain,*/*',
        'X-Requested-With': 'XMLHttpRequest',
        Origin: PORTAL,
        Referer: referer,
        Cookie: cookieJar,
      },
      body,
      signal: AbortSignal.timeout(60000),
    });
    if (!res.ok) {
      const txt = await res.text().catch(() => '');
      throw new Error(`POST p${page} ${res.status}: ${txt.slice(0, 200)}`);
    }
    const json = (await res.json()) as { d?: string };
    if (typeof json.d !== 'string') throw new Error(`bad payload for p${page}: ${JSON.stringify(json).slice(0, 200)}`);
    if (json.d.startsWith('Object reference')) {
      // session expired, re-login once
      if (attempt === 1) {
        cookieJar = '';
        await ensureCookies();
        return fetchPage(page, attempt + 1);
      }
      throw new Error(`session error: ${json.d}`);
    }
    return json.d;
  } catch (e) {
    const maxAttempts = 4;
    if (attempt >= maxAttempts) throw e;
    const backoffMs = 2000 * attempt;
    console.warn(
      `[cnsc] p${page} attempt ${attempt} failed: ${e instanceof Error ? e.message : e}; retry in ${backoffMs}ms`
    );
    await new Promise((r) => setTimeout(r, backoffMs));
    return fetchPage(page, attempt + 1);
  }
}

// ── Parser ──────────────────────────────────────────────────────────────────
type ListingRow = {
  decision_no: number;
  decision_year: number;
  registration_no_cnsc: string | null;
  registration_date: string | null;
  contestator_name_raw: string | null;
  contestator_names: string[];
  contestator_cui_raw: string | null;
  contestator_cuis: string[];
  authority_name: string | null;
  authority_cui_raw: string | null;
  authority_cuis: string[];
  pdf_filename: string | null;
  pdf_docuid_b64: string | null;
  pdf_url: string | null;
};

/** Picks data rows out of the <tbody>: skip the header row (which has <th>
 *  not <td>), keep <tr>s with exactly 9 <td>s. */
function parseListing(html: string): ListingRow[] {
  const out: ListingRow[] = [];

  // Find the <tbody> block (CNSC always wraps decisions in a single tbody)
  const tbodyMatch = html.match(/<tbody[^>]*>([\s\S]*?)<\/tbody>/i);
  if (!tbodyMatch) return out;
  const tbody = tbodyMatch[1];

  const rowRe = /<tr[^>]*>([\s\S]*?)<\/tr>/gi;
  let rm: RegExpExecArray | null;
  while ((rm = rowRe.exec(tbody)) !== null) {
    const tr = rm[1];
    const cells: string[] = [];
    const cellRe = /<td[^>]*>([\s\S]*?)<\/td>/gi;
    let cm: RegExpExecArray | null;
    while ((cm = cellRe.exec(tr)) !== null) cells.push(cm[1]);
    if (cells.length !== 9) continue;

    const decisionNoStr = textify(cells[0]);
    const decisionNo = parseInt(decisionNoStr, 10);
    if (!Number.isFinite(decisionNo)) continue;

    const yearStr = textify(cells[6]);
    const year = parseInt(yearStr, 10);
    if (!Number.isFinite(year)) continue;

    const contestatorNameRaw = textify(cells[1]) || null;
    const regNoRaw = textify(cells[2]) || null;
    const authorityName = textify(cells[3]) || null;
    const contestatorCuiRaw = textify(cells[4]) || null;
    const authorityCuiRaw = textify(cells[5]) || null;
    const regDateRaw = textify(cells[7]) || null;
    const regDate = parseDmyDate(regDateRaw);

    // PDF link extraction from cell 8: ShowDialog(...,"<filename>","<inline_url>","<attach_url>")
    const cell8 = cells[8];
    let pdfFilename: string | null = null;
    let pdfDocuidB64: string | null = null;
    let pdfUrl: string | null = null;
    const fnMatch = cell8.match(/&quot;([^&]+\.pdf)&quot;/i);
    if (fnMatch) pdfFilename = fnMatch[1];
    const inlineUrlMatch = cell8.match(
      /&quot;(\/sivadoc\/download\.aspx\?docUID=[^&]+(?:&amp;[^&"]+)*?&amp;action=aW5saW5l)&quot;/
    );
    if (inlineUrlMatch) {
      const decoded = htmlDecode(inlineUrlMatch[1]);
      pdfUrl = `${PORTAL}${decoded}`;
      const dm = decoded.match(/docUID=([A-Za-z0-9+/=_-]+)/);
      if (dm) pdfDocuidB64 = dm[1].replace(/=+$/, '');
    }

    out.push({
      decision_no: decisionNo,
      decision_year: year,
      registration_no_cnsc: regNoRaw,
      registration_date: regDate,
      contestator_name_raw: contestatorNameRaw,
      contestator_names: splitNames(contestatorNameRaw),
      contestator_cui_raw: contestatorCuiRaw,
      contestator_cuis: splitCuis(contestatorCuiRaw),
      authority_name: authorityName,
      authority_cui_raw: authorityCuiRaw,
      authority_cuis: splitCuis(authorityCuiRaw),
      pdf_filename: pdfFilename,
      pdf_docuid_b64: pdfDocuidB64,
      pdf_url: pdfUrl,
    });
  }
  return out;
}

/** Extracts the highest page number from the pagination block.
 *  CNSC always shows the last page link (e.g. "617") even on page 1.
 *  The HTML has both `?page=N` (first link) and `&amp;page=N` (later links),
 *  so we accept `?`, `&`, or `;` (the trailing char of `&amp;`) as separator. */
function parseTotalPages(html: string): number {
  const re = /[?&;]page=(\d+)\b/g;
  let max = 1;
  let m: RegExpExecArray | null;
  while ((m = re.exec(html)) !== null) {
    const n = parseInt(m[1], 10);
    if (Number.isFinite(n) && n > max) max = n;
  }
  return max;
}

// ── Ingest ──────────────────────────────────────────────────────────────────
type IngestStats = { seen: number; inserted: number; updated: number; skipped: number };

async function ingestRows(rows: ListingRow[], sourcePage: number): Promise<IngestStats> {
  const stats: IngestStats = { seen: rows.length, inserted: 0, updated: 0, skipped: 0 };
  const client = await pool.connect();
  try {
    for (const r of rows) {
      try {
        const result = await client.query(
          `INSERT INTO cnsc.decizii (
             decision_no, decision_year,
             registration_no_cnsc, registration_date,
             contestator_name_raw, contestator_names,
             contestator_cui_raw, contestator_cuis,
             authority_name, authority_cui_raw, authority_cuis,
             pdf_filename, pdf_docuid_b64, pdf_url,
             source_page, fetched_at
           ) VALUES (
             $1, $2,
             $3, $4,
             $5, $6,
             $7, $8,
             $9, $10, $11,
             $12, $13, $14,
             $15, now()
           )
           ON CONFLICT (decision_no, decision_year) DO UPDATE SET
             registration_no_cnsc = EXCLUDED.registration_no_cnsc,
             registration_date    = EXCLUDED.registration_date,
             contestator_name_raw = EXCLUDED.contestator_name_raw,
             contestator_names    = EXCLUDED.contestator_names,
             contestator_cui_raw  = EXCLUDED.contestator_cui_raw,
             contestator_cuis     = EXCLUDED.contestator_cuis,
             authority_name       = EXCLUDED.authority_name,
             authority_cui_raw    = EXCLUDED.authority_cui_raw,
             authority_cuis       = EXCLUDED.authority_cuis,
             pdf_filename         = EXCLUDED.pdf_filename,
             pdf_docuid_b64       = EXCLUDED.pdf_docuid_b64,
             pdf_url              = EXCLUDED.pdf_url,
             source_page          = EXCLUDED.source_page,
             fetched_at           = now()
           RETURNING (xmax = 0) AS inserted`,
          [
            r.decision_no,
            r.decision_year,
            trimOrNull(r.registration_no_cnsc),
            r.registration_date,
            trimOrNull(r.contestator_name_raw),
            r.contestator_names,
            trimOrNull(r.contestator_cui_raw),
            r.contestator_cuis,
            trimOrNull(r.authority_name),
            trimOrNull(r.authority_cui_raw),
            r.authority_cuis,
            trimOrNull(r.pdf_filename),
            trimOrNull(r.pdf_docuid_b64),
            trimOrNull(r.pdf_url),
            sourcePage,
          ]
        );
        if (result.rows[0]?.inserted) stats.inserted++;
        else stats.updated++;
      } catch (e) {
        console.warn(
          `[cnsc] skip row no=${r.decision_no} y=${r.decision_year}: ${
            e instanceof Error ? e.message : e
          }`
        );
        stats.skipped++;
      }
    }
  } finally {
    client.release();
  }
  return stats;
}

async function logScrape(
  scraper: string,
  pageFrom: number,
  pageTo: number,
  stats: IngestStats,
  durationMs: number,
  startedAt: Date,
  error?: string
): Promise<void> {
  await pool.query(
    `INSERT INTO cnsc.scrape_log
       (scraper, page_from, page_to, rows_seen, rows_inserted, rows_updated, rows_skipped,
        duration_ms, started_at, error)
     VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`,
    [
      scraper,
      pageFrom,
      pageTo,
      stats.seen,
      stats.inserted,
      stats.updated,
      stats.skipped,
      durationMs,
      startedAt,
      error || null,
    ]
  );
}

// ── Main ────────────────────────────────────────────────────────────────────
async function main(): Promise<void> {
  console.log(
    `[cnsc] start-page=${argStartPage} max-pages=${argMaxPages || 'all'}`
  );
  await ensureCookies();

  // First fetch tells us total pages.
  const startedAt = new Date();
  const t0 = Date.now();
  let totalPages = 1;
  const totals: IngestStats = { seen: 0, inserted: 0, updated: 0, skipped: 0 };
  let lastPage = argStartPage;
  let hardError: string | undefined;
  const missedPages: number[] = [];

  try {
    const firstHtml = await fetchPage(argStartPage);
    totalPages = parseTotalPages(firstHtml);
    console.log(`[cnsc] total pages ≈ ${totalPages}`);

    const cap = argMaxPages > 0
      ? Math.min(argStartPage + argMaxPages - 1, totalPages)
      : totalPages;

    let firstRows = parseListing(firstHtml);
    console.log(`[cnsc] p${argStartPage}: parsed ${firstRows.length} rows`);
    let s = await ingestRows(firstRows, argStartPage);
    totals.seen += s.seen;
    totals.inserted += s.inserted;
    totals.updated += s.updated;
    totals.skipped += s.skipped;
    console.log(
      `[cnsc] p${argStartPage}: ins=${s.inserted} upd=${s.updated} skip=${s.skipped}`
    );

    for (let p = argStartPage + 1; p <= cap; p++) {
      lastPage = p;
      try {
        const html = await fetchPage(p);
        const rows = parseListing(html);
        s = await ingestRows(rows, p);
        totals.seen += s.seen;
        totals.inserted += s.inserted;
        totals.updated += s.updated;
        totals.skipped += s.skipped;
        if (p % 10 === 0 || p === cap) {
          console.log(
            `[cnsc]   p${p}/${cap}: ${rows.length} rows; running totals seen=${totals.seen} ins=${totals.inserted} upd=${totals.updated} skip=${totals.skipped}`
          );
        }
      } catch (e) {
        const msg = e instanceof Error ? e.message : String(e);
        console.warn(`[cnsc] HARD SKIP page ${p}: ${msg}`);
        missedPages.push(p);
      }
      // Be polite: 250ms between pages → ~150 pages/min, full crawl ~7 min.
      await new Promise((r) => setTimeout(r, 250));
    }
  } catch (e) {
    hardError = e instanceof Error ? e.message : String(e);
    console.error(`[cnsc] ERROR: ${hardError}`);
  }

  const dur = Date.now() - t0;
  const errStr = [
    hardError,
    missedPages.length
      ? `missed_pages=${missedPages.length}: ${missedPages.slice(0, 20).join(',')}`
      : null,
  ]
    .filter(Boolean)
    .join(' | ') || undefined;
  await logScrape('listing', argStartPage, lastPage, totals, dur, startedAt, errStr);

  console.log(
    `[cnsc] done in ${(dur / 1000).toFixed(1)}s seen=${totals.seen} ins=${
      totals.inserted
    } upd=${totals.updated} skip=${totals.skipped}${
      missedPages.length ? ` MISSED=${missedPages.length}` : ''
    }${hardError ? ' ERROR' : ''}`
  );

  // Refresh MVs (cheap; will fail concurrently on first run → fall back).
  console.log('[cnsc] refreshing materialized views...');
  for (const mv of ['cnsc.mv_per_authority_cui', 'cnsc.mv_per_contestator_cui']) {
    try {
      await pool.query(`REFRESH MATERIALIZED VIEW CONCURRENTLY ${mv}`);
    } catch {
      try {
        await pool.query(`REFRESH MATERIALIZED VIEW ${mv}`);
      } catch (e) {
        console.warn(`[cnsc]   skip refresh ${mv}: ${e instanceof Error ? e.message : e}`);
      }
    }
  }
  if (hardError) throw new Error(hardError);
}

main()
  .catch((e) => {
    console.error(e);
    process.exit(1);
  })
  .finally(() => pool.end());
