/**
 * ANCOM — Autoritatea Națională pentru Administrare și Reglementare în
 * Comunicații. Scrapes the public registry of authorized providers of
 * electronic communications networks and services.
 *
 * Source list (server-rendered HTML, paginated 10/page, ~57 pages → ~570 rows):
 *   https://www.ancom.ro/reglementare-ro/comunicatii-electronice/
 *     furnizori-comunicatii-electronice/
 *     lista-furnizorilor-de-retele-si-servicii-de-comunicatii-autorizati/
 *
 * Pagination: POST `paged=N` (form id="ms_form").
 *
 * Each list row links to a detail page:
 *   https://www.ancom.ro/sablon/furnizorinew_23/?id={ancom_id}&pid=4186
 *
 * Detail page contains:
 *   • <h2>{titular}</h2>
 *   • <p><strong>Adresa:</strong> ...</p>
 *   • <p><strong>Oras/Comuna:</strong> ...</p>
 *   • <p><strong>Judet/Sector:</strong> ...</p>
 *   • <p><strong>Cod unic de înregistrare:</strong> {CUI}</p>
 *   • <p><strong>Identificator unic la nivel european (EUID):</strong> ...</p>
 *   • <p><strong>R{1..11} – ...</strong><br/>
 *      <strong>Data nasterii dreptului</strong> DD-MM-YYYY</p>
 *   • <p><strong>S{1..12} – ...</strong><br/>
 *      <strong>Data nasterii dreptului</strong> DD-MM-YYYY</p>
 *
 * Idempotent on ancom_id (PK from registry URL).
 *
 * Run:
 *   npx tsx src/scrape-ancom.ts                    # full
 *   npx tsx src/scrape-ancom.ts --limit=20         # smoke test (first 20 ids)
 *   npx tsx src/scrape-ancom.ts --max-pages=2      # only 2 list pages = 20 ids
 */
import { Pool } from 'pg';
import { createHash } from 'crypto';

const BASE = 'https://www.ancom.ro';
const LIST_URL =
  `${BASE}/reglementare-ro/comunicatii-electronice/furnizori-comunicatii-electronice/lista-furnizorilor-de-retele-si-servicii-de-comunicatii-autorizati/`;
const DETAIL_URL = (id: number) => `${BASE}/sablon/furnizorinew_23/?id=${id}&pid=4186`;
const USER_AGENT = 'gov-agreg/1.0 vreaudigital.ro (contact: m.tarau@beletage.ro)';

const args = process.argv.slice(2);
const argLimit = parseInt(args.find((a) => a.startsWith('--limit='))?.split('=')[1] || '0');
const argMaxPages = parseInt(args.find((a) => a.startsWith('--max-pages='))?.split('=')[1] || '0');

const pool = new Pool({ connectionString: process.env.DATABASE_URL, max: 4 });

interface ListRow {
  ancom_id: number;
  titular_name: string;
  list_judet: string;
}

interface DetailFields {
  titular_name: string;
  adresa: string | null;
  oras: string | null;
  judet: string | null;
  cui: string | null;
  euid: string | null;
  drepturi: Array<{ cod: string; tip: 'retea' | 'serviciu'; descriere: string; data_nasterii: string | null }>;
  raw_hash: string;
}

interface IngestStats {
  seen: number;
  inserted: number;
  updated: number;
  skipped: number;
  drepturi: number;
}

// ── Helpers ─────────────────────────────────────────────────────────────────
function sha1Hex(s: string): string {
  return createHash('sha1').update(s).digest('hex');
}

function trimOrNull(v: any): string | null {
  if (v == null) return null;
  const s = String(v).trim();
  return s === '' ? null : s;
}

function decodeHtml(s: string): string {
  return s
    .replace(/&nbsp;/g, ' ')
    .replace(/&amp;/g, '&')
    .replace(/&lt;/g, '<')
    .replace(/&gt;/g, '>')
    .replace(/&quot;/g, '"')
    .replace(/&#39;/g, "'")
    .replace(/&#8211;/g, '-')
    .replace(/&#8212;/g, '-')
    .replace(/&#8217;/g, "'")
    .replace(/&hellip;/g, '...')
    .replace(/&#(\d+);/g, (_, n) => String.fromCharCode(parseInt(n, 10)));
}

/** Parses "DD-MM-YYYY" or "DD.MM.YYYY" into "YYYY-MM-DD". */
function parseDateRoDmy(v: string | null | undefined): string | null {
  if (!v) return null;
  const s = v.trim();
  const m = s.match(/^(\d{1,2})[-./](\d{1,2})[-./](\d{4})$/);
  if (!m) return null;
  const dd = m[1].padStart(2, '0');
  const mm = m[2].padStart(2, '0');
  const yyyy = m[3];
  const d = parseInt(dd, 10);
  const mo = parseInt(mm, 10);
  if (mo < 1 || mo > 12 || d < 1 || d > 31) return null;
  return `${yyyy}-${mm}-${dd}`;
}

async function sleep(ms: number): Promise<void> {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

// ── HTTP fetch with retries ────────────────────────────────────────────────
async function fetchWithRetry(
  url: string,
  init: RequestInit = {},
  attempt = 1,
): Promise<string> {
  const maxAttempts = 4;
  try {
    const res = await fetch(url, {
      ...init,
      headers: {
        'User-Agent': USER_AGENT,
        Accept: 'text/html,application/xhtml+xml,*/*',
        ...(init.headers || {}),
      },
      signal: AbortSignal.timeout(45000),
    });
    if (!res.ok) {
      throw new Error(`HTTP ${res.status} ${url}`);
    }
    return await res.text();
  } catch (e) {
    if (attempt >= maxAttempts) throw e;
    const backoff = 1500 * attempt;
    console.warn(`[ancom] fetch ${url} attempt ${attempt} failed: ${e instanceof Error ? e.message : e}; retry in ${backoff}ms`);
    await sleep(backoff);
    return fetchWithRetry(url, init, attempt + 1);
  }
}

// ── List page parser ───────────────────────────────────────────────────────
/**
 * Extracts ListRow entries from a list page HTML.
 * Looks for <a href="...?id={N}&pid=4186"> links inside the data table and
 * pulls denumire from the preceding <td>, judet from the <td> before that.
 */
function parseListPage(html: string): { rows: ListRow[]; totalPages: number } {
  const rows: ListRow[] = [];

  // Match pattern: <td>NUME</td><td>JUDET</td><td><a href="...?id=N&pid=4186"...
  const rowRe = /<tr>\s*<td>\s*\d+\s*<\/td>\s*<td>([^<]+)<\/td>\s*<td>([^<]+)<\/td>\s*<td>\s*<a[^>]*href="[^"]*\?id=(\d+)&(?:amp;)?pid=4186"/gi;
  let m: RegExpExecArray | null;
  while ((m = rowRe.exec(html)) !== null) {
    const name = decodeHtml(m[1]).trim();
    const judet = decodeHtml(m[2]).trim();
    const id = parseInt(m[3], 10);
    if (!Number.isFinite(id) || id <= 0 || !name) continue;
    rows.push({ ancom_id: id, titular_name: name, list_judet: judet });
  }

  // Pagination footer: ...<button ... value="57" ...> then next button.
  // Pull the largest page number visible.
  let totalPages = 1;
  const pageBtnRe = /name="paged"\s+value="(\d+)"/gi;
  let pm: RegExpExecArray | null;
  while ((pm = pageBtnRe.exec(html)) !== null) {
    const n = parseInt(pm[1], 10);
    if (Number.isFinite(n) && n > totalPages) totalPages = n;
  }

  return { rows, totalPages };
}

async function fetchListPage(page: number): Promise<{ rows: ListRow[]; totalPages: number }> {
  // Page 1 = GET. Page N = POST paged=N (via the form's submit button).
  let html: string;
  if (page === 1) {
    html = await fetchWithRetry(LIST_URL);
  } else {
    html = await fetchWithRetry(LIST_URL, {
      method: 'POST',
      headers: {
        'Content-Type': 'application/x-www-form-urlencoded',
      },
      body: `paged=${page}`,
    });
  }
  return parseListPage(html);
}

// ── Detail page parser ─────────────────────────────────────────────────────
/**
 * Extracts the field block: the article body between `<h2>{name}</h2>` and the
 * trailing inline <script>. Detail pages are short (~130 KB) and structurally
 * simple — `<p><strong>Label:</strong> value</p>` blocks plus R/S sections.
 */
function parseDetail(html: string, ancom_id: number): DetailFields | null {
  // Slice out the article body. Marker: first <h2>...</h2> after the breadcrumb.
  const h2 = html.match(/<h2>([^<]+)<\/h2>([\s\S]*?)<script>/i);
  if (!h2) return null;
  const titular_name = decodeHtml(h2[1]).trim();
  const body = h2[2];
  if (!titular_name) return null;

  // Field extractors
  function pickAfterStrong(label: string): string | null {
    // <p><strong>LABEL:</strong> VALUE</p>
    const re = new RegExp(
      `<p>\\s*<strong>\\s*${label.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\$&')}\\s*:?\\s*<\\/strong>\\s*([\\s\\S]*?)<\\/p>`,
      'i',
    );
    const m = body.match(re);
    if (!m) return null;
    return decodeHtml(m[1].replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ')).trim() || null;
  }

  const adresa = pickAfterStrong('Adresa');
  const oras = pickAfterStrong('Oras\\/Comuna');
  const judet = pickAfterStrong('Judet\\/Sector');
  let cui = pickAfterStrong('Cod unic de [iî]nregistrare');
  // Normalize CUI: digits only (drop "RO" prefix if present)
  if (cui) {
    const digits = cui.replace(/[^0-9]/g, '');
    cui = digits || null;
  }
  const euid = pickAfterStrong('Identificator unic la nivel european \\(EUID\\)');

  // R/S blocks: <p><strong>R3 – Fibră optică</strong><br />
  //              <strong>Data nasterii dreptului</strong> 23-12-2002</p>
  // The dash is `&#8211;` (EN DASH) in raw HTML — decode entities before matching.
  const drepturi: DetailFields['drepturi'] = [];
  const bodyDecoded = decodeHtml(body);
  const drRe =
    /<p>\s*<strong>\s*([RS])(\d{1,2})\s*[–—\-]\s*([^<]+?)<\/strong>\s*<br\s*\/?>\s*<strong>\s*Data\s+nasterii\s+dreptului\s*<\/strong>\s*([0-9.\-/]+)\s*<\/p>/gi;
  let dm: RegExpExecArray | null;
  while ((dm = drRe.exec(bodyDecoded)) !== null) {
    const letter = dm[1].toUpperCase();
    const num = parseInt(dm[2], 10);
    if (!Number.isFinite(num)) continue;
    const cod = `${letter}${num}`;
    const descriere = dm[3].trim();
    const data_nasterii = parseDateRoDmy(dm[4]);
    drepturi.push({
      cod,
      tip: letter === 'R' ? 'retea' : 'serviciu',
      descriere,
      data_nasterii,
    });
  }

  // Stable hash — used for change detection. Hash the structured payload, not
  // the surrounding chrome (which changes with theme tweaks etc.).
  const raw_hash = sha1Hex(
    JSON.stringify({
      ancom_id,
      titular_name,
      cui,
      euid,
      adresa,
      oras,
      judet,
      drepturi: drepturi.map((d) => ({ ...d })).sort((a, b) => a.cod.localeCompare(b.cod)),
    }),
  );

  return { titular_name, adresa, oras, judet, cui, euid, drepturi, raw_hash };
}

async function fetchDetail(ancom_id: number): Promise<DetailFields | null> {
  const html = await fetchWithRetry(DETAIL_URL(ancom_id));
  return parseDetail(html, ancom_id);
}

// ── Ingest ──────────────────────────────────────────────────────────────────
async function ingestOne(
  list: ListRow,
  detail: DetailFields,
  stats: IngestStats,
): Promise<void> {
  const client = await pool.connect();
  try {
    // Use detail name when present (canonical), fall back to list.
    const titular = detail.titular_name || list.titular_name;
    const judet = detail.judet || list.list_judet || null;

    // Stage A direct match — if cui present from detail, mark cui_match_method='direct'.
    const cui = detail.cui || null;
    const cui_method = cui ? 'direct' : null;
    const cui_score = cui ? 1.0 : null;

    // Compute name_norm via firms.normalize_company_name() — deferred to matcher
    // for rows where cui is NULL. Store NULL initially.
    const result = await client.query(
      `INSERT INTO ancom.operatori (
         ancom_id, titular_name, titular_cui, cui_match_method, cui_match_score, matched_at,
         euid, adresa, oras, judet, list_judet, detail_url, status, raw_html_hash, fetched_at
       )
       VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,'autorizat',$13,now())
       ON CONFLICT (ancom_id) DO UPDATE SET
         titular_name = EXCLUDED.titular_name,
         titular_cui = COALESCE(EXCLUDED.titular_cui, ancom.operatori.titular_cui),
         cui_match_method = CASE
           WHEN EXCLUDED.titular_cui IS NOT NULL THEN 'direct'
           ELSE ancom.operatori.cui_match_method
         END,
         cui_match_score = CASE
           WHEN EXCLUDED.titular_cui IS NOT NULL THEN 1.0
           ELSE ancom.operatori.cui_match_score
         END,
         matched_at = CASE
           WHEN EXCLUDED.titular_cui IS NOT NULL THEN now()
           ELSE ancom.operatori.matched_at
         END,
         euid = EXCLUDED.euid,
         adresa = EXCLUDED.adresa,
         oras = EXCLUDED.oras,
         judet = EXCLUDED.judet,
         list_judet = EXCLUDED.list_judet,
         detail_url = EXCLUDED.detail_url,
         status = 'autorizat',
         raw_html_hash = EXCLUDED.raw_html_hash,
         fetched_at = now()
       RETURNING (xmax = 0) AS inserted`,
      [
        list.ancom_id,
        titular,
        cui,
        cui_method,
        cui_score,
        cui ? new Date() : null,
        detail.euid,
        detail.adresa,
        detail.oras,
        judet,
        list.list_judet || null,
        DETAIL_URL(list.ancom_id),
        detail.raw_hash,
      ],
    );
    if (result.rows[0]?.inserted) stats.inserted++;
    else stats.updated++;

    // Replace drepturi (clean re-load is cheaper than diffing for ~10 rows/operator).
    await client.query('DELETE FROM ancom.drepturi WHERE ancom_id = $1', [list.ancom_id]);
    for (const d of detail.drepturi) {
      await client.query(
        `INSERT INTO ancom.drepturi (ancom_id, cod, tip, descriere, data_nasterii)
         VALUES ($1,$2,$3,$4,$5)
         ON CONFLICT (ancom_id, cod) DO UPDATE SET
           tip = EXCLUDED.tip,
           descriere = EXCLUDED.descriere,
           data_nasterii = EXCLUDED.data_nasterii`,
        [list.ancom_id, d.cod, d.tip, d.descriere, d.data_nasterii],
      );
      stats.drepturi++;
    }
  } finally {
    client.release();
  }
}

async function logScrape(
  scraper: string,
  sourceUrl: string,
  stats: IngestStats,
  durationMs: number,
  startedAt: Date,
  error?: string,
): Promise<void> {
  await pool.query(
    `INSERT INTO ancom.scrape_log (scraper, source_url, rows_seen, rows_inserted, rows_updated, rows_skipped, duration_ms, started_at, error)
     VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)`,
    [
      scraper,
      sourceUrl,
      stats.seen,
      stats.inserted,
      stats.updated,
      stats.skipped,
      durationMs,
      startedAt,
      error || null,
    ],
  );
}

// ── Main ───────────────────────────────────────────────────────────────────
async function main(): Promise<void> {
  console.log(`[ancom] start limit=${argLimit || 'none'} maxPages=${argMaxPages || 'all'}`);
  const startedAt = new Date();
  const t0 = Date.now();
  const stats: IngestStats = { seen: 0, inserted: 0, updated: 0, skipped: 0, drepturi: 0 };
  let error: string | undefined;
  let isHardError = false;

  try {
    // Phase 1: enumerate list pages
    const first = await fetchListPage(1);
    let totalPages = first.totalPages;
    if (argMaxPages > 0) totalPages = Math.min(totalPages, argMaxPages);
    console.log(`[ancom] list: page 1/${totalPages} → ${first.rows.length} rows; expected total pages ${first.totalPages}`);

    let allRows: ListRow[] = [...first.rows];
    for (let p = 2; p <= totalPages; p++) {
      const lp = await fetchListPage(p);
      allRows = allRows.concat(lp.rows);
      if (p % 10 === 0 || p === totalPages) {
        console.log(`[ancom] list: p${p}/${totalPages} (${allRows.length} rows so far)`);
      }
      // Respect: small jitter between pages.
      await sleep(120);
    }

    // Dedupe by ancom_id (paranoia — if list shifts mid-scrape).
    const seen = new Set<number>();
    const uniqueRows: ListRow[] = [];
    for (const r of allRows) {
      if (seen.has(r.ancom_id)) continue;
      seen.add(r.ancom_id);
      uniqueRows.push(r);
    }
    let target = uniqueRows;
    if (argLimit > 0) target = uniqueRows.slice(0, argLimit);
    console.log(`[ancom] list done: ${uniqueRows.length} unique operators (processing ${target.length})`);
    stats.seen = target.length;

    // Phase 2: detail fetch + ingest, sequential (be polite, ~570 reqs).
    for (let i = 0; i < target.length; i++) {
      const row = target[i];
      try {
        const det = await fetchDetail(row.ancom_id);
        if (!det) {
          stats.skipped++;
          console.warn(`[ancom] skip id=${row.ancom_id} (no detail body)`);
          continue;
        }
        await ingestOne(row, det, stats);
      } catch (e) {
        stats.skipped++;
        console.warn(`[ancom] skip id=${row.ancom_id}: ${e instanceof Error ? e.message : e}`);
      }
      if ((i + 1) % 25 === 0 || i === target.length - 1) {
        console.log(
          `[ancom] detail: ${i + 1}/${target.length} ins=${stats.inserted} upd=${stats.updated} skip=${stats.skipped} drepturi=${stats.drepturi}`,
        );
      }
      await sleep(150);
    }
  } catch (e) {
    error = e instanceof Error ? e.message : String(e);
    isHardError = true;
    console.error(`[ancom] ERROR: ${error}`);
  }

  const dur = Date.now() - t0;
  await logScrape('autorizati', LIST_URL, stats, dur, startedAt, error);
  console.log(
    `[ancom] done in ${(dur / 1000).toFixed(1)}s seen=${stats.seen} ins=${stats.inserted} upd=${stats.updated} skip=${stats.skipped} drepturi=${stats.drepturi}${error ? ' ERROR' : ''}`,
  );

  // Refresh MV
  console.log(`[ancom] refreshing materialized views...`);
  try {
    await pool.query('REFRESH MATERIALIZED VIEW CONCURRENTLY ancom.mv_operatori_per_cui');
  } catch (e) {
    await pool.query('REFRESH MATERIALIZED VIEW ancom.mv_operatori_per_cui');
  }
  console.log(`[ancom] done.`);

  if (isHardError) throw new Error(error);
}

main()
  .catch((e) => {
    console.error(e);
    process.exit(1);
  })
  .finally(() => pool.end());
