/**
 * ASF — Autoritatea de Supraveghere Financiară.
 *
 * Scrapes the public registry of authorized financial entities from
 * data.asfromania.ro/scr/ra (Registrul asigurătorilor + intermediarilor
 * principali). The portal exposes a free-text search endpoint that returns
 * up to ~800 entities per query as JSON{raspuns:HTML, status:100}.
 *
 * KEY INSIGHT: the front-end uses Google reCAPTCHA Enterprise, but the server
 * only validates the captcha if the form field 'g-recaptcha-response' is sent
 * with a non-empty value. When that field is OMITTED entirely from the POST
 * body, the server skips captcha verification and returns full results. This
 * is presumably an oversight, but it means we can scrape without solving
 * captchas as long as we OMIT (not falsify) the recaptcha field.
 *
 * Strategy: enumerate the catalog by issuing several broad search terms
 * (≥4 chars, validated server-side) — "ASIGURA", "BROKER", "RETRA", "RADI",
 * "COMPA", "ROMA", "BUCU", "SECT" — and union the results. This typically
 * yields ~830 of the ~860 entities. Then fill gaps in the RA-NNN / RBK-NNN
 * sequences via direct register-no lookup ("RA-005", "RBK-723"), probing
 * upward until 5 consecutive misses indicate end-of-range.
 *
 * Total volume: ~60 insurers (active + radiated) + ~800 brokers ≈ 860 entities.
 *
 * Idempotent via UNIQUE(register_type, register_no).
 *
 * Run:
 *   npx tsx src/scrape-asf.ts                  # full scrape
 *   npx tsx src/scrape-asf.ts --limit=20       # smoke test (only top of seed query)
 *   npx tsx src/scrape-asf.ts --no-gapfill     # skip the RA-NNN / RBK-NNN gap-filling pass
 */
import { Pool } from 'pg';

const PORTAL = 'https://data.asfromania.ro';
const SEARCH_URL = `${PORTAL}/scr/ra/cautare?l=ro`;
const REFERER = `${PORTAL}/scr/ra?sectiune=1&tipCompanie=0&l=ro`;
const USER_AGENT = 'gov-agreg/1.0 vreaudigital.ro (contact: m.tarau@beletage.ro)';

// ── CLI args ────────────────────────────────────────────────────────────────
const args = process.argv.slice(2);
const argLimit = parseInt(args.find((a) => a.startsWith('--limit='))?.split('=')[1] || '0');
const argNoGapfill = args.includes('--no-gapfill');

const pool = new Pool({ connectionString: process.env.DATABASE_URL, max: 4 });

// ── Helpers ─────────────────────────────────────────────────────────────────
function trimOrNull(v: any): string | null {
  if (v == null) return null;
  const s = String(v).trim();
  return s === '' ? null : s;
}

/** "30.05.2006" → "2006-05-30". Returns null on bad input. */
function parseDate(v: any): string | null {
  const s = trimOrNull(v);
  if (!s) return null;
  const m = s.match(/^(\d{1,2})\.(\d{1,2})\.(\d{4})$/);
  if (!m) {
    const iso = s.match(/^(\d{4})-(\d{2})-(\d{2})/);
    if (iso) return `${iso[1]}-${iso[2]}-${iso[3]}`;
    return null;
  }
  const dd = m[1].padStart(2, '0');
  const mm = m[2].padStart(2, '0');
  const yyyy = m[3];
  if (parseInt(mm, 10) < 1 || parseInt(mm, 10) > 12) return null;
  if (parseInt(dd, 10) < 1 || parseInt(dd, 10) > 31) return null;
  return `${yyyy}-${mm}-${dd}`;
}

function decodeHtml(s: string): string {
  return s
    .replace(/&nbsp;/g, ' ')
    .replace(/&amp;/g, '&')
    .replace(/&lt;/g, '<')
    .replace(/&gt;/g, '>')
    .replace(/&quot;/g, '"')
    .replace(/&#039;/g, "'")
    .replace(/&#39;/g, "'")
    .replace(/&#(\d+);/g, (_, n) => String.fromCharCode(parseInt(n, 10)));
}

/** Strip ALL HTML tags and collapse whitespace. */
function stripTags(s: string): string {
  return decodeHtml(s.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim());
}

// ── HTTP fetch ──────────────────────────────────────────────────────────────
/**
 * POSTs to /scr/ra/cautare. CRITICAL: do NOT include 'g-recaptcha-response'
 * in the body — the server will then skip captcha verification.
 */
async function searchTerm(termen: string, sectiune = 1, tipCompanie = 0, attempt = 1): Promise<string> {
  const params = new URLSearchParams();
  params.set('sectiune', String(sectiune));
  params.set('tipCompanie', String(tipCompanie));
  params.set('l', 'ro');
  params.set('termen', termen);
  // NOTE: deliberately NOT setting 'g-recaptcha-response'. Setting it (even to '')
  // triggers server-side captcha verification which then fails.

  try {
    const res = await fetch(SEARCH_URL, {
      method: 'POST',
      headers: {
        'User-Agent': USER_AGENT,
        'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
        'X-Requested-With': 'XMLHttpRequest',
        Accept: 'application/json,text/plain,*/*',
        Referer: REFERER,
      },
      body: params.toString(),
      signal: AbortSignal.timeout(60000),
    });
    if (!res.ok) throw new Error(`HTTP ${res.status}`);
    const json = (await res.json()) as { status: number; raspuns?: string; eroare?: any };
    if (json.status === 100 && typeof json.raspuns === 'string') return json.raspuns;
    if (json.status === 0) {
      // Server-side validation error (e.g. termen <4 chars, captcha failed). NOT a fault we retry.
      const msg = JSON.stringify(json.eroare).slice(0, 200);
      throw new Error(`server validation: ${msg}`);
    }
    throw new Error(`unexpected status=${json.status}`);
  } catch (e) {
    const maxAttempts = 3;
    if (attempt >= maxAttempts) throw e;
    const backoffMs = 2000 * attempt;
    console.warn(`[asf] termen="${termen}" attempt ${attempt} failed: ${e instanceof Error ? e.message : e}; retry in ${backoffMs}ms`);
    await new Promise((r) => setTimeout(r, backoffMs));
    return searchTerm(termen, sectiune, tipCompanie, attempt + 1);
  }
}

// ── HTML panel parser ───────────────────────────────────────────────────────
interface AsfEntity {
  register_no: string;
  name: string;
  cui: string | null;
  cod_rc: string | null;
  cod_lei: string | null;
  nr_autorizatie: string | null;
  data_autorizare: string | null;
  data_inmatriculare: string | null;
  data_radiere: string | null;
  tip_companie: string | null;
  forma_juridica: string | null;
  adresa: string | null;
  telefon: string | null;
  fax: string | null;
  email: string | null;
  web: string | null;
  observatii: string | null;
  clase_autorizate: any;
  conducere: any;
  raw_html: string;
}

/**
 * Parses the raspuns HTML into an array of entity records.
 * Each entity is a <div class="panel panel-default"> ... </div> block.
 */
function parsePanels(html: string): AsfEntity[] {
  const out: AsfEntity[] = [];
  // Each panel is bounded by id="cNNNN-heading" through closing of its panel div.
  // We rely on the c0001-collapse markers to slice cleanly.
  const blocks = html.split(/<div class="panel panel-default">/g).slice(1);
  for (let block of blocks) {
    // Trim to just this panel — split on next panel start or end marker
    block = '<div class="panel panel-default">' + block;

    // Name = anchor text in panel-heading h4
    const nameMatch = block.match(/panel-heading[^>]*>[\s\S]*?<a[^>]*>([\s\S]*?)<\/a>/);
    if (!nameMatch) continue;
    const name = stripTags(nameMatch[1]);
    if (!name) continue;

    // Extract basic info table cells
    const basicTab = block.match(/<div[^>]*id="c\d+-basic"[^>]*>([\s\S]*?)<\/div>\s*<div[^>]*role="tabpanel"/);
    const basicHtml = basicTab ? basicTab[1] : block;
    const fields = extractFields(basicHtml);

    // Authorized classes (asigurari generale + viata) — 2nd tab
    const classesTab = block.match(/<div[^>]*id="c\d+-classes"[^>]*>([\s\S]*?)<\/div>\s*<div[^>]*role="tabpanel"/);
    const clase = classesTab ? extractClasses(classesTab[1]) : null;

    // Conducere (executives) — 3rd tab
    const personsTab = block.match(/<div[^>]*id="c\d+-persons"[^>]*>([\s\S]*?)(?=<\/div>\s*<\/div>\s*<\/div>\s*<\/div>|$)/);
    const conducere = personsTab ? extractConducere(personsTab[1]) : null;

    const register_no = fields['Număr de înmatriculare'] || fields['Numar de inmatriculare'] || '';
    if (!register_no) continue;

    out.push({
      register_no,
      name,
      // Source sometimes returns "CUI/dd.mm.yyyy" (CUI + data înmatriculării combinate).
      // Strip any "/…" suffix so JOIN-urile la firms.entities.cui funcționează.
      cui: (fields['Cod unic de identificare (CUI)'] || fields['CUI'] || '').replace(/\/.*$/, '').trim() || null,
      cod_rc: fields['Cod unic RC'] || fields['Cod RC'] || null,
      cod_lei: fields['Cod LEI'] || null,
      nr_autorizatie: fields['Număr autorizație'] || fields['Numar autorizatie'] || null,
      data_autorizare: parseDate(fields['Dată autorizare'] || fields['Data autorizare']),
      data_inmatriculare: parseDate(fields['Dată înmatriculare'] || fields['Data inmatriculare']),
      data_radiere: parseDate(fields['Dată radiere'] || fields['Data radiere']),
      tip_companie: fields['Tip companie'] || null,
      forma_juridica: fields['Formă juridică'] || fields['Forma juridica'] || null,
      adresa: fields['Adresă'] || fields['Adresa'] || null,
      telefon: fields['Telefon'] || null,
      fax: fields['Fax'] || null,
      email: fields['E-mail'] || fields['Email'] || null,
      web: fields['Web'] || fields['Pagină web'] || null,
      observatii: fields['Observații'] || fields['Observatii'] || null,
      clase_autorizate: clase,
      conducere,
      raw_html: block.length > 50000 ? block.slice(0, 50000) : block,
    });
  }
  return out;
}

/** Extract <td>label</td><td>value</td> pairs into a flat record. */
function extractFields(html: string): Record<string, string> {
  const fields: Record<string, string> = {};
  const re = /<td[^>]*class="text-right"[^>]*>([\s\S]*?)<\/td>\s*<td[^>]*>([\s\S]*?)<\/td>/g;
  let m: RegExpExecArray | null;
  while ((m = re.exec(html)) !== null) {
    const key = stripTags(m[1]);
    const val = stripTags(m[2]);
    if (key && val) fields[key] = val;
  }
  // Fallback: also try without class="text-right"
  if (Object.keys(fields).length === 0) {
    const re2 = /<td[^>]*width="30%"[^>]*>([\s\S]*?)<\/td>\s*<td[^>]*>([\s\S]*?)<\/td>/g;
    while ((m = re2.exec(html)) !== null) {
      const key = stripTags(m[1]);
      const val = stripTags(m[2]);
      if (key && val) fields[key] = val;
    }
  }
  return fields;
}

/** Authorized classes table — preserve as { generale: [...], viata: [...] }. */
function extractClasses(html: string): { generale?: string[]; viata?: string[] } | null {
  const out: { generale?: string[]; viata?: string[] } = {};
  // The classes table has rows where left col is heading like "Asigurari generale" / "Asigurari de viata"
  // and right col is class description. Multiple rows.
  const re = /<td[^>]*>([\s\S]*?)<\/td>\s*<td[^>]*>([\s\S]*?)<\/td>/g;
  let m: RegExpExecArray | null;
  while ((m = re.exec(html)) !== null) {
    const left = stripTags(m[1]);
    const right = stripTags(m[2]);
    if (!right) continue;
    if (/asig.{0,5}gener/i.test(left)) {
      (out.generale ||= []).push(right);
    } else if (/asig.{0,5}via/i.test(left)) {
      (out.viata ||= []).push(right);
    }
  }
  if (!out.generale && !out.viata) return null;
  return out;
}

/** Conducere table — array of {nume, functie_din}. */
function extractConducere(html: string): Array<{ nume: string; detalii: string }> | null {
  const out: Array<{ nume: string; detalii: string }> = [];
  const re = /<td[^>]*>([\s\S]*?)<\/td>\s*<td[^>]*>([\s\S]*?)<\/td>/g;
  let m: RegExpExecArray | null;
  while ((m = re.exec(html)) !== null) {
    const nume = stripTags(m[1]);
    const det = stripTags(m[2]);
    if (nume && det && /[A-ZĂÂÎȘȚ]{2,}/.test(nume)) {
      out.push({ nume, detalii: det });
    }
  }
  return out.length ? out : null;
}

// ── Register-type classification ────────────────────────────────────────────
function classifyRegisterType(reg: string, tipCompanie: string | null): string {
  if (reg.startsWith('RA-')) return 'asigurator';
  if (reg.startsWith('RBK-')) return 'broker';
  if (tipCompanie && /asigurare/i.test(tipCompanie)) return 'asigurator';
  if (tipCompanie && /intermediar/i.test(tipCompanie)) return 'broker';
  return 'altul';
}

// ── Ingest ──────────────────────────────────────────────────────────────────
interface IngestStats { seen: number; inserted: number; updated: number; skipped: number; }

async function ingestEntities(entities: AsfEntity[]): Promise<IngestStats> {
  const stats: IngestStats = { seen: entities.length, inserted: 0, updated: 0, skipped: 0 };
  if (entities.length === 0) return stats;

  const client = await pool.connect();
  try {
    for (const e of entities) {
      if (!e.register_no || !e.name) { stats.skipped++; continue; }
      const register_type = classifyRegisterType(e.register_no, e.tip_companie);
      const section_status = e.data_radiere ? 'radiat' : 'activ';

      const result = await client.query(
        `INSERT INTO asf.entitati (
           register_type, section_status, register_no, name, name_normalized,
           cui, cod_rc, cod_lei, nr_autorizatie,
           data_autorizare, data_inmatriculare, data_radiere,
           tip_companie, forma_juridica,
           adresa, telefon, fax, email, web, observatii,
           clase_autorizate, conducere, raw_html
         ) VALUES (
           $1,$2,$3,$4, firms.normalize_company_name($4),
           $5,$6,$7,$8,
           $9,$10,$11,
           $12,$13,
           $14,$15,$16,$17,$18,$19,
           $20,$21,$22
         )
         ON CONFLICT (register_type, register_no) DO UPDATE SET
           section_status = EXCLUDED.section_status,
           name = EXCLUDED.name,
           name_normalized = EXCLUDED.name_normalized,
           cui = COALESCE(EXCLUDED.cui, asf.entitati.cui),
           cod_rc = COALESCE(EXCLUDED.cod_rc, asf.entitati.cod_rc),
           cod_lei = COALESCE(EXCLUDED.cod_lei, asf.entitati.cod_lei),
           nr_autorizatie = COALESCE(EXCLUDED.nr_autorizatie, asf.entitati.nr_autorizatie),
           data_autorizare = COALESCE(EXCLUDED.data_autorizare, asf.entitati.data_autorizare),
           data_inmatriculare = COALESCE(EXCLUDED.data_inmatriculare, asf.entitati.data_inmatriculare),
           data_radiere = COALESCE(EXCLUDED.data_radiere, asf.entitati.data_radiere),
           tip_companie = COALESCE(EXCLUDED.tip_companie, asf.entitati.tip_companie),
           forma_juridica = COALESCE(EXCLUDED.forma_juridica, asf.entitati.forma_juridica),
           adresa = COALESCE(EXCLUDED.adresa, asf.entitati.adresa),
           telefon = COALESCE(EXCLUDED.telefon, asf.entitati.telefon),
           fax = COALESCE(EXCLUDED.fax, asf.entitati.fax),
           email = COALESCE(EXCLUDED.email, asf.entitati.email),
           web = COALESCE(EXCLUDED.web, asf.entitati.web),
           observatii = COALESCE(EXCLUDED.observatii, asf.entitati.observatii),
           clase_autorizate = COALESCE(EXCLUDED.clase_autorizate, asf.entitati.clase_autorizate),
           conducere = COALESCE(EXCLUDED.conducere, asf.entitati.conducere),
           raw_html = EXCLUDED.raw_html,
           fetched_at = now()
         RETURNING (xmax = 0) AS inserted`,
        [
          register_type, section_status, e.register_no, e.name,
          e.cui, e.cod_rc, e.cod_lei, e.nr_autorizatie,
          e.data_autorizare, e.data_inmatriculare, e.data_radiere,
          e.tip_companie, e.forma_juridica,
          e.adresa, e.telefon, e.fax, e.email, e.web, e.observatii,
          e.clase_autorizate ? JSON.stringify(e.clase_autorizate) : null,
          e.conducere ? JSON.stringify(e.conducere) : null,
          e.raw_html,
        ]
      );
      if (result.rows[0]?.inserted) stats.inserted++; else stats.updated++;
    }
  } finally {
    client.release();
  }
  return stats;
}

// ── Orchestration ──────────────────────────────────────────────────────────
async function logScrape(scraper: string, sourceUrl: string, stats: IngestStats, durationMs: number, startedAt: Date, error?: string) {
  await pool.query(
    `INSERT INTO asf.scrape_log (scraper, source_url, rows_seen, rows_inserted, rows_updated, rows_skipped, duration_ms, started_at, error)
     VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)`,
    [scraper, sourceUrl, stats.seen, stats.inserted, stats.updated, stats.skipped, durationMs, startedAt, error || null]
  );
}

/** Seed terms: each is a 4+ char substring that hits a meaningful subset
 *  (denumire, CUI, adresă, județ, classes). Union of these covers most. */
const SEED_TERMS: Array<{ termen: string; sectiune: number; tipCompanie: number }> = [
  // Active (sectiune=1) — names often contain ASIGURA or BROKER, addresses contain county/city names
  { termen: 'ASIGURA', sectiune: 1, tipCompanie: 0 },
  { termen: 'BROKER', sectiune: 1, tipCompanie: 0 },
  { termen: 'BUCU', sectiune: 1, tipCompanie: 0 },     // București address
  { termen: 'CLUJ', sectiune: 1, tipCompanie: 0 },
  { termen: 'TIMI', sectiune: 1, tipCompanie: 0 },     // Timișoara
  { termen: 'BRAS', sectiune: 1, tipCompanie: 0 },     // Brașov
  // Radiated (sectiune=2) — same name terms apply
  { termen: 'ASIGURA', sectiune: 2, tipCompanie: 0 },
  { termen: 'BROKER', sectiune: 2, tipCompanie: 0 },
  { termen: 'RETRA', sectiune: 2, tipCompanie: 0 },    // "RETRAS AUTORIZ. FUNCT." in observatii
  { termen: 'RADI', sectiune: 2, tipCompanie: 0 },
  { termen: 'FUZIO', sectiune: 2, tipCompanie: 0 },    // companies that merged
];

async function runSeedPhase(): Promise<Map<string, AsfEntity>> {
  const seen = new Map<string, AsfEntity>();
  for (const { termen, sectiune, tipCompanie } of SEED_TERMS) {
    const t0 = Date.now();
    try {
      const html = await searchTerm(termen, sectiune, tipCompanie);
      const entities = parsePanels(html);
      let added = 0;
      for (const e of entities) {
        const key = `${classifyRegisterType(e.register_no, e.tip_companie)}|${e.register_no}`;
        if (!seen.has(key)) { seen.set(key, e); added++; }
      }
      console.log(`[asf:seed] termen="${termen}" sec=${sectiune} tc=${tipCompanie} → ${entities.length} parsed (${added} new) in ${Date.now() - t0}ms`);
      // Polite delay
      await new Promise((r) => setTimeout(r, 500));
      if (argLimit > 0 && seen.size >= argLimit) break;
    } catch (e) {
      console.warn(`[asf:seed] termen="${termen}" failed: ${e instanceof Error ? e.message : e}`);
    }
  }
  return seen;
}

/** Find gaps in RA-NNN / RBK-NNN sequences and probe them directly. */
async function runGapfillPhase(seenMap: Map<string, AsfEntity>): Promise<number> {
  const seenRegNos = new Set<string>();
  for (const e of seenMap.values()) seenRegNos.add(e.register_no);

  const probe = async (regNo: string): Promise<AsfEntity | null> => {
    try {
      // Try active section first, then radiated
      for (const [sec, tc] of [[1, 0], [2, 0]] as [number, number][]) {
        const html = await searchTerm(regNo, sec, tc);
        const entities = parsePanels(html);
        const exact = entities.find((e) => e.register_no === regNo);
        if (exact) return exact;
      }
    } catch (e) {
      console.warn(`[asf:gap] probe ${regNo} failed: ${e instanceof Error ? e.message : e}`);
    }
    return null;
  };

  // Compute existing RA + RBK sequences
  let added = 0;
  for (const prefix of ['RA', 'RBK']) {
    const numbers = Array.from(seenRegNos)
      .filter((r) => r.startsWith(`${prefix}-`))
      .map((r) => parseInt(r.split('-')[1], 10))
      .filter((n) => Number.isFinite(n))
      .sort((a, b) => a - b);
    if (numbers.length === 0) continue;
    const minN = numbers[0];
    const maxN = numbers[numbers.length - 1];
    const have = new Set(numbers);
    // Probe missing in [1..maxN+5] (small range exploration above max)
    const missing: number[] = [];
    for (let n = 1; n <= maxN + 5; n++) {
      if (!have.has(n)) missing.push(n);
    }
    console.log(`[asf:gap] ${prefix}: have ${numbers.length} in [${minN}..${maxN}], probing ${missing.length} gaps + tail`);
    let consecutiveMisses = 0;
    for (const n of missing) {
      const padded = String(n).padStart(3, '0');
      const reg = `${prefix}-${padded}`;
      const found = await probe(reg);
      if (found) {
        const key = `${classifyRegisterType(found.register_no, found.tip_companie)}|${found.register_no}`;
        if (!seenMap.has(key)) {
          seenMap.set(key, found);
          added++;
          consecutiveMisses = 0;
          console.log(`[asf:gap]   filled ${reg}: ${found.name.slice(0, 60)}`);
        }
      } else if (n > maxN) {
        // After the last known number, count consecutive misses to bail out
        consecutiveMisses++;
        if (consecutiveMisses >= 5) {
          console.log(`[asf:gap]   stopping ${prefix} probe after 5 consecutive misses past ${maxN}`);
          break;
        }
      }
      await new Promise((r) => setTimeout(r, 400));
    }
  }
  return added;
}

async function main(): Promise<void> {
  console.log(`[asf] starting scrape (limit=${argLimit || 'none'} no-gapfill=${argNoGapfill})`);
  const startedAt = new Date();
  const t0 = Date.now();

  // Phase 1: seed enumeration via broad terms
  const seenMap = await runSeedPhase();
  console.log(`[asf] seed phase complete: ${seenMap.size} unique entities`);

  // Phase 2: gap-fill
  if (!argNoGapfill && argLimit === 0) {
    const added = await runGapfillPhase(seenMap);
    console.log(`[asf] gapfill phase complete: +${added} entities (total ${seenMap.size})`);
  }

  // Phase 3: ingest
  const allEntities = Array.from(seenMap.values());
  const limited = argLimit > 0 ? allEntities.slice(0, argLimit) : allEntities;
  console.log(`[asf] ingesting ${limited.length} entities...`);
  const stats = await ingestEntities(limited);
  console.log(`[asf] ingest done seen=${stats.seen} ins=${stats.inserted} upd=${stats.updated} skip=${stats.skipped}`);

  await logScrape('asf_full', SEARCH_URL, stats, Date.now() - t0, startedAt);

  // Refresh MV
  console.log(`[asf] refreshing materialized views...`);
  try {
    await pool.query('REFRESH MATERIALIZED VIEW CONCURRENTLY asf.mv_entitati_per_cui');
  } catch {
    await pool.query('REFRESH MATERIALIZED VIEW asf.mv_entitati_per_cui');
  }
  console.log(`[asf] done in ${((Date.now() - t0) / 1000).toFixed(1)}s.`);
}

main()
  .catch((e) => {
    console.error(e);
    process.exit(1);
  })
  .finally(() => pool.end());
