/**
 * Scraper pentru anunțuri achiziții ale beneficiarilor PRIVAȚI ai fondurilor EU.
 * Source: https://beneficiar.fonduri-ue.ro:8080/anunturi
 *
 * The portal serves Joomla HTML (no JSON API). We hit each detail page directly
 * and parse the embedded structured fields. Detail URLs look like:
 *   /anunturi/details/2/{ID}/{slug}
 *
 * Strategy:
 *   1. Walk list pages (/anunturi?start=N) once to harvest the maximum known ID
 *      and refresh titles + slugs (cheap dedupe).
 *   2. For every ID we don't yet have in fonduri.beneficiar_anunt, fetch the
 *      detail page (concurrent workers, polite per-worker rate limit).
 *   3. Parse the page (regex + minimal HTML walking — DOM parser would be heavy
 *      for the simple structured patterns this site uses) into the schema in
 *      sql/018_fonduri_beneficiar_privat.sql.
 *   4. Bulk INSERT … ON CONFLICT DO UPDATE per batch (anunț + its loturi).
 *
 * Idempotent. Re-runnable daily for delta (just hits new/updated IDs).
 *
 * Run: npx tsx src/scrape-beneficiar-privat.ts [--limit=N] [--concurrency=N]
 *      [--start-id=N] [--end-id=N] [--list-only]
 */
import { Pool } from 'pg';

const BASE = 'https://beneficiar.fonduri-ue.ro:8080';
const USER_AGENT = 'gov-agreg/1.0 vreaudigital.ro (contact: m.tarau@beletage.ro)';

const args = process.argv.slice(2);
const argLimit = parseInt(args.find((a) => a.startsWith('--limit='))?.split('=')[1] || '0');
const argConc = Math.max(1, parseInt(args.find((a) => a.startsWith('--concurrency='))?.split('=')[1] || '3'));
const argStartId = parseInt(args.find((a) => a.startsWith('--start-id='))?.split('=')[1] || '0');
const argEndId = parseInt(args.find((a) => a.startsWith('--end-id='))?.split('=')[1] || '0');
const argListOnly = args.includes('--list-only');
const argSleepMs = parseInt(args.find((a) => a.startsWith('--sleep-ms='))?.split('=')[1] || '700');

const pool = new Pool({
  connectionString: process.env.DATABASE_URL,
  max: Math.max(4, argConc * 2),
});

// ── HTTP helper with retry + rate awareness ──────────────────────────────
async function fetchHtml(url: string, retries = 3): Promise<string | null> {
  for (let i = 0; i <= retries; i++) {
    try {
      const res = await fetch(url, {
        headers: { 'User-Agent': USER_AGENT, 'Accept': 'text/html' },
        signal: AbortSignal.timeout(15000),
      });
      if (res.status === 404) return null;
      if (!res.ok) {
        if (i < retries) {
          await new Promise((r) => setTimeout(r, 2000 * (i + 1)));
          continue;
        }
        return null;
      }
      return await res.text();
    } catch {
      if (i < retries) await new Promise((r) => setTimeout(r, 2000 * (i + 1)));
    }
  }
  return null;
}

// ── HTML parsing helpers ─────────────────────────────────────────────────
function decode(s: string | null): string | null {
  if (s == null) return null;
  return s
    .replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>')
    .replace(/&quot;/g, '"').replace(/&nbsp;/g, ' ').replace(/&#039;/g, "'")
    .replace(/&Acirc;/g, 'Â').replace(/&acirc;/g, 'â')
    .replace(/&Icirc;/g, 'Î').replace(/&icirc;/g, 'î')
    .replace(/&Atilde;/g, 'Ã').replace(/&Aring;/g, 'Å')
    .replace(/&Cedil;/g, '¸').replace(/&Scedil;/g, 'Ş').replace(/&scedil;/g, 'ş')
    .replace(/&Tcedil;/g, 'Ţ').replace(/&tcedil;/g, 'ţ')
    .replace(/&hellip;/g, '…')
    .replace(/<[^>]+>/g, '')   // strip remaining HTML tags
    .replace(/\s+/g, ' ').trim();
}

function parseRoDate(s: string | null): string | null {
  if (!s) return null;
  const m = s.trim().match(/^(\d{2})\.(\d{2})\.(\d{4})/);
  return m ? `${m[3]}-${m[2]}-${m[1]}` : null;
}

interface ListEntry { id: number; title: string; slug: string }

function parseListPage(html: string): ListEntry[] {
  const out: ListEntry[] = [];
  // Pattern: href="/anunturi/details/2/{ID}/{slug}"
  const re = /href="\/anunturi\/details\/(\d+)\/(\d+)\/([^"]+)"/g;
  for (const m of html.matchAll(re)) {
    out.push({ id: parseInt(m[2]), title: decode(m[3]) || '', slug: m[3] });
  }
  return out;
}

interface AnuntDetail {
  id: number;
  smis_proiect_id: number | null;
  smis_proiect_type: number | null;
  smis_proiect_code: string | null;
  smis_proiect_name: string | null;
  beneficiar_name: string | null;
  beneficiar_program_tag: string | null;
  beneficiar_adresa: string | null;
  beneficiar_contact: string | null;
  beneficiar_telefon: string | null;
  beneficiar_regiune: string | null;
  beneficiar_judet: string | null;
  beneficiar_localitate: string | null;
  procedura_status: string | null;
  data_publicare: string | null;
  data_limita_oferta: string | null;
  ora_limita_oferta: string | null;
  judet: string | null;
  tip_contract: string | null;
  versiune_specificatii: string | null;
  loturi: Array<{
    lot_no: number;
    lot_label: string | null;
    descriere_url: string | null;
    durata_contract: string | null;
    buget_lei: number | null;
    cpv_cod: string | null;
    spec_url: string | null;
  }>;
}

// Romanian numeric: "1.234.567,89" → 1234567.89 ; "26240,00" → 26240.00 ;
// "313930" → 313930 ; "313.930" → 313930 (we assume "." is thousands sep here).
function parseRoNumber(s: string | null): number | null {
  if (!s) return null;
  const cleaned = s.replace(/\s/g, '').replace(/\./g, '').replace(/,/g, '.');
  const v = parseFloat(cleaned);
  return Number.isFinite(v) ? v : null;
}

function parseDetailPage(id: number, html: string): AnuntDetail | null {
  const out: AnuntDetail = {
    id,
    smis_proiect_id: null, smis_proiect_type: null, smis_proiect_code: null, smis_proiect_name: null,
    beneficiar_name: null, beneficiar_program_tag: null,
    beneficiar_adresa: null, beneficiar_contact: null, beneficiar_telefon: null,
    beneficiar_regiune: null, beneficiar_judet: null, beneficiar_localitate: null,
    procedura_status: null,
    data_publicare: null, data_limita_oferta: null, ora_limita_oferta: null,
    judet: null, tip_contract: null, versiune_specificatii: null,
    loturi: [],
  };

  // Empty/error page detection
  if (!html.includes('contact_benef') || !html.includes('divwrapdetails')) return null;

  // Beneficiar block: <ul>NAME (TAG)<hr/><li>Adresa: ...</li>...
  const benefBlock = html.match(/<ul>([\s\S]*?)<\/ul>/);
  if (benefBlock) {
    const block = benefBlock[1];
    const nameMatch = block.match(/^([^<\n]+?)\s*(?:\(([^)]+)\))?\s*<hr/);
    if (nameMatch) {
      out.beneficiar_name = decode(nameMatch[1]);
      out.beneficiar_program_tag = nameMatch[2] ? decode(nameMatch[2]) : null;
    }
    const adresa = block.match(/Adresa:\s*([^<]+)/);
    out.beneficiar_adresa = decode(adresa?.[1] || null);
    const contact = block.match(/Contact:\s*([^<]+)/);
    out.beneficiar_contact = decode(contact?.[1] || null);
    const tel = block.match(/Telefon:\s*([^<]+)/);
    out.beneficiar_telefon = decode(tel?.[1] || null);
    const reg = block.match(/Regiune:\s*([^<]+)/);
    out.beneficiar_regiune = decode(reg?.[1] || null);
    const jud = block.match(/Judet:\s*([^<]+)/);
    out.beneficiar_judet = decode(jud?.[1] || null);
    const loc = block.match(/Localitate:\s*([^<]+)/);
    out.beneficiar_localitate = decode(loc?.[1] || null);
  }

  // Project link: <em>Proiect [319946]: <a href="/proiecte/details/1/15088"
  const projMatch = html.match(/Proiect\s*\[([^\]]+)\]:\s*<a\s+href="\/proiecte\/details\/(\d+)\/(\d+)"[^>]*>([^<]+)<\/a>/);
  if (projMatch) {
    out.smis_proiect_code = decode(projMatch[1]);
    out.smis_proiect_type = parseInt(projMatch[2]);
    out.smis_proiect_id = parseInt(projMatch[3]);
    out.smis_proiect_name = decode(projMatch[4]);
  }

  // Procedura status (usually "Procedură în curs de ofertare" or "Procedură închisă")
  const procStatus = html.match(/<b>(Procedur[ăa][^<]*?)<\/b>/);
  if (procStatus) out.procedura_status = decode(procStatus[1]);

  // Field-value table — a sequence of <strong class="list-title">FIELD</strong> ... <span ...>VALUE</span>
  const fieldRows = html.matchAll(
    /<strong class="list-title"[^>]*>(?:<strong[^>]*>)?([^<]+?)(?:<\/strong>)?<\/strong>[\s\S]*?<td>(?:&nbsp;|\s)*<span[^>]*>(?:<span[^>]*>)?\s*([^<]+?)\s*(?:<\/span>)?<\/span>/g,
  );
  for (const m of fieldRows) {
    const field = decode(m[1])?.toLowerCase().trim() || '';
    const value = decode(m[2]) || '';
    if (field.includes('data publicare')) out.data_publicare = parseRoDate(value);
    else if (field.includes('data limit')) out.data_limita_oferta = parseRoDate(value);
    else if (field.includes('ora limit')) out.ora_limita_oferta = value;
    else if (field === 'județ' || field === 'judet') out.judet = value;
    else if (field.includes('tip contract')) out.tip_contract = value;
  }

  // Versiune specificatii
  const verMatch = html.match(/Data\/Ora celei mai recente versiuni a specifica[ţt]iilor:[\s\S]*?<strong[^>]*>([^<]+)<\/strong>/);
  if (verMatch) out.versiune_specificatii = decode(verMatch[1]);

  // Loturi — table-like sections. The lot heading can be a number ("1", "2") OR
  // free text ("Achiziție licențe"). We capture whatever the label is and assign
  // an ordinal lot_no by matching order. Each lot then has the metadata we want.
  const lotBlocks = [...html.matchAll(
    /<div style="background:#444;color:#eee;font-weight: bold;width:100%;padding: 0 5px 0 5px;">([\s\S]*?)<\/div>([\s\S]*?)(?=<div style="background:#444|<\/div><\/div><\/div>)/g,
  )];
  lotBlocks.forEach((lm, idx) => {
    const label = decode(lm[1]);
    const block = lm[2];
    const descLink = block.match(/href="(\/desc-lot[^"]+)"/);
    const durata = block.match(/Durat[ăa] contract:\s*([^<]+)/);
    const buget = block.match(/Buget contract:\s*([\d.,\s]+)\s*lei/);
    const cpv = block.match(/CPV[^:]*:\s*<[^>]*>?\s*(\d{8})/i);
    const specLink = block.match(/href="(\/dwl-spec[^"]+)"/);
    out.loturi.push({
      lot_no: idx + 1,
      lot_label: label,
      descriere_url: descLink?.[1] ? `${BASE}${descLink[1]}` : null,
      durata_contract: decode(durata?.[1] || null),
      buget_lei: parseRoNumber(buget?.[1] || null),
      cpv_cod: cpv?.[1] || null,
      spec_url: specLink?.[1] ? `${BASE}${specLink[1]}` : null,
    });
  });

  return out;
}

// ── DB persistence ────────────────────────────────────────────────────────
async function persist(detail: AnuntDetail) {
  const client = await pool.connect();
  try {
    await client.query('BEGIN');
    await client.query(
      `
      INSERT INTO fonduri.beneficiar_anunt (
        id, smis_proiect_id, smis_proiect_type, smis_proiect_code, smis_proiect_name,
        beneficiar_name, beneficiar_program_tag, beneficiar_adresa, beneficiar_contact, beneficiar_telefon,
        beneficiar_regiune, beneficiar_judet, beneficiar_localitate,
        procedura_status, data_publicare, data_limita_oferta, ora_limita_oferta,
        judet, tip_contract, versiune_specificatii
      ) VALUES (
        $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15::date, $16::date, $17, $18, $19, $20
      )
      ON CONFLICT (id) DO UPDATE SET
        smis_proiect_id = EXCLUDED.smis_proiect_id,
        smis_proiect_type = EXCLUDED.smis_proiect_type,
        smis_proiect_code = EXCLUDED.smis_proiect_code,
        smis_proiect_name = EXCLUDED.smis_proiect_name,
        beneficiar_name = EXCLUDED.beneficiar_name,
        beneficiar_program_tag = EXCLUDED.beneficiar_program_tag,
        beneficiar_adresa = EXCLUDED.beneficiar_adresa,
        beneficiar_contact = EXCLUDED.beneficiar_contact,
        beneficiar_telefon = EXCLUDED.beneficiar_telefon,
        beneficiar_regiune = EXCLUDED.beneficiar_regiune,
        beneficiar_judet = EXCLUDED.beneficiar_judet,
        beneficiar_localitate = EXCLUDED.beneficiar_localitate,
        procedura_status = EXCLUDED.procedura_status,
        data_publicare = EXCLUDED.data_publicare,
        data_limita_oferta = EXCLUDED.data_limita_oferta,
        ora_limita_oferta = EXCLUDED.ora_limita_oferta,
        judet = EXCLUDED.judet,
        tip_contract = EXCLUDED.tip_contract,
        versiune_specificatii = EXCLUDED.versiune_specificatii,
        fetched_at = now();
      `,
      [
        detail.id, detail.smis_proiect_id, detail.smis_proiect_type, detail.smis_proiect_code, detail.smis_proiect_name,
        detail.beneficiar_name, detail.beneficiar_program_tag, detail.beneficiar_adresa, detail.beneficiar_contact, detail.beneficiar_telefon,
        detail.beneficiar_regiune, detail.beneficiar_judet, detail.beneficiar_localitate,
        detail.procedura_status, detail.data_publicare, detail.data_limita_oferta, detail.ora_limita_oferta,
        detail.judet, detail.tip_contract, detail.versiune_specificatii,
      ],
    );
    // Replace lots on each ingest (lots are append-only on portal but easier to TRUNCATE+INSERT)
    await client.query(`DELETE FROM fonduri.beneficiar_anunt_lot WHERE anunt_id = $1`, [detail.id]);
    for (const lot of detail.loturi) {
      await client.query(
        `INSERT INTO fonduri.beneficiar_anunt_lot
         (anunt_id, lot_no, lot_label, descriere_url, durata_contract, buget_lei, cpv_cod, spec_url)
         VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
         ON CONFLICT (anunt_id, lot_no) DO NOTHING`,
        [detail.id, lot.lot_no, lot.lot_label, lot.descriere_url, lot.durata_contract, lot.buget_lei, lot.cpv_cod, lot.spec_url],
      );
    }
    await client.query('COMMIT');
  } catch (err) {
    await client.query('ROLLBACK').catch(() => {});
    throw err;
  } finally {
    client.release();
  }
}

// ── ID discovery ──────────────────────────────────────────────────────────
async function discoverIdRange(): Promise<{ min: number; max: number }> {
  // First page = newest, last page = oldest. Find max via first page; min via
  // walking last page (start = pageCount * 10 — 10).
  const firstHtml = await fetchHtml(`${BASE}/anunturi`);
  if (!firstHtml) throw new Error('cannot load /anunturi');
  const entries = parseListPage(firstHtml);
  const max = Math.max(...entries.map((e) => e.id));

  // Find pagination — look for the highest start=N
  const startRe = /[?&]start=(\d+)/g;
  const starts = [...firstHtml.matchAll(startRe)].map((m) => parseInt(m[1]));
  const maxStart = Math.max(0, ...starts);

  // Get last page for min ID
  const lastHtml = maxStart > 0 ? await fetchHtml(`${BASE}/anunturi?start=${maxStart}`) : firstHtml;
  if (!lastHtml) return { min: 1, max };
  const lastEntries = parseListPage(lastHtml);
  const min = lastEntries.length > 0 ? Math.min(...lastEntries.map((e) => e.id)) : 1;

  console.log(`[BEN] ID range discovered: min=${min} max=${max}, last list page start=${maxStart}`);
  return { min, max };
}

// ── Main ──────────────────────────────────────────────────────────────────
async function main() {
  const startTs = Date.now();

  let { min, max } = await discoverIdRange();
  if (argStartId > 0) min = argStartId;
  if (argEndId > 0) max = argEndId;

  if (argListOnly) {
    console.log(`[BEN] --list-only: range ${min}..${max} = ${max - min + 1} candidates`);
    await pool.end();
    return;
  }

  // Pull existing IDs to skip (idempotency)
  const existingQ = await pool.query<{ id: number }>(
    `SELECT id FROM fonduri.beneficiar_anunt WHERE id BETWEEN $1 AND $2`,
    [min, max],
  );
  const existing = new Set(existingQ.rows.map((r) => r.id));
  console.log(`[BEN] Already have ${existing.size} of [${min}..${max}]`);

  // Build queue (newest first to surface fresh data quickly)
  const queue: number[] = [];
  for (let id = max; id >= min; id--) {
    if (!existing.has(id)) queue.push(id);
    if (argLimit > 0 && queue.length >= argLimit) break;
  }
  const total = queue.length;
  console.log(`[BEN] To fetch: ${total} (concurrency=${argConc}, sleep=${argSleepMs}ms/worker)`);

  let done = 0;
  let ok = 0;
  let notFound = 0;
  let failed = 0;

  async function worker(id: number) {
    while (true) {
      const next = queue.shift();
      if (next === undefined) break;
      const tStart = Date.now();
      try {
        const html = await fetchHtml(`${BASE}/anunturi/details/2/${next}/`);
        if (html === null) {
          notFound++;
        } else {
          const detail = parseDetailPage(next, html);
          if (detail && detail.beneficiar_name) {
            await persist(detail);
            ok++;
          } else {
            notFound++;
          }
        }
      } catch (err: any) {
        failed++;
        console.error(`[BEN w${id}] id=${next} error: ${err.message}`);
      }
      done++;
      if (done % 50 === 0 || done === total) {
        const elapsed = (Date.now() - startTs) / 1000;
        const rate = done / elapsed;
        const eta = (total - done) / Math.max(rate, 0.001);
        console.log(
          `[BEN] ${done}/${total} (${(done / total * 100).toFixed(1)}%) · ` +
          `ok=${ok} 404=${notFound} fail=${failed} · ${rate.toFixed(1)} req/s · ` +
          `ETA ${(eta / 60).toFixed(0)}min`,
        );
      }
      const elapsed = Date.now() - tStart;
      const remain = argSleepMs - elapsed;
      if (remain > 0) await new Promise((r) => setTimeout(r, remain));
    }
  }

  const workers: Promise<void>[] = [];
  for (let i = 0; i < argConc; i++) {
    const stagger = (i * argSleepMs) / argConc;
    workers.push(new Promise((r) => setTimeout(r, stagger)).then(() => worker(i + 1)));
  }
  await Promise.all(workers);

  console.log(`[BEN] DONE in ${((Date.now() - startTs) / 60000).toFixed(1)}min: ok=${ok} 404=${notFound} fail=${failed}`);
  await pool.end();
}

main().catch((err) => {
  console.error(err);
  process.exit(1);
});
