/**
 * Scraper pentru proiectele EU referențiate de fonduri.beneficiar_anunt.
 * Source: https://beneficiar.fonduri-ue.ro:8080/proiecte/details/{type}/{id}
 *
 * The portal exposes only 7 fields per project page (NOT valoare/contributie_ue
 * like one might hope from MFE bulk data). See sql/020_fonduri_proiect_v2.sql
 * for the actual schema.
 *
 * Strategy:
 *   1. Pull distinct (smis_proiect_type, smis_proiect_id) from beneficiar_anunt
 *      — these are the projects we know exist (referenced by ≥1 announcement).
 *   2. Skip ones already in beneficiar_proiect (idempotent).
 *   3. Fetch /proiecte/details/{type}/{id}, parse, INSERT … ON CONFLICT.
 *   4. Polite rate limit (server is slow, ~3-11s/req sustained).
 *
 * Run: npx tsx src/scrape-beneficiar-proiect.ts [--limit=N] [--concurrency=N]
 */
import { Pool } from 'pg';
import { createHash } from 'crypto';

const BASE = 'https://beneficiar.fonduri-ue.ro:8080';
const USER_AGENT = 'gov-agreg/1.0 vreaudigital.ro (contact: m.tarau@beletage.ro)';

const args = process.argv.slice(2);
const argLimit = parseInt(args.find((a) => a.startsWith('--limit='))?.split('=')[1] || '0');
const argConc = Math.max(1, parseInt(args.find((a) => a.startsWith('--concurrency='))?.split('=')[1] || '3'));
const argSleepMs = parseInt(args.find((a) => a.startsWith('--sleep-ms='))?.split('=')[1] || '700');
const argRefetch = args.includes('--refetch');

const pool = new Pool({
  connectionString: process.env.DATABASE_URL,
  max: Math.max(4, argConc * 2),
});

// ── HTTP helper with retry ───────────────────────────────────────────────
async function fetchHtml(url: string, retries = 3): Promise<string | null> {
  for (let i = 0; i <= retries; i++) {
    try {
      const res = await fetch(url, {
        headers: { 'User-Agent': USER_AGENT, Accept: 'text/html' },
        signal: AbortSignal.timeout(30000),
      });
      if (res.status === 404) return null;
      if (!res.ok) {
        if (i < retries) {
          await new Promise((r) => setTimeout(r, 2000 * (i + 1)));
          continue;
        }
        return null;
      }
      return await res.text();
    } catch {
      if (i < retries) await new Promise((r) => setTimeout(r, 2000 * (i + 1)));
    }
  }
  return null;
}

// ── HTML decode ──────────────────────────────────────────────────────────
function decode(s: string | null): string | null {
  if (s == null) return null;
  return s
    .replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>')
    .replace(/&quot;/g, '"').replace(/&nbsp;/g, ' ').replace(/&#039;/g, "'")
    .replace(/&#40;/g, '(').replace(/&#41;/g, ')')
    .replace(/&Acirc;/g, 'Â').replace(/&acirc;/g, 'â')
    .replace(/&Icirc;/g, 'Î').replace(/&icirc;/g, 'î')
    .replace(/&Atilde;/g, 'Ã').replace(/&Aring;/g, 'Å')
    .replace(/&Cedil;/g, '¸').replace(/&Scedil;/g, 'Ş').replace(/&scedil;/g, 'ş')
    .replace(/&Tcedil;/g, 'Ţ').replace(/&tcedil;/g, 'ţ')
    .replace(/&ndash;/g, '–').replace(/&mdash;/g, '—').replace(/&hellip;/g, '…')
    .replace(/<[^>]+>/g, '')
    .replace(/\s+/g, ' ')
    .trim();
}

function parseRoDate(s: string | null): string | null {
  if (!s) return null;
  const m = s.trim().match(/^(\d{2})\.(\d{2})\.(\d{4})/);
  return m ? `${m[3]}-${m[2]}-${m[1]}` : null;
}

// "PRNE Program Regional Nord-Est" → { cod: "PRNE", text: "Program Regional Nord-Est" }
// "PRNE_P1 P1.P1. Nord-Est – O regiune…" → { cod: "PRNE_P1", text: "P1.P1. Nord-Est…" }
function splitCodeText(s: string | null): { cod: string | null; text: string | null } {
  if (!s) return { cod: null, text: null };
  const trimmed = s.trim();
  const idx = trimmed.search(/\s/);
  if (idx < 0) return { cod: trimmed, text: null };
  return { cod: trimmed.slice(0, idx), text: trimmed.slice(idx + 1).trim() || null };
}

// ── Detail parser ────────────────────────────────────────────────────────
interface ProiectDetail {
  id: number;
  proiect_type: number;
  smis_code: string | null;
  titlu: string | null;
  beneficiar_name: string | null;
  program_op_cod: string | null;
  program_op_text: string | null;
  axa_cod: string | null;
  axa_text: string | null;
  domeniul_cod: string | null;
  domeniul_text: string | null;
  operatiune_cod: string | null;
  operatiune_text: string | null;
  data_contract: string | null;
  raw_html_sha256: string;
}

function parseDetailPage(id: number, type: number, html: string): ProiectDetail | null {
  if (!html.includes('divwrapdetails')) return null;

  const out: ProiectDetail = {
    id, proiect_type: type,
    smis_code: null, titlu: null, beneficiar_name: null,
    program_op_cod: null, program_op_text: null,
    axa_cod: null, axa_text: null,
    domeniul_cod: null, domeniul_text: null,
    operatiune_cod: null, operatiune_text: null,
    data_contract: null,
    raw_html_sha256: createHash('sha256').update(html).digest('hex'),
  };

  // Title from <title>Proiecte [achiziții beneficiari privați]: NAME</title>
  const titleMatch = html.match(/<title>[^<:]*:\s*([^<]+?)<\/title>/);
  if (titleMatch) out.titlu = decode(titleMatch[1]);

  // Field-value pairs. Each row:
  //   <li class="cat-list-row0"><strong class="list-title">FIELD</strong>
  //   <div>VALUE</div></li>
  const rowRe = /<strong class="list-title">([^<]+)<\/strong>\s*<div>([^<]*(?:<[^>]+>[^<]*)*?)<\/div>/g;
  const rowMatches = [...html.matchAll(rowRe)];
  for (const m of rowMatches) {
    const field = decode(m[1])?.toLowerCase().trim() || '';
    const value = decode(m[2]);
    if (!value) continue;
    if (field.includes('cod smis')) {
      out.smis_code = value;
    } else if (field.includes('program opera')) {
      const { cod, text } = splitCodeText(value);
      out.program_op_cod = cod;
      out.program_op_text = text;
    } else if (field === 'axa') {
      const { cod, text } = splitCodeText(value);
      out.axa_cod = cod;
      out.axa_text = text;
    } else if (field.includes('domeniul')) {
      const { cod, text } = splitCodeText(value);
      out.domeniul_cod = cod;
      out.domeniul_text = text;
    } else if (field.includes('opera') && field.includes('iune')) {
      // "Operațiune"
      const { cod, text } = splitCodeText(value);
      out.operatiune_cod = cod;
      out.operatiune_text = text;
    } else if (field === 'beneficiar') {
      out.beneficiar_name = value;
    } else if (field.includes('data contract')) {
      out.data_contract = parseRoDate(value);
    }
  }

  // Page must at minimum yield a SMIS code or beneficiar name to count as parsed
  if (!out.smis_code && !out.beneficiar_name) return null;
  return out;
}

// ── DB persistence ───────────────────────────────────────────────────────
async function persist(detail: ProiectDetail): Promise<void> {
  await pool.query(
    `
    INSERT INTO fonduri.beneficiar_proiect (
      id, proiect_type, smis_code, titlu, beneficiar_name,
      program_op_cod, program_op_text, axa_cod, axa_text,
      domeniul_cod, domeniul_text, operatiune_cod, operatiune_text,
      data_contract, raw_html_sha256
    ) VALUES (
      $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14::date, $15
    )
    ON CONFLICT (id) DO UPDATE SET
      smis_code        = EXCLUDED.smis_code,
      titlu            = EXCLUDED.titlu,
      beneficiar_name  = EXCLUDED.beneficiar_name,
      program_op_cod   = EXCLUDED.program_op_cod,
      program_op_text  = EXCLUDED.program_op_text,
      axa_cod          = EXCLUDED.axa_cod,
      axa_text         = EXCLUDED.axa_text,
      domeniul_cod     = EXCLUDED.domeniul_cod,
      domeniul_text    = EXCLUDED.domeniul_text,
      operatiune_cod   = EXCLUDED.operatiune_cod,
      operatiune_text  = EXCLUDED.operatiune_text,
      data_contract    = EXCLUDED.data_contract,
      raw_html_sha256  = EXCLUDED.raw_html_sha256,
      fetched_at       = now()
    `,
    [
      detail.id, detail.proiect_type, detail.smis_code, detail.titlu, detail.beneficiar_name,
      detail.program_op_cod, detail.program_op_text, detail.axa_cod, detail.axa_text,
      detail.domeniul_cod, detail.domeniul_text, detail.operatiune_cod, detail.operatiune_text,
      detail.data_contract, detail.raw_html_sha256,
    ],
  );
}

// ── Main ────────────────────────────────────────────────────────────────
async function main(): Promise<void> {
  const startTs = Date.now();

  // Fetch the work list: distinct (type, id) pairs from beneficiar_anunt
  const work = await pool.query<{ type: number; id: number }>(
    `
    SELECT DISTINCT smis_proiect_type AS type, smis_proiect_id AS id
    FROM fonduri.beneficiar_anunt
    WHERE smis_proiect_id IS NOT NULL AND smis_proiect_type IS NOT NULL
    ORDER BY smis_proiect_id DESC
    `,
  );

  // Filter out already-fetched (unless --refetch)
  let candidates = work.rows;
  if (!argRefetch) {
    const existing = await pool.query<{ id: number }>(
      `SELECT id FROM fonduri.beneficiar_proiect`,
    );
    const have = new Set(existing.rows.map((r) => r.id));
    candidates = candidates.filter((r) => !have.has(r.id));
  }
  if (argLimit > 0) candidates = candidates.slice(0, argLimit);

  const total = candidates.length;
  console.log(
    `[PROIECT] To fetch: ${total} ` +
    `(concurrency=${argConc}, sleep=${argSleepMs}ms/worker, refetch=${argRefetch})`,
  );
  if (total === 0) {
    await pool.end();
    return;
  }

  const queue = candidates.slice();
  let done = 0, ok = 0, notFound = 0, failed = 0;

  async function worker(wid: number): Promise<void> {
    while (true) {
      const next = queue.shift();
      if (!next) break;
      const tStart = Date.now();
      try {
        const html = await fetchHtml(`${BASE}/proiecte/details/${next.type}/${next.id}`);
        if (html === null) {
          notFound++;
        } else {
          const detail = parseDetailPage(next.id, next.type, html);
          if (detail) {
            await persist(detail);
            ok++;
          } else {
            notFound++;
          }
        }
      } catch (err: any) {
        failed++;
        console.error(`[PROIECT w${wid}] id=${next.id} error: ${err.message}`);
      }
      done++;
      if (done % 50 === 0 || done === total) {
        const elapsed = (Date.now() - startTs) / 1000;
        const rate = done / elapsed;
        const eta = (total - done) / Math.max(rate, 0.001);
        console.log(
          `[PROIECT] ${done}/${total} (${(done / total * 100).toFixed(1)}%) · ` +
          `ok=${ok} 404=${notFound} fail=${failed} · ${rate.toFixed(2)} req/s · ` +
          `ETA ${(eta / 60).toFixed(0)}min`,
        );
      }
      const elapsed = Date.now() - tStart;
      const remain = argSleepMs - elapsed;
      if (remain > 0) await new Promise((r) => setTimeout(r, remain));
    }
  }

  const workers: Promise<void>[] = [];
  for (let i = 0; i < argConc; i++) {
    const stagger = (i * argSleepMs) / argConc;
    workers.push(new Promise((r) => setTimeout(r, stagger)).then(() => worker(i + 1)));
  }
  await Promise.all(workers);

  console.log(
    `[PROIECT] DONE in ${((Date.now() - startTs) / 60000).toFixed(1)}min: ` +
    `ok=${ok} 404=${notFound} fail=${failed}`,
  );
  await pool.end();
}

main().catch((err) => {
  console.error(err);
  process.exit(1);
});
