/**
 * Curtea de Conturi a României — Stage 1: listing-page harvest.
 *
 * Walks the public listing pages of:
 *   - rapoarte-audit-financiar     (~127 pages × 15 = ~1,890 reports)
 *   - rapoarte-conformitate        (~173 pages × 15 = ~2,580 reports)
 *   - rapoarte-audit-performanta   (~9 pages × 15 = ~135 reports)
 *
 * Per listing card extracts:
 *   - slug + detail URL
 *   - title (full, raw)
 *   - <time datetime="YYYY-MM-DD"> publication date
 *   - audit_type (mapped from category slug)
 *   - parsed audit_year, doc_number, doc_date, audited_entity_name from title
 *
 * Idempotent UPSERT on (category, slug). Numeric download_id + PDF metadata
 * are populated in Stage 2 (see services/seap-scraper/CURTEACONT-PLAN.md).
 *
 * Run:
 *   npx tsx src/scrape-curteacont.ts --source=financiar --limit=500
 *   npx tsx src/scrape-curteacont.ts --source=conformitate
 *   npx tsx src/scrape-curteacont.ts --source=performanta
 *   npx tsx src/scrape-curteacont.ts --source=all
 *
 * CLI args:
 *   --source=financiar|conformitate|performanta|all   (default: all)
 *   --limit=N                                          stop after N rows total (0 = no limit)
 *   --start-page=N                                     resume mid-walk (default: 1)
 */
import { Pool } from 'pg';
import { createHash } from 'crypto';

const PORTAL = 'https://www.curteadeconturi.ro';
const USER_AGENT = 'gov-agreg/1.0 vreaudigital.ro (contact: m.tarau@beletage.ro)';

type Source = 'financiar' | 'conformitate' | 'performanta';

const CATEGORIES: Record<Source, { categoryPath: string; auditType: string }> = {
  financiar: {
    categoryPath: 'rapoarte-audit-financiar',
    auditType: 'financiar',
  },
  conformitate: {
    categoryPath: 'rapoarte-conformitate',
    auditType: 'conformitate',
  },
  performanta: {
    categoryPath: 'rapoarte-audit-performanta',
    auditType: 'performanta',
  },
};

// ── CLI args ────────────────────────────────────────────────────────────────
const args = process.argv.slice(2);
const argSource = (args.find((a) => a.startsWith('--source='))?.split('=')[1] || 'all') as Source | 'all';
const argLimit = parseInt(args.find((a) => a.startsWith('--limit='))?.split('=')[1] || '0', 10);
const argStartPage = parseInt(args.find((a) => a.startsWith('--start-page='))?.split('=')[1] || '1', 10);

const pool = new Pool({ connectionString: process.env.DATABASE_URL, max: 4 });

// ── Helpers ─────────────────────────────────────────────────────────────────
function sha1Hex(s: string): string {
  return createHash('sha1').update(s).digest('hex');
}

function trimOrNull(v: any): string | null {
  if (v == null) return null;
  const s = String(v).trim();
  return s === '' ? null : s;
}

/** "07.04.2026" → "2026-04-07" (ISO date string). */
function parseDmy(s: string): string | null {
  const m = s.match(/^(\d{1,2})\.(\d{1,2})\.(\d{4})$/);
  if (!m) return null;
  const dd = m[1].padStart(2, '0');
  const mm = m[2].padStart(2, '0');
  const yyyy = m[3];
  const month = parseInt(mm, 10);
  const day = parseInt(dd, 10);
  if (month < 1 || month > 12 || day < 1 || day > 31) return null;
  return `${yyyy}-${mm}-${dd}`;
}

/**
 * Parse one title to extract structured fields.
 *
 * Examples seen:
 *   "Raport de audit financiar pentru anul 2024, nr.27500/07.04.2026, MINISTERUL ENERGIEI"
 *   "Raport de audit financiar pentru anul 2025, nr.28580/16.04.2026, UATC BIERTAN"
 *   "Raport de audit al performanței privind tratamentul consumatorilor de droguri"
 *   "Sinteza raportului de audit al performantei riscul de inundatii ..."
 *
 * Strategy:
 *   - audit_year: regex /pentru anul (\d{4})/i
 *   - doc_number / doc_date: /nr\.?\s*(\d+)\/(\d{2}\.\d{2}\.\d{4})/
 *   - audited_entity_name: text after the last comma if doc_date matched, else null
 *   - follow-up detection: title contains "follow-up" or "follow up"
 */
function parseTitle(title: string): {
  audit_year: number | null;
  doc_number: string | null;
  doc_date: string | null;
  audited_entity_name: string | null;
  is_follow_up: boolean;
} {
  const out = {
    audit_year: null as number | null,
    doc_number: null as string | null,
    doc_date: null as string | null,
    audited_entity_name: null as string | null,
    is_follow_up: /follow[-\s]?up/i.test(title),
  };

  const yearM = title.match(/pentru anul (\d{4})/i);
  if (yearM) {
    const y = parseInt(yearM[1], 10);
    if (y >= 1990 && y <= 2100) out.audit_year = y;
  }

  // Handle source typos: nr.NNNN / DD.MM.YYYY  but separator can be /, \, .,
  // and date separator can be ., /, , or even mixed (".." double-dot, "/" mid-date).
  const nrM = title.match(/nr\.?\s*(\d+)\s*[\/\\\.,]\s*(\d{1,2}[\.\/,]+\d{1,2}[\.\/,]+\d{4})/i);
  if (nrM) {
    out.doc_number = nrM[1];
    // Normalize ALL non-digit separators to "." before parsing.
    const dateNorm = nrM[2].replace(/[^0-9]+/g, '.');
    out.doc_date = parseDmy(dateNorm);
    // Audited entity = text AFTER the doc_date in the title, stripped of
    // leading comma/whitespace.
    const idx = title.indexOf(nrM[0]) + nrM[0].length;
    if (idx < title.length) {
      const tail = title.slice(idx).replace(/^[\s,]+/, '').trim();
      if (tail.length > 0) out.audited_entity_name = tail;
    }
  }

  return out;
}

/** Decode HTML entities we care about (&amp;, &#39;, &quot;, &amp;quot;, etc). */
function decodeHtmlEntities(s: string): string {
  return s
    .replace(/&amp;/g, '&')
    .replace(/&quot;/g, '"')
    .replace(/&#39;/g, "'")
    .replace(/&apos;/g, "'")
    .replace(/&lt;/g, '<')
    .replace(/&gt;/g, '>')
    .replace(/&nbsp;/g, ' ');
}

// ── HTTP fetch ──────────────────────────────────────────────────────────────
async function fetchListingPage(categoryPath: string, page: number, attempt = 1): Promise<string> {
  const url = `${PORTAL}/rapoarte-audit/${categoryPath}?page=${page}`;
  try {
    const res = await fetch(url, {
      headers: {
        'User-Agent': USER_AGENT,
        'Accept': 'text/html,application/xhtml+xml',
        'Accept-Language': 'ro-RO,ro;q=0.9,en;q=0.8',
      },
      signal: AbortSignal.timeout(30000),
    });
    if (!res.ok) {
      throw new Error(`HTTP ${res.status} on ${url}`);
    }
    return await res.text();
  } catch (err: any) {
    if (attempt < 3) {
      const delay = 1500 * attempt;
      console.error(`  [retry ${attempt}] ${url} — ${err.message} — waiting ${delay}ms`);
      await new Promise((r) => setTimeout(r, delay));
      return fetchListingPage(categoryPath, page, attempt + 1);
    }
    throw err;
  }
}

// ── HTML parsing ────────────────────────────────────────────────────────────
type Card = {
  slug: string;
  detail_url: string;
  title: string;
  publication_date: string | null;
};

/**
 * Extract listing cards from a category page HTML.
 *
 * Each card has the shape:
 *   <div class="listing-item row">
 *     ...
 *     <time datetime="2026-04-24">24 aprilie 2026</time>
 *     ...
 *     <a class="text-body-xl ..." href="/rapoarte-audit/{cat}/{slug}">Title text</a>
 *
 * The Detalii anchor repeats the same href, so we de-dup by href.
 * One time element appears per card immediately before the title anchor.
 */
function parseListingCards(html: string, categoryPath: string): Card[] {
  // Split on listing-item rows — rough but works for this template.
  const cardChunks = html.split('<div class="listing-item row">').slice(1);
  const cards: Card[] = [];

  const linkRe = new RegExp(
    `href="/rapoarte-audit/${categoryPath}/([^"]+)"[^>]*>([^<]+)</a>`,
    'i'
  );
  const timeRe = /<time datetime="(\d{4}-\d{2}-\d{2})"/;

  for (const chunk of cardChunks) {
    const linkM = chunk.match(linkRe);
    if (!linkM) continue;

    const slug = linkM[1];
    let title = decodeHtmlEntities(linkM[2]).trim();
    if (!title) continue;

    const timeM = chunk.match(timeRe);
    const pubDate = timeM ? timeM[1] : null;

    cards.push({
      slug,
      detail_url: `${PORTAL}/rapoarte-audit/${categoryPath}/${slug}`,
      title,
      publication_date: pubDate,
    });
  }

  // De-dup by slug (Detalii anchor + title anchor produce same card).
  const seen = new Set<string>();
  return cards.filter((c) => {
    if (seen.has(c.slug)) return false;
    seen.add(c.slug);
    return true;
  });
}

/** Detect the highest page number from the pagy nav block. */
function parseMaxPage(html: string): number {
  const matches = Array.from(html.matchAll(/[?&]page=(\d+)/g));
  let max = 1;
  for (const m of matches) {
    const n = parseInt(m[1], 10);
    if (n > max) max = n;
  }
  return max;
}

// ── DB upsert ───────────────────────────────────────────────────────────────
type UpsertResult = { inserted: boolean; updated: boolean };

async function upsertCard(
  source: Source,
  card: Card
): Promise<UpsertResult> {
  const cat = CATEGORIES[source];
  const slugId = sha1Hex(`${cat.categoryPath}|${card.slug}`);
  const parsed = parseTitle(card.title);
  const auditType = parsed.is_follow_up ? 'follow-up' : cat.auditType;

  const sql = `
    INSERT INTO curteacont.rapoarte (
      slug_id, category, slug, detail_url,
      title, audit_type, audit_year, doc_number, doc_date,
      audited_entity_name, publication_date, fetched_at
    ) VALUES (
      $1, $2, $3, $4,
      $5, $6, $7, $8, $9,
      $10, $11, now()
    )
    ON CONFLICT (slug_id) DO UPDATE SET
      title = EXCLUDED.title,
      audit_type = EXCLUDED.audit_type,
      audit_year = EXCLUDED.audit_year,
      doc_number = EXCLUDED.doc_number,
      doc_date = EXCLUDED.doc_date,
      audited_entity_name = EXCLUDED.audited_entity_name,
      publication_date = EXCLUDED.publication_date,
      fetched_at = now()
    RETURNING (xmax = 0) AS inserted;
  `;
  const params = [
    slugId,
    cat.categoryPath,
    card.slug,
    card.detail_url,
    card.title,
    auditType,
    parsed.audit_year,
    parsed.doc_number,
    parsed.doc_date,
    parsed.audited_entity_name,
    card.publication_date,
  ];
  const res = await pool.query(sql, params);
  const inserted = res.rows[0]?.inserted === true;
  return { inserted, updated: !inserted };
}

// ── Main scrape loop ────────────────────────────────────────────────────────
async function scrapeSource(source: Source, globalLimit: number): Promise<{ rows: number; pages: number; inserted: number; updated: number }> {
  const cat = CATEGORIES[source];
  console.log(`\n=== [${source}] /rapoarte-audit/${cat.categoryPath} ===`);

  const runRes = await pool.query(
    `INSERT INTO curteacont.scrape_runs (category) VALUES ($1) RETURNING id`,
    [cat.categoryPath]
  );
  const runId = runRes.rows[0].id as number;

  let page = argStartPage;
  let totalRows = 0;
  let inserted = 0;
  let updated = 0;
  let pagesVisited = 0;
  let lastError: string | null = null;
  let maxPage = 1;

  try {
    while (true) {
      if (globalLimit > 0 && totalRows >= globalLimit) {
        console.log(`  reached global limit=${globalLimit}, stopping`);
        break;
      }

      const html = await fetchListingPage(cat.categoryPath, page);
      pagesVisited++;
      const cards = parseListingCards(html, cat.categoryPath);
      const pageMax = parseMaxPage(html);
      if (pageMax > maxPage) maxPage = pageMax;

      if (cards.length === 0) {
        console.log(`  page=${page} empty — stopping (max seen: ${maxPage})`);
        break;
      }

      let pageInserted = 0;
      let pageUpdated = 0;
      for (const card of cards) {
        if (globalLimit > 0 && totalRows >= globalLimit) break;
        try {
          const r = await upsertCard(source, card);
          if (r.inserted) {
            inserted++;
            pageInserted++;
          } else {
            updated++;
            pageUpdated++;
          }
          totalRows++;
        } catch (e: any) {
          console.error(`  upsert err slug=${card.slug}: ${e.message}`);
        }
      }

      console.log(
        `  page=${page} cards=${cards.length} ins=${pageInserted} upd=${pageUpdated} (running: ${totalRows} rows)`
      );

      page++;
      // gentle pacing — site is on shared infra, no need to hammer.
      await new Promise((r) => setTimeout(r, 600));
    }
  } catch (err: any) {
    lastError = err.message;
    console.error(`  FATAL on page=${page}: ${err.message}`);
  }

  await pool.query(
    `UPDATE curteacont.scrape_runs
     SET finished_at = now(),
         pages_visited = $2,
         rows_inserted = $3,
         rows_updated = $4,
         last_error = $5
     WHERE id = $1`,
    [runId, pagesVisited, inserted, updated, lastError]
  );

  return { rows: totalRows, pages: pagesVisited, inserted, updated };
}

async function main() {
  const sources: Source[] =
    argSource === 'all'
      ? ['financiar', 'conformitate', 'performanta']
      : [argSource as Source];

  const summary: Array<{ source: Source; rows: number; inserted: number; updated: number; pages: number }> = [];

  let remaining = argLimit;
  for (const s of sources) {
    if (argLimit > 0 && remaining <= 0) {
      console.log(`global limit hit, skipping ${s}`);
      continue;
    }
    const r = await scrapeSource(s, remaining);
    summary.push({ source: s, ...r });
    if (argLimit > 0) remaining -= r.rows;
  }

  console.log('\n=== summary ===');
  for (const s of summary) {
    console.log(
      `  ${s.source.padEnd(13)} pages=${String(s.pages).padStart(4)}  rows=${String(s.rows).padStart(5)}  ins=${String(s.inserted).padStart(5)}  upd=${String(s.updated).padStart(5)}`
    );
  }

  await pool.end();
}

main().catch(async (err) => {
  console.error('FATAL:', err);
  try {
    await pool.end();
  } catch {}
  process.exit(1);
});
