/**
 * AAAS — Autoritatea pentru Administrarea Activelor Statului.
 *
 * Scrapes the public portfolio of companies under AAAS authority from
 * https://www.aaas.gov.ro/.
 *
 * Approach (HTML scraping; AAAS publishes no Excel/JSON):
 *   1. Fetch index page /1-9-3-companii-sub-autoritatea-aaas/ → parse list of
 *      company subpage URLs (anchors matching the 1-9-3- slug prefix).
 *   2. For each subpage, fetch + extract a key/value bag from the entry
 *      content paragraph. Stable tokens observed on every company page:
 *        "CUI: NNN / J.."           → cui + reg_number
 *        "Adresa: ..."              → address
 *        "Telefon: ..."             → phone
 *        "Site: ..."                → website
 *        "Email:..."                → email
 *        "Participatie AAAS: NN%"   → state_share_pct
 *      Boilerplate "dummy text" / nav menus are stripped.
 *   3. UPSERT on cui → aaas.firme with status='active_holding' (since this
 *      list is exactly the active-holding portfolio). cui_match_method =
 *      'aaas_published' when AAAS itself published the CUI.
 *   4. After insert, populate name_normalized via firms.normalize_company_name
 *      and run a verification join against firms.entities to report match
 *      rate.
 *
 * Idempotent on PK cui. Re-runs UPSERT (overwrite). Refreshes MV at end.
 *
 * AAAS sections under construction (vanzari_actiuni / valorificare_creante)
 * are noted in the log; only the portfolio source produces rows today.
 *
 * Run:
 *   npx tsx src/scrape-aaas.ts
 *   npx tsx src/scrape-aaas.ts --limit=5    # smoke test
 *   npx tsx src/scrape-aaas.ts --dry-run    # parse, do not write
 */
import { Pool } from 'pg';

const PORTAL = 'https://www.aaas.gov.ro';
const PORTFOLIO_INDEX =
  '/despre-aaas/1-9-guvernanta-corporativa-aaas/1-9-3-companii-sub-autoritatea-aaas/';
const VANZARI_ACTIUNI = '/4-oferta-a-a-a-s/4-2-vanzari-actiuni/';
const VANZARI_CREANTE = '/4-oferta-a-a-a-s/4-3-valorificare-creante/';

const USER_AGENT =
  'gov-agreg/1.0 vreaudigital.ro (contact: m.tarau@beletage.ro)';

// ── CLI args ───────────────────────────────────────────────────────────────
const args = process.argv.slice(2);
const argLimit = parseInt(
  args.find((a) => a.startsWith('--limit='))?.split('=')[1] || '0',
  10,
);
const argDryRun = args.includes('--dry-run');

const pool = new Pool({ connectionString: process.env.DATABASE_URL, max: 4 });

// ── Helpers ────────────────────────────────────────────────────────────────
function trimOrNull(v: any): string | null {
  if (v == null) return null;
  const s = String(v).replace(/\s+/g, ' ').trim();
  return s === '' ? null : s;
}

/** Strip HTML tags and decode common entities into plain text. */
function htmlToText(html: string): string {
  return html
    .replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, ' ')
    .replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, ' ')
    .replace(/<\/(p|div|li|tr|br|h[1-6])>/gi, '\n')
    .replace(/<[^>]+>/g, ' ')
    .replace(/&nbsp;/g, ' ')
    .replace(/&amp;/g, '&')
    .replace(/&lt;/g, '<')
    .replace(/&gt;/g, '>')
    .replace(/&quot;/g, '"')
    .replace(/&#039;|&apos;/g, "'")
    .replace(/&#x27;/g, "'")
    .replace(/&hellip;/g, '...')
    .replace(/&([a-z]+|#\d+|#x[0-9a-f]+);/gi, ' ')
    .replace(/[ \t]+/g, ' ')
    .replace(/\n{2,}/g, '\n');
}

async function fetchHtml(url: string): Promise<string> {
  const res = await fetch(url, {
    headers: {
      'User-Agent': USER_AGENT,
      Accept:
        'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
      'Accept-Language': 'ro-RO,ro;q=0.9,en;q=0.6',
    },
  });
  if (!res.ok) throw new Error(`HTTP ${res.status} fetching ${url}`);
  return res.text();
}

// ── Index parser: list company subpage URLs from the portfolio page ────────
function extractCompanyLinks(indexHtml: string): string[] {
  // Anchors matching the 1-9-3- slug prefix but NOT the index itself.
  const re = new RegExp(
    'href="(' +
      PORTAL.replace(/\./g, '\\.') +
      PORTFOLIO_INDEX.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') +
      '1-9-3-[^"]+/)"',
    'g',
  );
  const out = new Set<string>();
  let m;
  while ((m = re.exec(indexHtml)) !== null) {
    const url = m[1];
    if (!url.endsWith(PORTFOLIO_INDEX)) out.add(url);
  }
  return [...out];
}

// ── Subpage parser: extract the content paragraph + key/value bag ──────────
type ParsedCompany = {
  source_url: string;
  name: string;
  cui: string | null;
  reg_number: string | null;
  address: string | null;
  phone: string | null;
  website: string | null;
  email: string | null;
  state_share_pct: number | null;
  raw: Record<string, string>;
};

function parseCompanyPage(url: string, html: string): ParsedCompany | null {
  const text = htmlToText(html);

  // The body has a breadcrumb chain ending with the company title:
  //   "... Companii sub autoritatea AAAS 1.9.3 <NAME> <NAME> CUI: NNN / Jxx Adresa: ... Email: ... Participatie AAAS: NN% Hotararile adunarilor ..."
  // We anchor on "CUI: <digits> / <reg-number>" (highly specific) and walk
  // backwards for the company name (last "1.9.3 ..." segment before CUI:),
  // then forwards for address/contact/share fields until "Hotararile adunarilor"
  // (or "AAAS este institutia" as fallback).
  const cuiMatch = text.match(
    /CUI:\s*(\d{2,12})\s*\/\s*([A-Z]?\d+\/\d+\/\d+)/,
  );
  if (!cuiMatch) return null;
  const cui = cuiMatch[1].trim();
  const regNumber = cuiMatch[2].trim();

  const cuiIdx = cuiMatch.index ?? text.indexOf(cuiMatch[0]);
  const before = text.slice(0, cuiIdx);

  // Last "1.9.3 ..." segment before CUI: take the last occurrence (skips
  // the breadcrumb-link "1.9.3 Companii sub autoritatea AAAS").
  const lastBreadcrumb = before.lastIndexOf('1.9.3');
  const rawName = lastBreadcrumb >= 0
    ? before.slice(lastBreadcrumb + 5).trim()
    : before.trim();

  // The page renders the title twice in a row ("ACTIVE CONEXE ACTIVE CONEXE"
  // or "BLUE AIR TEHNIC SA BLUE AIR TEHNIC S.A."). De-duplicate by treating
  // the upper-half (without dots) as the prefix that may be repeated.
  let name = rawName.replace(/\s+/g, ' ').trim();
  // Strip trailing/leading orphaned tokens like "S.A." with no name body.
  // Try literal half-split first (works for "ACTIVE CONEXE ACTIVE CONEXE").
  const halfDedup = (s: string): string => {
    const ws = s.split(/\s+/);
    if (ws.length >= 2 && ws.length % 2 === 0) {
      const h = ws.length / 2;
      if (ws.slice(0, h).join(' ') === ws.slice(h).join(' ')) {
        return ws.slice(0, h).join(' ');
      }
    }
    return s;
  };
  name = halfDedup(name);
  // For "BLUE AIR TEHNIC SA BLUE AIR TEHNIC S.A." pattern: both halves are
  // the same firm with different forma-juridica spelling. Detect by checking
  // if name (stripped of S.A./S.R.L./SA/SRL tokens) has a duplicated body.
  const stripped = name
    .replace(/\bS\.A\.|\bS\.R\.L\./gi, ' ')
    .replace(/\bSA\b|\bSRL\b/gi, ' ')
    .replace(/\./g, ' ')
    .replace(/\s+/g, ' ')
    .trim();
  const strippedDedup = halfDedup(stripped);
  if (strippedDedup !== stripped && strippedDedup.length > 0) {
    // Prefer dotted form if it appears in the original.
    const dotted = name.match(new RegExp(strippedDedup.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '\\s+(?:S\\.A\\.|S\\.R\\.L\\.)', 'i'));
    name = dotted ? dotted[0].trim() : strippedDedup;
  }
  name = name.replace(/\s+/g, ' ').trim();

  // Tail: from CUI line to "Hotararile adunarilor" (or fallback).
  const after = text.slice(cuiIdx + cuiMatch[0].length);
  const tailEnd =
    after.search(/Hotararile adunarilor|AAAS este institu/i);
  const tail = tailEnd > 0 ? after.slice(0, tailEnd) : after.slice(0, 2000);

  const grab = (label: RegExp): string | null => {
    const m = tail.match(label);
    return m ? trimOrNull(m[1]) : null;
  };

  const address = grab(/Adresa:\s*([^\n]+?)(?=\s*(?:Telefon:|Site:|Email|Participatie|$))/);
  const phone = grab(/Telefon:\s*([^\n]+?)(?=\s*(?:Site:|Email|Participatie|Adresa:|$))/);
  const website = grab(/Site:\s*([^\s]+)/);
  const email = grab(/Email\s*:?\s*([^\s]+@[^\s]+)/);
  const sharePctStr = grab(/Participatie AAAS:\s*(\d+(?:[.,]\d+)?)\s*%/);
  const state_share_pct =
    sharePctStr == null ? null : parseFloat(sharePctStr.replace(',', '.'));

  return {
    source_url: url,
    name,
    cui,
    reg_number: regNumber,
    address,
    phone,
    website,
    email,
    state_share_pct,
    raw: {
      raw_name_block: rawName,
      cui,
      reg_number: regNumber,
      ...(address ? { address } : {}),
      ...(phone ? { phone } : {}),
      ...(website ? { website } : {}),
      ...(email ? { email } : {}),
      ...(sharePctStr ? { state_share_pct_raw: sharePctStr } : {}),
    },
  };
}

// ── Vanzari acțiuni / creanțe — currently teaser pages, log only ───────────
async function probeOfferPage(
  scraper: 'vanzari_actiuni' | 'vanzari_creante',
  path: string,
  startedAt: Date,
): Promise<void> {
  const url = `${PORTAL}${path}`;
  let note = 'unknown';
  try {
    const html = await fetchHtml(url);
    const text = htmlToText(html);
    if (/SECTIUNE IN CONSTRUCTIE|SECȚIUNE ÎN CONSTRUCȚIE/i.test(text)) {
      note = 'section_under_construction';
    } else {
      note = 'has_content_recheck_parser';
    }
  } catch (e: any) {
    note = `fetch_error: ${e.message}`;
  }
  console.log(`  [${scraper}] ${url} → ${note}`);
  if (!argDryRun) {
    await pool.query(
      `INSERT INTO aaas.scrape_log
         (scraper, source_url, rows_seen, rows_inserted, rows_updated, rows_skipped,
          duration_ms, started_at, finished_at, error)
       VALUES ($1,$2,0,0,0,0,$3,$4, now(), $5)`,
      [
        scraper,
        url,
        Date.now() - startedAt.getTime(),
        startedAt,
        note === 'section_under_construction' ? null : note,
      ],
    );
  }
}

// ── Main: scrape portfolio ─────────────────────────────────────────────────
async function scrapePortfolio(): Promise<{
  seen: number;
  inserted: number;
  updated: number;
  skipped: number;
}> {
  const startedAt = new Date();
  const indexUrl = `${PORTAL}${PORTFOLIO_INDEX}`;
  console.log(`[portfolio] index ${indexUrl}`);
  const indexHtml = await fetchHtml(indexUrl);
  const links = extractCompanyLinks(indexHtml);
  console.log(`[portfolio] found ${links.length} company subpages`);

  const limit = argLimit > 0 ? Math.min(argLimit, links.length) : links.length;
  let seen = 0,
    inserted = 0,
    updated = 0,
    skipped = 0;
  const parsed: ParsedCompany[] = [];

  for (let i = 0; i < limit; i++) {
    const url = links[i];
    seen++;
    try {
      const html = await fetchHtml(url);
      const c = parseCompanyPage(url, html);
      if (!c || !c.cui) {
        console.log(`  SKIP (no cui parsed): ${url}`);
        skipped++;
        continue;
      }
      parsed.push(c);
      console.log(
        `  parsed cui=${c.cui} pct=${c.state_share_pct ?? '?'} ${c.name}`,
      );
    } catch (e: any) {
      console.log(`  ERR fetching ${url}: ${e.message}`);
      skipped++;
    }
    // be polite — small delay between requests
    await new Promise((r) => setTimeout(r, 350));
  }

  if (argDryRun) {
    console.log(`[portfolio] DRY RUN — would upsert ${parsed.length} rows`);
    return { seen, inserted: 0, updated: 0, skipped };
  }

  // UPSERT
  for (const c of parsed) {
    const r = await pool.query<{ inserted: boolean }>(
      `INSERT INTO aaas.firme (
         cui, name, name_normalized, reg_number, aaas_status,
         state_share_pct, address, phone, email, website,
         source_url, raw, cui_match_score, cui_match_method, fetched_at
       ) VALUES (
         $1, $2, firms.normalize_company_name($2), $3, 'active_holding',
         $4, $5, $6, $7, $8,
         $9, $10::jsonb, 1.000, 'aaas_published', now()
       )
       ON CONFLICT (cui) DO UPDATE SET
         name              = EXCLUDED.name,
         name_normalized   = EXCLUDED.name_normalized,
         reg_number        = COALESCE(EXCLUDED.reg_number, aaas.firme.reg_number),
         aaas_status       = EXCLUDED.aaas_status,
         state_share_pct   = EXCLUDED.state_share_pct,
         address           = COALESCE(EXCLUDED.address, aaas.firme.address),
         phone             = COALESCE(EXCLUDED.phone, aaas.firme.phone),
         email             = COALESCE(EXCLUDED.email, aaas.firme.email),
         website           = COALESCE(EXCLUDED.website, aaas.firme.website),
         source_url        = EXCLUDED.source_url,
         raw               = EXCLUDED.raw,
         cui_match_score   = EXCLUDED.cui_match_score,
         cui_match_method  = EXCLUDED.cui_match_method,
         fetched_at        = now()
       RETURNING (xmax = 0) AS inserted`,
      [
        c.cui,
        c.name,
        c.reg_number,
        c.state_share_pct,
        c.address,
        c.phone,
        c.email,
        c.website,
        c.source_url,
        JSON.stringify(c.raw),
      ],
    );
    if (r.rows[0]?.inserted) inserted++;
    else updated++;
  }

  await pool.query(
    `INSERT INTO aaas.scrape_log
       (scraper, source_url, rows_seen, rows_inserted, rows_updated,
        rows_skipped, duration_ms, started_at, finished_at)
     VALUES ('portfolio', $1, $2, $3, $4, $5, $6, $7, now())`,
    [
      indexUrl,
      seen,
      inserted,
      updated,
      skipped,
      Date.now() - startedAt.getTime(),
      startedAt,
    ],
  );

  return { seen, inserted, updated, skipped };
}

// ── Match-rate report ──────────────────────────────────────────────────────
async function reportMatchRate(): Promise<void> {
  const r = await pool.query<{
    total: string;
    matched: string;
    pct: string;
  }>(
    `SELECT
       COUNT(*)::text AS total,
       COUNT(*) FILTER (
         WHERE EXISTS (SELECT 1 FROM firms.entities e WHERE e.cui = aaas.firme.cui)
       )::text AS matched,
       ROUND(
         100.0 * COUNT(*) FILTER (
           WHERE EXISTS (SELECT 1 FROM firms.entities e WHERE e.cui = aaas.firme.cui)
         ) / NULLIF(COUNT(*), 0),
         1
       )::text AS pct
     FROM aaas.firme`,
  );
  const row = r.rows[0];
  console.log(
    `[match] aaas.firme rows=${row.total} matched_in_firms.entities=${row.matched} (${row.pct}%)`,
  );

  const breakdown = await pool.query<{ aaas_status: string; n: string }>(
    `SELECT aaas_status, COUNT(*)::text AS n
       FROM aaas.firme GROUP BY aaas_status ORDER BY COUNT(*) DESC`,
  );
  for (const b of breakdown.rows) {
    console.log(`  status=${b.aaas_status}  n=${b.n}`);
  }
}

// ── Entry ──────────────────────────────────────────────────────────────────
async function main() {
  const t0 = Date.now();
  console.log('=== AAAS scrape starting ===');
  const stats = await scrapePortfolio();
  console.log(
    `[portfolio] seen=${stats.seen} inserted=${stats.inserted} updated=${stats.updated} skipped=${stats.skipped}`,
  );

  console.log('--- probing offer pages (currently under construction) ---');
  await probeOfferPage('vanzari_actiuni', VANZARI_ACTIUNI, new Date());
  await probeOfferPage('vanzari_creante', VANZARI_CREANTE, new Date());

  if (!argDryRun) {
    try {
      await pool.query('REFRESH MATERIALIZED VIEW aaas.mv_per_cui');
      console.log('[mv] refreshed aaas.mv_per_cui');
    } catch (e: any) {
      console.log(`[mv] refresh error: ${e.message}`);
    }
    await reportMatchRate();
  }

  console.log(`=== AAAS scrape done in ${Date.now() - t0}ms ===`);
  await pool.end();
}

main().catch((e) => {
  console.error('FATAL:', e);
  pool.end().finally(() => process.exit(1));
});
