feat(parcel-sync): sync-first architecture — DB as ground truth

- Rewrite export-bundle to sync-first: check freshness -> sync layers -> enrich (magic) -> build GPKG/CSV from local DB - Rewrite export-layer-gpkg to sync-first: sync if stale -> export from DB - Create enrich-service.ts: extracted magic enrichment logic (CF, owners, addresses) with DB storage - Add enrichment + enrichedAt columns to GisFeature schema - Update PostGIS views to include enrichment data - UI: update button labels for sync-first semantics, refresh sync status after exports - Smart caching: skip sync if data is fresh (168h / 1 week default)
2026-03-07 11:12:54 +02:00
parent 0d0b1f8c9f
commit b0927ee075
7 changed files with 898 additions and 698 deletions
@@ -0,0 +1,535 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+/**
+ * Enrich service — fetches CF/owner/address/building data from eTerra
+ * and stores it in GisFeature.enrichment JSON column.
+ *
+ * Called after sync to add "magic" data to parcels.
+ * Idempotent: re-running overwrites previous enrichment.
+ */
+
+import { Prisma } from "@prisma/client";
+import { prisma } from "@/core/storage/prisma";
+import { EterraClient } from "./eterra-client";
+import {
+  setProgress,
+  clearProgress,
+  type SyncProgress,
+} from "./progress-store";
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+export type EnrichResult = {
+  siruta: string;
+  enrichedCount: number;
+  buildingCrossRefs: number;
+  status: "done" | "error";
+  error?: string;
+};
+
+/* ── Helpers (extracted from export-bundle) ──────────────────── */
+
+const formatNumber = (value: number) =>
+  Number.isFinite(value) ? value.toFixed(2).replace(/\.00$/, "") : "";
+
+const normalizeId = (value: unknown) => {
+  if (value === null || value === undefined) return "";
+  const text = String(value).trim();
+  if (!text) return "";
+  return text.replace(/\.0$/, "");
+};
+
+const normalizeCadRef = (value: unknown) =>
+  normalizeId(value).replace(/\s+/g, "").toUpperCase();
+
+const baseCadRef = (value: unknown) => {
+  const ref = normalizeCadRef(value);
+  if (!ref) return "";
+  return ref.includes("-") ? ref.split("-")[0]! : ref;
+};
+
+const makeWorkspaceKey = (workspaceId: unknown, immovableId: unknown) => {
+  const ws = normalizeId(workspaceId);
+  const im = normalizeId(immovableId);
+  if (!ws || !im) return "";
+  return `${ws}:${im}`;
+};
+
+const isRetryable = (error: unknown) => {
+  const err = error as { response?: { status?: number }; code?: string };
+  const status = err?.response?.status ?? 0;
+  if ([429, 500, 502, 503, 504].includes(status)) return true;
+  return err?.code === "ECONNRESET" || err?.code === "ETIMEDOUT";
+};
+
+const normalizeIntravilan = (values: string[]) => {
+  const normalized = values
+    .map((v) =>
+      String(v ?? "")
+        .trim()
+        .toLowerCase(),
+    )
+    .filter(Boolean);
+  const unique = new Set(normalized);
+  if (!unique.size) return "-";
+  if (unique.size === 1)
+    return unique.has("da") ? "Da" : unique.has("nu") ? "Nu" : "Mixt";
+  return "Mixt";
+};
+
+const formatCategories = (entries: any[]) => {
+  const map = new Map<string, number>();
+  for (const entry of entries) {
+    const key = String(entry?.categorieFolosinta ?? "").trim();
+    if (!key) continue;
+    const area = Number(entry?.suprafata ?? 0);
+    map.set(key, (map.get(key) ?? 0) + (Number.isFinite(area) ? area : 0));
+  }
+  return Array.from(map.entries())
+    .map(([k, a]) => `${k}:${formatNumber(a)}`)
+    .join("; ");
+};
+
+const formatAddress = (item?: any) => {
+  const address = item?.immovableAddresses?.[0]?.address ?? null;
+  if (!address) return "-";
+  const parts: string[] = [];
+  if (address.addressDescription) parts.push(address.addressDescription);
+  if (address.street) parts.push(`Str. ${address.street}`);
+  if (address.buildingNo) parts.push(`Nr. ${address.buildingNo}`);
+  if (address.locality?.name) parts.push(address.locality.name);
+  return parts.length ? parts.join(", ") : "-";
+};
+
+const pickApplication = (entries: any[], applicationId?: number) => {
+  if (!entries.length) return null;
+  if (applicationId) {
+    const match = entries.find(
+      (entry: any) => entry?.applicationId === applicationId,
+    );
+    if (match) return match;
+  }
+  return (
+    entries
+      .filter((entry: any) => entry?.dataCerere)
+      .sort((a: any, b: any) => (b.dataCerere ?? 0) - (a.dataCerere ?? 0))[0] ??
+    entries[0]
+  );
+};
+
+/**
+ * Enrichment data stored per-feature in the `enrichment` JSON column.
+ */
+export type FeatureEnrichment = {
+  NR_CAD: string;
+  NR_CF: string;
+  NR_CF_VECHI: string;
+  NR_TOPO: string;
+  ADRESA: string;
+  PROPRIETARI: string;
+  SUPRAFATA_2D: number | string;
+  SUPRAFATA_R: number | string;
+  SOLICITANT: string;
+  INTRAVILAN: string;
+  CATEGORIE_FOLOSINTA: string;
+  HAS_BUILDING: number;
+  BUILD_LEGAL: number;
+};
+
+/**
+ * Enrich all TERENURI_ACTIVE features for a given UAT.
+ *
+ * Reads features from DB, fetches extra data from eTerra (immovable list,
+ * documentation, owners, folosinte), cross-references with CLADIRI_ACTIVE,
+ * and stores the enrichment JSON on each GisFeature.
+ */
+export async function enrichFeatures(
+  client: EterraClient,
+  siruta: string,
+  options?: {
+    jobId?: string;
+    onProgress?: (done: number, total: number, phase: string) => void;
+  },
+): Promise<EnrichResult> {
+  const jobId = options?.jobId;
+  const push = (partial: Partial<SyncProgress>) => {
+    if (!jobId) return;
+    setProgress({
+      jobId,
+      downloaded: 0,
+      status: "running",
+      ...partial,
+    } as SyncProgress);
+  };
+
+  try {
+    // Load terenuri and cladiri from DB
+    const terenuri = await prisma.gisFeature.findMany({
+      where: { layerId: "TERENURI_ACTIVE", siruta },
+      select: {
+        id: true,
+        objectId: true,
+        attributes: true,
+        cadastralRef: true,
+      },
+    });
+
+    const cladiri = await prisma.gisFeature.findMany({
+      where: { layerId: "CLADIRI_ACTIVE", siruta },
+      select: { attributes: true },
+    });
+
+    if (terenuri.length === 0) {
+      return {
+        siruta,
+        enrichedCount: 0,
+        buildingCrossRefs: 0,
+        status: "done",
+      };
+    }
+
+    push({
+      phase: "Pregătire îmbogățire",
+      downloaded: 0,
+      total: terenuri.length,
+    });
+
+    // ── Throttled request helper ──
+    let lastRequest = 0;
+    const minInterval = 250;
+    const throttled = async <T>(fn: () => Promise<T>) => {
+      let attempt = 0;
+      while (true) {
+        const now = Date.now();
+        const wait = Math.max(0, lastRequest + minInterval - now);
+        if (wait > 0) await sleep(wait);
+        try {
+          const result = await fn();
+          lastRequest = Date.now();
+          return result;
+        } catch (error) {
+          if (!isRetryable(error) || attempt >= 2) throw error;
+          attempt += 1;
+          const backoff = Math.min(5000, 1000 * attempt);
+          await sleep(backoff);
+        }
+      }
+    };
+
+    // ── Building cross-ref map (from local DB cladiri) ──
+    const buildingMap = new Map<string, { has: boolean; legal: boolean }>();
+    for (const feature of cladiri) {
+      const attrs = feature.attributes as Record<string, unknown>;
+      const immovableId = attrs.IMMOVABLE_ID ?? attrs.IMOVABLE_ID ?? null;
+      const workspaceId = attrs.WORKSPACE_ID ?? null;
+      const baseRef = baseCadRef(attrs.NATIONAL_CADASTRAL_REFERENCE ?? "");
+      const isLegal =
+        Number(attrs.IS_LEGAL ?? 0) === 1 ||
+        String(attrs.IS_LEGAL ?? "").toLowerCase() === "true";
+      const add = (key: string) => {
+        if (!key) return;
+        const existing = buildingMap.get(key) ?? { has: false, legal: false };
+        existing.has = true;
+        if (isLegal) existing.legal = true;
+        buildingMap.set(key, existing);
+      };
+      const immKey = normalizeId(immovableId);
+      const wKey = makeWorkspaceKey(workspaceId, immovableId);
+      if (immKey) add(immKey);
+      if (wKey) add(wKey);
+      if (baseRef) add(baseRef);
+    }
+
+    // ── Fetch immovable list from eTerra ──
+    push({ phase: "Descărcare listă imobile", downloaded: 0 });
+    const immovableListById = new Map<string, any>();
+    const immovableListByCad = new Map<string, any>();
+    const ownersByLandbook = new Map<string, Set<string>>();
+
+    const addOwner = (landbook: string, name: string) => {
+      if (!landbook || !name) return;
+      const existing = ownersByLandbook.get(landbook) ?? new Set<string>();
+      existing.add(name);
+      ownersByLandbook.set(landbook, existing);
+    };
+
+    let listPage = 0;
+    let listTotalPages = 1;
+    let includeInscrisCF = true;
+    while (listPage < listTotalPages) {
+      const listResponse = await throttled(() =>
+        client.fetchImmovableListByAdminUnit(
+          65,
+          siruta,
+          listPage,
+          200,
+          includeInscrisCF,
+        ),
+      );
+      if (
+        listPage === 0 &&
+        !(listResponse?.content ?? []).length &&
+        includeInscrisCF
+      ) {
+        includeInscrisCF = false;
+        listPage = 0;
+        listTotalPages = 1;
+        continue;
+      }
+      listTotalPages =
+        typeof listResponse?.totalPages === "number"
+          ? listResponse.totalPages
+          : listTotalPages;
+      (listResponse?.content ?? []).forEach((item: any) => {
+        const idKey = normalizeId(item?.immovablePk);
+        if (idKey) immovableListById.set(idKey, item);
+        const cadKey = normalizeCadRef(item?.identifierDetails ?? "");
+        if (cadKey) immovableListByCad.set(cadKey, item);
+      });
+      listPage += 1;
+    }
+
+    // ── Fetch documentation/owner data ──
+    push({ phase: "Descărcare documentații CF" });
+    const docByImmovable = new Map<string, any>();
+    const immovableIds = Array.from(immovableListById.keys());
+    const docBatchSize = 50;
+    for (let i = 0; i < immovableIds.length; i += docBatchSize) {
+      const batch = immovableIds.slice(i, i + docBatchSize);
+      const docResponse = await throttled(() =>
+        client.fetchDocumentationData(65, batch),
+      );
+      (docResponse?.immovables ?? []).forEach((item: any) => {
+        const idKey = normalizeId(item?.immovablePk);
+        if (idKey) docByImmovable.set(idKey, item);
+      });
+      (docResponse?.partTwoRegs ?? []).forEach((item: any) => {
+        if (
+          String(item?.nodeType ?? "").toUpperCase() === "P" &&
+          item?.landbookIE
+        ) {
+          const name = String(item?.nodeName ?? "").trim();
+          if (name) addOwner(String(item.landbookIE), name);
+        }
+      });
+    }
+
+    // ── Enrich each teren feature ──
+    push({
+      phase: "Îmbogățire parcele",
+      downloaded: 0,
+      total: terenuri.length,
+    });
+    const immAppsCache = new Map<string, any[]>();
+    const folCache = new Map<string, any[]>();
+    let enrichedCount = 0;
+    let buildingCrossRefs = 0;
+    const now = new Date();
+
+    for (let index = 0; index < terenuri.length; index += 1) {
+      const feature = terenuri[index]!;
+      const attrs = feature.attributes as Record<string, unknown>;
+      const immovableId = attrs.IMMOVABLE_ID ?? "";
+      const workspaceId = attrs.WORKSPACE_ID ?? "";
+      const applicationId = (attrs.APPLICATION_ID as number) ?? null;
+
+      let solicitant = "-";
+      let intravilan = "-";
+      let categorie = "-";
+      let proprietari = "-";
+      let nrCF = "-";
+      let nrCFVechi = "-";
+      let nrTopo = "-";
+      let addressText = "-";
+
+      if (immovableId && workspaceId) {
+        const appKey = `${workspaceId}:${immovableId}`;
+        let apps = immAppsCache.get(appKey);
+        if (!apps) {
+          apps = await throttled(() =>
+            client.fetchImmAppsByImmovable(
+              immovableId as string | number,
+              workspaceId as string | number,
+            ),
+          );
+          immAppsCache.set(appKey, apps);
+        }
+        const chosen = pickApplication(apps, Number(applicationId ?? 0));
+        const appId =
+          chosen?.applicationId ??
+          (applicationId ? Number(applicationId) : null);
+        solicitant = chosen?.solicitant ?? chosen?.deponent ?? solicitant;
+
+        if (appId) {
+          const folKey = `${workspaceId}:${immovableId}:${appId}`;
+          let fol = folCache.get(folKey);
+          if (!fol) {
+            fol = await throttled(() =>
+              client.fetchParcelFolosinte(
+                workspaceId as string | number,
+                immovableId as string | number,
+                appId,
+              ),
+            );
+            folCache.set(folKey, fol);
+          }
+          intravilan = normalizeIntravilan(
+            fol.map((item: any) => item?.intravilan ?? ""),
+          );
+          categorie = formatCategories(fol);
+        }
+      }
+
+      const cadRefRaw = (attrs.NATIONAL_CADASTRAL_REFERENCE ?? "") as string;
+      const cadRef = normalizeCadRef(cadRefRaw);
+      const immKey = normalizeId(immovableId);
+      const listItem =
+        (immKey ? immovableListById.get(immKey) : undefined) ??
+        (cadRef ? immovableListByCad.get(cadRef) : undefined);
+      const docKey = listItem?.immovablePk
+        ? normalizeId(listItem.immovablePk)
+        : "";
+      const docItem = docKey ? docByImmovable.get(docKey) : undefined;
+      const landbookIE = docItem?.landbookIE ?? "";
+      const owners =
+        landbookIE && ownersByLandbook.get(String(landbookIE))
+          ? Array.from(ownersByLandbook.get(String(landbookIE)) ?? [])
+          : [];
+      const ownersByCad =
+        cadRefRaw && ownersByLandbook.get(String(cadRefRaw))
+          ? Array.from(ownersByLandbook.get(String(cadRefRaw)) ?? [])
+          : [];
+      proprietari =
+        Array.from(new Set([...owners, ...ownersByCad])).join("; ") ||
+        proprietari;
+
+      nrCF =
+        docItem?.landbookIE ||
+        listItem?.paperLbNo ||
+        listItem?.paperCadNo ||
+        nrCF;
+      const nrCFVechiRaw = listItem?.paperLbNo || listItem?.paperCadNo || "";
+      nrCFVechi =
+        docItem?.landbookIE && nrCFVechiRaw !== nrCF ? nrCFVechiRaw : nrCFVechi;
+      nrTopo =
+        listItem?.topNo || docItem?.topNo || listItem?.paperCadNo || nrTopo;
+      addressText = listItem ? formatAddress(listItem) : addressText;
+
+      const parcelRef = baseCadRef(cadRefRaw);
+      const wKey = makeWorkspaceKey(workspaceId, immovableId);
+      const build = (immKey ? buildingMap.get(immKey) : undefined) ??
+        (wKey ? buildingMap.get(wKey) : undefined) ??
+        (parcelRef ? buildingMap.get(parcelRef) : undefined) ?? {
+          has: false,
+          legal: false,
+        };
+      const hasBuilding = build.has ? 1 : 0;
+      const buildLegal = build.has ? (build.legal ? 1 : 0) : 0;
+      if (hasBuilding) buildingCrossRefs += 1;
+
+      const areaValue =
+        typeof attrs.AREA_VALUE === "number" ? attrs.AREA_VALUE : null;
+
+      const enrichment: FeatureEnrichment = {
+        NR_CAD: cadRefRaw,
+        NR_CF: nrCF,
+        NR_CF_VECHI: nrCFVechi,
+        NR_TOPO: nrTopo,
+        ADRESA: addressText,
+        PROPRIETARI: proprietari,
+        SUPRAFATA_2D: areaValue !== null ? Number(areaValue.toFixed(2)) : "",
+        SUPRAFATA_R: areaValue !== null ? Math.round(areaValue) : "",
+        SOLICITANT: solicitant,
+        INTRAVILAN: intravilan,
+        CATEGORIE_FOLOSINTA: categorie,
+        HAS_BUILDING: hasBuilding,
+        BUILD_LEGAL: buildLegal,
+      };
+
+      // Store enrichment in DB
+      await prisma.gisFeature.update({
+        where: { id: feature.id },
+        data: {
+          enrichment: enrichment as unknown as Prisma.InputJsonValue,
+          enrichedAt: now,
+        },
+      });
+
+      enrichedCount += 1;
+      if (index % 10 === 0) {
+        push({
+          phase: "Îmbogățire parcele",
+          downloaded: index + 1,
+          total: terenuri.length,
+        });
+        options?.onProgress?.(index + 1, terenuri.length, "Îmbogățire parcele");
+      }
+    }
+
+    push({
+      phase: "Îmbogățire completă",
+      status: "done",
+      downloaded: terenuri.length,
+      total: terenuri.length,
+    });
+    if (jobId) setTimeout(() => clearProgress(jobId), 60_000);
+
+    return {
+      siruta,
+      enrichedCount,
+      buildingCrossRefs,
+      status: "done",
+    };
+  } catch (error) {
+    const msg = error instanceof Error ? error.message : "Unknown error";
+    push({ phase: "Eroare îmbogățire", status: "error", message: msg });
+    if (jobId) setTimeout(() => clearProgress(jobId), 60_000);
+    return {
+      siruta,
+      enrichedCount: 0,
+      buildingCrossRefs: 0,
+      status: "error",
+      error: msg,
+    };
+  }
+}
+
+/**
+ * Check data freshness for a UAT + layer.
+ * Returns the most recent sync run's completedAt, or null if never synced.
+ */
+export async function getLayerFreshness(
+  siruta: string,
+  layerId: string,
+): Promise<{
+  lastSynced: Date | null;
+  featureCount: number;
+  enrichedCount: number;
+}> {
+  const lastRun = await prisma.gisSyncRun.findFirst({
+    where: { siruta, layerId, status: "done" },
+    orderBy: { completedAt: "desc" },
+    select: { completedAt: true },
+  });
+
+  const featureCount = await prisma.gisFeature.count({
+    where: { siruta, layerId },
+  });
+
+  const enrichedCount = await prisma.gisFeature.count({
+    where: { siruta, layerId, enrichedAt: { not: null } },
+  });
+
+  return {
+    lastSynced: lastRun?.completedAt ?? null,
+    featureCount,
+    enrichedCount,
+  };
+}
+
+/**
+ * Check if layer data is "fresh enough" (synced within maxAgeHours).
+ */
+export function isFresh(lastSynced: Date | null, maxAgeHours = 168): boolean {
+  if (!lastSynced) return false;
+  const ageMs = Date.now() - lastSynced.getTime();
+  return ageMs < maxAgeHours * 60 * 60 * 1000;
+}