fix: scan math consistency + stale enrichment detection + re-enrichment

- withGeometry = matched immovable count (not GIS feature count) — numbers always add up - Added remoteGisCount to show raw GIS layer count separately - Enrichment completeness check: ENRICHMENT_REQUIRED_KEYS 7-field schema - localDbEnrichedComplete vs localDbEnriched detects stale enrichment - UI: orange warning when enrichment incomplete (missing PROPRIETARI_VECHI) - UI: workflow preview uses enrichedComplete for accurate time estimate - UI: note when GIS feature count differs from matched immovable count - enrich-service: re-enriches features with incomplete schema instead of skipping
2026-03-07 18:29:03 +02:00
parent ba579d75c1
commit 53914c7fc3
4 changed files with 157 additions and 46 deletions
@@ -389,11 +389,13 @@ export function ParcelSyncModule() {
  const [noGeomScan, setNoGeomScan] = useState<{
    totalImmovables: number;
    withGeometry: number;
+    remoteGisCount: number;
    noGeomCount: number;
    localDbTotal: number;
    localDbWithGeom: number;
    localDbNoGeom: number;
    localDbEnriched: number;
+    localDbEnrichedComplete: number;
    localSyncFresh: boolean;
  } | null>(null);
  const [noGeomScanSiruta, setNoGeomScanSiruta] = useState(""); // siruta for which scan was done
@@ -694,11 +696,13 @@ export function ParcelSyncModule() {
      const emptyResult = {
        totalImmovables: 0,
        withGeometry: 0,
+        remoteGisCount: 0,
        noGeomCount: 0,
        localDbTotal: 0,
        localDbWithGeom: 0,
        localDbNoGeom: 0,
        localDbEnriched: 0,
+        localDbEnrichedComplete: 0,
        localSyncFresh: false,
      };
      try {
@@ -718,11 +722,13 @@ export function ParcelSyncModule() {
          setNoGeomScan({
            totalImmovables: Number(data.totalImmovables ?? 0),
            withGeometry: Number(data.withGeometry ?? 0),
+            remoteGisCount: Number(data.remoteGisCount ?? 0),
            noGeomCount: Number(data.noGeomCount ?? 0),
            localDbTotal: Number(data.localDbTotal ?? 0),
            localDbWithGeom: Number(data.localDbWithGeom ?? 0),
            localDbNoGeom: Number(data.localDbNoGeom ?? 0),
            localDbEnriched: Number(data.localDbEnriched ?? 0),
+            localDbEnrichedComplete: Number(data.localDbEnrichedComplete ?? 0),
            localSyncFresh: Boolean(data.localSyncFresh),
          });
        }
@@ -2400,39 +2406,65 @@ export function ParcelSyncModule() {
              );

            // Helper: local DB status line
+            const staleEnrichment =
+              scanDone &&
+              noGeomScan.localDbEnriched > 0 &&
+              noGeomScan.localDbEnrichedComplete < noGeomScan.localDbEnriched;
+            const staleCount = scanDone
+              ? noGeomScan.localDbEnriched - noGeomScan.localDbEnrichedComplete
+              : 0;
+
            const localDbLine = scanDone && noGeomScan.localDbTotal > 0 && (
-              <div className="flex items-center gap-1.5 flex-wrap text-[11px] text-muted-foreground mt-1">
-                <Database className="h-3 w-3 shrink-0" />
-                <span>
-                  Baza de date locală:{" "}
-                  <span className="font-medium text-foreground">
-                    {noGeomScan.localDbWithGeom.toLocaleString("ro-RO")}
-                  </span>{" "}
-                  cu geometrie
-                  {noGeomScan.localDbNoGeom > 0 && (
-                    <>
-                      {" + "}
-                      <span className="font-medium text-amber-600 dark:text-amber-400">
-                        {noGeomScan.localDbNoGeom.toLocaleString("ro-RO")}
-                      </span>{" "}
-                      fără geometrie
-                    </>
-                  )}
-                  {noGeomScan.localDbEnriched > 0 && (
-                    <>
-                      {" · "}
-                      <span className="font-medium text-teal-600 dark:text-teal-400">
-                        {noGeomScan.localDbEnriched.toLocaleString("ro-RO")}
-                      </span>{" "}
-                      îmbogățite
-                    </>
-                  )}
-                  {noGeomScan.localSyncFresh && (
-                    <span className="text-emerald-600 dark:text-emerald-400 ml-1">
-                      (proaspăt)
+              <div className="space-y-0.5 mt-1">
+                <div className="flex items-center gap-1.5 flex-wrap text-[11px] text-muted-foreground">
+                  <Database className="h-3 w-3 shrink-0" />
+                  <span>
+                    Baza de date locală:{" "}
+                    <span className="font-medium text-foreground">
+                      {noGeomScan.localDbWithGeom.toLocaleString("ro-RO")}
+                    </span>{" "}
+                    cu geometrie
+                    {noGeomScan.localDbNoGeom > 0 && (
+                      <>
+                        {" + "}
+                        <span className="font-medium text-amber-600 dark:text-amber-400">
+                          {noGeomScan.localDbNoGeom.toLocaleString("ro-RO")}
+                        </span>{" "}
+                        fără geometrie
+                      </>
+                    )}
+                    {noGeomScan.localDbEnriched > 0 && (
+                      <>
+                        {" · "}
+                        <span className="font-medium text-teal-600 dark:text-teal-400">
+                          {noGeomScan.localDbEnriched.toLocaleString("ro-RO")}
+                        </span>{" "}
+                        îmbogățite
+                        {staleEnrichment && (
+                          <span className="text-orange-600 dark:text-orange-400">
+                            {" "}
+                            ({staleCount.toLocaleString("ro-RO")} incomplete)
+                          </span>
+                        )}
+                      </>
+                    )}
+                    {noGeomScan.localSyncFresh && (
+                      <span className="text-emerald-600 dark:text-emerald-400 ml-1">
+                        (proaspăt)
+                      </span>
+                    )}
+                  </span>
+                </div>
+                {staleEnrichment && (
+                  <div className="flex items-center gap-1.5 text-[11px] text-orange-600 dark:text-orange-400 ml-[18px]">
+                    <AlertTriangle className="h-3 w-3 shrink-0" />
+                    <span>
+                      {staleCount.toLocaleString("ro-RO")} parcele au îmbogățire
+                      veche (lipsă PROPRIETARI_VECHI). Vor fi re-îmbogățite la
+                      următorul export Magic.
                    </span>
-                  )}
-                </span>
+                  </div>
+                )}
              </div>
            );

@@ -2491,8 +2523,11 @@ export function ParcelSyncModule() {
                                  noGeomScan.localDbNoGeom,
                              )
                            : 0);
+                        // Use enrichedComplete (not enriched) — stale
+                        // enrichment (missing PROPRIETARI_VECHI etc.)
+                        // will be re-processed
                        const remaining =
-                          totalToEnrich - noGeomScan.localDbEnriched;
+                          totalToEnrich - noGeomScan.localDbEnrichedComplete;
                        return remaining > 0
                          ? `~${remaining.toLocaleString("ro-RO")} de procesat (~${Math.ceil((remaining * 0.25) / 60)} min)`
                          : "deja îmbogățite";
@@ -2535,6 +2570,19 @@ export function ParcelSyncModule() {
                          </span>{" "}
                          <span className="font-medium">fără geometrie</span>
                        </p>
+                        {noGeomScan.remoteGisCount > 0 &&
+                          noGeomScan.remoteGisCount !==
+                            noGeomScan.withGeometry && (
+                            <p className="text-[10px] text-muted-foreground/70 mt-0.5">
+                              Layerul GIS are{" "}
+                              {noGeomScan.remoteGisCount.toLocaleString(
+                                "ro-RO",
+                              )}{" "}
+                              features, dar doar{" "}
+                              {noGeomScan.withGeometry.toLocaleString("ro-RO")}{" "}
+                              se potrivesc cu lista de imobile
+                            </p>
+                          )}
                        <p className="text-[11px] text-muted-foreground mt-0.5">
                          Cele fără geometrie există în baza de date eTerra dar
                          nu au contur desenat în layerul GIS.
@@ -2595,6 +2643,13 @@ export function ParcelSyncModule() {
                              în DB local
                              {noGeomScan.localDbEnriched > 0 &&
                                `, ${noGeomScan.localDbEnriched.toLocaleString("ro-RO")} îmbogățite`}
+                              {noGeomScan.localDbEnriched > 0 &&
+                                noGeomScan.localDbEnrichedComplete <
+                                  noGeomScan.localDbEnriched && (
+                                  <span className="text-orange-600 dark:text-orange-400">
+                                    {` (${(noGeomScan.localDbEnriched - noGeomScan.localDbEnrichedComplete).toLocaleString("ro-RO")} incomplete)`}
+                                  </span>
+                                )}
                              {noGeomScan.localSyncFresh && ", proaspăt"})
                            </span>
                          )}
@@ -171,6 +171,7 @@ export async function enrichFeatures(
        attributes: true,
        cadastralRef: true,
        enrichedAt: true,
+        enrichment: true,
      },
    });

@@ -400,17 +401,34 @@ export async function enrichFeatures(
      const feature = terenuri[index]!;
      const attrs = feature.attributes as Record<string, unknown>;

-      // Skip features already enriched (resume after crash/interruption)
+      // Skip features with complete enrichment (resume after crash/interruption).
+      // Re-enrich if enrichment schema is incomplete (e.g., missing PROPRIETARI_VECHI
+      // added in a later version).
      if (feature.enrichedAt != null) {
-        enrichedCount += 1;
-        if (index % 50 === 0) {
-          options?.onProgress?.(
-            index + 1,
-            terenuri.length,
-            "Îmbogățire parcele (skip enriched)",
-          );
+        const enrichJson = feature.enrichment as Record<string, unknown> | null;
+        const isComplete =
+          enrichJson != null &&
+          [
+            "NR_CAD",
+            "NR_CF",
+            "PROPRIETARI",
+            "PROPRIETARI_VECHI",
+            "ADRESA",
+            "CATEGORIE_FOLOSINTA",
+            "HAS_BUILDING",
+          ].every((k) => k in enrichJson && enrichJson[k] !== undefined);
+        if (isComplete) {
+          enrichedCount += 1;
+          if (index % 50 === 0) {
+            options?.onProgress?.(
+              index + 1,
+              terenuri.length,
+              "Îmbogățire parcele (skip enriched)",
+            );
+          }
+          continue;
        }
-        continue;
+        // Stale enrichment — will be re-enriched below
      }

      const immovableId = attrs.IMMOVABLE_ID ?? "";
@@ -62,7 +62,9 @@ export const buildGpkg = async (options: GpkgBuildOptions): Promise<Buffer> => {
      for (const layer of options.layers) {
        // Split: spatial features go first (define the geometry column),
        // then null-geometry features are appended as rows without geom.
-        const spatialFeatures = layer.features.filter((f) => f.geometry != null);
+        const spatialFeatures = layer.features.filter(
+          (f) => f.geometry != null,
+        );
        const nullGeomFeatures = layer.includeNullGeometry
          ? layer.features.filter((f) => f.geometry == null)
          : [];
@@ -93,8 +93,10 @@ const normalizeCadRef = (value: unknown) =>

 export type NoGeomScanResult = {
  totalImmovables: number;
-  /** Features present in the remote ArcGIS TERENURI_ACTIVE layer (have geometry) */
+  /** Immovables that matched a remote GIS feature (have geometry) */
  withGeometry: number;
+  /** Total features in the remote ArcGIS TERENURI_ACTIVE layer */
+  remoteGisCount: number;
  noGeomCount: number;
  /** Sample of immovable identifiers without geometry */
  samples: Array<{
@@ -111,6 +113,8 @@ export type NoGeomScanResult = {
  localDbNoGeom: number;
  /** How many are already enriched (magic) in local DB */
  localDbEnriched: number;
+  /** How many enriched features have complete/current enrichment schema */
+  localDbEnrichedComplete: number;
  /** Whether local sync is fresh (< 7 days) */
  localSyncFresh: boolean;
  /** Error message if workspace couldn't be resolved */
@@ -149,12 +153,14 @@ export async function scanNoGeometryParcels(
    return {
      totalImmovables: 0,
      withGeometry: 0,
+      remoteGisCount: 0,
      noGeomCount: 0,
      samples: [],
      localDbTotal: 0,
      localDbWithGeom: 0,
      localDbNoGeom: 0,
      localDbEnriched: 0,
+      localDbEnrichedComplete: 0,
      localSyncFresh: false,
      error: `Nu s-a putut determina workspace-ul (județul) pentru SIRUTA ${siruta}`,
    };
@@ -222,7 +228,19 @@ export async function scanNoGeometryParcels(
  }

  // 4. Query local DB for context (what's already synced/imported)
-  const [localTotal, localNoGeom, localEnriched, lastSyncRun] =
+  //    Also check enrichment completeness — do enriched features have
+  //    the current schema? (e.g., PROPRIETARI_VECHI added later)
+  const ENRICHMENT_REQUIRED_KEYS = [
+    "NR_CAD",
+    "NR_CF",
+    "PROPRIETARI",
+    "PROPRIETARI_VECHI",
+    "ADRESA",
+    "CATEGORIE_FOLOSINTA",
+    "HAS_BUILDING",
+  ];
+
+  const [localTotal, localNoGeom, enrichedFeatures, lastSyncRun] =
    await Promise.all([
      prisma.gisFeature.count({
        where: { layerId: "TERENURI_ACTIVE", siruta },
@@ -234,12 +252,13 @@ export async function scanNoGeometryParcels(
          geometrySource: "NO_GEOMETRY",
        },
      }),
-      prisma.gisFeature.count({
+      prisma.gisFeature.findMany({
        where: {
          layerId: "TERENURI_ACTIVE",
          siruta,
          enrichedAt: { not: null },
        },
+        select: { enrichment: true },
      }),
      prisma.gisSyncRun.findFirst({
        where: { siruta, layerId: "TERENURI_ACTIVE", status: "done" },
@@ -248,20 +267,37 @@ export async function scanNoGeometryParcels(
      }),
    ]);

+  const localEnriched = enrichedFeatures.length;
+  let localEnrichedComplete = 0;
+  for (const f of enrichedFeatures) {
+    const e = f.enrichment as Record<string, unknown> | null;
+    if (
+      e &&
+      ENRICHMENT_REQUIRED_KEYS.every((k) => k in e && e[k] !== undefined)
+    ) {
+      localEnrichedComplete++;
+    }
+  }
+
  const localWithGeom = localTotal - localNoGeom;
  const syncFresh = lastSyncRun?.completedAt
    ? Date.now() - lastSyncRun.completedAt.getTime() < 168 * 60 * 60 * 1000
    : false;

+  // withGeometry = immovables that MATCHED a GIS feature (always adds up)
+  const matchedCount = allImmovables.length - noGeomItems.length;
+
  return {
    totalImmovables: allImmovables.length,
-    withGeometry: remoteFeatures.length,
+    withGeometry: matchedCount,
+    remoteGisCount: remoteFeatures.length,
    noGeomCount: noGeomItems.length,
    samples: noGeomItems.slice(0, 20),
    localDbTotal: localTotal,
    localDbWithGeom: localWithGeom,
    localDbNoGeom: localNoGeom,
    localDbEnriched: localEnriched,
+    localDbEnrichedComplete: localEnrichedComplete,
    localSyncFresh: syncFresh,
  };
 }