fix: scan math consistency + stale enrichment detection + re-enrichment

- withGeometry = matched immovable count (not GIS feature count) — numbers always add up
- Added remoteGisCount to show raw GIS layer count separately
- Enrichment completeness check: ENRICHMENT_REQUIRED_KEYS 7-field schema
- localDbEnrichedComplete vs localDbEnriched detects stale enrichment
- UI: orange warning when enrichment incomplete (missing PROPRIETARI_VECHI)
- UI: workflow preview uses enrichedComplete for accurate time estimate
- UI: note when GIS feature count differs from matched immovable count
- enrich-service: re-enriches features with incomplete schema instead of skipping
This commit is contained in:
AI Assistant
2026-03-07 18:29:03 +02:00
parent ba579d75c1
commit 53914c7fc3
4 changed files with 157 additions and 46 deletions
@@ -389,11 +389,13 @@ export function ParcelSyncModule() {
const [noGeomScan, setNoGeomScan] = useState<{ const [noGeomScan, setNoGeomScan] = useState<{
totalImmovables: number; totalImmovables: number;
withGeometry: number; withGeometry: number;
remoteGisCount: number;
noGeomCount: number; noGeomCount: number;
localDbTotal: number; localDbTotal: number;
localDbWithGeom: number; localDbWithGeom: number;
localDbNoGeom: number; localDbNoGeom: number;
localDbEnriched: number; localDbEnriched: number;
localDbEnrichedComplete: number;
localSyncFresh: boolean; localSyncFresh: boolean;
} | null>(null); } | null>(null);
const [noGeomScanSiruta, setNoGeomScanSiruta] = useState(""); // siruta for which scan was done const [noGeomScanSiruta, setNoGeomScanSiruta] = useState(""); // siruta for which scan was done
@@ -694,11 +696,13 @@ export function ParcelSyncModule() {
const emptyResult = { const emptyResult = {
totalImmovables: 0, totalImmovables: 0,
withGeometry: 0, withGeometry: 0,
remoteGisCount: 0,
noGeomCount: 0, noGeomCount: 0,
localDbTotal: 0, localDbTotal: 0,
localDbWithGeom: 0, localDbWithGeom: 0,
localDbNoGeom: 0, localDbNoGeom: 0,
localDbEnriched: 0, localDbEnriched: 0,
localDbEnrichedComplete: 0,
localSyncFresh: false, localSyncFresh: false,
}; };
try { try {
@@ -718,11 +722,13 @@ export function ParcelSyncModule() {
setNoGeomScan({ setNoGeomScan({
totalImmovables: Number(data.totalImmovables ?? 0), totalImmovables: Number(data.totalImmovables ?? 0),
withGeometry: Number(data.withGeometry ?? 0), withGeometry: Number(data.withGeometry ?? 0),
remoteGisCount: Number(data.remoteGisCount ?? 0),
noGeomCount: Number(data.noGeomCount ?? 0), noGeomCount: Number(data.noGeomCount ?? 0),
localDbTotal: Number(data.localDbTotal ?? 0), localDbTotal: Number(data.localDbTotal ?? 0),
localDbWithGeom: Number(data.localDbWithGeom ?? 0), localDbWithGeom: Number(data.localDbWithGeom ?? 0),
localDbNoGeom: Number(data.localDbNoGeom ?? 0), localDbNoGeom: Number(data.localDbNoGeom ?? 0),
localDbEnriched: Number(data.localDbEnriched ?? 0), localDbEnriched: Number(data.localDbEnriched ?? 0),
localDbEnrichedComplete: Number(data.localDbEnrichedComplete ?? 0),
localSyncFresh: Boolean(data.localSyncFresh), localSyncFresh: Boolean(data.localSyncFresh),
}); });
} }
@@ -2400,39 +2406,65 @@ export function ParcelSyncModule() {
); );
// Helper: local DB status line // Helper: local DB status line
const staleEnrichment =
scanDone &&
noGeomScan.localDbEnriched > 0 &&
noGeomScan.localDbEnrichedComplete < noGeomScan.localDbEnriched;
const staleCount = scanDone
? noGeomScan.localDbEnriched - noGeomScan.localDbEnrichedComplete
: 0;
const localDbLine = scanDone && noGeomScan.localDbTotal > 0 && ( const localDbLine = scanDone && noGeomScan.localDbTotal > 0 && (
<div className="flex items-center gap-1.5 flex-wrap text-[11px] text-muted-foreground mt-1"> <div className="space-y-0.5 mt-1">
<Database className="h-3 w-3 shrink-0" /> <div className="flex items-center gap-1.5 flex-wrap text-[11px] text-muted-foreground">
<span> <Database className="h-3 w-3 shrink-0" />
Baza de date locală:{" "} <span>
<span className="font-medium text-foreground"> Baza de date locală:{" "}
{noGeomScan.localDbWithGeom.toLocaleString("ro-RO")} <span className="font-medium text-foreground">
</span>{" "} {noGeomScan.localDbWithGeom.toLocaleString("ro-RO")}
cu geometrie </span>{" "}
{noGeomScan.localDbNoGeom > 0 && ( cu geometrie
<> {noGeomScan.localDbNoGeom > 0 && (
{" + "} <>
<span className="font-medium text-amber-600 dark:text-amber-400"> {" + "}
{noGeomScan.localDbNoGeom.toLocaleString("ro-RO")} <span className="font-medium text-amber-600 dark:text-amber-400">
</span>{" "} {noGeomScan.localDbNoGeom.toLocaleString("ro-RO")}
fără geometrie </span>{" "}
</> fără geometrie
)} </>
{noGeomScan.localDbEnriched > 0 && ( )}
<> {noGeomScan.localDbEnriched > 0 && (
{" · "} <>
<span className="font-medium text-teal-600 dark:text-teal-400"> {" · "}
{noGeomScan.localDbEnriched.toLocaleString("ro-RO")} <span className="font-medium text-teal-600 dark:text-teal-400">
</span>{" "} {noGeomScan.localDbEnriched.toLocaleString("ro-RO")}
îmbogățite </span>{" "}
</> îmbogățite
)} {staleEnrichment && (
{noGeomScan.localSyncFresh && ( <span className="text-orange-600 dark:text-orange-400">
<span className="text-emerald-600 dark:text-emerald-400 ml-1"> {" "}
(proaspăt) ({staleCount.toLocaleString("ro-RO")} incomplete)
</span>
)}
</>
)}
{noGeomScan.localSyncFresh && (
<span className="text-emerald-600 dark:text-emerald-400 ml-1">
(proaspăt)
</span>
)}
</span>
</div>
{staleEnrichment && (
<div className="flex items-center gap-1.5 text-[11px] text-orange-600 dark:text-orange-400 ml-[18px]">
<AlertTriangle className="h-3 w-3 shrink-0" />
<span>
{staleCount.toLocaleString("ro-RO")} parcele au îmbogățire
veche (lipsă PROPRIETARI_VECHI). Vor fi re-îmbogățite la
următorul export Magic.
</span> </span>
)} </div>
</span> )}
</div> </div>
); );
@@ -2491,8 +2523,11 @@ export function ParcelSyncModule() {
noGeomScan.localDbNoGeom, noGeomScan.localDbNoGeom,
) )
: 0); : 0);
// Use enrichedComplete (not enriched) — stale
// enrichment (missing PROPRIETARI_VECHI etc.)
// will be re-processed
const remaining = const remaining =
totalToEnrich - noGeomScan.localDbEnriched; totalToEnrich - noGeomScan.localDbEnrichedComplete;
return remaining > 0 return remaining > 0
? `~${remaining.toLocaleString("ro-RO")} de procesat (~${Math.ceil((remaining * 0.25) / 60)} min)` ? `~${remaining.toLocaleString("ro-RO")} de procesat (~${Math.ceil((remaining * 0.25) / 60)} min)`
: "deja îmbogățite"; : "deja îmbogățite";
@@ -2535,6 +2570,19 @@ export function ParcelSyncModule() {
</span>{" "} </span>{" "}
<span className="font-medium">fără geometrie</span> <span className="font-medium">fără geometrie</span>
</p> </p>
{noGeomScan.remoteGisCount > 0 &&
noGeomScan.remoteGisCount !==
noGeomScan.withGeometry && (
<p className="text-[10px] text-muted-foreground/70 mt-0.5">
Layerul GIS are{" "}
{noGeomScan.remoteGisCount.toLocaleString(
"ro-RO",
)}{" "}
features, dar doar{" "}
{noGeomScan.withGeometry.toLocaleString("ro-RO")}{" "}
se potrivesc cu lista de imobile
</p>
)}
<p className="text-[11px] text-muted-foreground mt-0.5"> <p className="text-[11px] text-muted-foreground mt-0.5">
Cele fără geometrie există în baza de date eTerra dar Cele fără geometrie există în baza de date eTerra dar
nu au contur desenat în layerul GIS. nu au contur desenat în layerul GIS.
@@ -2595,6 +2643,13 @@ export function ParcelSyncModule() {
în DB local în DB local
{noGeomScan.localDbEnriched > 0 && {noGeomScan.localDbEnriched > 0 &&
`, ${noGeomScan.localDbEnriched.toLocaleString("ro-RO")} îmbogățite`} `, ${noGeomScan.localDbEnriched.toLocaleString("ro-RO")} îmbogățite`}
{noGeomScan.localDbEnriched > 0 &&
noGeomScan.localDbEnrichedComplete <
noGeomScan.localDbEnriched && (
<span className="text-orange-600 dark:text-orange-400">
{` (${(noGeomScan.localDbEnriched - noGeomScan.localDbEnrichedComplete).toLocaleString("ro-RO")} incomplete)`}
</span>
)}
{noGeomScan.localSyncFresh && ", proaspăt"}) {noGeomScan.localSyncFresh && ", proaspăt"})
</span> </span>
)} )}
@@ -171,6 +171,7 @@ export async function enrichFeatures(
attributes: true, attributes: true,
cadastralRef: true, cadastralRef: true,
enrichedAt: true, enrichedAt: true,
enrichment: true,
}, },
}); });
@@ -400,17 +401,34 @@ export async function enrichFeatures(
const feature = terenuri[index]!; const feature = terenuri[index]!;
const attrs = feature.attributes as Record<string, unknown>; const attrs = feature.attributes as Record<string, unknown>;
// Skip features already enriched (resume after crash/interruption) // Skip features with complete enrichment (resume after crash/interruption).
// Re-enrich if enrichment schema is incomplete (e.g., missing PROPRIETARI_VECHI
// added in a later version).
if (feature.enrichedAt != null) { if (feature.enrichedAt != null) {
enrichedCount += 1; const enrichJson = feature.enrichment as Record<string, unknown> | null;
if (index % 50 === 0) { const isComplete =
options?.onProgress?.( enrichJson != null &&
index + 1, [
terenuri.length, "NR_CAD",
"Îmbogățire parcele (skip enriched)", "NR_CF",
); "PROPRIETARI",
"PROPRIETARI_VECHI",
"ADRESA",
"CATEGORIE_FOLOSINTA",
"HAS_BUILDING",
].every((k) => k in enrichJson && enrichJson[k] !== undefined);
if (isComplete) {
enrichedCount += 1;
if (index % 50 === 0) {
options?.onProgress?.(
index + 1,
terenuri.length,
"Îmbogățire parcele (skip enriched)",
);
}
continue;
} }
continue; // Stale enrichment — will be re-enriched below
} }
const immovableId = attrs.IMMOVABLE_ID ?? ""; const immovableId = attrs.IMMOVABLE_ID ?? "";
@@ -62,7 +62,9 @@ export const buildGpkg = async (options: GpkgBuildOptions): Promise<Buffer> => {
for (const layer of options.layers) { for (const layer of options.layers) {
// Split: spatial features go first (define the geometry column), // Split: spatial features go first (define the geometry column),
// then null-geometry features are appended as rows without geom. // then null-geometry features are appended as rows without geom.
const spatialFeatures = layer.features.filter((f) => f.geometry != null); const spatialFeatures = layer.features.filter(
(f) => f.geometry != null,
);
const nullGeomFeatures = layer.includeNullGeometry const nullGeomFeatures = layer.includeNullGeometry
? layer.features.filter((f) => f.geometry == null) ? layer.features.filter((f) => f.geometry == null)
: []; : [];
@@ -93,8 +93,10 @@ const normalizeCadRef = (value: unknown) =>
export type NoGeomScanResult = { export type NoGeomScanResult = {
totalImmovables: number; totalImmovables: number;
/** Features present in the remote ArcGIS TERENURI_ACTIVE layer (have geometry) */ /** Immovables that matched a remote GIS feature (have geometry) */
withGeometry: number; withGeometry: number;
/** Total features in the remote ArcGIS TERENURI_ACTIVE layer */
remoteGisCount: number;
noGeomCount: number; noGeomCount: number;
/** Sample of immovable identifiers without geometry */ /** Sample of immovable identifiers without geometry */
samples: Array<{ samples: Array<{
@@ -111,6 +113,8 @@ export type NoGeomScanResult = {
localDbNoGeom: number; localDbNoGeom: number;
/** How many are already enriched (magic) in local DB */ /** How many are already enriched (magic) in local DB */
localDbEnriched: number; localDbEnriched: number;
/** How many enriched features have complete/current enrichment schema */
localDbEnrichedComplete: number;
/** Whether local sync is fresh (< 7 days) */ /** Whether local sync is fresh (< 7 days) */
localSyncFresh: boolean; localSyncFresh: boolean;
/** Error message if workspace couldn't be resolved */ /** Error message if workspace couldn't be resolved */
@@ -149,12 +153,14 @@ export async function scanNoGeometryParcels(
return { return {
totalImmovables: 0, totalImmovables: 0,
withGeometry: 0, withGeometry: 0,
remoteGisCount: 0,
noGeomCount: 0, noGeomCount: 0,
samples: [], samples: [],
localDbTotal: 0, localDbTotal: 0,
localDbWithGeom: 0, localDbWithGeom: 0,
localDbNoGeom: 0, localDbNoGeom: 0,
localDbEnriched: 0, localDbEnriched: 0,
localDbEnrichedComplete: 0,
localSyncFresh: false, localSyncFresh: false,
error: `Nu s-a putut determina workspace-ul (județul) pentru SIRUTA ${siruta}`, error: `Nu s-a putut determina workspace-ul (județul) pentru SIRUTA ${siruta}`,
}; };
@@ -222,7 +228,19 @@ export async function scanNoGeometryParcels(
} }
// 4. Query local DB for context (what's already synced/imported) // 4. Query local DB for context (what's already synced/imported)
const [localTotal, localNoGeom, localEnriched, lastSyncRun] = // Also check enrichment completeness — do enriched features have
// the current schema? (e.g., PROPRIETARI_VECHI added later)
const ENRICHMENT_REQUIRED_KEYS = [
"NR_CAD",
"NR_CF",
"PROPRIETARI",
"PROPRIETARI_VECHI",
"ADRESA",
"CATEGORIE_FOLOSINTA",
"HAS_BUILDING",
];
const [localTotal, localNoGeom, enrichedFeatures, lastSyncRun] =
await Promise.all([ await Promise.all([
prisma.gisFeature.count({ prisma.gisFeature.count({
where: { layerId: "TERENURI_ACTIVE", siruta }, where: { layerId: "TERENURI_ACTIVE", siruta },
@@ -234,12 +252,13 @@ export async function scanNoGeometryParcels(
geometrySource: "NO_GEOMETRY", geometrySource: "NO_GEOMETRY",
}, },
}), }),
prisma.gisFeature.count({ prisma.gisFeature.findMany({
where: { where: {
layerId: "TERENURI_ACTIVE", layerId: "TERENURI_ACTIVE",
siruta, siruta,
enrichedAt: { not: null }, enrichedAt: { not: null },
}, },
select: { enrichment: true },
}), }),
prisma.gisSyncRun.findFirst({ prisma.gisSyncRun.findFirst({
where: { siruta, layerId: "TERENURI_ACTIVE", status: "done" }, where: { siruta, layerId: "TERENURI_ACTIVE", status: "done" },
@@ -248,20 +267,37 @@ export async function scanNoGeometryParcels(
}), }),
]); ]);
const localEnriched = enrichedFeatures.length;
let localEnrichedComplete = 0;
for (const f of enrichedFeatures) {
const e = f.enrichment as Record<string, unknown> | null;
if (
e &&
ENRICHMENT_REQUIRED_KEYS.every((k) => k in e && e[k] !== undefined)
) {
localEnrichedComplete++;
}
}
const localWithGeom = localTotal - localNoGeom; const localWithGeom = localTotal - localNoGeom;
const syncFresh = lastSyncRun?.completedAt const syncFresh = lastSyncRun?.completedAt
? Date.now() - lastSyncRun.completedAt.getTime() < 168 * 60 * 60 * 1000 ? Date.now() - lastSyncRun.completedAt.getTime() < 168 * 60 * 60 * 1000
: false; : false;
// withGeometry = immovables that MATCHED a GIS feature (always adds up)
const matchedCount = allImmovables.length - noGeomItems.length;
return { return {
totalImmovables: allImmovables.length, totalImmovables: allImmovables.length,
withGeometry: remoteFeatures.length, withGeometry: matchedCount,
remoteGisCount: remoteFeatures.length,
noGeomCount: noGeomItems.length, noGeomCount: noGeomItems.length,
samples: noGeomItems.slice(0, 20), samples: noGeomItems.slice(0, 20),
localDbTotal: localTotal, localDbTotal: localTotal,
localDbWithGeom: localWithGeom, localDbWithGeom: localWithGeom,
localDbNoGeom: localNoGeom, localDbNoGeom: localNoGeom,
localDbEnriched: localEnriched, localDbEnriched: localEnriched,
localDbEnrichedComplete: localEnrichedComplete,
localSyncFresh: syncFresh, localSyncFresh: syncFresh,
}; };
} }