fix: scan math consistency + stale enrichment detection + re-enrichment
- withGeometry = matched immovable count (not GIS feature count) — numbers always add up - Added remoteGisCount to show raw GIS layer count separately - Enrichment completeness check: ENRICHMENT_REQUIRED_KEYS 7-field schema - localDbEnrichedComplete vs localDbEnriched detects stale enrichment - UI: orange warning when enrichment incomplete (missing PROPRIETARI_VECHI) - UI: workflow preview uses enrichedComplete for accurate time estimate - UI: note when GIS feature count differs from matched immovable count - enrich-service: re-enriches features with incomplete schema instead of skipping
This commit is contained in:
@@ -389,11 +389,13 @@ export function ParcelSyncModule() {
|
|||||||
const [noGeomScan, setNoGeomScan] = useState<{
|
const [noGeomScan, setNoGeomScan] = useState<{
|
||||||
totalImmovables: number;
|
totalImmovables: number;
|
||||||
withGeometry: number;
|
withGeometry: number;
|
||||||
|
remoteGisCount: number;
|
||||||
noGeomCount: number;
|
noGeomCount: number;
|
||||||
localDbTotal: number;
|
localDbTotal: number;
|
||||||
localDbWithGeom: number;
|
localDbWithGeom: number;
|
||||||
localDbNoGeom: number;
|
localDbNoGeom: number;
|
||||||
localDbEnriched: number;
|
localDbEnriched: number;
|
||||||
|
localDbEnrichedComplete: number;
|
||||||
localSyncFresh: boolean;
|
localSyncFresh: boolean;
|
||||||
} | null>(null);
|
} | null>(null);
|
||||||
const [noGeomScanSiruta, setNoGeomScanSiruta] = useState(""); // siruta for which scan was done
|
const [noGeomScanSiruta, setNoGeomScanSiruta] = useState(""); // siruta for which scan was done
|
||||||
@@ -694,11 +696,13 @@ export function ParcelSyncModule() {
|
|||||||
const emptyResult = {
|
const emptyResult = {
|
||||||
totalImmovables: 0,
|
totalImmovables: 0,
|
||||||
withGeometry: 0,
|
withGeometry: 0,
|
||||||
|
remoteGisCount: 0,
|
||||||
noGeomCount: 0,
|
noGeomCount: 0,
|
||||||
localDbTotal: 0,
|
localDbTotal: 0,
|
||||||
localDbWithGeom: 0,
|
localDbWithGeom: 0,
|
||||||
localDbNoGeom: 0,
|
localDbNoGeom: 0,
|
||||||
localDbEnriched: 0,
|
localDbEnriched: 0,
|
||||||
|
localDbEnrichedComplete: 0,
|
||||||
localSyncFresh: false,
|
localSyncFresh: false,
|
||||||
};
|
};
|
||||||
try {
|
try {
|
||||||
@@ -718,11 +722,13 @@ export function ParcelSyncModule() {
|
|||||||
setNoGeomScan({
|
setNoGeomScan({
|
||||||
totalImmovables: Number(data.totalImmovables ?? 0),
|
totalImmovables: Number(data.totalImmovables ?? 0),
|
||||||
withGeometry: Number(data.withGeometry ?? 0),
|
withGeometry: Number(data.withGeometry ?? 0),
|
||||||
|
remoteGisCount: Number(data.remoteGisCount ?? 0),
|
||||||
noGeomCount: Number(data.noGeomCount ?? 0),
|
noGeomCount: Number(data.noGeomCount ?? 0),
|
||||||
localDbTotal: Number(data.localDbTotal ?? 0),
|
localDbTotal: Number(data.localDbTotal ?? 0),
|
||||||
localDbWithGeom: Number(data.localDbWithGeom ?? 0),
|
localDbWithGeom: Number(data.localDbWithGeom ?? 0),
|
||||||
localDbNoGeom: Number(data.localDbNoGeom ?? 0),
|
localDbNoGeom: Number(data.localDbNoGeom ?? 0),
|
||||||
localDbEnriched: Number(data.localDbEnriched ?? 0),
|
localDbEnriched: Number(data.localDbEnriched ?? 0),
|
||||||
|
localDbEnrichedComplete: Number(data.localDbEnrichedComplete ?? 0),
|
||||||
localSyncFresh: Boolean(data.localSyncFresh),
|
localSyncFresh: Boolean(data.localSyncFresh),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -2400,39 +2406,65 @@ export function ParcelSyncModule() {
|
|||||||
);
|
);
|
||||||
|
|
||||||
// Helper: local DB status line
|
// Helper: local DB status line
|
||||||
|
const staleEnrichment =
|
||||||
|
scanDone &&
|
||||||
|
noGeomScan.localDbEnriched > 0 &&
|
||||||
|
noGeomScan.localDbEnrichedComplete < noGeomScan.localDbEnriched;
|
||||||
|
const staleCount = scanDone
|
||||||
|
? noGeomScan.localDbEnriched - noGeomScan.localDbEnrichedComplete
|
||||||
|
: 0;
|
||||||
|
|
||||||
const localDbLine = scanDone && noGeomScan.localDbTotal > 0 && (
|
const localDbLine = scanDone && noGeomScan.localDbTotal > 0 && (
|
||||||
<div className="flex items-center gap-1.5 flex-wrap text-[11px] text-muted-foreground mt-1">
|
<div className="space-y-0.5 mt-1">
|
||||||
<Database className="h-3 w-3 shrink-0" />
|
<div className="flex items-center gap-1.5 flex-wrap text-[11px] text-muted-foreground">
|
||||||
<span>
|
<Database className="h-3 w-3 shrink-0" />
|
||||||
Baza de date locală:{" "}
|
<span>
|
||||||
<span className="font-medium text-foreground">
|
Baza de date locală:{" "}
|
||||||
{noGeomScan.localDbWithGeom.toLocaleString("ro-RO")}
|
<span className="font-medium text-foreground">
|
||||||
</span>{" "}
|
{noGeomScan.localDbWithGeom.toLocaleString("ro-RO")}
|
||||||
cu geometrie
|
</span>{" "}
|
||||||
{noGeomScan.localDbNoGeom > 0 && (
|
cu geometrie
|
||||||
<>
|
{noGeomScan.localDbNoGeom > 0 && (
|
||||||
{" + "}
|
<>
|
||||||
<span className="font-medium text-amber-600 dark:text-amber-400">
|
{" + "}
|
||||||
{noGeomScan.localDbNoGeom.toLocaleString("ro-RO")}
|
<span className="font-medium text-amber-600 dark:text-amber-400">
|
||||||
</span>{" "}
|
{noGeomScan.localDbNoGeom.toLocaleString("ro-RO")}
|
||||||
fără geometrie
|
</span>{" "}
|
||||||
</>
|
fără geometrie
|
||||||
)}
|
</>
|
||||||
{noGeomScan.localDbEnriched > 0 && (
|
)}
|
||||||
<>
|
{noGeomScan.localDbEnriched > 0 && (
|
||||||
{" · "}
|
<>
|
||||||
<span className="font-medium text-teal-600 dark:text-teal-400">
|
{" · "}
|
||||||
{noGeomScan.localDbEnriched.toLocaleString("ro-RO")}
|
<span className="font-medium text-teal-600 dark:text-teal-400">
|
||||||
</span>{" "}
|
{noGeomScan.localDbEnriched.toLocaleString("ro-RO")}
|
||||||
îmbogățite
|
</span>{" "}
|
||||||
</>
|
îmbogățite
|
||||||
)}
|
{staleEnrichment && (
|
||||||
{noGeomScan.localSyncFresh && (
|
<span className="text-orange-600 dark:text-orange-400">
|
||||||
<span className="text-emerald-600 dark:text-emerald-400 ml-1">
|
{" "}
|
||||||
(proaspăt)
|
({staleCount.toLocaleString("ro-RO")} incomplete)
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
{noGeomScan.localSyncFresh && (
|
||||||
|
<span className="text-emerald-600 dark:text-emerald-400 ml-1">
|
||||||
|
(proaspăt)
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
{staleEnrichment && (
|
||||||
|
<div className="flex items-center gap-1.5 text-[11px] text-orange-600 dark:text-orange-400 ml-[18px]">
|
||||||
|
<AlertTriangle className="h-3 w-3 shrink-0" />
|
||||||
|
<span>
|
||||||
|
{staleCount.toLocaleString("ro-RO")} parcele au îmbogățire
|
||||||
|
veche (lipsă PROPRIETARI_VECHI). Vor fi re-îmbogățite la
|
||||||
|
următorul export Magic.
|
||||||
</span>
|
</span>
|
||||||
)}
|
</div>
|
||||||
</span>
|
)}
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -2491,8 +2523,11 @@ export function ParcelSyncModule() {
|
|||||||
noGeomScan.localDbNoGeom,
|
noGeomScan.localDbNoGeom,
|
||||||
)
|
)
|
||||||
: 0);
|
: 0);
|
||||||
|
// Use enrichedComplete (not enriched) — stale
|
||||||
|
// enrichment (missing PROPRIETARI_VECHI etc.)
|
||||||
|
// will be re-processed
|
||||||
const remaining =
|
const remaining =
|
||||||
totalToEnrich - noGeomScan.localDbEnriched;
|
totalToEnrich - noGeomScan.localDbEnrichedComplete;
|
||||||
return remaining > 0
|
return remaining > 0
|
||||||
? `~${remaining.toLocaleString("ro-RO")} de procesat (~${Math.ceil((remaining * 0.25) / 60)} min)`
|
? `~${remaining.toLocaleString("ro-RO")} de procesat (~${Math.ceil((remaining * 0.25) / 60)} min)`
|
||||||
: "deja îmbogățite";
|
: "deja îmbogățite";
|
||||||
@@ -2535,6 +2570,19 @@ export function ParcelSyncModule() {
|
|||||||
</span>{" "}
|
</span>{" "}
|
||||||
<span className="font-medium">fără geometrie</span>
|
<span className="font-medium">fără geometrie</span>
|
||||||
</p>
|
</p>
|
||||||
|
{noGeomScan.remoteGisCount > 0 &&
|
||||||
|
noGeomScan.remoteGisCount !==
|
||||||
|
noGeomScan.withGeometry && (
|
||||||
|
<p className="text-[10px] text-muted-foreground/70 mt-0.5">
|
||||||
|
Layerul GIS are{" "}
|
||||||
|
{noGeomScan.remoteGisCount.toLocaleString(
|
||||||
|
"ro-RO",
|
||||||
|
)}{" "}
|
||||||
|
features, dar doar{" "}
|
||||||
|
{noGeomScan.withGeometry.toLocaleString("ro-RO")}{" "}
|
||||||
|
se potrivesc cu lista de imobile
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
<p className="text-[11px] text-muted-foreground mt-0.5">
|
<p className="text-[11px] text-muted-foreground mt-0.5">
|
||||||
Cele fără geometrie există în baza de date eTerra dar
|
Cele fără geometrie există în baza de date eTerra dar
|
||||||
nu au contur desenat în layerul GIS.
|
nu au contur desenat în layerul GIS.
|
||||||
@@ -2595,6 +2643,13 @@ export function ParcelSyncModule() {
|
|||||||
în DB local
|
în DB local
|
||||||
{noGeomScan.localDbEnriched > 0 &&
|
{noGeomScan.localDbEnriched > 0 &&
|
||||||
`, ${noGeomScan.localDbEnriched.toLocaleString("ro-RO")} îmbogățite`}
|
`, ${noGeomScan.localDbEnriched.toLocaleString("ro-RO")} îmbogățite`}
|
||||||
|
{noGeomScan.localDbEnriched > 0 &&
|
||||||
|
noGeomScan.localDbEnrichedComplete <
|
||||||
|
noGeomScan.localDbEnriched && (
|
||||||
|
<span className="text-orange-600 dark:text-orange-400">
|
||||||
|
{` (${(noGeomScan.localDbEnriched - noGeomScan.localDbEnrichedComplete).toLocaleString("ro-RO")} incomplete)`}
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
{noGeomScan.localSyncFresh && ", proaspăt"})
|
{noGeomScan.localSyncFresh && ", proaspăt"})
|
||||||
</span>
|
</span>
|
||||||
)}
|
)}
|
||||||
|
|||||||
@@ -171,6 +171,7 @@ export async function enrichFeatures(
|
|||||||
attributes: true,
|
attributes: true,
|
||||||
cadastralRef: true,
|
cadastralRef: true,
|
||||||
enrichedAt: true,
|
enrichedAt: true,
|
||||||
|
enrichment: true,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -400,17 +401,34 @@ export async function enrichFeatures(
|
|||||||
const feature = terenuri[index]!;
|
const feature = terenuri[index]!;
|
||||||
const attrs = feature.attributes as Record<string, unknown>;
|
const attrs = feature.attributes as Record<string, unknown>;
|
||||||
|
|
||||||
// Skip features already enriched (resume after crash/interruption)
|
// Skip features with complete enrichment (resume after crash/interruption).
|
||||||
|
// Re-enrich if enrichment schema is incomplete (e.g., missing PROPRIETARI_VECHI
|
||||||
|
// added in a later version).
|
||||||
if (feature.enrichedAt != null) {
|
if (feature.enrichedAt != null) {
|
||||||
enrichedCount += 1;
|
const enrichJson = feature.enrichment as Record<string, unknown> | null;
|
||||||
if (index % 50 === 0) {
|
const isComplete =
|
||||||
options?.onProgress?.(
|
enrichJson != null &&
|
||||||
index + 1,
|
[
|
||||||
terenuri.length,
|
"NR_CAD",
|
||||||
"Îmbogățire parcele (skip enriched)",
|
"NR_CF",
|
||||||
);
|
"PROPRIETARI",
|
||||||
|
"PROPRIETARI_VECHI",
|
||||||
|
"ADRESA",
|
||||||
|
"CATEGORIE_FOLOSINTA",
|
||||||
|
"HAS_BUILDING",
|
||||||
|
].every((k) => k in enrichJson && enrichJson[k] !== undefined);
|
||||||
|
if (isComplete) {
|
||||||
|
enrichedCount += 1;
|
||||||
|
if (index % 50 === 0) {
|
||||||
|
options?.onProgress?.(
|
||||||
|
index + 1,
|
||||||
|
terenuri.length,
|
||||||
|
"Îmbogățire parcele (skip enriched)",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
continue;
|
||||||
}
|
}
|
||||||
continue;
|
// Stale enrichment — will be re-enriched below
|
||||||
}
|
}
|
||||||
|
|
||||||
const immovableId = attrs.IMMOVABLE_ID ?? "";
|
const immovableId = attrs.IMMOVABLE_ID ?? "";
|
||||||
|
|||||||
@@ -62,7 +62,9 @@ export const buildGpkg = async (options: GpkgBuildOptions): Promise<Buffer> => {
|
|||||||
for (const layer of options.layers) {
|
for (const layer of options.layers) {
|
||||||
// Split: spatial features go first (define the geometry column),
|
// Split: spatial features go first (define the geometry column),
|
||||||
// then null-geometry features are appended as rows without geom.
|
// then null-geometry features are appended as rows without geom.
|
||||||
const spatialFeatures = layer.features.filter((f) => f.geometry != null);
|
const spatialFeatures = layer.features.filter(
|
||||||
|
(f) => f.geometry != null,
|
||||||
|
);
|
||||||
const nullGeomFeatures = layer.includeNullGeometry
|
const nullGeomFeatures = layer.includeNullGeometry
|
||||||
? layer.features.filter((f) => f.geometry == null)
|
? layer.features.filter((f) => f.geometry == null)
|
||||||
: [];
|
: [];
|
||||||
|
|||||||
@@ -93,8 +93,10 @@ const normalizeCadRef = (value: unknown) =>
|
|||||||
|
|
||||||
export type NoGeomScanResult = {
|
export type NoGeomScanResult = {
|
||||||
totalImmovables: number;
|
totalImmovables: number;
|
||||||
/** Features present in the remote ArcGIS TERENURI_ACTIVE layer (have geometry) */
|
/** Immovables that matched a remote GIS feature (have geometry) */
|
||||||
withGeometry: number;
|
withGeometry: number;
|
||||||
|
/** Total features in the remote ArcGIS TERENURI_ACTIVE layer */
|
||||||
|
remoteGisCount: number;
|
||||||
noGeomCount: number;
|
noGeomCount: number;
|
||||||
/** Sample of immovable identifiers without geometry */
|
/** Sample of immovable identifiers without geometry */
|
||||||
samples: Array<{
|
samples: Array<{
|
||||||
@@ -111,6 +113,8 @@ export type NoGeomScanResult = {
|
|||||||
localDbNoGeom: number;
|
localDbNoGeom: number;
|
||||||
/** How many are already enriched (magic) in local DB */
|
/** How many are already enriched (magic) in local DB */
|
||||||
localDbEnriched: number;
|
localDbEnriched: number;
|
||||||
|
/** How many enriched features have complete/current enrichment schema */
|
||||||
|
localDbEnrichedComplete: number;
|
||||||
/** Whether local sync is fresh (< 7 days) */
|
/** Whether local sync is fresh (< 7 days) */
|
||||||
localSyncFresh: boolean;
|
localSyncFresh: boolean;
|
||||||
/** Error message if workspace couldn't be resolved */
|
/** Error message if workspace couldn't be resolved */
|
||||||
@@ -149,12 +153,14 @@ export async function scanNoGeometryParcels(
|
|||||||
return {
|
return {
|
||||||
totalImmovables: 0,
|
totalImmovables: 0,
|
||||||
withGeometry: 0,
|
withGeometry: 0,
|
||||||
|
remoteGisCount: 0,
|
||||||
noGeomCount: 0,
|
noGeomCount: 0,
|
||||||
samples: [],
|
samples: [],
|
||||||
localDbTotal: 0,
|
localDbTotal: 0,
|
||||||
localDbWithGeom: 0,
|
localDbWithGeom: 0,
|
||||||
localDbNoGeom: 0,
|
localDbNoGeom: 0,
|
||||||
localDbEnriched: 0,
|
localDbEnriched: 0,
|
||||||
|
localDbEnrichedComplete: 0,
|
||||||
localSyncFresh: false,
|
localSyncFresh: false,
|
||||||
error: `Nu s-a putut determina workspace-ul (județul) pentru SIRUTA ${siruta}`,
|
error: `Nu s-a putut determina workspace-ul (județul) pentru SIRUTA ${siruta}`,
|
||||||
};
|
};
|
||||||
@@ -222,7 +228,19 @@ export async function scanNoGeometryParcels(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 4. Query local DB for context (what's already synced/imported)
|
// 4. Query local DB for context (what's already synced/imported)
|
||||||
const [localTotal, localNoGeom, localEnriched, lastSyncRun] =
|
// Also check enrichment completeness — do enriched features have
|
||||||
|
// the current schema? (e.g., PROPRIETARI_VECHI added later)
|
||||||
|
const ENRICHMENT_REQUIRED_KEYS = [
|
||||||
|
"NR_CAD",
|
||||||
|
"NR_CF",
|
||||||
|
"PROPRIETARI",
|
||||||
|
"PROPRIETARI_VECHI",
|
||||||
|
"ADRESA",
|
||||||
|
"CATEGORIE_FOLOSINTA",
|
||||||
|
"HAS_BUILDING",
|
||||||
|
];
|
||||||
|
|
||||||
|
const [localTotal, localNoGeom, enrichedFeatures, lastSyncRun] =
|
||||||
await Promise.all([
|
await Promise.all([
|
||||||
prisma.gisFeature.count({
|
prisma.gisFeature.count({
|
||||||
where: { layerId: "TERENURI_ACTIVE", siruta },
|
where: { layerId: "TERENURI_ACTIVE", siruta },
|
||||||
@@ -234,12 +252,13 @@ export async function scanNoGeometryParcels(
|
|||||||
geometrySource: "NO_GEOMETRY",
|
geometrySource: "NO_GEOMETRY",
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
prisma.gisFeature.count({
|
prisma.gisFeature.findMany({
|
||||||
where: {
|
where: {
|
||||||
layerId: "TERENURI_ACTIVE",
|
layerId: "TERENURI_ACTIVE",
|
||||||
siruta,
|
siruta,
|
||||||
enrichedAt: { not: null },
|
enrichedAt: { not: null },
|
||||||
},
|
},
|
||||||
|
select: { enrichment: true },
|
||||||
}),
|
}),
|
||||||
prisma.gisSyncRun.findFirst({
|
prisma.gisSyncRun.findFirst({
|
||||||
where: { siruta, layerId: "TERENURI_ACTIVE", status: "done" },
|
where: { siruta, layerId: "TERENURI_ACTIVE", status: "done" },
|
||||||
@@ -248,20 +267,37 @@ export async function scanNoGeometryParcels(
|
|||||||
}),
|
}),
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
const localEnriched = enrichedFeatures.length;
|
||||||
|
let localEnrichedComplete = 0;
|
||||||
|
for (const f of enrichedFeatures) {
|
||||||
|
const e = f.enrichment as Record<string, unknown> | null;
|
||||||
|
if (
|
||||||
|
e &&
|
||||||
|
ENRICHMENT_REQUIRED_KEYS.every((k) => k in e && e[k] !== undefined)
|
||||||
|
) {
|
||||||
|
localEnrichedComplete++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const localWithGeom = localTotal - localNoGeom;
|
const localWithGeom = localTotal - localNoGeom;
|
||||||
const syncFresh = lastSyncRun?.completedAt
|
const syncFresh = lastSyncRun?.completedAt
|
||||||
? Date.now() - lastSyncRun.completedAt.getTime() < 168 * 60 * 60 * 1000
|
? Date.now() - lastSyncRun.completedAt.getTime() < 168 * 60 * 60 * 1000
|
||||||
: false;
|
: false;
|
||||||
|
|
||||||
|
// withGeometry = immovables that MATCHED a GIS feature (always adds up)
|
||||||
|
const matchedCount = allImmovables.length - noGeomItems.length;
|
||||||
|
|
||||||
return {
|
return {
|
||||||
totalImmovables: allImmovables.length,
|
totalImmovables: allImmovables.length,
|
||||||
withGeometry: remoteFeatures.length,
|
withGeometry: matchedCount,
|
||||||
|
remoteGisCount: remoteFeatures.length,
|
||||||
noGeomCount: noGeomItems.length,
|
noGeomCount: noGeomItems.length,
|
||||||
samples: noGeomItems.slice(0, 20),
|
samples: noGeomItems.slice(0, 20),
|
||||||
localDbTotal: localTotal,
|
localDbTotal: localTotal,
|
||||||
localDbWithGeom: localWithGeom,
|
localDbWithGeom: localWithGeom,
|
||||||
localDbNoGeom: localNoGeom,
|
localDbNoGeom: localNoGeom,
|
||||||
localDbEnriched: localEnriched,
|
localDbEnriched: localEnriched,
|
||||||
|
localDbEnrichedComplete: localEnrichedComplete,
|
||||||
localSyncFresh: syncFresh,
|
localSyncFresh: syncFresh,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user