feat(parcel-sync): quality gate filter for no-geom import + diagnostic endpoint
- Filter no-geom items before import: must have identification (cadRef/CF/paperCad/paperLb) OR area - Multi-field area extraction: area, measuredArea, areaValue, suprafata - Scan quality breakdown: withCadRef, withPaperCf, withPaperCad, withArea, useful, empty - Added paperLbNo to quality analysis and samples - UI: quality breakdown grid in scan card - UI: filtered count in workflow preview (shows useful, not total) - UI: enrichment estimate uses useful count - New diagnostic endpoint /api/eterra/no-geom-debug for field inspection
This commit is contained in:
@@ -0,0 +1,173 @@
|
|||||||
|
/**
|
||||||
|
* POST /api/eterra/no-geom-debug
|
||||||
|
*
|
||||||
|
* Diagnostic endpoint: fetches a small sample of no-geometry immovables
|
||||||
|
* and returns ALL their raw fields, so we can understand what data is
|
||||||
|
* available for quality filtering.
|
||||||
|
*
|
||||||
|
* Body: { siruta: string, sampleSize?: number }
|
||||||
|
* Returns: { sample: [...raw items], allFields: [...field names], fieldStats: {...} }
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { NextResponse } from "next/server";
|
||||||
|
import { EterraClient } from "@/modules/parcel-sync/services/eterra-client";
|
||||||
|
import { getSessionCredentials } from "@/modules/parcel-sync/services/session-store";
|
||||||
|
|
||||||
|
export const runtime = "nodejs";
|
||||||
|
export const dynamic = "force-dynamic";
|
||||||
|
|
||||||
|
export async function POST(req: Request) {
|
||||||
|
try {
|
||||||
|
const body = (await req.json()) as {
|
||||||
|
siruta?: string;
|
||||||
|
sampleSize?: number;
|
||||||
|
};
|
||||||
|
const siruta = String(body.siruta ?? "").trim();
|
||||||
|
if (!/^\d+$/.test(siruta)) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: "SIRUTA must be numeric" },
|
||||||
|
{ status: 400 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const session = getSessionCredentials();
|
||||||
|
const username = String(
|
||||||
|
session?.username || process.env.ETERRA_USERNAME || "",
|
||||||
|
).trim();
|
||||||
|
const password = String(
|
||||||
|
session?.password || process.env.ETERRA_PASSWORD || "",
|
||||||
|
).trim();
|
||||||
|
if (!username || !password) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: "Nu ești conectat la eTerra" },
|
||||||
|
{ status: 401 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const client = await EterraClient.create(username, password);
|
||||||
|
|
||||||
|
// Fetch first page of immovables
|
||||||
|
const response = await client.fetchImmovableListByAdminUnit(
|
||||||
|
// We need workspace PK — try to resolve
|
||||||
|
0, // placeholder, we'll get from the sample
|
||||||
|
siruta,
|
||||||
|
0,
|
||||||
|
20,
|
||||||
|
true, // inscrisCF = -1
|
||||||
|
);
|
||||||
|
|
||||||
|
// If workspace 0 doesn't work, try to resolve it
|
||||||
|
let items = response?.content ?? [];
|
||||||
|
if (items.length === 0) {
|
||||||
|
// Try without inscrisCF filter
|
||||||
|
const response2 = await client.fetchImmovableListByAdminUnit(
|
||||||
|
0,
|
||||||
|
siruta,
|
||||||
|
0,
|
||||||
|
20,
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
items = response2?.content ?? [];
|
||||||
|
}
|
||||||
|
|
||||||
|
// If still empty, try to get workspace from GIS layer
|
||||||
|
if (items.length === 0) {
|
||||||
|
const features = await client.fetchAllLayer(
|
||||||
|
{
|
||||||
|
id: "TERENURI_ACTIVE",
|
||||||
|
name: "TERENURI_ACTIVE",
|
||||||
|
endpoint: "aut" as const,
|
||||||
|
whereTemplate: "{{adminField}}={{siruta}} AND IS_ACTIVE=1",
|
||||||
|
},
|
||||||
|
siruta,
|
||||||
|
{ returnGeometry: false, outFields: "WORKSPACE_ID", pageSize: 1 },
|
||||||
|
);
|
||||||
|
const wsId = features?.[0]?.attributes?.WORKSPACE_ID;
|
||||||
|
if (wsId) {
|
||||||
|
const response3 = await client.fetchImmovableListByAdminUnit(
|
||||||
|
Number(wsId),
|
||||||
|
siruta,
|
||||||
|
0,
|
||||||
|
20,
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
items = response3?.content ?? [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (items.length === 0) {
|
||||||
|
return NextResponse.json({
|
||||||
|
error: "Nu s-au găsit imobile",
|
||||||
|
raw: response,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Analyze ALL fields across all items
|
||||||
|
const allFields = new Set<string>();
|
||||||
|
const fieldStats: Record<
|
||||||
|
string,
|
||||||
|
{ present: number; nonNull: number; nonEmpty: number; sample: unknown }
|
||||||
|
> = {};
|
||||||
|
|
||||||
|
for (const item of items) {
|
||||||
|
if (typeof item !== "object" || item == null) continue;
|
||||||
|
for (const [key, value] of Object.entries(item)) {
|
||||||
|
allFields.add(key);
|
||||||
|
if (!fieldStats[key]) {
|
||||||
|
fieldStats[key] = {
|
||||||
|
present: 0,
|
||||||
|
nonNull: 0,
|
||||||
|
nonEmpty: 0,
|
||||||
|
sample: undefined,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const stat = fieldStats[key]!;
|
||||||
|
stat.present++;
|
||||||
|
if (value != null) {
|
||||||
|
stat.nonNull++;
|
||||||
|
if (stat.sample === undefined) stat.sample = value;
|
||||||
|
}
|
||||||
|
if (value != null && value !== "" && value !== 0) stat.nonEmpty++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Also try to get GIS features for comparison
|
||||||
|
const gisFeatures = await client.fetchAllLayer(
|
||||||
|
{
|
||||||
|
id: "TERENURI_ACTIVE",
|
||||||
|
name: "TERENURI_ACTIVE",
|
||||||
|
endpoint: "aut" as const,
|
||||||
|
whereTemplate: "{{adminField}}={{siruta}} AND IS_ACTIVE=1",
|
||||||
|
},
|
||||||
|
siruta,
|
||||||
|
{
|
||||||
|
returnGeometry: false,
|
||||||
|
outFields:
|
||||||
|
"OBJECTID,NATIONAL_CADASTRAL_REFERENCE,IMMOVABLE_ID,AREA_VALUE",
|
||||||
|
pageSize: 5,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
const gisFieldStats: Record<string, unknown> = {};
|
||||||
|
if (gisFeatures.length > 0) {
|
||||||
|
const sample = gisFeatures[0]!.attributes;
|
||||||
|
for (const [key, value] of Object.entries(sample ?? {})) {
|
||||||
|
gisFieldStats[key] = value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const sampleSize = Math.min(body.sampleSize ?? 5, items.length);
|
||||||
|
return NextResponse.json({
|
||||||
|
totalItems: response?.totalElements ?? items.length,
|
||||||
|
totalPages: response?.totalPages ?? 1,
|
||||||
|
sampleCount: sampleSize,
|
||||||
|
sample: items.slice(0, sampleSize),
|
||||||
|
allFields: Array.from(allFields).sort(),
|
||||||
|
fieldStats,
|
||||||
|
gisSampleFields: gisFieldStats,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
const message = error instanceof Error ? error.message : "Eroare server";
|
||||||
|
return NextResponse.json({ error: message }, { status: 500 });
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2525,12 +2525,19 @@ export function ParcelSyncModule() {
|
|||||||
Import parcele fără geometrie —{" "}
|
Import parcele fără geometrie —{" "}
|
||||||
<span className="font-medium text-amber-600 dark:text-amber-400">
|
<span className="font-medium text-amber-600 dark:text-amber-400">
|
||||||
{(() => {
|
{(() => {
|
||||||
|
// Only useful items will be imported (quality filter)
|
||||||
|
const usefulNoGeom =
|
||||||
|
noGeomScan.qualityBreakdown.useful;
|
||||||
const newNoGeom = Math.max(
|
const newNoGeom = Math.max(
|
||||||
0,
|
0,
|
||||||
noGeomScan.noGeomCount - noGeomScan.localDbNoGeom,
|
usefulNoGeom - noGeomScan.localDbNoGeom,
|
||||||
);
|
);
|
||||||
|
const filtered = noGeomScan.qualityBreakdown.empty;
|
||||||
return newNoGeom > 0
|
return newNoGeom > 0
|
||||||
? `~${newNoGeom.toLocaleString("ro-RO")} noi de importat`
|
? `~${newNoGeom.toLocaleString("ro-RO")} noi de importat` +
|
||||||
|
(filtered > 0
|
||||||
|
? ` (${filtered.toLocaleString("ro-RO")} filtrate)`
|
||||||
|
: "")
|
||||||
: "deja importate";
|
: "deja importate";
|
||||||
})()}
|
})()}
|
||||||
</span>
|
</span>
|
||||||
@@ -2540,13 +2547,13 @@ export function ParcelSyncModule() {
|
|||||||
Îmbogățire CF, proprietari, adrese —{" "}
|
Îmbogățire CF, proprietari, adrese —{" "}
|
||||||
<span className="font-medium text-teal-600 dark:text-teal-400">
|
<span className="font-medium text-teal-600 dark:text-teal-400">
|
||||||
{(() => {
|
{(() => {
|
||||||
|
const usefulNoGeom = noGeomScan.qualityBreakdown.useful;
|
||||||
const totalToEnrich =
|
const totalToEnrich =
|
||||||
noGeomScan.localDbTotal +
|
noGeomScan.localDbTotal +
|
||||||
(includeNoGeom
|
(includeNoGeom
|
||||||
? Math.max(
|
? Math.max(
|
||||||
0,
|
0,
|
||||||
noGeomScan.noGeomCount -
|
usefulNoGeom - noGeomScan.localDbNoGeom,
|
||||||
noGeomScan.localDbNoGeom,
|
|
||||||
)
|
)
|
||||||
: 0);
|
: 0);
|
||||||
// Use enrichedComplete (not enriched) — stale
|
// Use enrichedComplete (not enriched) — stale
|
||||||
@@ -2704,9 +2711,10 @@ export function ParcelSyncModule() {
|
|||||||
)}
|
)}
|
||||||
{includeNoGeom && (
|
{includeNoGeom && (
|
||||||
<p className="text-[11px] text-muted-foreground ml-7">
|
<p className="text-[11px] text-muted-foreground ml-7">
|
||||||
Vor fi importate în DB și incluse în CSV + Magic GPKG
|
{noGeomScan.qualityBreakdown.empty > 0
|
||||||
(coloana HAS_GEOMETRY=0/1). În GPKG de bază apar doar
|
? `Din ${noGeomScan.noGeomCount.toLocaleString("ro-RO")} fără geometrie, ~${noGeomScan.qualityBreakdown.useful.toLocaleString("ro-RO")} vor fi importate (cele cu identificare sau suprafață). ${noGeomScan.qualityBreakdown.empty.toLocaleString("ro-RO")} fără nicio dată de identificare vor fi filtrate.`
|
||||||
cele cu geometrie.
|
: "Vor fi importate în DB și incluse în CSV + Magic GPKG (coloana HAS_GEOMETRY=0/1)."}{" "}
|
||||||
|
În GPKG de bază apar doar cele cu geometrie.
|
||||||
</p>
|
</p>
|
||||||
)}
|
)}
|
||||||
{workflowPreview}
|
{workflowPreview}
|
||||||
|
|||||||
@@ -122,6 +122,7 @@ export type NoGeomScanResult = {
|
|||||||
identifierDetails: string;
|
identifierDetails: string;
|
||||||
paperCadNo?: string;
|
paperCadNo?: string;
|
||||||
paperCfNo?: string;
|
paperCfNo?: string;
|
||||||
|
paperLbNo?: string;
|
||||||
}>;
|
}>;
|
||||||
/** Total features already in local DB (geometry + no-geom) */
|
/** Total features already in local DB (geometry + no-geom) */
|
||||||
localDbTotal: number;
|
localDbTotal: number;
|
||||||
@@ -232,6 +233,7 @@ export async function scanNoGeometryParcels(
|
|||||||
identifierDetails: string;
|
identifierDetails: string;
|
||||||
paperCadNo?: string;
|
paperCadNo?: string;
|
||||||
paperCfNo?: string;
|
paperCfNo?: string;
|
||||||
|
paperLbNo?: string;
|
||||||
}> = [];
|
}> = [];
|
||||||
|
|
||||||
for (const item of allImmovables) {
|
for (const item of allImmovables) {
|
||||||
@@ -250,6 +252,7 @@ export async function scanNoGeometryParcels(
|
|||||||
identifierDetails: String(item.identifierDetails ?? ""),
|
identifierDetails: String(item.identifierDetails ?? ""),
|
||||||
paperCadNo: item.paperCadNo ?? undefined,
|
paperCadNo: item.paperCadNo ?? undefined,
|
||||||
paperCfNo: item.paperCfNo ?? undefined,
|
paperCfNo: item.paperCfNo ?? undefined,
|
||||||
|
paperLbNo: item.paperLbNo ?? undefined,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -314,12 +317,20 @@ export async function scanNoGeometryParcels(
|
|||||||
const matchedCount = allImmovables.length - noGeomItems.length;
|
const matchedCount = allImmovables.length - noGeomItems.length;
|
||||||
|
|
||||||
// Quality analysis of no-geom items
|
// Quality analysis of no-geom items
|
||||||
// Build a quick lookup for area data from the immovable list
|
// Build a quick lookup for area data from the immovable list (any area field)
|
||||||
const areaByPk = new Map<number, number>();
|
const areaByPk = new Map<number, number>();
|
||||||
for (const item of allImmovables) {
|
for (const item of allImmovables) {
|
||||||
const pk = Number(item.immovablePk ?? 0);
|
const pk = Number(item.immovablePk ?? 0);
|
||||||
if (pk > 0 && typeof item.area === "number" && item.area > 0) {
|
const areaVal = [
|
||||||
areaByPk.set(pk, item.area);
|
item.area,
|
||||||
|
item.measuredArea,
|
||||||
|
item.areaValue,
|
||||||
|
item.suprafata,
|
||||||
|
]
|
||||||
|
.map((v) => (typeof v === "number" && v > 0 ? v : null))
|
||||||
|
.find((v) => v != null);
|
||||||
|
if (pk > 0 && areaVal != null) {
|
||||||
|
areaByPk.set(pk, areaVal);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -333,12 +344,16 @@ export async function scanNoGeometryParcels(
|
|||||||
const hasCad = !!item.identifierDetails?.trim();
|
const hasCad = !!item.identifierDetails?.trim();
|
||||||
const hasPaperCad = !!item.paperCadNo?.trim();
|
const hasPaperCad = !!item.paperCadNo?.trim();
|
||||||
const hasPaperCf = !!item.paperCfNo?.trim();
|
const hasPaperCf = !!item.paperCfNo?.trim();
|
||||||
|
const hasPaperLb = !!item.paperLbNo?.trim();
|
||||||
const hasArea = areaByPk.has(item.immovablePk);
|
const hasArea = areaByPk.has(item.immovablePk);
|
||||||
if (hasCad) qWithCadRef++;
|
if (hasCad) qWithCadRef++;
|
||||||
if (hasPaperCad) qWithPaperCad++;
|
if (hasPaperCad) qWithPaperCad++;
|
||||||
if (hasPaperCf) qWithPaperCf++;
|
if (hasPaperCf) qWithPaperCf++;
|
||||||
if (hasArea) qWithArea++;
|
if (hasArea) qWithArea++;
|
||||||
if (hasCad || (hasPaperCad && hasPaperCf)) qUseful++;
|
// "Useful" = has any form of identification OR area
|
||||||
|
// Matches the import quality gate: !hasIdentification && !hasArea → filter out
|
||||||
|
const hasIdentification = hasCad || hasPaperCf || hasPaperLb || hasPaperCad;
|
||||||
|
if (hasIdentification || hasArea) qUseful++;
|
||||||
else qEmpty++;
|
else qEmpty++;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -409,17 +424,50 @@ export async function syncNoGeometryParcels(
|
|||||||
existingObjIds.add(f.objectId);
|
existingObjIds.add(f.objectId);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 3. Filter to only those not yet in DB
|
// 3. Filter: not yet in DB + quality gate
|
||||||
|
// Quality: Must have EITHER a valid cadRef/CF number AND area > 0.
|
||||||
|
// Items with no identification AND no area are noise — skip them.
|
||||||
|
let filteredOut = 0;
|
||||||
const candidates = allImmovables.filter((item) => {
|
const candidates = allImmovables.filter((item) => {
|
||||||
const cadRef = normalizeCadRef(item.identifierDetails ?? "");
|
const cadRef = normalizeCadRef(item.identifierDetails ?? "");
|
||||||
const immPk = Number(item.immovablePk ?? 0);
|
const immPk = Number(item.immovablePk ?? 0);
|
||||||
|
|
||||||
|
// Already in DB? → skip
|
||||||
if (cadRef && existingCadRefs.has(cadRef)) return false;
|
if (cadRef && existingCadRefs.has(cadRef)) return false;
|
||||||
if (immPk > 0 && existingObjIds.has(-immPk)) return false;
|
if (immPk > 0 && existingObjIds.has(-immPk)) return false;
|
||||||
|
|
||||||
|
// Quality gate: must have CF/identification reference
|
||||||
|
const hasCadRef = !!cadRef;
|
||||||
|
const hasPaperCf = !!(item.paperCfNo ?? "").toString().trim();
|
||||||
|
const hasPaperLb = !!(item.paperLbNo ?? "").toString().trim();
|
||||||
|
const hasPaperCad = !!(item.paperCadNo ?? "").toString().trim();
|
||||||
|
const hasIdentification =
|
||||||
|
hasCadRef || hasPaperCf || hasPaperLb || hasPaperCad;
|
||||||
|
|
||||||
|
// Quality gate: must have area from any field
|
||||||
|
const areaFields = [
|
||||||
|
item.area,
|
||||||
|
item.measuredArea,
|
||||||
|
item.areaValue,
|
||||||
|
item.suprafata,
|
||||||
|
];
|
||||||
|
const hasArea = areaFields.some((v) => typeof v === "number" && v > 0);
|
||||||
|
|
||||||
|
if (!hasIdentification && !hasArea) {
|
||||||
|
filteredOut++;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
});
|
});
|
||||||
|
|
||||||
if (candidates.length === 0) {
|
if (candidates.length === 0) {
|
||||||
return { imported: 0, skipped: 0, errors: 0, status: "done" };
|
return {
|
||||||
|
imported: 0,
|
||||||
|
skipped: filteredOut,
|
||||||
|
errors: 0,
|
||||||
|
status: "done",
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
// 4. Import candidates in batches with retry
|
// 4. Import candidates in batches with retry
|
||||||
@@ -446,7 +494,11 @@ export async function syncNoGeometryParcels(
|
|||||||
}
|
}
|
||||||
|
|
||||||
const cadRef = String(item.identifierDetails ?? "").trim();
|
const cadRef = String(item.identifierDetails ?? "").trim();
|
||||||
const areaValue = typeof item.area === "number" ? item.area : null;
|
// Extract area from any available field
|
||||||
|
const areaValue =
|
||||||
|
[item.area, item.measuredArea, item.areaValue, item.suprafata]
|
||||||
|
.map((v) => (typeof v === "number" && v > 0 ? v : null))
|
||||||
|
.find((v) => v != null) ?? null;
|
||||||
|
|
||||||
const attributes: Record<string, unknown> = {
|
const attributes: Record<string, unknown> = {
|
||||||
OBJECTID: -immPk,
|
OBJECTID: -immPk,
|
||||||
@@ -527,7 +579,7 @@ export async function syncNoGeometryParcels(
|
|||||||
options?.onProgress?.(done, total, "Import parcele fără geometrie");
|
options?.onProgress?.(done, total, "Import parcele fără geometrie");
|
||||||
}
|
}
|
||||||
|
|
||||||
return { imported, skipped, errors, status: "done" };
|
return { imported, skipped: skipped + filteredOut, errors, status: "done" };
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
const msg = error instanceof Error ? error.message : "Unknown error";
|
const msg = error instanceof Error ? error.message : "Unknown error";
|
||||||
return { imported: 0, skipped: 0, errors: 0, status: "error", error: msg };
|
return { imported: 0, skipped: 0, errors: 0, status: "error", error: msg };
|
||||||
|
|||||||
Reference in New Issue
Block a user