feat(parcel-sync): quality gate filter for no-geom import + diagnostic endpoint
- Filter no-geom items before import: must have identification (cadRef/CF/paperCad/paperLb) OR area - Multi-field area extraction: area, measuredArea, areaValue, suprafata - Scan quality breakdown: withCadRef, withPaperCf, withPaperCad, withArea, useful, empty - Added paperLbNo to quality analysis and samples - UI: quality breakdown grid in scan card - UI: filtered count in workflow preview (shows useful, not total) - UI: enrichment estimate uses useful count - New diagnostic endpoint /api/eterra/no-geom-debug for field inspection
This commit is contained in:
AI Assistant
@@ -0,0 +1,173 @@
|
||||
/**
|
||||
* POST /api/eterra/no-geom-debug
|
||||
*
|
||||
* Diagnostic endpoint: fetches a small sample of no-geometry immovables
|
||||
* and returns ALL their raw fields, so we can understand what data is
|
||||
* available for quality filtering.
|
||||
*
|
||||
* Body: { siruta: string, sampleSize?: number }
|
||||
* Returns: { sample: [...raw items], allFields: [...field names], fieldStats: {...} }
|
||||
*/
|
||||
|
||||
import { NextResponse } from "next/server";
|
||||
import { EterraClient } from "@/modules/parcel-sync/services/eterra-client";
|
||||
import { getSessionCredentials } from "@/modules/parcel-sync/services/session-store";
|
||||
|
||||
export const runtime = "nodejs";
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
export async function POST(req: Request) {
|
||||
try {
|
||||
const body = (await req.json()) as {
|
||||
siruta?: string;
|
||||
sampleSize?: number;
|
||||
};
|
||||
const siruta = String(body.siruta ?? "").trim();
|
||||
if (!/^\d+$/.test(siruta)) {
|
||||
return NextResponse.json(
|
||||
{ error: "SIRUTA must be numeric" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
const session = getSessionCredentials();
|
||||
const username = String(
|
||||
session?.username || process.env.ETERRA_USERNAME || "",
|
||||
).trim();
|
||||
const password = String(
|
||||
session?.password || process.env.ETERRA_PASSWORD || "",
|
||||
).trim();
|
||||
if (!username || !password) {
|
||||
return NextResponse.json(
|
||||
{ error: "Nu ești conectat la eTerra" },
|
||||
{ status: 401 },
|
||||
);
|
||||
}
|
||||
|
||||
const client = await EterraClient.create(username, password);
|
||||
|
||||
// Fetch first page of immovables
|
||||
const response = await client.fetchImmovableListByAdminUnit(
|
||||
// We need workspace PK — try to resolve
|
||||
0, // placeholder, we'll get from the sample
|
||||
siruta,
|
||||
0,
|
||||
20,
|
||||
true, // inscrisCF = -1
|
||||
);
|
||||
|
||||
// If workspace 0 doesn't work, try to resolve it
|
||||
let items = response?.content ?? [];
|
||||
if (items.length === 0) {
|
||||
// Try without inscrisCF filter
|
||||
const response2 = await client.fetchImmovableListByAdminUnit(
|
||||
0,
|
||||
siruta,
|
||||
0,
|
||||
20,
|
||||
false,
|
||||
);
|
||||
items = response2?.content ?? [];
|
||||
}
|
||||
|
||||
// If still empty, try to get workspace from GIS layer
|
||||
if (items.length === 0) {
|
||||
const features = await client.fetchAllLayer(
|
||||
{
|
||||
id: "TERENURI_ACTIVE",
|
||||
name: "TERENURI_ACTIVE",
|
||||
endpoint: "aut" as const,
|
||||
whereTemplate: "{{adminField}}={{siruta}} AND IS_ACTIVE=1",
|
||||
},
|
||||
siruta,
|
||||
{ returnGeometry: false, outFields: "WORKSPACE_ID", pageSize: 1 },
|
||||
);
|
||||
const wsId = features?.[0]?.attributes?.WORKSPACE_ID;
|
||||
if (wsId) {
|
||||
const response3 = await client.fetchImmovableListByAdminUnit(
|
||||
Number(wsId),
|
||||
siruta,
|
||||
0,
|
||||
20,
|
||||
true,
|
||||
);
|
||||
items = response3?.content ?? [];
|
||||
}
|
||||
}
|
||||
|
||||
if (items.length === 0) {
|
||||
return NextResponse.json({
|
||||
error: "Nu s-au găsit imobile",
|
||||
raw: response,
|
||||
});
|
||||
}
|
||||
|
||||
// Analyze ALL fields across all items
|
||||
const allFields = new Set<string>();
|
||||
const fieldStats: Record<
|
||||
string,
|
||||
{ present: number; nonNull: number; nonEmpty: number; sample: unknown }
|
||||
> = {};
|
||||
|
||||
for (const item of items) {
|
||||
if (typeof item !== "object" || item == null) continue;
|
||||
for (const [key, value] of Object.entries(item)) {
|
||||
allFields.add(key);
|
||||
if (!fieldStats[key]) {
|
||||
fieldStats[key] = {
|
||||
present: 0,
|
||||
nonNull: 0,
|
||||
nonEmpty: 0,
|
||||
sample: undefined,
|
||||
};
|
||||
}
|
||||
const stat = fieldStats[key]!;
|
||||
stat.present++;
|
||||
if (value != null) {
|
||||
stat.nonNull++;
|
||||
if (stat.sample === undefined) stat.sample = value;
|
||||
}
|
||||
if (value != null && value !== "" && value !== 0) stat.nonEmpty++;
|
||||
}
|
||||
}
|
||||
|
||||
// Also try to get GIS features for comparison
|
||||
const gisFeatures = await client.fetchAllLayer(
|
||||
{
|
||||
id: "TERENURI_ACTIVE",
|
||||
name: "TERENURI_ACTIVE",
|
||||
endpoint: "aut" as const,
|
||||
whereTemplate: "{{adminField}}={{siruta}} AND IS_ACTIVE=1",
|
||||
},
|
||||
siruta,
|
||||
{
|
||||
returnGeometry: false,
|
||||
outFields:
|
||||
"OBJECTID,NATIONAL_CADASTRAL_REFERENCE,IMMOVABLE_ID,AREA_VALUE",
|
||||
pageSize: 5,
|
||||
},
|
||||
);
|
||||
|
||||
const gisFieldStats: Record<string, unknown> = {};
|
||||
if (gisFeatures.length > 0) {
|
||||
const sample = gisFeatures[0]!.attributes;
|
||||
for (const [key, value] of Object.entries(sample ?? {})) {
|
||||
gisFieldStats[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
const sampleSize = Math.min(body.sampleSize ?? 5, items.length);
|
||||
return NextResponse.json({
|
||||
totalItems: response?.totalElements ?? items.length,
|
||||
totalPages: response?.totalPages ?? 1,
|
||||
sampleCount: sampleSize,
|
||||
sample: items.slice(0, sampleSize),
|
||||
allFields: Array.from(allFields).sort(),
|
||||
fieldStats,
|
||||
gisSampleFields: gisFieldStats,
|
||||
});
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "Eroare server";
|
||||
return NextResponse.json({ error: message }, { status: 500 });
|
||||
}
|
||||
}
|
||||
@@ -2525,12 +2525,19 @@ export function ParcelSyncModule() {
|
||||
Import parcele fără geometrie —{" "}
|
||||
<span className="font-medium text-amber-600 dark:text-amber-400">
|
||||
{(() => {
|
||||
// Only useful items will be imported (quality filter)
|
||||
const usefulNoGeom =
|
||||
noGeomScan.qualityBreakdown.useful;
|
||||
const newNoGeom = Math.max(
|
||||
0,
|
||||
noGeomScan.noGeomCount - noGeomScan.localDbNoGeom,
|
||||
usefulNoGeom - noGeomScan.localDbNoGeom,
|
||||
);
|
||||
const filtered = noGeomScan.qualityBreakdown.empty;
|
||||
return newNoGeom > 0
|
||||
? `~${newNoGeom.toLocaleString("ro-RO")} noi de importat`
|
||||
? `~${newNoGeom.toLocaleString("ro-RO")} noi de importat` +
|
||||
(filtered > 0
|
||||
? ` (${filtered.toLocaleString("ro-RO")} filtrate)`
|
||||
: "")
|
||||
: "deja importate";
|
||||
})()}
|
||||
</span>
|
||||
@@ -2540,13 +2547,13 @@ export function ParcelSyncModule() {
|
||||
Îmbogățire CF, proprietari, adrese —{" "}
|
||||
<span className="font-medium text-teal-600 dark:text-teal-400">
|
||||
{(() => {
|
||||
const usefulNoGeom = noGeomScan.qualityBreakdown.useful;
|
||||
const totalToEnrich =
|
||||
noGeomScan.localDbTotal +
|
||||
(includeNoGeom
|
||||
? Math.max(
|
||||
0,
|
||||
noGeomScan.noGeomCount -
|
||||
noGeomScan.localDbNoGeom,
|
||||
usefulNoGeom - noGeomScan.localDbNoGeom,
|
||||
)
|
||||
: 0);
|
||||
// Use enrichedComplete (not enriched) — stale
|
||||
@@ -2704,9 +2711,10 @@ export function ParcelSyncModule() {
|
||||
)}
|
||||
{includeNoGeom && (
|
||||
<p className="text-[11px] text-muted-foreground ml-7">
|
||||
Vor fi importate în DB și incluse în CSV + Magic GPKG
|
||||
(coloana HAS_GEOMETRY=0/1). În GPKG de bază apar doar
|
||||
cele cu geometrie.
|
||||
{noGeomScan.qualityBreakdown.empty > 0
|
||||
? `Din ${noGeomScan.noGeomCount.toLocaleString("ro-RO")} fără geometrie, ~${noGeomScan.qualityBreakdown.useful.toLocaleString("ro-RO")} vor fi importate (cele cu identificare sau suprafață). ${noGeomScan.qualityBreakdown.empty.toLocaleString("ro-RO")} fără nicio dată de identificare vor fi filtrate.`
|
||||
: "Vor fi importate în DB și incluse în CSV + Magic GPKG (coloana HAS_GEOMETRY=0/1)."}{" "}
|
||||
În GPKG de bază apar doar cele cu geometrie.
|
||||
</p>
|
||||
)}
|
||||
{workflowPreview}
|
||||
|
||||
@@ -122,6 +122,7 @@ export type NoGeomScanResult = {
|
||||
identifierDetails: string;
|
||||
paperCadNo?: string;
|
||||
paperCfNo?: string;
|
||||
paperLbNo?: string;
|
||||
}>;
|
||||
/** Total features already in local DB (geometry + no-geom) */
|
||||
localDbTotal: number;
|
||||
@@ -232,6 +233,7 @@ export async function scanNoGeometryParcels(
|
||||
identifierDetails: string;
|
||||
paperCadNo?: string;
|
||||
paperCfNo?: string;
|
||||
paperLbNo?: string;
|
||||
}> = [];
|
||||
|
||||
for (const item of allImmovables) {
|
||||
@@ -250,6 +252,7 @@ export async function scanNoGeometryParcels(
|
||||
identifierDetails: String(item.identifierDetails ?? ""),
|
||||
paperCadNo: item.paperCadNo ?? undefined,
|
||||
paperCfNo: item.paperCfNo ?? undefined,
|
||||
paperLbNo: item.paperLbNo ?? undefined,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -314,12 +317,20 @@ export async function scanNoGeometryParcels(
|
||||
const matchedCount = allImmovables.length - noGeomItems.length;
|
||||
|
||||
// Quality analysis of no-geom items
|
||||
// Build a quick lookup for area data from the immovable list
|
||||
// Build a quick lookup for area data from the immovable list (any area field)
|
||||
const areaByPk = new Map<number, number>();
|
||||
for (const item of allImmovables) {
|
||||
const pk = Number(item.immovablePk ?? 0);
|
||||
if (pk > 0 && typeof item.area === "number" && item.area > 0) {
|
||||
areaByPk.set(pk, item.area);
|
||||
const areaVal = [
|
||||
item.area,
|
||||
item.measuredArea,
|
||||
item.areaValue,
|
||||
item.suprafata,
|
||||
]
|
||||
.map((v) => (typeof v === "number" && v > 0 ? v : null))
|
||||
.find((v) => v != null);
|
||||
if (pk > 0 && areaVal != null) {
|
||||
areaByPk.set(pk, areaVal);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -333,12 +344,16 @@ export async function scanNoGeometryParcels(
|
||||
const hasCad = !!item.identifierDetails?.trim();
|
||||
const hasPaperCad = !!item.paperCadNo?.trim();
|
||||
const hasPaperCf = !!item.paperCfNo?.trim();
|
||||
const hasPaperLb = !!item.paperLbNo?.trim();
|
||||
const hasArea = areaByPk.has(item.immovablePk);
|
||||
if (hasCad) qWithCadRef++;
|
||||
if (hasPaperCad) qWithPaperCad++;
|
||||
if (hasPaperCf) qWithPaperCf++;
|
||||
if (hasArea) qWithArea++;
|
||||
if (hasCad || (hasPaperCad && hasPaperCf)) qUseful++;
|
||||
// "Useful" = has any form of identification OR area
|
||||
// Matches the import quality gate: !hasIdentification && !hasArea → filter out
|
||||
const hasIdentification = hasCad || hasPaperCf || hasPaperLb || hasPaperCad;
|
||||
if (hasIdentification || hasArea) qUseful++;
|
||||
else qEmpty++;
|
||||
}
|
||||
|
||||
@@ -409,17 +424,50 @@ export async function syncNoGeometryParcels(
|
||||
existingObjIds.add(f.objectId);
|
||||
}
|
||||
|
||||
// 3. Filter to only those not yet in DB
|
||||
// 3. Filter: not yet in DB + quality gate
|
||||
// Quality: Must have EITHER a valid cadRef/CF number AND area > 0.
|
||||
// Items with no identification AND no area are noise — skip them.
|
||||
let filteredOut = 0;
|
||||
const candidates = allImmovables.filter((item) => {
|
||||
const cadRef = normalizeCadRef(item.identifierDetails ?? "");
|
||||
const immPk = Number(item.immovablePk ?? 0);
|
||||
|
||||
// Already in DB? → skip
|
||||
if (cadRef && existingCadRefs.has(cadRef)) return false;
|
||||
if (immPk > 0 && existingObjIds.has(-immPk)) return false;
|
||||
|
||||
// Quality gate: must have CF/identification reference
|
||||
const hasCadRef = !!cadRef;
|
||||
const hasPaperCf = !!(item.paperCfNo ?? "").toString().trim();
|
||||
const hasPaperLb = !!(item.paperLbNo ?? "").toString().trim();
|
||||
const hasPaperCad = !!(item.paperCadNo ?? "").toString().trim();
|
||||
const hasIdentification =
|
||||
hasCadRef || hasPaperCf || hasPaperLb || hasPaperCad;
|
||||
|
||||
// Quality gate: must have area from any field
|
||||
const areaFields = [
|
||||
item.area,
|
||||
item.measuredArea,
|
||||
item.areaValue,
|
||||
item.suprafata,
|
||||
];
|
||||
const hasArea = areaFields.some((v) => typeof v === "number" && v > 0);
|
||||
|
||||
if (!hasIdentification && !hasArea) {
|
||||
filteredOut++;
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
});
|
||||
|
||||
if (candidates.length === 0) {
|
||||
return { imported: 0, skipped: 0, errors: 0, status: "done" };
|
||||
return {
|
||||
imported: 0,
|
||||
skipped: filteredOut,
|
||||
errors: 0,
|
||||
status: "done",
|
||||
};
|
||||
}
|
||||
|
||||
// 4. Import candidates in batches with retry
|
||||
@@ -446,7 +494,11 @@ export async function syncNoGeometryParcels(
|
||||
}
|
||||
|
||||
const cadRef = String(item.identifierDetails ?? "").trim();
|
||||
const areaValue = typeof item.area === "number" ? item.area : null;
|
||||
// Extract area from any available field
|
||||
const areaValue =
|
||||
[item.area, item.measuredArea, item.areaValue, item.suprafata]
|
||||
.map((v) => (typeof v === "number" && v > 0 ? v : null))
|
||||
.find((v) => v != null) ?? null;
|
||||
|
||||
const attributes: Record<string, unknown> = {
|
||||
OBJECTID: -immPk,
|
||||
@@ -527,7 +579,7 @@ export async function syncNoGeometryParcels(
|
||||
options?.onProgress?.(done, total, "Import parcele fără geometrie");
|
||||
}
|
||||
|
||||
return { imported, skipped, errors, status: "done" };
|
||||
return { imported, skipped: skipped + filteredOut, errors, status: "done" };
|
||||
} catch (error) {
|
||||
const msg = error instanceof Error ? error.message : "Unknown error";
|
||||
return { imported: 0, skipped: 0, errors: 0, status: "error", error: msg };
|
||||
|
||||
Reference in New Issue
Block a user