/* eslint-disable @typescript-eslint/no-explicit-any */ /** * No-geometry sync — imports eTerra immovables that have NO geometry * in the GIS layer (TERENURI_ACTIVE). * * These are parcels that exist in the eTerra immovable database but * have no spatial representation in the ArcGIS layer. They are stored * in GisFeature with: * - geometry = null * - geometrySource = "NO_GEOMETRY" * - objectId = negative immovablePk (to avoid collisions with real OBJECTIDs) * * The cross-reference works by: * 1. Fetch full immovable list from eTerra for the UAT (paginated) * 2. Load all existing cadastralRefs from DB for TERENURI_ACTIVE + siruta * 3. Immovables whose cadastralRef is NOT in DB → candidates * 4. Store each candidate as a GisFeature with geometry=null */ import { Prisma, PrismaClient } from "@prisma/client"; import { EterraClient } from "./eterra-client"; const prisma = new PrismaClient(); /* ------------------------------------------------------------------ */ /* Workspace resolution (county → eTerra workspace PK) */ /* ------------------------------------------------------------------ */ /** * Resolve the eTerra workspace PK for a SIRUTA. * Chain: explicit param → GisUat DB row → ArcGIS layer query → null. */ async function resolveWorkspacePk( client: EterraClient, siruta: string, explicitPk?: number | null, ): Promise { // 1. Explicit param if (explicitPk && Number.isFinite(explicitPk) && explicitPk > 0) { return explicitPk; } // 2. DB lookup try { const row = await prisma.gisUat.findUnique({ where: { siruta }, select: { workspacePk: true }, }); if (row?.workspacePk && row.workspacePk > 0) return row.workspacePk; } catch { /* ignore */ } // 3. ArcGIS layer query — fetch 1 feature from TERENURI_ACTIVE for this siruta try { const features = await client.listLayer( { id: "TERENURI_ACTIVE", name: "TERENURI_ACTIVE", endpoint: "aut", whereTemplate: "{{adminField}}={{siruta}} AND IS_ACTIVE=1", }, siruta, { limit: 1, outFields: "WORKSPACE_ID" }, ); const wsId = features?.[0]?.attributes?.WORKSPACE_ID; if (wsId != null) { const num = Number(wsId); if (Number.isFinite(num) && num > 0) { // Persist for future lookups prisma.gisUat .update({ where: { siruta }, data: { workspacePk: num } }) .catch(() => {}); return num; } } } catch { /* ignore */ } return null; } const normalizeId = (value: unknown) => { if (value === null || value === undefined) return ""; const text = String(value).trim(); if (!text) return ""; return text.replace(/\.0$/, ""); }; const normalizeCadRef = (value: unknown) => normalizeId(value).replace(/\s+/g, "").toUpperCase(); /** Quality breakdown of no-geometry immovables from scan */ export type NoGeomQuality = { /** Have electronic cadRef (identifierDetails non-empty) */ withCadRef: number; /** Have paper cadastral number */ withPaperCad: number; /** Have paper LB / CF (carte funciară) number — field is paperLbNo in API */ withPaperLb: number; /** Have hasLandbook=1 flag from eTerra */ withLandbook: number; /** Have area > 0 (measuredArea or legalArea) */ withArea: number; /** status=1 (active) in eTerra */ withActiveStatus: number; /** "Useful" = active AND has identification or area */ useful: number; /** Filtered out: inactive, or no identification AND no area */ empty: number; }; export type NoGeomScanResult = { totalImmovables: number; /** Immovables that matched a remote GIS feature (cross-ref, may vary) */ withGeometry: number; /** Total features in the remote ArcGIS TERENURI_ACTIVE layer (stable) */ remoteGisCount: number; /** Total features in the remote ArcGIS CLADIRI_ACTIVE layer (stable) */ remoteCladiriCount: number; noGeomCount: number; /** Match quality: how many matched by cadastral ref vs immovable ID */ matchedByRef: number; matchedById: number; /** Quality breakdown of no-geometry items */ qualityBreakdown: NoGeomQuality; /** Sample of immovable identifiers without geometry */ samples: Array<{ immovablePk: number; identifierDetails: string; paperCadNo?: string; paperLbNo?: string; status?: number; hasLandbook?: number; measuredArea?: number; }>; /** Total features already in local DB (geometry + no-geom) */ localDbTotal: number; /** Geometry features already synced in local DB */ localDbWithGeom: number; /** No-geometry features already imported in local DB */ localDbNoGeom: number; /** How many are already enriched (magic) in local DB */ localDbEnriched: number; /** How many enriched features have complete/current enrichment schema */ localDbEnrichedComplete: number; /** Whether local sync is fresh (< 7 days) */ localSyncFresh: boolean; /** Timestamp of the scan (for audit trail) */ scannedAt: string; /** Error message if workspace couldn't be resolved */ error?: string; }; export type NoGeomSyncResult = { imported: number; skipped: number; cleaned: number; errors: number; status: "done" | "error"; error?: string; }; /** * Scan: count how many eTerra immovables for this UAT have no geometry * in the remote GIS layer (TERENURI_ACTIVE). * * Cross-references the eTerra immovable list against the REMOTE ArcGIS * layer (lightweight fetch, no geometry download). This works correctly * regardless of whether the user has synced to local DB yet. * * This does NOT write anything — it's a read-only operation. */ export async function scanNoGeometryParcels( client: EterraClient, siruta: string, options?: { onProgress?: (page: number, totalPages: number) => void; workspacePk?: number | null; }, ): Promise { // 0. Resolve workspace const wsPk = await resolveWorkspacePk(client, siruta, options?.workspacePk); if (!wsPk) { return { totalImmovables: 0, withGeometry: 0, remoteGisCount: 0, remoteCladiriCount: 0, noGeomCount: 0, matchedByRef: 0, matchedById: 0, qualityBreakdown: { withCadRef: 0, withPaperCad: 0, withPaperLb: 0, withLandbook: 0, withArea: 0, withActiveStatus: 0, useful: 0, empty: 0, }, samples: [], localDbTotal: 0, localDbWithGeom: 0, localDbNoGeom: 0, localDbEnriched: 0, localDbEnrichedComplete: 0, localSyncFresh: false, scannedAt: new Date().toISOString(), error: `Nu s-a putut determina workspace-ul (județul) pentru SIRUTA ${siruta}`, }; } // 1. Fetch all immovables from eTerra immovable list API const allImmovables = await fetchAllImmovables( client, siruta, wsPk, options?.onProgress, ); // 2. Fetch remote GIS cadastral refs (lightweight — no geometry) // This is the source of truth for "has geometry" regardless of local DB state. // Count first so pagination knows the total and doesn't stop early. const terenuriLayer = { id: "TERENURI_ACTIVE", name: "TERENURI_ACTIVE", endpoint: "aut" as const, whereTemplate: "{{adminField}}={{siruta}} AND IS_ACTIVE=1", }; const [terenuriCount, cladiriCount] = await Promise.all([ client.countLayer(terenuriLayer, siruta).catch(() => 0), client .countLayer( { id: "CLADIRI_ACTIVE", name: "CLADIRI_ACTIVE", endpoint: "aut" as const, whereTemplate: "{{adminField}}={{siruta}} AND IS_ACTIVE=1", }, siruta, ) .catch(() => 0), ]); const remoteFeatures = await client.fetchAllLayer(terenuriLayer, siruta, { returnGeometry: false, outFields: "OBJECTID,NATIONAL_CADASTRAL_REFERENCE,IMMOVABLE_ID", pageSize: 1000, total: terenuriCount > 0 ? terenuriCount : undefined, }); // 2b. Also fetch CLADIRI_ACTIVE features (lightweight, just OBJECTID) const cladiriLayer = { id: "CLADIRI_ACTIVE", name: "CLADIRI_ACTIVE", endpoint: "aut" as const, whereTemplate: "{{adminField}}={{siruta}} AND IS_ACTIVE=1", }; let remoteCladiriCount = cladiriCount; if (remoteCladiriCount === 0) { try { const cladiriFeatures = await client.fetchAllLayer(cladiriLayer, siruta, { returnGeometry: false, outFields: "OBJECTID", pageSize: 1000, }); remoteCladiriCount = cladiriFeatures.length; } catch { // Non-fatal — just won't show clădiri count } } const remoteCadRefs = new Set(); const remoteImmIds = new Set(); for (const f of remoteFeatures) { const cadRef = normalizeCadRef( f.attributes?.NATIONAL_CADASTRAL_REFERENCE ?? "", ); if (cadRef) remoteCadRefs.add(cadRef); const immId = normalizeId(f.attributes?.IMMOVABLE_ID); if (immId) remoteImmIds.add(immId); } // 3. Cross-reference: immovables NOT in remote GIS = no geometry const noGeomItems: Array<{ immovablePk: number; identifierDetails: string; paperCadNo?: string; paperLbNo?: string; status?: number; hasLandbook?: number; measuredArea?: number; legalArea?: number; }> = []; let matchedByRef = 0; let matchedById = 0; for (const item of allImmovables) { const cadRef = normalizeCadRef(item.identifierDetails ?? ""); const immPk = Number(item.immovablePk ?? 0); const immId = normalizeId(item.immovablePk); // Present in remote GIS layer by cadastral ref? → has geometry if (cadRef && remoteCadRefs.has(cadRef)) { matchedByRef++; continue; } // Present in remote GIS layer by IMMOVABLE_ID? → has geometry if (immId && remoteImmIds.has(immId)) { matchedById++; continue; } noGeomItems.push({ immovablePk: immPk, identifierDetails: String(item.identifierDetails ?? ""), paperCadNo: item.paperCadNo ?? undefined, paperLbNo: item.paperLbNo ?? undefined, status: typeof item.status === "number" ? item.status : undefined, hasLandbook: typeof item.hasLandbook === "number" ? item.hasLandbook : undefined, measuredArea: typeof item.measuredArea === "number" ? item.measuredArea : undefined, legalArea: typeof item.legalArea === "number" ? item.legalArea : undefined, }); } // 4. Query local DB for context (what's already synced/imported) // Also check enrichment completeness — do enriched features have // the current schema? (e.g., PROPRIETARI_VECHI added later) const ENRICHMENT_REQUIRED_KEYS = [ "NR_CAD", "NR_CF", "PROPRIETARI", "PROPRIETARI_VECHI", "ADRESA", "CATEGORIE_FOLOSINTA", "HAS_BUILDING", ]; const [localTotal, localNoGeom, enrichedFeatures, lastSyncRun] = await Promise.all([ prisma.gisFeature.count({ where: { layerId: "TERENURI_ACTIVE", siruta }, }), prisma.gisFeature.count({ where: { layerId: "TERENURI_ACTIVE", siruta, geometrySource: "NO_GEOMETRY", }, }), prisma.gisFeature.findMany({ where: { layerId: "TERENURI_ACTIVE", siruta, enrichedAt: { not: null }, }, select: { enrichment: true }, }), prisma.gisSyncRun.findFirst({ where: { siruta, layerId: "TERENURI_ACTIVE", status: "done" }, orderBy: { completedAt: "desc" }, select: { completedAt: true }, }), ]); const localEnriched = enrichedFeatures.length; let localEnrichedComplete = 0; for (const f of enrichedFeatures) { const e = f.enrichment as Record | null; if ( e && ENRICHMENT_REQUIRED_KEYS.every((k) => k in e && e[k] !== undefined) ) { localEnrichedComplete++; } } const localWithGeom = localTotal - localNoGeom; const syncFresh = lastSyncRun?.completedAt ? Date.now() - lastSyncRun.completedAt.getTime() < 168 * 60 * 60 * 1000 : false; // withGeometry = immovables that MATCHED a GIS feature (always adds up) const matchedCount = allImmovables.length - noGeomItems.length; console.log( `[no-geom-scan] Match quality: ${matchedCount} total (${matchedByRef} by cadRef, ${matchedById} by immId)` + ` | GIS layer: ${remoteFeatures.length} features | Immovables: ${allImmovables.length}` + ` | Unmatched GIS: ${remoteFeatures.length - matchedCount}`, ); // Quality analysis of no-geom items let qWithCadRef = 0; let qWithPaperCad = 0; let qWithPaperLb = 0; let qWithLandbook = 0; let qWithArea = 0; let qWithActiveStatus = 0; let qUseful = 0; let qEmpty = 0; for (const item of noGeomItems) { const hasCad = !!item.identifierDetails?.trim(); const hasPaperCad = !!item.paperCadNo?.trim(); const hasPaperLb = !!item.paperLbNo?.trim(); const hasArea = (item.measuredArea != null && item.measuredArea > 0) || (item.legalArea != null && item.legalArea > 0); const isActive = item.status === 1; const hasLb = item.hasLandbook === 1; if (hasCad) qWithCadRef++; if (hasPaperCad) qWithPaperCad++; if (hasPaperLb) qWithPaperLb++; if (hasLb) qWithLandbook++; if (hasArea) qWithArea++; if (isActive) qWithActiveStatus++; // "Useful" = ACTIVE + HAS_LANDBOOK (imobil electronic) + has identification OR area // Matches the import quality gate — only IE items are worth importing const hasIdentification = hasCad || hasPaperLb || hasPaperCad; if (isActive && hasLb && (hasIdentification || hasArea)) qUseful++; else qEmpty++; } return { totalImmovables: allImmovables.length, withGeometry: matchedCount, remoteGisCount: remoteFeatures.length, remoteCladiriCount, noGeomCount: noGeomItems.length, matchedByRef, matchedById, qualityBreakdown: { withCadRef: qWithCadRef, withPaperCad: qWithPaperCad, withPaperLb: qWithPaperLb, withLandbook: qWithLandbook, withArea: qWithArea, withActiveStatus: qWithActiveStatus, useful: qUseful, empty: qEmpty, }, samples: noGeomItems.slice(0, 20), localDbTotal: localTotal, localDbWithGeom: localWithGeom, localDbNoGeom: localNoGeom, localDbEnriched: localEnriched, localDbEnrichedComplete: localEnrichedComplete, localSyncFresh: syncFresh, scannedAt: new Date().toISOString(), }; } /** * Import: store no-geometry immovables as GisFeature records. * * Uses negative immovablePk as objectId to avoid collision with * real OBJECTID values from the GIS layer (always positive). */ export async function syncNoGeometryParcels( client: EterraClient, siruta: string, options?: { onProgress?: (done: number, total: number, phase: string) => void; workspacePk?: number | null; }, ): Promise { try { // 0. Resolve workspace const wsPk = await resolveWorkspacePk(client, siruta, options?.workspacePk); if (!wsPk) { return { imported: 0, skipped: 0, cleaned: 0, errors: 0, status: "error", error: `Nu s-a putut determina workspace-ul pentru SIRUTA ${siruta}`, }; } // 1. Fetch all immovables options?.onProgress?.(0, 1, "Descărcare listă imobile (fără geometrie)"); const allImmovables = await fetchAllImmovables(client, siruta, wsPk); // 2. Cleanup: remove stale/orphan no-geom records from DB // Build a set of valid immovablePks from the fresh immovable list. // Any NO_GEOMETRY record whose immovablePk is NOT in the fresh list // (or whose immovable is inactive/empty) gets deleted. const validImmPks = new Set(); for (const item of allImmovables) { const pk = Number(item.immovablePk ?? 0); if (pk > 0) { const status = typeof item.status === "number" ? item.status : 1; const cadRef = (item.identifierDetails ?? "").toString().trim(); const hasPaperLb = !!(item.paperLbNo ?? "").toString().trim(); const hasPaperCad = !!(item.paperCadNo ?? "").toString().trim(); const hasLandbook = typeof item.hasLandbook === "number" ? item.hasLandbook : 0; const hasArea = (typeof item.measuredArea === "number" && item.measuredArea > 0) || (typeof item.legalArea === "number" && item.legalArea > 0); const hasIdentification = !!cadRef || hasPaperLb || hasPaperCad; // Only keep items that pass the quality gate (active + hasLandbook + identification/area) if ( status === 1 && hasLandbook === 1 && (hasIdentification || hasArea) ) { validImmPks.add(pk); } } } // Find stale no-geom records: objectId < 0 means no-geom (objectId = -immPk) const existingNoGeom = await prisma.gisFeature.findMany({ where: { layerId: "TERENURI_ACTIVE", siruta, geometrySource: "NO_GEOMETRY", }, select: { id: true, objectId: true }, }); const staleIds: string[] = []; for (const f of existingNoGeom) { const immPk = -f.objectId; // objectId = -immovablePk for no-geom if (!validImmPks.has(immPk)) { staleIds.push(f.id); } } if (staleIds.length > 0) { const BATCH = 30_000; for (let i = 0; i < staleIds.length; i += BATCH) { await prisma.gisFeature.deleteMany({ where: { id: { in: staleIds.slice(i, i + BATCH) } }, }); } console.log( `[no-geom-sync] Cleanup: removed ${staleIds.length} stale/invalid no-geom records`, ); } // 3. Get existing features from DB (after cleanup) const existingFeatures = await prisma.gisFeature.findMany({ where: { layerId: "TERENURI_ACTIVE", siruta }, select: { cadastralRef: true, objectId: true }, }); const existingCadRefs = new Set(); const existingObjIds = new Set(); for (const f of existingFeatures) { if (f.cadastralRef) existingCadRefs.add(normalizeCadRef(f.cadastralRef)); existingObjIds.add(f.objectId); } // 4. Filter: not yet in DB + quality gate // Quality: must be ACTIVE (status=1) AND hasLandbook=1 (IE) AND have identification OR area. // Items without landbook are not electronic immovables — no CF data to extract. let filteredOut = 0; const candidates = allImmovables.filter((item) => { const cadRef = normalizeCadRef(item.identifierDetails ?? ""); const immPk = Number(item.immovablePk ?? 0); // Already in DB? → skip (not counted as filtered) if (cadRef && existingCadRefs.has(cadRef)) return false; if (immPk > 0 && existingObjIds.has(-immPk)) return false; // Quality gate 1: must be active (status=1) const status = typeof item.status === "number" ? item.status : 1; if (status !== 1) { filteredOut++; return false; } // Quality gate 2: must be an electronic immovable (hasLandbook=1) const hasLandbook = typeof item.hasLandbook === "number" ? item.hasLandbook : 0; if (hasLandbook !== 1) { filteredOut++; return false; } // Quality gate 3: must have identification OR area const hasCadRef = !!cadRef; const hasPaperLb = !!(item.paperLbNo ?? "").toString().trim(); const hasPaperCad = !!(item.paperCadNo ?? "").toString().trim(); const hasIdentification = hasCadRef || hasPaperLb || hasPaperCad; // Area: measuredArea and legalArea are the only area fields on immovable/list const hasArea = (typeof item.measuredArea === "number" && item.measuredArea > 0) || (typeof item.legalArea === "number" && item.legalArea > 0); if (!hasIdentification && !hasArea) { filteredOut++; return false; } return true; }); if (candidates.length === 0) { return { imported: 0, skipped: filteredOut, cleaned: staleIds.length, errors: 0, status: "done", }; } // 4. Import candidates in batches with retry let imported = 0; let skipped = 0; let errors = 0; const total = candidates.length; const BATCH_SIZE = 50; const MAX_RETRIES = 3; for ( let batchStart = 0; batchStart < candidates.length; batchStart += BATCH_SIZE ) { const batch = candidates.slice(batchStart, batchStart + BATCH_SIZE); const ops: Array> = []; for (const item of batch) { const immPk = Number(item.immovablePk ?? 0); if (immPk <= 0) { skipped++; continue; } const cadRef = String(item.identifierDetails ?? "").trim(); // Extract area — on immovable/list, the real fields are measuredArea and legalArea const areaValue = (typeof item.measuredArea === "number" && item.measuredArea > 0 ? item.measuredArea : null) ?? (typeof item.legalArea === "number" && item.legalArea > 0 ? item.legalArea : null); const attributes: Record = { OBJECTID: -immPk, IMMOVABLE_ID: immPk, WORKSPACE_ID: item.workspace?.nomenPk ?? wsPk, APPLICATION_ID: item.applicationId ?? null, NATIONAL_CADASTRAL_REFERENCE: cadRef, AREA_VALUE: areaValue, IS_ACTIVE: item.status === 1 ? 1 : 0, ADMIN_UNIT_ID: Number(siruta), PAPER_CAD_NO: item.paperCadNo ?? null, PAPER_LB_NO: item.paperLbNo ?? null, HAS_LANDBOOK: item.hasLandbook ?? null, TOP_NO: item.topNo ?? null, IMMOVABLE_TYPE: item.immovableType ?? "P", MEASURED_AREA: item.measuredArea ?? null, LEGAL_AREA: item.legalArea ?? null, NO_GEOMETRY_SOURCE: "ETERRA_IMMOVABLE_LIST", }; ops.push( prisma.gisFeature.upsert({ where: { layerId_objectId: { layerId: "TERENURI_ACTIVE", objectId: -immPk, }, }, create: { layerId: "TERENURI_ACTIVE", siruta, objectId: -immPk, cadastralRef: cadRef || null, areaValue, isActive: true, attributes: attributes as Prisma.InputJsonValue, geometry: Prisma.JsonNull, geometrySource: "NO_GEOMETRY", }, update: { cadastralRef: cadRef || null, areaValue, attributes: attributes as Prisma.InputJsonValue, geometrySource: "NO_GEOMETRY", updatedAt: new Date(), }, }), ); } // Execute batch with retry if (ops.length > 0) { let attempt = 0; while (attempt < MAX_RETRIES) { try { await prisma.$transaction(ops); imported += ops.length; break; } catch (err) { attempt++; if (attempt >= MAX_RETRIES) { // Fall back to individual upserts for this batch for (const op of ops) { try { await op; imported++; } catch { errors++; } } } else { // Wait before retry (exponential backoff) await new Promise((r) => setTimeout(r, 500 * attempt)); } } } } const done = Math.min(batchStart + BATCH_SIZE, total); options?.onProgress?.(done, total, "Import parcele fără geometrie"); } return { imported, skipped: skipped + filteredOut, cleaned: staleIds.length, errors, status: "done", }; } catch (error) { const msg = error instanceof Error ? error.message : "Unknown error"; return { imported: 0, skipped: 0, cleaned: 0, errors: 0, status: "error", error: msg, }; } } /** * Fetch all immovables from the eTerra immovable list for a UAT. * Paginated — fetches all pages. */ async function fetchAllImmovables( client: EterraClient, siruta: string, workspaceId: number, onProgress?: (page: number, totalPages: number) => void, ): Promise { const all: any[] = []; let page = 0; let totalPages = 1; let includeInscrisCF = true; while (page < totalPages) { const response = await client.fetchImmovableListByAdminUnit( workspaceId, siruta, page, 200, includeInscrisCF, ); // Retry without CF filter if first page is empty if (page === 0 && !(response?.content ?? []).length && includeInscrisCF) { includeInscrisCF = false; page = 0; totalPages = 1; continue; } totalPages = typeof response?.totalPages === "number" ? response.totalPages : totalPages; const content = response?.content ?? []; all.push(...content); page++; if (onProgress) { onProgress(page, totalPages); } } return all; }