diff --git a/prisma/postgis-setup.sql b/prisma/postgis-setup.sql index a33bf12..1553627 100644 --- a/prisma/postgis-setup.sql +++ b/prisma/postgis-setup.sql @@ -78,6 +78,8 @@ SELECT "areaValue" AS area_value, "isActive" AS is_active, attributes, + enrichment, + "enrichedAt" AS enriched_at, "projectId" AS project_id, "createdAt" AS created_at, "updatedAt" AS updated_at, diff --git a/prisma/schema.prisma b/prisma/schema.prisma index f9b82a7..400ad51 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -34,6 +34,8 @@ model GisFeature { geometry Json? // GeoJSON geometry (Polygon/MultiPolygon) // NOTE: native PostGIS column 'geom' is managed via SQL trigger (see prisma/postgis-setup.sql) // Prisma doesn't need to know about it — trigger auto-populates from geometry JSON + enrichment Json? // magic data: CF, owners, address, categories, etc. + enrichedAt DateTime? // when enrichment was last fetched syncRunId String? projectId String? // link to project tag createdAt DateTime @default(now()) diff --git a/src/app/api/eterra/export-bundle/route.ts b/src/app/api/eterra/export-bundle/route.ts index 6e8e8cf..e70b6e0 100644 --- a/src/app/api/eterra/export-bundle/route.ts +++ b/src/app/api/eterra/export-bundle/route.ts @@ -1,10 +1,27 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ +/** + * POST /api/eterra/export-bundle (v2 — sync-first) + * + * Flow: + * 1. Sync TERENURI_ACTIVE + CLADIRI_ACTIVE to local DB (skip if fresh) + * 2. Enrich parcels with CF/owner/address data (magic mode only, skip if enriched) + * 3. Build GPKG + CSV from local DB + * 4. Return ZIP + * + * Body: { siruta, jobId?, mode?: "base"|"magic", forceSync?: boolean } + */ import JSZip from "jszip"; +import { prisma } from "@/core/storage/prisma"; import { EterraClient } from "@/modules/parcel-sync/services/eterra-client"; -import { findLayerById } from "@/modules/parcel-sync/services/eterra-layers"; -import { esriToGeojson } from "@/modules/parcel-sync/services/esri-geojson"; import { getEpsg3844Wkt } from "@/modules/parcel-sync/services/reproject"; import { buildGpkg } from "@/modules/parcel-sync/services/gpkg-export"; +import { syncLayer } from "@/modules/parcel-sync/services/sync-service"; +import { + enrichFeatures, + getLayerFreshness, + isFresh, + type FeatureEnrichment, +} from "@/modules/parcel-sync/services/enrich-service"; import { clearProgress, setProgress, @@ -14,6 +31,7 @@ import { registerJob, unregisterJob, } from "@/modules/parcel-sync/services/session-store"; +import type { GeoJsonFeature } from "@/modules/parcel-sync/services/esri-geojson"; export const runtime = "nodejs"; export const dynamic = "force-dynamic"; @@ -24,10 +42,10 @@ type ExportBundleRequest = { siruta?: string | number; jobId?: string; mode?: "base" | "magic"; + forceSync?: boolean; }; const validate = (body: ExportBundleRequest) => { - // Priority: request body > session store > env vars const session = getSessionCredentials(); const username = String( body.username || session?.username || process.env.ETERRA_USERNAME || "", @@ -38,58 +56,29 @@ const validate = (body: ExportBundleRequest) => { const siruta = String(body.siruta ?? "").trim(); const jobId = body.jobId ? String(body.jobId).trim() : undefined; const mode = body.mode === "magic" ? "magic" : "base"; + const forceSync = body.forceSync === true; if (!username) throw new Error("Email is required"); if (!password) throw new Error("Password is required"); if (!/^\d+$/.test(siruta)) throw new Error("SIRUTA must be numeric"); - return { username, password, siruta, jobId, mode }; + return { username, password, siruta, jobId, mode, forceSync }; }; -const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); - const scheduleClear = (jobId?: string) => { if (!jobId) return; setTimeout(() => clearProgress(jobId), 60_000); }; -const isRetryable = (error: unknown) => { - const err = error as { response?: { status?: number }; code?: string }; - const status = err?.response?.status ?? 0; - if ([429, 500, 502, 503, 504].includes(status)) return true; - return err?.code === "ECONNRESET" || err?.code === "ETIMEDOUT"; -}; - -const formatNumber = (value: number) => - Number.isFinite(value) ? value.toFixed(2).replace(/\.00$/, "") : ""; - -const normalizeId = (value: unknown) => { - if (value === null || value === undefined) return ""; - const text = String(value).trim(); - if (!text) return ""; - return text.replace(/\.0$/, ""); -}; - -const normalizeCadRef = (value: unknown) => - normalizeId(value).replace(/\s+/g, "").toUpperCase(); - -const baseCadRef = (value: unknown) => { - const ref = normalizeCadRef(value); - if (!ref) return ""; - return ref.includes("-") ? ref.split("-")[0]! : ref; -}; - -const makeWorkspaceKey = (workspaceId: unknown, immovableId: unknown) => { - const ws = normalizeId(workspaceId); - const im = normalizeId(immovableId); - if (!ws || !im) return ""; - return `${ws}:${im}`; +const csvEscape = (val: unknown) => { + const s = String(val ?? "").replace(/"/g, '""'); + return `"${s}"`; }; export async function POST(req: Request) { let jobId: string | undefined; let message: string | undefined; - let phase = "Initializare"; + let phase = "Inițializare"; let note: string | undefined; let status: "running" | "done" | "error" = "running"; let downloaded = 0; @@ -116,8 +105,7 @@ export async function POST(req: Request) { const updateOverall = (fraction = 0) => { const overall = completedWeight + currentWeight * fraction; - const clipped = Math.min(100, Math.max(0, overall)); - downloaded = Number(clipped.toFixed(1)); + downloaded = Number(Math.min(100, Math.max(0, overall)).toFixed(1)); total = 100; pushProgress(); }; @@ -172,312 +160,197 @@ export async function POST(req: Request) { if (jobId) registerJob(jobId); pushProgress(); - const terenuriLayer = findLayerById("TERENURI_ACTIVE"); - const cladiriLayer = findLayerById("CLADIRI_ACTIVE"); - if (!terenuriLayer || !cladiriLayer) - throw new Error("Missing layer configuration"); - const weights = validated.mode === "magic" - ? { - auth: 3, - count: 2, - terenuri: 23, - cladiri: 13, - detalii: 34, - gpkgT: 8, - gpkgC: 7, - gpkgM: 5, - zip: 5, - } - : { - auth: 3, - count: 2, - terenuri: 40, - cladiri: 22, - detalii: 0, - gpkgT: 10, - gpkgC: 10, - gpkgM: 0, - zip: 13, - }; + ? { sync: 40, enrich: 35, gpkg: 15, zip: 10 } + : { sync: 55, enrich: 0, gpkg: 30, zip: 15 }; - /* Auth */ - setPhaseState("Autentificare", weights.auth, 1); - const client = await EterraClient.create( - validated.username, - validated.password, - { timeoutMs: 120_000 }, - ); - updatePhaseProgress(1, 1); - finishPhase(); + /* ══════════════════════════════════════════════════════════ */ + /* Phase 1: Sync layers to local DB */ + /* ══════════════════════════════════════════════════════════ */ + setPhaseState("Verificare date locale", weights.sync, 2); - /* Count */ - const safeCount = async (layerId: "terenuri" | "cladiri") => { - try { - return await client.countLayer( - layerId === "terenuri" ? terenuriLayer : cladiriLayer, + const terenuriLayerId = "TERENURI_ACTIVE"; + const cladiriLayerId = "CLADIRI_ACTIVE"; + + const [terenuriStatus, cladiriStatus] = await Promise.all([ + getLayerFreshness(validated.siruta, terenuriLayerId), + getLayerFreshness(validated.siruta, cladiriLayerId), + ]); + + const terenuriNeedsSync = + validated.forceSync || + !isFresh(terenuriStatus.lastSynced) || + terenuriStatus.featureCount === 0; + const cladiriNeedsSync = + validated.forceSync || + !isFresh(cladiriStatus.lastSynced) || + cladiriStatus.featureCount === 0; + + if (terenuriNeedsSync || cladiriNeedsSync) { + if (terenuriNeedsSync) { + phase = "Sincronizare terenuri"; + note = + terenuriStatus.featureCount > 0 + ? "Re-sync (date expirate)" + : "Sync inițial"; + pushProgress(); + + await syncLayer( + validated.username, + validated.password, validated.siruta, + terenuriLayerId, + { forceFullSync: validated.forceSync }, ); - } catch (error) { - const msg = - error instanceof Error ? error.message : "Unexpected server error"; - if (msg.toLowerCase().includes("count unavailable")) return undefined; - throw error; } - }; + updatePhaseProgress(1, 2); - setPhaseState("Numarare", weights.count, 2); - const terenuriCount = await safeCount("terenuri"); - updatePhaseProgress(1, 2); - const cladiriCount = await safeCount("cladiri"); - updatePhaseProgress(2, 2); + if (cladiriNeedsSync) { + phase = "Sincronizare clădiri"; + note = + cladiriStatus.featureCount > 0 + ? "Re-sync (date expirate)" + : "Sync inițial"; + pushProgress(); + + await syncLayer( + validated.username, + validated.password, + validated.siruta, + cladiriLayerId, + { forceFullSync: validated.forceSync }, + ); + } + updatePhaseProgress(2, 2); + } else { + note = "Date proaspete în baza de date — skip sync"; + pushProgress(); + updatePhaseProgress(2, 2); + } finishPhase(); - const calcPageSize = (cnt?: number) => { - if (!cnt || cnt <= 0) return 1000; - if (cnt <= 200) return Math.max(50, Math.ceil(cnt / 2)); - return Math.min(1000, Math.max(200, Math.ceil(cnt / 8))); - }; + /* ══════════════════════════════════════════════════════════ */ + /* Phase 2: Enrich (magic mode only) */ + /* ══════════════════════════════════════════════════════════ */ + if (validated.mode === "magic") { + setPhaseState("Verificare îmbogățire", weights.enrich, 1); - /* Download terenuri */ - setPhaseState("Descarcare terenuri", weights.terenuri, terenuriCount); - const terenuriFeatures = await client.fetchAllLayer( - terenuriLayer, - validated.siruta, - { - total: terenuriCount, - pageSize: calcPageSize(terenuriCount), - delayMs: 250, - onProgress: (count, totalCount) => - updatePhaseProgress(count, totalCount), - }, - ); - finishPhase(); + const enrichStatus = await getLayerFreshness( + validated.siruta, + terenuriLayerId, + ); + const needsEnrich = + validated.forceSync || + enrichStatus.enrichedCount === 0 || + enrichStatus.enrichedCount < enrichStatus.featureCount; - /* Download cladiri */ - setPhaseState("Descarcare cladiri", weights.cladiri, cladiriCount); - const cladiriFeatures = await client.fetchAllLayer( - cladiriLayer, - validated.siruta, - { - total: cladiriCount, - pageSize: calcPageSize(cladiriCount), - delayMs: 250, - onProgress: (count, totalCount) => - updatePhaseProgress(count, totalCount), - }, - ); - finishPhase(); + if (needsEnrich) { + phase = "Îmbogățire parcele (CF, proprietari, adrese)"; + note = undefined; + pushProgress(); - const terenuriGeo = esriToGeojson(terenuriFeatures); - const cladiriGeo = esriToGeojson(cladiriFeatures); + const client = await EterraClient.create( + validated.username, + validated.password, + { timeoutMs: 120_000 }, + ); + + await enrichFeatures(client, validated.siruta, { + onProgress: (done, tot, ph) => { + phase = ph; + updatePhaseProgress(done, tot); + }, + }); + } else { + note = "Îmbogățire existentă — skip"; + pushProgress(); + } + updatePhaseProgress(1, 1); + finishPhase(); + } + + /* ══════════════════════════════════════════════════════════ */ + /* Phase 3: Build GPKGs from local DB */ + /* ══════════════════════════════════════════════════════════ */ + setPhaseState("Generare GPKG din baza de date", weights.gpkg, 3); const srsWkt = getEpsg3844Wkt(); - const terenuriFields = await client.getLayerFieldNames(terenuriLayer); - const cladiriFields = await client.getLayerFieldNames(cladiriLayer); - let terenuriGpkg: Buffer | null = null; - let cladiriGpkg: Buffer | null = null; + // Load features from DB + const dbTerenuri = await prisma.gisFeature.findMany({ + where: { layerId: terenuriLayerId, siruta: validated.siruta }, + select: { attributes: true, geometry: true, enrichment: true }, + }); + + const dbCladiri = await prisma.gisFeature.findMany({ + where: { layerId: cladiriLayerId, siruta: validated.siruta }, + select: { attributes: true, geometry: true }, + }); + + // Convert DB records to GeoJSON features + const toGeoFeatures = ( + records: { attributes: unknown; geometry: unknown }[], + ): GeoJsonFeature[] => + records + .filter((r) => r.geometry != null) + .map((r) => ({ + type: "Feature" as const, + geometry: r.geometry as GeoJsonFeature["geometry"], + properties: r.attributes as Record, + })); + + const terenuriGeoFeatures = toGeoFeatures(dbTerenuri); + const cladiriGeoFeatures = toGeoFeatures(dbCladiri); + + const terenuriFields = + terenuriGeoFeatures.length > 0 + ? Object.keys(terenuriGeoFeatures[0]!.properties) + : []; + const cladiriFields = + cladiriGeoFeatures.length > 0 + ? Object.keys(cladiriGeoFeatures[0]!.properties) + : []; + + // GPKG terenuri + const terenuriGpkg = await withHeartbeat(() => + buildGpkg({ + srsId: 3844, + srsWkt, + layers: [ + { + name: "TERENURI_ACTIVE", + fields: terenuriFields, + features: terenuriGeoFeatures, + }, + ], + }), + ); + updatePhaseProgress(1, 3); + + // GPKG cladiri + const cladiriGpkg = await withHeartbeat(() => + buildGpkg({ + srsId: 3844, + srsWkt, + layers: [ + { + name: "CLADIRI_ACTIVE", + fields: cladiriFields, + features: cladiriGeoFeatures, + }, + ], + }), + ); + updatePhaseProgress(2, 3); + + // Magic: GPKG with enrichment + CSV let magicGpkg: Buffer | null = null; let csvContent: string | null = null; let hasBuildingCount = 0; let legalBuildingCount = 0; if (validated.mode === "magic") { - /* ── Magic mode: enrich parcels ─────────────────────────── */ - setPhaseState("Detalii parcele", weights.detalii, terenuriCount); - const immAppsCache = new Map(); - const folCache = new Map(); - - let lastRequest = 0; - const minInterval = 250; - const throttled = async (fn: () => Promise) => { - let attempt = 0; - while (true) { - const now = Date.now(); - const wait = Math.max(0, lastRequest + minInterval - now); - if (wait > 0) { - note = `Throttling ${Math.ceil(wait)}ms`; - pushProgress(); - await sleep(wait); - } - try { - const result = await fn(); - lastRequest = Date.now(); - note = undefined; - pushProgress(); - return result; - } catch (error) { - if (!isRetryable(error) || attempt >= 2) throw error; - attempt += 1; - const backoff = Math.min(5000, 1000 * attempt); - note = `Backoff ${backoff}ms (retry ${attempt})`; - pushProgress(); - await sleep(backoff); - } - } - }; - - const pickApplication = (entries: any[], applicationId?: number) => { - if (!entries.length) return null; - if (applicationId) { - const match = entries.find( - (entry: any) => entry?.applicationId === applicationId, - ); - if (match) return match; - } - return ( - entries - .filter((entry: any) => entry?.dataCerere) - .sort( - (a: any, b: any) => (b.dataCerere ?? 0) - (a.dataCerere ?? 0), - )[0] ?? entries[0] - ); - }; - - const normalizeIntravilan = (values: string[]) => { - const normalized = values - .map((v) => - String(v ?? "") - .trim() - .toLowerCase(), - ) - .filter(Boolean); - const unique = new Set(normalized); - if (!unique.size) return "-"; - if (unique.size === 1) - return unique.has("da") ? "Da" : unique.has("nu") ? "Nu" : "Mixt"; - return "Mixt"; - }; - - const formatCategories = (entries: any[]) => { - const map = new Map(); - for (const entry of entries) { - const key = String(entry?.categorieFolosinta ?? "").trim(); - if (!key) continue; - const area = Number(entry?.suprafata ?? 0); - map.set( - key, - (map.get(key) ?? 0) + (Number.isFinite(area) ? area : 0), - ); - } - return Array.from(map.entries()) - .map(([k, a]) => `${k}:${formatNumber(a)}`) - .join("; "); - }; - - const formatAddress = (item?: any) => { - const address = item?.immovableAddresses?.[0]?.address ?? null; - if (!address) return "-"; - const parts: string[] = []; - if (address.addressDescription) parts.push(address.addressDescription); - if (address.street) parts.push(`Str. ${address.street}`); - if (address.buildingNo) parts.push(`Nr. ${address.buildingNo}`); - if (address.locality?.name) parts.push(address.locality.name); - return parts.length ? parts.join(", ") : "-"; - }; - - /* Building cross-ref map */ - const buildingMap = new Map(); - for (const feature of cladiriFeatures) { - const attrs = feature.attributes ?? {}; - const immovableId = attrs.IMMOVABLE_ID ?? attrs.IMOVABLE_ID ?? null; - const workspaceId = attrs.WORKSPACE_ID ?? null; - const baseRef = baseCadRef(attrs.NATIONAL_CADASTRAL_REFERENCE ?? ""); - const isLegal = - Number(attrs.IS_LEGAL ?? 0) === 1 || - String(attrs.IS_LEGAL ?? "").toLowerCase() === "true"; - const add = (key: string) => { - if (!key) return; - const existing = buildingMap.get(key) ?? { - has: false, - legal: false, - }; - existing.has = true; - if (isLegal) existing.legal = true; - buildingMap.set(key, existing); - }; - const immKey = normalizeId(immovableId); - const wKey = makeWorkspaceKey(workspaceId, immovableId); - if (immKey) add(immKey); - if (wKey) add(wKey); - if (baseRef) add(baseRef); - } - - /* Fetch immovable list */ - const immovableListById = new Map(); - const immovableListByCad = new Map(); - const docByImmovable = new Map(); - const ownersByLandbook = new Map>(); - - const addOwner = (landbook: string, name: string) => { - if (!landbook || !name) return; - const existing = ownersByLandbook.get(landbook) ?? new Set(); - existing.add(name); - ownersByLandbook.set(landbook, existing); - }; - - let listPage = 0; - let listTotalPages = 1; - let includeInscrisCF = true; - while (listPage < listTotalPages) { - const listResponse = await throttled(() => - client.fetchImmovableListByAdminUnit( - 65, - validated.siruta, - listPage, - 200, - includeInscrisCF, - ), - ); - if ( - listPage === 0 && - !(listResponse?.content ?? []).length && - includeInscrisCF - ) { - includeInscrisCF = false; - listPage = 0; - listTotalPages = 1; - continue; - } - listTotalPages = - typeof listResponse?.totalPages === "number" - ? listResponse.totalPages - : listTotalPages; - (listResponse?.content ?? []).forEach((item: any) => { - const idKey = normalizeId(item?.immovablePk); - if (idKey) immovableListById.set(idKey, item); - const cadKey = normalizeCadRef(item?.identifierDetails ?? ""); - if (cadKey) immovableListByCad.set(cadKey, item); - }); - listPage += 1; - } - - /* Fetch documentation data */ - const immovableIds = Array.from(immovableListById.keys()); - const docBatchSize = 50; - for (let i = 0; i < immovableIds.length; i += docBatchSize) { - const batch = immovableIds.slice(i, i + docBatchSize); - const docResponse = await throttled(() => - client.fetchDocumentationData(65, batch), - ); - (docResponse?.immovables ?? []).forEach((item: any) => { - const idKey = normalizeId(item?.immovablePk); - if (idKey) docByImmovable.set(idKey, item); - }); - (docResponse?.partTwoRegs ?? []).forEach((item: any) => { - if ( - String(item?.nodeType ?? "").toUpperCase() === "P" && - item?.landbookIE - ) { - const name = String(item?.nodeName ?? "").trim(); - if (name) addOwner(String(item.landbookIE), name); - } - }); - } - - /* Build CSV + detail map */ - const csvRows: string[] = []; + // Build CSV const headers = [ "OBJECTID", "IMMOVABLE_ID", @@ -498,202 +371,9 @@ export async function POST(req: Request) { "HAS_BUILDING", "BUILD_LEGAL", ]; - csvRows.push(headers.join(",")); + const csvRows: string[] = [headers.map(csvEscape).join(",")]; - const detailsByObjectId = new Map>(); - - for (let index = 0; index < terenuriFeatures.length; index += 1) { - const feature = terenuriFeatures[index]!; - const attrs = feature.attributes ?? {}; - const objectId = attrs.OBJECTID ?? ""; - const immovableId = attrs.IMMOVABLE_ID ?? ""; - const workspaceId = attrs.WORKSPACE_ID ?? ""; - const applicationId = (attrs.APPLICATION_ID as number) ?? null; - - let solicitant = "-"; - let intravilan = "-"; - let categorie = "-"; - let proprietari = "-"; - let nrCF = "-"; - let nrCFVechi = "-"; - let nrTopo = "-"; - let addressText = "-"; - - if (immovableId && workspaceId) { - const appKey = `${workspaceId}:${immovableId}`; - let apps = immAppsCache.get(appKey); - if (!apps) { - apps = await throttled(() => - client.fetchImmAppsByImmovable( - immovableId as string | number, - workspaceId as string | number, - ), - ); - immAppsCache.set(appKey, apps); - } - const chosen = pickApplication(apps, Number(applicationId ?? 0)); - const appId = - chosen?.applicationId ?? - (applicationId ? Number(applicationId) : null); - solicitant = chosen?.solicitant ?? chosen?.deponent ?? solicitant; - - if (appId) { - const folKey = `${workspaceId}:${immovableId}:${appId}`; - let fol = folCache.get(folKey); - if (!fol) { - fol = await throttled(() => - client.fetchParcelFolosinte( - workspaceId as string | number, - immovableId as string | number, - appId, - ), - ); - folCache.set(folKey, fol); - } - intravilan = normalizeIntravilan( - fol.map((item: any) => item?.intravilan ?? ""), - ); - categorie = formatCategories(fol); - } - } - - const cadRefRaw = (attrs.NATIONAL_CADASTRAL_REFERENCE ?? "") as string; - const cadRef = normalizeCadRef(cadRefRaw); - const immKey = normalizeId(immovableId); - const listItem = - (immKey ? immovableListById.get(immKey) : undefined) ?? - (cadRef ? immovableListByCad.get(cadRef) : undefined); - const docKey = listItem?.immovablePk - ? normalizeId(listItem.immovablePk) - : ""; - const docItem = docKey ? docByImmovable.get(docKey) : undefined; - const landbookIE = docItem?.landbookIE ?? ""; - const owners = - landbookIE && ownersByLandbook.get(String(landbookIE)) - ? Array.from(ownersByLandbook.get(String(landbookIE)) ?? []) - : []; - const ownersByCad = - cadRefRaw && ownersByLandbook.get(String(cadRefRaw)) - ? Array.from(ownersByLandbook.get(String(cadRefRaw)) ?? []) - : []; - proprietari = - Array.from(new Set([...owners, ...ownersByCad])).join("; ") || - proprietari; - - nrCF = - docItem?.landbookIE || - listItem?.paperLbNo || - listItem?.paperCadNo || - nrCF; - const nrCFVechiRaw = listItem?.paperLbNo || listItem?.paperCadNo || ""; - nrCFVechi = - docItem?.landbookIE && nrCFVechiRaw !== nrCF - ? nrCFVechiRaw - : nrCFVechi; - nrTopo = - listItem?.topNo || docItem?.topNo || listItem?.paperCadNo || nrTopo; - addressText = listItem ? formatAddress(listItem) : addressText; - - const parcelRef = baseCadRef(cadRefRaw); - const wKey = makeWorkspaceKey(workspaceId, immovableId); - const build = (immKey ? buildingMap.get(immKey) : undefined) ?? - (wKey ? buildingMap.get(wKey) : undefined) ?? - (parcelRef ? buildingMap.get(parcelRef) : undefined) ?? { - has: false, - legal: false, - }; - const hasBuilding = build.has ? 1 : 0; - const buildLegal = build.has ? (build.legal ? 1 : 0) : 0; - if (hasBuilding) hasBuildingCount += 1; - if (buildLegal === 1) legalBuildingCount += 1; - - const areaValue = - typeof attrs.AREA_VALUE === "number" ? attrs.AREA_VALUE : null; - const detailRecord = { - NR_CAD: cadRefRaw, - NR_CF: nrCF, - NR_CF_VECHI: nrCFVechi, - NR_TOPO: nrTopo, - ADRESA: addressText, - PROPRIETARI: proprietari, - SUPRAFATA_2D: areaValue !== null ? Number(areaValue.toFixed(2)) : "", - SUPRAFATA_R: areaValue !== null ? Math.round(areaValue) : "", - SOLICITANT: solicitant, - INTRAVILAN: intravilan, - CATEGORIE_FOLOSINTA: categorie, - HAS_BUILDING: hasBuilding, - BUILD_LEGAL: buildLegal, - }; - detailsByObjectId.set(String(objectId), detailRecord); - - const row = [ - objectId, - immovableId, - applicationId ?? "", - attrs.NATIONAL_CADASTRAL_REFERENCE ?? "", - cadRefRaw, - attrs.AREA_VALUE ?? "", - nrCF, - nrCFVechi, - nrTopo, - `"${String(addressText).replace(/"/g, '""')}"`, - `"${String(proprietari).replace(/"/g, '""')}"`, - areaValue !== null ? areaValue.toFixed(2) : "", - areaValue !== null ? Math.round(areaValue) : "", - `"${String(solicitant).replace(/"/g, '""')}"`, - intravilan, - `"${String(categorie).replace(/"/g, '""')}"`, - hasBuilding, - buildLegal, - ]; - csvRows.push(row.join(",")); - - if (index % 10 === 0) updatePhaseProgress(index + 1, terenuriCount); - } - - updatePhaseProgress(terenuriFeatures.length, terenuriCount); - finishPhase(); - - csvContent = csvRows.join("\n"); - - /* GPKG terenuri */ - setPhaseState("GPKG terenuri", weights.gpkgT, 1); - terenuriGpkg = await withHeartbeat(() => - buildGpkg({ - srsId: 3844, - srsWkt, - layers: [ - { - name: "TERENURI_ACTIVE", - fields: terenuriFields, - features: terenuriGeo.features, - }, - ], - }), - ); - updatePhaseProgress(1, 1); - finishPhase(); - - /* GPKG cladiri */ - setPhaseState("GPKG cladiri", weights.gpkgC, 1); - cladiriGpkg = await withHeartbeat(() => - buildGpkg({ - srsId: 3844, - srsWkt, - layers: [ - { - name: "CLADIRI_ACTIVE", - fields: cladiriFields, - features: cladiriGeo.features, - }, - ], - }), - ); - updatePhaseProgress(1, 1); - finishPhase(); - - /* GPKG magic */ - setPhaseState("GPKG magic", weights.gpkgM, 1); + const magicFeatures: GeoJsonFeature[] = []; const magicFields = Array.from( new Set([ ...terenuriFields, @@ -712,14 +392,51 @@ export async function POST(req: Request) { "BUILD_LEGAL", ]), ); - const magicFeatures = terenuriGeo.features.map((feature) => { - const objectId = String(feature.properties?.OBJECTID ?? ""); - const extra = detailsByObjectId.get(objectId) ?? {}; - return { - ...feature, - properties: { ...feature.properties, ...extra }, - }; - }); + + for (const record of dbTerenuri) { + const attrs = record.attributes as Record; + const enrichment = + (record.enrichment as FeatureEnrichment | null) ?? + ({} as Partial); + const geom = record.geometry as GeoJsonFeature["geometry"] | null; + + const e = enrichment as Partial; + if (Number(e.HAS_BUILDING ?? 0)) hasBuildingCount += 1; + if (Number(e.BUILD_LEGAL ?? 0)) legalBuildingCount += 1; + + const row = [ + attrs.OBJECTID ?? "", + attrs.IMMOVABLE_ID ?? "", + attrs.APPLICATION_ID ?? "", + attrs.NATIONAL_CADASTRAL_REFERENCE ?? "", + e.NR_CAD ?? "", + attrs.AREA_VALUE ?? "", + e.NR_CF ?? "", + e.NR_CF_VECHI ?? "", + e.NR_TOPO ?? "", + e.ADRESA ?? "", + e.PROPRIETARI ?? "", + e.SUPRAFATA_2D ?? "", + e.SUPRAFATA_R ?? "", + e.SOLICITANT ?? "", + e.INTRAVILAN ?? "", + e.CATEGORIE_FOLOSINTA ?? "", + e.HAS_BUILDING ?? 0, + e.BUILD_LEGAL ?? 0, + ]; + csvRows.push(row.map(csvEscape).join(",")); + + if (geom) { + magicFeatures.push({ + type: "Feature", + geometry: geom, + properties: { ...attrs, ...e }, + }); + } + } + + csvContent = csvRows.join("\n"); + magicGpkg = await withHeartbeat(() => buildGpkg({ srsId: 3844, @@ -733,65 +450,30 @@ export async function POST(req: Request) { ], }), ); - updatePhaseProgress(1, 1); - finishPhase(); - } else { - /* ── Base mode ──────────────────────────────────────────── */ - setPhaseState("GPKG terenuri", weights.gpkgT, 1); - terenuriGpkg = await withHeartbeat(() => - buildGpkg({ - srsId: 3844, - srsWkt, - layers: [ - { - name: "TERENURI_ACTIVE", - fields: terenuriFields, - features: terenuriGeo.features, - }, - ], - }), - ); - updatePhaseProgress(1, 1); - finishPhase(); - - setPhaseState("GPKG cladiri", weights.gpkgC, 1); - cladiriGpkg = await withHeartbeat(() => - buildGpkg({ - srsId: 3844, - srsWkt, - layers: [ - { - name: "CLADIRI_ACTIVE", - fields: cladiriFields, - features: cladiriGeo.features, - }, - ], - }), - ); - updatePhaseProgress(1, 1); - finishPhase(); } + updatePhaseProgress(3, 3); + finishPhase(); - /* ZIP */ + /* ══════════════════════════════════════════════════════════ */ + /* Phase 4: ZIP */ + /* ══════════════════════════════════════════════════════════ */ setPhaseState("Comprimare ZIP", weights.zip, 1); const zip = new JSZip(); - if (!terenuriGpkg || !cladiriGpkg) - throw new Error("Failed to build GeoPackage files"); zip.file("terenuri.gpkg", terenuriGpkg); zip.file("cladiri.gpkg", cladiriGpkg); const report: Record = { siruta: validated.siruta, generatedAt: new Date().toISOString(), - terenuri: { - count: terenuriFeatures.length, - expected: terenuriCount ?? null, - }, - cladiri: { - count: cladiriFeatures.length, - expected: cladiriCount ?? null, + source: "local-db (sync-first)", + terenuri: { count: terenuriGeoFeatures.length }, + cladiri: { count: cladiriGeoFeatures.length }, + syncSkipped: { + terenuri: !terenuriNeedsSync, + cladiri: !cladiriNeedsSync, }, }; + if (validated.mode === "magic" && magicGpkg && csvContent) { zip.file("terenuri_magic.gpkg", magicGpkg); zip.file("terenuri_complet.csv", csvContent); @@ -802,6 +484,7 @@ export async function POST(req: Request) { }; } zip.file("export_report.json", JSON.stringify(report, null, 2)); + const zipBuffer = await withHeartbeat(() => zip.generateAsync({ type: "nodebuffer", compression: "STORE" }), ); @@ -809,15 +492,10 @@ export async function POST(req: Request) { finishPhase(); /* Done */ - const terenuriLabel = - typeof terenuriCount === "number" - ? `${terenuriFeatures.length}/${terenuriCount}` - : `${terenuriFeatures.length}/?`; - const cladiriLabel = - typeof cladiriCount === "number" - ? `${cladiriFeatures.length}/${cladiriCount}` - : `${cladiriFeatures.length}/?`; - message = `Finalizat 100% · Terenuri ${terenuriLabel} · Cladiri ${cladiriLabel}`; + message = `Finalizat 100% · Terenuri ${terenuriGeoFeatures.length} · Clădiri ${cladiriGeoFeatures.length}`; + if (!terenuriNeedsSync && !cladiriNeedsSync) { + message += " (din cache local)"; + } status = "done"; phase = "Finalizat"; note = undefined; diff --git a/src/app/api/eterra/export-layer-gpkg/route.ts b/src/app/api/eterra/export-layer-gpkg/route.ts index c546980..ca57faf 100644 --- a/src/app/api/eterra/export-layer-gpkg/route.ts +++ b/src/app/api/eterra/export-layer-gpkg/route.ts @@ -1,9 +1,23 @@ -import { EterraClient } from "@/modules/parcel-sync/services/eterra-client"; +/** + * POST /api/eterra/export-layer-gpkg (v2 — sync-first) + * + * Flow: + * 1. Check local DB freshness for the requested layer + * 2. If stale/empty → sync from eTerra (stores in DB) + * 3. Build GPKG from local DB + * 4. Return GPKG + * + * Body: { username?, password?, siruta, layerId, jobId?, forceSync? } + */ +import { prisma } from "@/core/storage/prisma"; import { findLayerById } from "@/modules/parcel-sync/services/eterra-layers"; -import { esriToGeojson } from "@/modules/parcel-sync/services/esri-geojson"; import { getEpsg3844Wkt } from "@/modules/parcel-sync/services/reproject"; import { buildGpkg } from "@/modules/parcel-sync/services/gpkg-export"; -import { fetchUatGeometry } from "@/modules/parcel-sync/services/uat-geometry"; +import { syncLayer } from "@/modules/parcel-sync/services/sync-service"; +import { + getLayerFreshness, + isFresh, +} from "@/modules/parcel-sync/services/enrich-service"; import { clearProgress, setProgress, @@ -13,6 +27,7 @@ import { registerJob, unregisterJob, } from "@/modules/parcel-sync/services/session-store"; +import type { GeoJsonFeature } from "@/modules/parcel-sync/services/esri-geojson"; export const runtime = "nodejs"; export const dynamic = "force-dynamic"; @@ -23,10 +38,10 @@ type ExportLayerRequest = { siruta?: string | number; layerId?: string; jobId?: string; + forceSync?: boolean; }; const validate = (body: ExportLayerRequest) => { - // Priority: request body > session store > env vars const session = getSessionCredentials(); const username = String( body.username || session?.username || process.env.ETERRA_USERNAME || "", @@ -37,13 +52,14 @@ const validate = (body: ExportLayerRequest) => { const siruta = String(body.siruta ?? "").trim(); const layerId = String(body.layerId ?? "").trim(); const jobId = body.jobId ? String(body.jobId).trim() : undefined; + const forceSync = body.forceSync === true; if (!username) throw new Error("Email is required"); if (!password) throw new Error("Password is required"); if (!/^\d+$/.test(siruta)) throw new Error("SIRUTA must be numeric"); if (!layerId) throw new Error("Layer ID missing"); - return { username, password, siruta, layerId, jobId }; + return { username, password, siruta, layerId, jobId, forceSync }; }; const scheduleClear = (jobId?: string) => { @@ -54,7 +70,7 @@ const scheduleClear = (jobId?: string) => { export async function POST(req: Request) { let jobId: string | undefined; let message: string | undefined; - let phase = "Initializare"; + let phase = "Inițializare"; let note: string | undefined; let status: "running" | "done" | "error" = "running"; let downloaded = 0; @@ -139,121 +155,88 @@ export async function POST(req: Request) { const layer = findLayerById(validated.layerId); if (!layer) throw new Error("Layer not configured"); - const weights = { - auth: 5, - count: 5, - download: 70, - gpkg: 15, - finalize: 5, - }; + const weights = { sync: 60, gpkg: 30, finalize: 10 }; - /* Auth */ - setPhaseState("Autentificare", weights.auth, 1); - const client = await EterraClient.create( - validated.username, - validated.password, - { timeoutMs: 120_000 }, + /* ── Phase 1: Check freshness & sync if needed ── */ + setPhaseState("Verificare date locale", weights.sync, 1); + + const freshness = await getLayerFreshness( + validated.siruta, + validated.layerId, ); + const needsSync = + validated.forceSync || + !isFresh(freshness.lastSynced) || + freshness.featureCount === 0; + + let syncedFromCache = true; + + if (needsSync) { + syncedFromCache = false; + phase = `Sincronizare ${layer.name}`; + note = + freshness.featureCount > 0 + ? "Re-sync (date expirate)" + : "Sync inițial de la eTerra"; + pushProgress(); + + await syncLayer( + validated.username, + validated.password, + validated.siruta, + validated.layerId, + { forceFullSync: validated.forceSync }, + ); + } else { + note = "Date proaspete în baza de date — skip sync"; + pushProgress(); + } updatePhaseProgress(1, 1); finishPhase(); - /* Count */ - let geometry; - setPhaseState("Numarare", weights.count, 2); - let count: number | undefined; - try { - if (layer.spatialFilter) { - geometry = await fetchUatGeometry(client, validated.siruta); - count = await client.countLayerByGeometry(layer, geometry); - } else { - count = await client.countLayer(layer, validated.siruta); - } - } catch (error) { - const msg = error instanceof Error ? error.message : "Count error"; - if (!msg.toLowerCase().includes("count unavailable")) throw error; - } - updatePhaseProgress(2, 2); - finishPhase(); + /* ── Phase 2: Build GPKG from local DB ── */ + setPhaseState("Generare GPKG din baza de date", weights.gpkg, 1); - if (layer.spatialFilter && !geometry) { - geometry = await fetchUatGeometry(client, validated.siruta); + const features = await prisma.gisFeature.findMany({ + where: { layerId: validated.layerId, siruta: validated.siruta }, + select: { attributes: true, geometry: true }, + }); + + if (features.length === 0) { + throw new Error( + `Niciun feature în DB pentru ${layer.name} / SIRUTA ${validated.siruta}`, + ); } - const pageSize = - typeof count === "number" - ? Math.min(1000, Math.max(200, Math.ceil(count / 8))) - : 500; + const geoFeatures: GeoJsonFeature[] = features + .filter((f) => f.geometry != null) + .map((f) => ({ + type: "Feature" as const, + geometry: f.geometry as GeoJsonFeature["geometry"], + properties: f.attributes as Record, + })); - /* Download */ - setPhaseState("Descarcare", weights.download, count); - const features = layer.spatialFilter - ? await client.fetchAllLayerByGeometry(layer, geometry!, { - total: count, - pageSize, - delayMs: 250, - onProgress: (value, totalCount) => { - updatePhaseProgress( - value, - typeof totalCount === "number" ? totalCount : value + pageSize, - ); - }, - }) - : await client.fetchAllLayer(layer, validated.siruta, { - total: count, - pageSize, - delayMs: 250, - onProgress: (value, totalCount) => { - updatePhaseProgress( - value, - typeof totalCount === "number" ? totalCount : value + pageSize, - ); - }, - }); - updatePhaseProgress(features.length, count ?? features.length); - finishPhase(); + const fields = Object.keys(geoFeatures[0]?.properties ?? {}); - /* Fields */ - let fields: string[] = []; - try { - fields = await client.getLayerFieldNames(layer); - } catch (error) { - const msg = error instanceof Error ? error.message : ""; - if (!msg.toLowerCase().includes("returned no fields")) throw error; - } - if (!fields.length) { - fields = Object.keys(features[0]?.attributes ?? {}); - } - - /* GPKG */ - setPhaseState("GPKG", weights.gpkg, 1); - const geo = esriToGeojson(features); const gpkg = await withHeartbeat(() => buildGpkg({ srsId: 3844, srsWkt: getEpsg3844Wkt(), - layers: [ - { - name: layer.name, - fields, - features: geo.features, - }, - ], + layers: [{ name: layer.name, fields, features: geoFeatures }], }), ); updatePhaseProgress(1, 1); finishPhase(); - /* Finalize */ + /* ── Phase 3: Finalize ── */ setPhaseState("Finalizare", weights.finalize, 1); updatePhaseProgress(1, 1); finishPhase(); + const suffix = syncedFromCache ? " (din cache local)" : ""; status = "done"; phase = "Finalizat"; - message = - typeof count === "number" - ? `Finalizat 100% · ${features.length}/${count} elemente` - : `Finalizat 100% · ${features.length} elemente`; + message = `Finalizat 100% · ${geoFeatures.length} elemente${suffix}`; pushProgress(); scheduleClear(jobId); diff --git a/src/app/api/eterra/setup-postgis/route.ts b/src/app/api/eterra/setup-postgis/route.ts index 00a4f59..f8e86b3 100644 --- a/src/app/api/eterra/setup-postgis/route.ts +++ b/src/app/api/eterra/setup-postgis/route.ts @@ -93,6 +93,8 @@ export async function POST() { "areaValue" AS area_value, "isActive" AS is_active, attributes, + enrichment, + "enrichedAt" AS enriched_at, "projectId" AS project_id, "createdAt" AS created_at, "updatedAt" AS updated_at, diff --git a/src/modules/parcel-sync/components/parcel-sync-module.tsx b/src/modules/parcel-sync/components/parcel-sync-module.tsx index 5f0d793..8464b48 100644 --- a/src/modules/parcel-sync/components/parcel-sync-module.tsx +++ b/src/modules/parcel-sync/components/parcel-sync-module.tsx @@ -337,6 +337,7 @@ export function ParcelSyncModule() { const [syncingLayer, setSyncingLayer] = useState(null); const [syncProgress, setSyncProgress] = useState(""); const [exportingLocal, setExportingLocal] = useState(false); + const refreshSyncRef = useRef<(() => void) | null>(null); /* ── PostGIS setup ───────────────────────────────────────────── */ const [postgisRunning, setPostgisRunning] = useState(false); @@ -597,6 +598,8 @@ export function ParcelSyncModule() { pollingRef.current = null; } setExporting(false); + // Refresh sync status — data was synced to DB + refreshSyncRef.current?.(); }, [siruta, exporting, startPolling], ); @@ -699,6 +702,9 @@ export function ParcelSyncModule() { } }, [siruta]); + // Keep ref in sync so callbacks defined earlier can trigger refresh + refreshSyncRef.current = () => void fetchSyncStatus(); + // Auto-fetch sync status when siruta changes useEffect(() => { if (siruta && /^\d+$/.test(siruta)) { @@ -876,6 +882,8 @@ export function ParcelSyncModule() { pollingRef.current = null; } setDownloadingLayer(null); + // Refresh sync status — layer was synced to DB + refreshSyncRef.current?.(); }, [siruta, downloadingLayer, startPolling], ); @@ -1775,7 +1783,7 @@ export function ParcelSyncModule() { Sync - {/* Direct GPKG from eTerra */} + {/* GPKG (sync-first: syncs to DB if needed, then exports from DB) */} - {/* Export from local DB */} - {localCount > 0 && ( - - )} @@ -2033,7 +2031,7 @@ export function ParcelSyncModule() { Descarcă Terenuri și Clădiri
- GPKG — terenuri.gpkg + cladiri.gpkg + Sync + GPKG (din cache dacă e proaspăt)
@@ -2052,7 +2050,7 @@ export function ParcelSyncModule() {
Magic
- GPKG îmbogățit + CSV cu proprietari, CF, adresă + Sync + îmbogățire (CF, proprietari, adresă) + GPKG + CSV
diff --git a/src/modules/parcel-sync/services/enrich-service.ts b/src/modules/parcel-sync/services/enrich-service.ts new file mode 100644 index 0000000..f06ae52 --- /dev/null +++ b/src/modules/parcel-sync/services/enrich-service.ts @@ -0,0 +1,535 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +/** + * Enrich service — fetches CF/owner/address/building data from eTerra + * and stores it in GisFeature.enrichment JSON column. + * + * Called after sync to add "magic" data to parcels. + * Idempotent: re-running overwrites previous enrichment. + */ + +import { Prisma } from "@prisma/client"; +import { prisma } from "@/core/storage/prisma"; +import { EterraClient } from "./eterra-client"; +import { + setProgress, + clearProgress, + type SyncProgress, +} from "./progress-store"; +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +export type EnrichResult = { + siruta: string; + enrichedCount: number; + buildingCrossRefs: number; + status: "done" | "error"; + error?: string; +}; + +/* ── Helpers (extracted from export-bundle) ──────────────────── */ + +const formatNumber = (value: number) => + Number.isFinite(value) ? value.toFixed(2).replace(/\.00$/, "") : ""; + +const normalizeId = (value: unknown) => { + if (value === null || value === undefined) return ""; + const text = String(value).trim(); + if (!text) return ""; + return text.replace(/\.0$/, ""); +}; + +const normalizeCadRef = (value: unknown) => + normalizeId(value).replace(/\s+/g, "").toUpperCase(); + +const baseCadRef = (value: unknown) => { + const ref = normalizeCadRef(value); + if (!ref) return ""; + return ref.includes("-") ? ref.split("-")[0]! : ref; +}; + +const makeWorkspaceKey = (workspaceId: unknown, immovableId: unknown) => { + const ws = normalizeId(workspaceId); + const im = normalizeId(immovableId); + if (!ws || !im) return ""; + return `${ws}:${im}`; +}; + +const isRetryable = (error: unknown) => { + const err = error as { response?: { status?: number }; code?: string }; + const status = err?.response?.status ?? 0; + if ([429, 500, 502, 503, 504].includes(status)) return true; + return err?.code === "ECONNRESET" || err?.code === "ETIMEDOUT"; +}; + +const normalizeIntravilan = (values: string[]) => { + const normalized = values + .map((v) => + String(v ?? "") + .trim() + .toLowerCase(), + ) + .filter(Boolean); + const unique = new Set(normalized); + if (!unique.size) return "-"; + if (unique.size === 1) + return unique.has("da") ? "Da" : unique.has("nu") ? "Nu" : "Mixt"; + return "Mixt"; +}; + +const formatCategories = (entries: any[]) => { + const map = new Map(); + for (const entry of entries) { + const key = String(entry?.categorieFolosinta ?? "").trim(); + if (!key) continue; + const area = Number(entry?.suprafata ?? 0); + map.set(key, (map.get(key) ?? 0) + (Number.isFinite(area) ? area : 0)); + } + return Array.from(map.entries()) + .map(([k, a]) => `${k}:${formatNumber(a)}`) + .join("; "); +}; + +const formatAddress = (item?: any) => { + const address = item?.immovableAddresses?.[0]?.address ?? null; + if (!address) return "-"; + const parts: string[] = []; + if (address.addressDescription) parts.push(address.addressDescription); + if (address.street) parts.push(`Str. ${address.street}`); + if (address.buildingNo) parts.push(`Nr. ${address.buildingNo}`); + if (address.locality?.name) parts.push(address.locality.name); + return parts.length ? parts.join(", ") : "-"; +}; + +const pickApplication = (entries: any[], applicationId?: number) => { + if (!entries.length) return null; + if (applicationId) { + const match = entries.find( + (entry: any) => entry?.applicationId === applicationId, + ); + if (match) return match; + } + return ( + entries + .filter((entry: any) => entry?.dataCerere) + .sort((a: any, b: any) => (b.dataCerere ?? 0) - (a.dataCerere ?? 0))[0] ?? + entries[0] + ); +}; + +/** + * Enrichment data stored per-feature in the `enrichment` JSON column. + */ +export type FeatureEnrichment = { + NR_CAD: string; + NR_CF: string; + NR_CF_VECHI: string; + NR_TOPO: string; + ADRESA: string; + PROPRIETARI: string; + SUPRAFATA_2D: number | string; + SUPRAFATA_R: number | string; + SOLICITANT: string; + INTRAVILAN: string; + CATEGORIE_FOLOSINTA: string; + HAS_BUILDING: number; + BUILD_LEGAL: number; +}; + +/** + * Enrich all TERENURI_ACTIVE features for a given UAT. + * + * Reads features from DB, fetches extra data from eTerra (immovable list, + * documentation, owners, folosinte), cross-references with CLADIRI_ACTIVE, + * and stores the enrichment JSON on each GisFeature. + */ +export async function enrichFeatures( + client: EterraClient, + siruta: string, + options?: { + jobId?: string; + onProgress?: (done: number, total: number, phase: string) => void; + }, +): Promise { + const jobId = options?.jobId; + const push = (partial: Partial) => { + if (!jobId) return; + setProgress({ + jobId, + downloaded: 0, + status: "running", + ...partial, + } as SyncProgress); + }; + + try { + // Load terenuri and cladiri from DB + const terenuri = await prisma.gisFeature.findMany({ + where: { layerId: "TERENURI_ACTIVE", siruta }, + select: { + id: true, + objectId: true, + attributes: true, + cadastralRef: true, + }, + }); + + const cladiri = await prisma.gisFeature.findMany({ + where: { layerId: "CLADIRI_ACTIVE", siruta }, + select: { attributes: true }, + }); + + if (terenuri.length === 0) { + return { + siruta, + enrichedCount: 0, + buildingCrossRefs: 0, + status: "done", + }; + } + + push({ + phase: "Pregătire îmbogățire", + downloaded: 0, + total: terenuri.length, + }); + + // ── Throttled request helper ── + let lastRequest = 0; + const minInterval = 250; + const throttled = async (fn: () => Promise) => { + let attempt = 0; + while (true) { + const now = Date.now(); + const wait = Math.max(0, lastRequest + minInterval - now); + if (wait > 0) await sleep(wait); + try { + const result = await fn(); + lastRequest = Date.now(); + return result; + } catch (error) { + if (!isRetryable(error) || attempt >= 2) throw error; + attempt += 1; + const backoff = Math.min(5000, 1000 * attempt); + await sleep(backoff); + } + } + }; + + // ── Building cross-ref map (from local DB cladiri) ── + const buildingMap = new Map(); + for (const feature of cladiri) { + const attrs = feature.attributes as Record; + const immovableId = attrs.IMMOVABLE_ID ?? attrs.IMOVABLE_ID ?? null; + const workspaceId = attrs.WORKSPACE_ID ?? null; + const baseRef = baseCadRef(attrs.NATIONAL_CADASTRAL_REFERENCE ?? ""); + const isLegal = + Number(attrs.IS_LEGAL ?? 0) === 1 || + String(attrs.IS_LEGAL ?? "").toLowerCase() === "true"; + const add = (key: string) => { + if (!key) return; + const existing = buildingMap.get(key) ?? { has: false, legal: false }; + existing.has = true; + if (isLegal) existing.legal = true; + buildingMap.set(key, existing); + }; + const immKey = normalizeId(immovableId); + const wKey = makeWorkspaceKey(workspaceId, immovableId); + if (immKey) add(immKey); + if (wKey) add(wKey); + if (baseRef) add(baseRef); + } + + // ── Fetch immovable list from eTerra ── + push({ phase: "Descărcare listă imobile", downloaded: 0 }); + const immovableListById = new Map(); + const immovableListByCad = new Map(); + const ownersByLandbook = new Map>(); + + const addOwner = (landbook: string, name: string) => { + if (!landbook || !name) return; + const existing = ownersByLandbook.get(landbook) ?? new Set(); + existing.add(name); + ownersByLandbook.set(landbook, existing); + }; + + let listPage = 0; + let listTotalPages = 1; + let includeInscrisCF = true; + while (listPage < listTotalPages) { + const listResponse = await throttled(() => + client.fetchImmovableListByAdminUnit( + 65, + siruta, + listPage, + 200, + includeInscrisCF, + ), + ); + if ( + listPage === 0 && + !(listResponse?.content ?? []).length && + includeInscrisCF + ) { + includeInscrisCF = false; + listPage = 0; + listTotalPages = 1; + continue; + } + listTotalPages = + typeof listResponse?.totalPages === "number" + ? listResponse.totalPages + : listTotalPages; + (listResponse?.content ?? []).forEach((item: any) => { + const idKey = normalizeId(item?.immovablePk); + if (idKey) immovableListById.set(idKey, item); + const cadKey = normalizeCadRef(item?.identifierDetails ?? ""); + if (cadKey) immovableListByCad.set(cadKey, item); + }); + listPage += 1; + } + + // ── Fetch documentation/owner data ── + push({ phase: "Descărcare documentații CF" }); + const docByImmovable = new Map(); + const immovableIds = Array.from(immovableListById.keys()); + const docBatchSize = 50; + for (let i = 0; i < immovableIds.length; i += docBatchSize) { + const batch = immovableIds.slice(i, i + docBatchSize); + const docResponse = await throttled(() => + client.fetchDocumentationData(65, batch), + ); + (docResponse?.immovables ?? []).forEach((item: any) => { + const idKey = normalizeId(item?.immovablePk); + if (idKey) docByImmovable.set(idKey, item); + }); + (docResponse?.partTwoRegs ?? []).forEach((item: any) => { + if ( + String(item?.nodeType ?? "").toUpperCase() === "P" && + item?.landbookIE + ) { + const name = String(item?.nodeName ?? "").trim(); + if (name) addOwner(String(item.landbookIE), name); + } + }); + } + + // ── Enrich each teren feature ── + push({ + phase: "Îmbogățire parcele", + downloaded: 0, + total: terenuri.length, + }); + const immAppsCache = new Map(); + const folCache = new Map(); + let enrichedCount = 0; + let buildingCrossRefs = 0; + const now = new Date(); + + for (let index = 0; index < terenuri.length; index += 1) { + const feature = terenuri[index]!; + const attrs = feature.attributes as Record; + const immovableId = attrs.IMMOVABLE_ID ?? ""; + const workspaceId = attrs.WORKSPACE_ID ?? ""; + const applicationId = (attrs.APPLICATION_ID as number) ?? null; + + let solicitant = "-"; + let intravilan = "-"; + let categorie = "-"; + let proprietari = "-"; + let nrCF = "-"; + let nrCFVechi = "-"; + let nrTopo = "-"; + let addressText = "-"; + + if (immovableId && workspaceId) { + const appKey = `${workspaceId}:${immovableId}`; + let apps = immAppsCache.get(appKey); + if (!apps) { + apps = await throttled(() => + client.fetchImmAppsByImmovable( + immovableId as string | number, + workspaceId as string | number, + ), + ); + immAppsCache.set(appKey, apps); + } + const chosen = pickApplication(apps, Number(applicationId ?? 0)); + const appId = + chosen?.applicationId ?? + (applicationId ? Number(applicationId) : null); + solicitant = chosen?.solicitant ?? chosen?.deponent ?? solicitant; + + if (appId) { + const folKey = `${workspaceId}:${immovableId}:${appId}`; + let fol = folCache.get(folKey); + if (!fol) { + fol = await throttled(() => + client.fetchParcelFolosinte( + workspaceId as string | number, + immovableId as string | number, + appId, + ), + ); + folCache.set(folKey, fol); + } + intravilan = normalizeIntravilan( + fol.map((item: any) => item?.intravilan ?? ""), + ); + categorie = formatCategories(fol); + } + } + + const cadRefRaw = (attrs.NATIONAL_CADASTRAL_REFERENCE ?? "") as string; + const cadRef = normalizeCadRef(cadRefRaw); + const immKey = normalizeId(immovableId); + const listItem = + (immKey ? immovableListById.get(immKey) : undefined) ?? + (cadRef ? immovableListByCad.get(cadRef) : undefined); + const docKey = listItem?.immovablePk + ? normalizeId(listItem.immovablePk) + : ""; + const docItem = docKey ? docByImmovable.get(docKey) : undefined; + const landbookIE = docItem?.landbookIE ?? ""; + const owners = + landbookIE && ownersByLandbook.get(String(landbookIE)) + ? Array.from(ownersByLandbook.get(String(landbookIE)) ?? []) + : []; + const ownersByCad = + cadRefRaw && ownersByLandbook.get(String(cadRefRaw)) + ? Array.from(ownersByLandbook.get(String(cadRefRaw)) ?? []) + : []; + proprietari = + Array.from(new Set([...owners, ...ownersByCad])).join("; ") || + proprietari; + + nrCF = + docItem?.landbookIE || + listItem?.paperLbNo || + listItem?.paperCadNo || + nrCF; + const nrCFVechiRaw = listItem?.paperLbNo || listItem?.paperCadNo || ""; + nrCFVechi = + docItem?.landbookIE && nrCFVechiRaw !== nrCF ? nrCFVechiRaw : nrCFVechi; + nrTopo = + listItem?.topNo || docItem?.topNo || listItem?.paperCadNo || nrTopo; + addressText = listItem ? formatAddress(listItem) : addressText; + + const parcelRef = baseCadRef(cadRefRaw); + const wKey = makeWorkspaceKey(workspaceId, immovableId); + const build = (immKey ? buildingMap.get(immKey) : undefined) ?? + (wKey ? buildingMap.get(wKey) : undefined) ?? + (parcelRef ? buildingMap.get(parcelRef) : undefined) ?? { + has: false, + legal: false, + }; + const hasBuilding = build.has ? 1 : 0; + const buildLegal = build.has ? (build.legal ? 1 : 0) : 0; + if (hasBuilding) buildingCrossRefs += 1; + + const areaValue = + typeof attrs.AREA_VALUE === "number" ? attrs.AREA_VALUE : null; + + const enrichment: FeatureEnrichment = { + NR_CAD: cadRefRaw, + NR_CF: nrCF, + NR_CF_VECHI: nrCFVechi, + NR_TOPO: nrTopo, + ADRESA: addressText, + PROPRIETARI: proprietari, + SUPRAFATA_2D: areaValue !== null ? Number(areaValue.toFixed(2)) : "", + SUPRAFATA_R: areaValue !== null ? Math.round(areaValue) : "", + SOLICITANT: solicitant, + INTRAVILAN: intravilan, + CATEGORIE_FOLOSINTA: categorie, + HAS_BUILDING: hasBuilding, + BUILD_LEGAL: buildLegal, + }; + + // Store enrichment in DB + await prisma.gisFeature.update({ + where: { id: feature.id }, + data: { + enrichment: enrichment as unknown as Prisma.InputJsonValue, + enrichedAt: now, + }, + }); + + enrichedCount += 1; + if (index % 10 === 0) { + push({ + phase: "Îmbogățire parcele", + downloaded: index + 1, + total: terenuri.length, + }); + options?.onProgress?.(index + 1, terenuri.length, "Îmbogățire parcele"); + } + } + + push({ + phase: "Îmbogățire completă", + status: "done", + downloaded: terenuri.length, + total: terenuri.length, + }); + if (jobId) setTimeout(() => clearProgress(jobId), 60_000); + + return { + siruta, + enrichedCount, + buildingCrossRefs, + status: "done", + }; + } catch (error) { + const msg = error instanceof Error ? error.message : "Unknown error"; + push({ phase: "Eroare îmbogățire", status: "error", message: msg }); + if (jobId) setTimeout(() => clearProgress(jobId), 60_000); + return { + siruta, + enrichedCount: 0, + buildingCrossRefs: 0, + status: "error", + error: msg, + }; + } +} + +/** + * Check data freshness for a UAT + layer. + * Returns the most recent sync run's completedAt, or null if never synced. + */ +export async function getLayerFreshness( + siruta: string, + layerId: string, +): Promise<{ + lastSynced: Date | null; + featureCount: number; + enrichedCount: number; +}> { + const lastRun = await prisma.gisSyncRun.findFirst({ + where: { siruta, layerId, status: "done" }, + orderBy: { completedAt: "desc" }, + select: { completedAt: true }, + }); + + const featureCount = await prisma.gisFeature.count({ + where: { siruta, layerId }, + }); + + const enrichedCount = await prisma.gisFeature.count({ + where: { siruta, layerId, enrichedAt: { not: null } }, + }); + + return { + lastSynced: lastRun?.completedAt ?? null, + featureCount, + enrichedCount, + }; +} + +/** + * Check if layer data is "fresh enough" (synced within maxAgeHours). + */ +export function isFresh(lastSynced: Date | null, maxAgeHours = 168): boolean { + if (!lastSynced) return false; + const ageMs = Date.now() - lastSynced.getTime(); + return ageMs < maxAgeHours * 60 * 60 * 1000; +}