diff --git a/src/app/api/eterra/uats/route.ts b/src/app/api/eterra/uats/route.ts index 381e9bc..16af464 100644 --- a/src/app/api/eterra/uats/route.ts +++ b/src/app/api/eterra/uats/route.ts @@ -12,18 +12,22 @@ export const dynamic = "force-dynamic"; /* Types */ /* ------------------------------------------------------------------ */ -type EnrichedUat = { +type UatResponse = { siruta: string; name: string; county: string; workspacePk: number; + /** Number of GIS features synced locally for this UAT */ + localFeatures: number; }; /* ------------------------------------------------------------------ */ /* Helpers */ /* ------------------------------------------------------------------ */ -function populateWorkspaceCache(uats: EnrichedUat[]) { +function populateWorkspaceCache( + uats: Array<{ siruta: string; workspacePk: number }>, +) { const wsGlobal = globalThis as { __eterraWorkspaceCache?: Map; }; @@ -37,24 +41,109 @@ function populateWorkspaceCache(uats: EnrichedUat[]) { } } +/** Remove diacritics and uppercase for fuzzy name matching */ +function normalizeName(s: string): string { + return s + .normalize("NFD") + .replace(/[\u0300-\u036f]/g, "") + .toUpperCase() + .trim(); +} + +/** Title-case: "SATU MARE" → "Satu Mare" */ +function titleCase(s: string): string { + return s + .toLowerCase() + .replace(/(?:^|\s)\S/g, (ch) => ch.toUpperCase()); +} + +/** + * Extract a name from an eTerra nomenclature entry. + * Tries multiple possible field names. + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function extractName(entry: any): string { + if (!entry || typeof entry !== "object") return ""; + for (const key of ["name", "nomenName", "label", "denumire", "NAME"]) { + const val = entry[key]; + if (typeof val === "string" && val.trim()) return val.trim(); + } + return ""; +} + +/** + * Extract a SIRUTA code from an eTerra nomenclature entry. + * Tries multiple possible field names (nomenPk ≠ SIRUTA, but code might be). + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function extractCode(entry: any): string { + if (!entry || typeof entry !== "object") return ""; + for (const key of [ + "code", + "sirutaCode", + "siruta", + "externalCode", + "cod", + "CODE", + ]) { + const val = entry[key]; + if (val != null) { + const s = String(val).trim(); + if (s && /^\d+$/.test(s)) return s; + } + } + return ""; +} + +/** + * Unwrap a potentially nested response (Spring Boot Page format). + * eTerra sometimes returns {content: [...]} instead of flat arrays. + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function unwrapArray(data: any): any[] { + if (Array.isArray(data)) return data; + if (data && typeof data === "object") { + if (Array.isArray(data.content)) return data.content; + if (Array.isArray(data.data)) return data.data; + if (Array.isArray(data.items)) return data.items; + if (Array.isArray(data.results)) return data.results; + } + return []; +} + /* ------------------------------------------------------------------ */ /* GET /api/eterra/uats */ /* */ /* Always serves from local PostgreSQL (GisUat table). */ +/* Includes local GIS feature counts per UAT for the UI indicator. */ /* No eTerra credentials needed — instant response. */ /* ------------------------------------------------------------------ */ export async function GET() { try { - const rows = await prisma.gisUat.findMany({ - orderBy: { name: "asc" }, - }); + // Fetch UATs and local feature counts in parallel + const [rows, featureCounts] = await Promise.all([ + prisma.gisUat.findMany({ orderBy: { name: "asc" } }), + prisma.gisFeature + .groupBy({ + by: ["siruta"], + _count: { id: true }, + }) + .then((groups) => { + const map = new Map(); + for (const g of groups) { + map.set(g.siruta, g._count.id); + } + return map; + }), + ]); - const uats: EnrichedUat[] = rows.map((r) => ({ + const uats: UatResponse[] = rows.map((r) => ({ siruta: r.siruta, name: r.name, county: r.county ?? "", workspacePk: r.workspacePk ?? 0, + localFeatures: featureCounts.get(r.siruta) ?? 0, })); // Populate in-memory workspace cache for search route @@ -163,27 +252,11 @@ export async function POST() { /* Phase 1: For UATs that already have workspacePk resolved, */ /* use fetchCounties() → countyMap[workspacePk] → instant update. */ /* Phase 2: For remaining UATs, enumerate counties → */ -/* fetchAdminUnitsByCounty() per county → match by name. */ +/* fetchAdminUnitsByCounty() per county → match by code or name. */ /* */ /* Requires active eTerra session. */ /* ------------------------------------------------------------------ */ -/** Remove diacritics and lowercase for fuzzy name matching */ -function normalizeName(s: string): string { - return s - .normalize("NFD") - .replace(/[\u0300-\u036f]/g, "") - .toUpperCase() - .trim(); -} - -/** Title-case: "SATU MARE" → "Satu Mare" */ -function titleCase(s: string): string { - return s - .toLowerCase() - .replace(/(?:^|\s)\S/g, (ch) => ch.toUpperCase()); -} - export async function PATCH() { try { // 1. Get eTerra credentials from session @@ -205,20 +278,33 @@ export async function PATCH() { const client = await EterraClient.create(username, password); // 2. Fetch all counties from eTerra nomenclature - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const counties: any[] = await client.fetchCounties(); + const rawCounties = await client.fetchCounties(); + const counties = unwrapArray(rawCounties); const countyMap = new Map(); // nomenPk → county name for (const c of counties) { const pk = Number(c?.nomenPk ?? 0); - const name = String(c?.name ?? "").trim(); + const name = extractName(c); if (pk > 0 && name) { countyMap.set(pk, titleCase(name)); } } if (countyMap.size === 0) { + // Log raw response for debugging + console.error( + "[uats-patch] fetchCounties returned 0 counties. Raw sample:", + JSON.stringify(rawCounties).slice(0, 500), + ); return NextResponse.json( - { error: "Nu s-au putut obține județele din eTerra." }, + { + error: "Nu s-au putut obține județele din eTerra.", + debug: { + rawType: typeof rawCounties, + isArray: Array.isArray(rawCounties), + length: Array.isArray(rawCounties) ? rawCounties.length : null, + sample: JSON.stringify(rawCounties).slice(0, 300), + }, + }, { status: 502 }, ); } @@ -231,7 +317,7 @@ export async function PATCH() { }); // Phase 1: instant fill for UATs that already have workspacePk - let phase1Updated = 0; + const phase1Ops: Array> = []; const needsCounty: Array<{ siruta: string; name: string }> = []; for (const uat of allUats) { @@ -240,26 +326,37 @@ export async function PATCH() { if (uat.workspacePk && uat.workspacePk > 0) { const county = countyMap.get(uat.workspacePk); if (county) { - await prisma.gisUat.update({ - where: { siruta: uat.siruta }, - data: { county }, - }); - phase1Updated++; + phase1Ops.push( + prisma.gisUat.update({ + where: { siruta: uat.siruta }, + data: { county }, + }), + ); continue; } } needsCounty.push({ siruta: uat.siruta, name: uat.name }); } + // Execute phase 1 in batches + let phase1Updated = 0; + for (let i = 0; i < phase1Ops.length; i += 100) { + const batch = phase1Ops.slice(i, i + 100); + await prisma.$transaction(batch); + phase1Updated += batch.length; + } + console.log( `[uats-patch] Phase 1: ${phase1Updated} updated via workspacePk. ` + `${needsCounty.length} remaining.`, ); - // Phase 2: enumerate UATs per county from nomenclature, match by name - // Build lookup: normalized name → list of SIRUTAs (for same-name UATs) + // Phase 2: enumerate UATs per county from nomenclature, match by code or name + // Build lookups const nameToSirutas = new Map(); + const sirutaSet = new Set(); for (const u of needsCounty) { + sirutaSet.add(u.siruta); const key = normalizeName(u.name); const arr = nameToSirutas.get(key); if (arr) arr.push(u.siruta); @@ -267,60 +364,87 @@ export async function PATCH() { } let phase2Updated = 0; + let codeMatches = 0; + let nameMatches = 0; const matchedSirutas = new Set(); + let loggedSample = false; for (const [countyPk, countyName] of countyMap) { - if (nameToSirutas.size === 0) break; // all matched + if (matchedSirutas.size >= needsCounty.length) break; try { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const uats: any[] = await client.fetchAdminUnitsByCounty(countyPk); + const rawUats = await client.fetchAdminUnitsByCounty(countyPk); + const uats = unwrapArray(rawUats); + + // Log first county's first UAT for debugging + if (!loggedSample && uats.length > 0) { + console.log( + `[uats-patch] Sample UAT from ${countyName}:`, + JSON.stringify(uats[0]).slice(0, 500), + ); + loggedSample = true; + } for (const uat of uats) { - const eterraName = String(uat?.name ?? "").trim(); + // Strategy A: match by code (might be SIRUTA) + const code = extractCode(uat); + if (code && sirutaSet.has(code) && !matchedSirutas.has(code)) { + matchedSirutas.add(code); + await prisma.gisUat.update({ + where: { siruta: code }, + data: { county: countyName, workspacePk: countyPk }, + }); + phase2Updated++; + codeMatches++; + continue; + } + + // Strategy B: match by normalized name + const eterraName = extractName(uat); if (!eterraName) continue; const key = normalizeName(eterraName); const sirutas = nameToSirutas.get(key); if (!sirutas || sirutas.length === 0) continue; - // Pick the first unmatched SIRUTA with this name const siruta = sirutas.find((s) => !matchedSirutas.has(s)); if (!siruta) continue; matchedSirutas.add(siruta); - await prisma.gisUat.update({ where: { siruta }, data: { county: countyName, workspacePk: countyPk }, }); phase2Updated++; + nameMatches++; - // If all SIRUTAs for this name matched, remove the key if (sirutas.every((s) => matchedSirutas.has(s))) { nameToSirutas.delete(key); } } } catch (err) { console.warn( - `[uats-patch] Failed to fetch UATs for county ${countyName}:`, + `[uats-patch] Failed to fetch UATs for county ${countyName} (pk=${countyPk}):`, err instanceof Error ? err.message : err, ); } } const totalUpdated = phase1Updated + phase2Updated; + const unmatched = needsCounty.length - phase2Updated; console.log( - `[uats-patch] Phase 2: ${phase2Updated} updated via name match. ` + - `Total: ${totalUpdated}. Unmatched: ${needsCounty.length - phase2Updated}.`, + `[uats-patch] Phase 2: ${phase2Updated} (${codeMatches} by code, ${nameMatches} by name). ` + + `Total: ${totalUpdated}. Unmatched: ${unmatched}.`, ); return NextResponse.json({ updated: totalUpdated, phase1: phase1Updated, phase2: phase2Updated, + codeMatches, + nameMatches, totalCounties: countyMap.size, - unmatched: needsCounty.length - phase2Updated, + unmatched, }); } catch (error) { const message = error instanceof Error ? error.message : "Eroare server"; diff --git a/src/modules/parcel-sync/SKILLS.md b/src/modules/parcel-sync/SKILLS.md new file mode 100644 index 0000000..e53f1ca --- /dev/null +++ b/src/modules/parcel-sync/SKILLS.md @@ -0,0 +1,219 @@ +# ParcelSync / eTerra GIS — Skills & Context + +> This file is the **single source of truth** for any AI assistant working on +> the ParcelSync module, eTerra integration, or GIS features. Read this FIRST +> before touching any code in this area. + +--- + +## Module Overview + +ParcelSync connects to Romania's **eTerra / ANCPI** national cadastral system +to fetch, store, enrich, and export parcel (land + building) data. All GIS data +lives in a local **PostGIS** database via Prisma ORM. + +**Key paths:** +| Area | Path | +|------|------| +| Module root | `src/modules/parcel-sync/` | +| Components | `src/modules/parcel-sync/components/parcel-sync-module.tsx` (~4100 lines, single file) | +| Services | `src/modules/parcel-sync/services/` | +| API routes | `src/app/api/eterra/` (20+ route files) | +| Types | `src/modules/parcel-sync/types.ts` | +| DB models | `prisma/schema.prisma` → `GisFeature`, `GisSyncRun`, `GisUat` | +| Layer catalog | `services/eterra-layers.ts` — 23 layers, 4 categories | +| Static UAT list | `public/uat.json` (~3000 entries, SIRUTA + name only) | + +--- + +## Architecture + +### eTerra API Client (`services/eterra-client.ts`, ~1000 lines) + +- **Auth**: form-post login → JSESSIONID cookie jar (axios-cookiejar-support) +- **Session cache**: keyed by credential hash, 9-minute TTL, auto-relogin on 401/redirect +- **Two API surfaces**: + - **ArcGIS REST** (`/api/map/rest/{endpoint}/layer/{name}/query`) — spatial queries, geometry + - **Application API** (`/api/immovable/...`, `/api/immApps/...`, `/api/adm/nomen/...`) — business data +- **Pagination**: `maxRecordCount=1000`, fallback page sizes (500, 200) +- **Key methods**: + - `listLayer()` / `fetchAllLayer()` / `fetchAllLayerByWhere()` — ArcGIS layer queries + - `countLayer()` — feature count + - `getLayerFieldNames()` — discover available fields + - `searchImmovableByIdentifier()` — parcel search by cadastral number + - `fetchDocumentationData()` — CF, owners (partTwoRegs tree) + - `fetchImmovableParcelDetails()` — area, intravilan, use categories + - `fetchImmovableListByAdminUnit()` — all immovables in a UAT (paginated, Spring Boot Page format) + - `fetchCounties()` — all counties from nomenclature (`/api/adm/nomen/COUNTY/list`) + - `fetchAdminUnitsByCounty(nomenPk)` — UATs per county (`/api/adm/nomen/ADMINISTRATIVEUNIT/filterByParent/{pk}`) + - `fetchNomenByPk(pk)` — single nomenclature entry + +### Session Management (`services/session-store.ts`) + +- Global singleton (one shared session for the whole app) +- Stores username + password in memory only +- Tracks active job IDs (blocks disconnect while jobs run) +- `getSessionCredentials()` — used by all API routes + +### Health Check (`services/eterra-health.ts`) + +- Pings `eterra.ancpi.ro` every 3 minutes +- Detects maintenance by keywords in HTML response +- Blocks login attempts when down +- UI shows amber "Mentenanță" state + +--- + +## Database Models + +### GisUat (UAT registry) +``` +siruta String @id — SIRUTA code (NOT eTerra nomenPk!) +name String — UAT name (from uat.json) +county String? — County name (populated via PATCH /api/eterra/uats) +workspacePk Int? — eTerra county workspace ID (= county nomenPk) +``` + +**CRITICAL**: `eTerra nomenPk ≠ SIRUTA`. The nomenclature API uses `nomenPk` internally, +but our DB keys on SIRUTA codes from `uat.json`. Matching is done by name or by +resolving WORKSPACE_ID from ArcGIS layer features. + +### GisFeature (parcel/building data) +``` +id String @id @default(uuid()) +layerId String — e.g. "TERENURI_ACTIVE", "CLADIRI_ACTIVE" +siruta String — UAT this feature belongs to +objectId Int — eTerra OBJECTID (negative = no-geometry record) +cadastralRef String? — National cadastral reference +areaValue Float? — Area in sqm +isActive Boolean +attributes Json — All ArcGIS attributes (WORKSPACE_ID, ADMIN_UNIT_ID, etc.) +geometry Json? — EsriGeometry { rings: number[][][] } +geometrySource String? — "NO_GEOMETRY" for parcels without GIS geometry +enrichment Json? — Scraped data: NR_CAD, NR_CF, PROPRIETARI, INTRAVILAN, etc. +enrichedAt DateTime? — When enrichment was fetched +@@unique([layerId, objectId]) +``` + +### GisSyncRun (sync history) +``` +siruta, layerId, status, totalRemote, totalLocal, newFeatures, removedFeatures, etc. +``` + +--- + +## Key Flows + +### 1. UAT Selection & Autocomplete +- On mount: `GET /api/eterra/uats` → loads all UATs from DB (fast, no eTerra) +- If DB empty: seeds via `POST /api/eterra/uats` from `public/uat.json` +- Client-side filter: diacritics-insensitive, matches SIRUTA prefix or name/county substring +- Dropdown shows: **name – county** + local feature count badge + SIRUTA code +- On select: sets `siruta` + `workspacePk` state + +### 2. County Population (`PATCH /api/eterra/uats`) +- **Phase 1**: UATs with `workspacePk` already → instant lookup via `fetchCounties()` map +- **Phase 2**: Enumerate all counties → `fetchAdminUnitsByCounty()` per county → match by code then name +- Auto-triggered on first eTerra connection when >50% UATs lack county +- ~43 API calls total (1 counties + 42 per-county) +- Name matching: NFD normalize, strip diacritics, uppercase + +### 3. Workspace Resolution +- `workspacePk` = eTerra county identifier (= county nomenPk from nomenclature) +- Resolved lazily: query 1 feature from TERENURI_ACTIVE → read WORKSPACE_ID attribute +- Cached in-memory + persisted to GisUat.workspacePk +- Resolution chain: explicit param → GisUat DB → ArcGIS layer query + +### 4. Layer Sync (`services/sync-service.ts`) +- Background job: fetch all features from an ArcGIS layer for a SIRUTA +- Uses UAT boundary geometry (from LIMITE_UAT) as spatial filter for dynamic layers +- Stores features in GisFeature with full geometry +- Progress tracking via `progress-store.ts` (2s polling) + +### 5. Enrichment Pipeline (`services/enrich-service.ts`) +- Per-feature: hits eTerra `/api/immovable/list` to extract detailed data +- Stored in `GisFeature.enrichment` JSONB: + - `NR_CAD`, `NR_CF`, `NR_CF_VECHI`, `NR_TOPO` + - `PROPRIETARI`, `PROPRIETARI_VECHI` (semicolon-separated) + - `SUPRAFATA`, `INTRAVILAN`, `CATEGORIE_FOLOSINTA` + - `HAS_BUILDING`, `BUILD_LEGAL` + - `ADRESA`, `SOLICITANT` + +### 6. Parcel Search (`POST /api/eterra/search`) +- Input: SIRUTA + cadastral number(s) +- Resolves workspace, then calls: + 1. `searchImmovableByIdentifier()` — find immovable + 2. `fetchDocumentationData()` — CF, owners (active/cancelled via nodeStatus tree) + 3. `fetchImmovableParcelDetails()` — area, intravilan, use categories + 4. `fetchImmAppsByImmovable()` → `fetchParcelFolosinte()` — application data +- Returns `ParcelDetail[]` with full property info + +### 7. No-Geometry Sync (`services/no-geom-sync.ts`) +- Finds eTerra immovables that have NO GIS geometry (no polygon in TERENURI_ACTIVE) +- Cross-references immovable list against remote ArcGIS features +- Stores as GisFeature with `geometry=null`, `objectId=-immovablePk` +- Quality gate: must be active + hasLandbook + has identification/area + +### 8. Export +- **GeoPackage** (`services/gpkg-export.ts`): EPSG:3844 → EPSG:4326 reprojection +- **Export bundle** (`/api/eterra/export-bundle`): ZIP with CSV + GPKG +- **Local export** (`/api/eterra/export-local`): from local DB + +--- + +## API Routes Reference + +| Route | Method | Purpose | +|-------|--------|---------| +| `/api/eterra/uats` | GET | All UATs from DB (with local feature counts) | +| `/api/eterra/uats` | POST | Seed DB from uat.json | +| `/api/eterra/uats` | PATCH | Populate county from eTerra nomenclature | +| `/api/eterra/login` | POST | Connect to eTerra | +| `/api/eterra/session` | GET/DELETE | Session status / disconnect | +| `/api/eterra/health` | GET | eTerra platform health | +| `/api/eterra/search` | POST | Search parcels by cadastral number | +| `/api/eterra/search-owner` | POST | Search by owner name | +| `/api/eterra/layers` | GET | Available layers + field names | +| `/api/eterra/count` | POST | Count features in layer | +| `/api/eterra/sync` | POST | Start layer sync job | +| `/api/eterra/sync-background` | POST | Background sync | +| `/api/eterra/sync-status` | GET | Sync job progress | +| `/api/eterra/progress` | GET | Generic job progress | +| `/api/eterra/features` | GET | Query local features | +| `/api/eterra/db-summary` | GET | Aggregate stats all UATs | +| `/api/eterra/uat-dashboard` | GET | Per-UAT analytics | +| `/api/eterra/export-bundle` | POST | Export ZIP (CSV + GPKG) | +| `/api/eterra/export-layer-gpkg` | POST | Export single layer GPKG | +| `/api/eterra/export-local` | POST | Export from local DB | +| `/api/eterra/no-geom-scan` | POST | Scan for no-geometry parcels | +| `/api/eterra/no-geom-debug` | POST | Debug no-geom data | +| `/api/eterra/setup-postgis` | POST | Initialize PostGIS extensions | + +--- + +## Critical Gotchas + +1. **nomenPk ≠ SIRUTA**: eTerra's internal nomenclature PKs are NOT SIRUTA codes. Never assume they're the same. +2. **workspacePk = county nomenPk**: The WORKSPACE_ID from ArcGIS layers equals the county's nomenclature PK from `fetchCounties()`. +3. **Session TTL**: eTerra sessions expire after ~10 minutes. Client has 9-minute cache TTL + auto-relogin. +4. **Maintenance windows**: eTerra goes down regularly. Health check detects this and blocks operations. +5. **ArcGIS maxRecordCount=1000**: Always paginate with `resultOffset`/`resultRecordCount`. Page size fallbacks: 1000 → 500 → 200. +6. **Admin field auto-discovery**: Different layers use different field names for SIRUTA (ADMIN_UNIT_ID, SIRUTA, UAT_ID, etc.). Always use `findAdminField()` / `buildWhere()`. +7. **LIMITE_UAT endpoint is "all"**: Unlike other layers (endpoint "aut"), LIMITE_UAT uses the "all" endpoint — no workspace context needed. +8. **No-geometry parcels use negative objectId**: `objectId = -immovablePk` to avoid collision with real ArcGIS OBJECTIDs. +9. **Spring Boot Page responses**: Some eTerra APIs return `{content: [...], totalPages: N}` instead of flat arrays. Always use `unwrapArray()` helper. +10. **Geometry format**: EsriGeometry `{ rings: number[][][] }` in EPSG:3844 (Stereo70). Reproject to EPSG:4326 for GeoPackage export. +11. **Owner tree parsing**: Documentation API returns owners as a tree (C→A→I→P nodes). Check `nodeStatus === -1` up the parent chain to detect radiated/cancelled owners. +12. **Local UatEntry type**: The component defines its own `UatEntry` type at the top of `parcel-sync-module.tsx` — this shadows the one in `types.ts`. Keep both in sync. + +--- + +## Test UAT + +**Feleacu** — SIRUTA `57582`, ~30k immovables, ~8k GIS features. Used as the primary test UAT. + +--- + +## Last Updated + +2026-03-22 — Added county population via nomenclature API, local feature count in UAT dropdown. diff --git a/src/modules/parcel-sync/components/parcel-sync-module.tsx b/src/modules/parcel-sync/components/parcel-sync-module.tsx index 6cf54f1..0415e34 100644 --- a/src/modules/parcel-sync/components/parcel-sync-module.tsx +++ b/src/modules/parcel-sync/components/parcel-sync-module.tsx @@ -67,6 +67,7 @@ type UatEntry = { name: string; county?: string; workspacePk?: number; + localFeatures?: number; }; type SessionStatus = { @@ -1770,19 +1771,21 @@ export function ParcelSyncModule() { setSearchResults([]); }} > - + {item.name} - - ({item.siruta}) - {item.county && ( - ,{" "} + –{" "} {item.county} )} + {(item.localFeatures ?? 0) > 0 && ( + + {(item.localFeatures ?? 0).toLocaleString("ro")} local + + )} {item.siruta} diff --git a/src/modules/parcel-sync/types.ts b/src/modules/parcel-sync/types.ts index 96a907a..1a18d61 100644 --- a/src/modules/parcel-sync/types.ts +++ b/src/modules/parcel-sync/types.ts @@ -7,6 +7,8 @@ export type UatEntry = { name: string; county?: string; workspacePk?: number; + /** Number of GIS features synced locally for this UAT */ + localFeatures?: number; }; export type LayerSyncStatus = {