diff --git a/CLAUDE.md b/CLAUDE.md index 13063a4..e8d6293 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -94,7 +94,7 @@ legacy/ # Original HTML tools for reference --- -## Implemented Modules (14/14 — zero placeholders) +## Implemented Modules (16 total — 14 original + 2 new) | # | Module | Route | Version | Key Features | | --- | ---------------------- | --------------------- | ------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -112,6 +112,8 @@ legacy/ # Original HTML tools for reference | 12 | **Word Templates** | `/word-templates` | 0.1.0 | Template library, 8 categories, version tracking, .docx placeholder auto-detection | | 13 | **AI Chat** | `/ai-chat` | 0.2.0 | Multi-provider (OpenAI/Claude/Ollama/demo), **project linking via Tag Manager**, provider status badge | | 14 | **Hot Desk** | `/hot-desk` | 0.1.0 | 4 desks, week-ahead calendar, room layout (window+door), reserve/cancel | +| 15 | **ParcelSync** | `/parcel-sync` | 0.5.0 | eTerra ANCPI integration, **PostGIS database**, background sync, 23-layer catalog, enrichment pipeline, owner search, **per-UAT analytics dashboard**, **health check + maintenance detection** | +| 16 | **Visual Copilot** | `/visual-copilot` | 0.1.0 | AI-powered image analysis (placeholder/early stage) | ### Registratura — Legal Deadline Tracking (Termene Legale) @@ -133,6 +135,31 @@ Key files: - `components/deadline-dashboard.tsx` — Stats + filters + table - `components/deadline-add-dialog.tsx` — 3-step wizard (category → type → date preview) +### ParcelSync — eTerra ANCPI GIS Integration + +The ParcelSync module connects to Romania's national eTerra/ANCPI cadastral system: + +- **eTerra API client** (`eterra-client.ts`): form-post auth, JSESSIONID cookie jar, session caching (9min TTL), auto-relogin, paginated fetching with `maxRecordCount=1000` + fallback page sizes (500, 200) +- **23-layer catalog** (`eterra-layers.ts`): TERENURI_ACTIVE, CLADIRI_ACTIVE, LIMITE_UAT, etc. organized in 6 categories +- **PostGIS storage**: `GisFeature` model with geometry column, SIRUTA-based partitioning, `enrichment` JSONB field +- **Background sync**: long-running jobs via server singleton, progress polling (2s), phase tracking (fetch → save → enrich) +- **Enrichment pipeline** (`enrich-service.ts`): hits eTerra `/api/immovable/list` per parcel to extract NR_CAD, NR_CF, PROPRIETARI, SUPRAFATA, INTRAVILAN, CATEGORIE_FOLOSINTA, HAS_BUILDING, etc. +- **Owner search**: DB-first (ILIKE on enrichment JSON) with eTerra API fallback +- **Per-UAT dashboard**: SQL aggregates (area stats, intravilan/extravilan, land use, top owners), CSS-only visualizations (donut ring, bar charts) +- **Health check** (`eterra-health.ts`): pings `eterra.ancpi.ro` every 3min, detects maintenance by keywords in HTML response, blocks login when down, UI shows amber "Mentenanță" state +- **Test UAT**: Feleacu (SIRUTA 57582, ~30k immovables, ~8k GIS features) + +Key files: + +- `services/eterra-client.ts` — API client (~1000 lines), session cache, pagination, retry +- `services/eterra-layers.ts` — 23-layer catalog with categories +- `services/sync-service.ts` — Layer sync engine with progress tracking +- `services/enrich-service.ts` — Enrichment pipeline (FeatureEnrichment type) +- `services/eterra-health.ts` — Health check singleton, maintenance detection +- `services/session-store.ts` — Server-side session management +- `components/parcel-sync-module.tsx` — Main UI (~4100 lines), 4 tabs (Export/Layers/Search/DB) +- `components/uat-dashboard.tsx` — Per-UAT analytics dashboard (CSS-only charts) + --- ## Infrastructure @@ -187,6 +214,9 @@ git push origin main - `Record[key]` returns `T | undefined` — always guard with null check - Spread of possibly-undefined objects: `{ ...obj[key], field }` — check existence first - lucide-react Icons: cast through `unknown` → `React.ComponentType<{ className?: string }>` +- `arr[0]` is `T | undefined` even after `arr.length > 0` check — assign to const first: `const first = arr[0]; if (first) { ... }` +- Prisma `$queryRaw` returns `unknown[]` — always cast with `as Array<{ field: type }>` and guard access +- `?? ""` on an object field typed `{}` produces `{}` not `string` — use explicit `typeof x === 'string'` or `'number'` check ### Conventions @@ -224,6 +254,15 @@ src/modules// └── index.ts # Public exports ``` +### eTerra / External API Rules + +- **ArcGIS REST API** has `maxRecordCount=1000` — always paginate with `resultOffset`/`resultRecordCount` +- **eTerra sessions expire after ~10min** — session cache TTL is 9min, auto-relogin on 401/redirect +- **eTerra goes into maintenance regularly** — health check must detect and block login attempts +- **Never hardcode timeouts too low** — eTerra 1000-feature geometry pages can take 60-90s; default is 120s +- **CookieJar + axios-cookiejar-support** required for eTerra auth (JSESSIONID tracking) +- **Page size fallbacks**: if 1000 fails, retry with 500, then 200 + ### Before Pushing 1. `npx next build` — must pass with zero errors @@ -255,6 +294,8 @@ src/modules// | **Vault Encryption** | ✅ Active | AES-256-GCM server-side, `/api/vault`, ENCRYPTION_SECRET env | | **ManicTime Sync** | ✅ Implemented | `/api/manictime` — bidirectional Tags.txt sync, needs SMB mount | | **NAS Paths** | ✅ Active | `\\newamun` (10.10.10.10), drives A/O/P/T, hostname+IP fallback, `src/config/nas-paths.ts` | +| **eTerra ANCPI** | ✅ Active | ParcelSync module, `eterra-client.ts`, health check + maintenance detection | +| **PostGIS** | ✅ Active | `GisFeature` model, geometry storage, spatial queries, used by ParcelSync | | **N8N automations** | Webhook URL configured | For notifications, backups, workflows | --- diff --git a/ROADMAP.md b/ROADMAP.md index 8eb2d75..6653ecd 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -46,8 +46,10 @@ | 12 | Dashboard | 0.1.0 | COMPLETE | — | Custom dashboards per role | | 13 | AI Chat | 0.2.0 | COMPLETE | Needs API key env vars for real AI | Streaming, model selector, conversation templates | | 14 | Hot Desk | 0.1.0 | COMPLETE | — | — | +| 15 | ParcelSync | 0.5.0 | COMPLETE | Needs real-world UAT testing at scale | Map visualization, batch enrichment, export tools | +| 16 | Visual Copilot | 0.1.0 | STUB | Placeholder only | AI image analysis integration | -**Phases 1–3 COMPLETE (all 42 tasks).** Next: Phase 4 (Quality & Testing). +**Phases 1–3 COMPLETE (all 42 tasks).** Phase 7B (ParcelSync) COMPLETE. Next: Phase 4 (Quality & Testing) or module hardening. --- @@ -644,6 +646,51 @@ Env vars (hardcoded in docker-compose.yml for Portainer CE): --- +## PHASE 7B — ParcelSync / eTerra Module (NEW — 2026-03) + +> GIS integration with Romania's national eTerra/ANCPI cadastral system. + +### 7B.01 ✅ `[HEAVY]` ParcelSync Core — eTerra Client + Layer Sync (2026-03) + +**What:** eTerra API client with form-post auth, JSESSIONID cookie jar, session caching, paginated layer fetching. 23-layer catalog. Background sync with progress polling. PostGIS storage with GisFeature model. +**Status:** ✅ Done. `eterra-client.ts` (~1000 lines), `sync-service.ts`, `eterra-layers.ts`, `session-store.ts`. Pagination with `maxRecordCount=1000` + page size fallbacks (500, 200). Default timeout 120s. Background sync via server singleton. + +--- + +### 7B.02 ✅ `[HEAVY]` ParcelSync — Enrichment Pipeline (2026-03) + +**What:** Per-parcel enrichment via eTerra `/api/immovable/list`. Extracts NR_CAD, NR_CF, PROPRIETARI, SUPRAFATA, INTRAVILAN, CATEGORIE_FOLOSINTA, HAS_BUILDING, etc. +**Status:** ✅ Done. `enrich-service.ts` with `FeatureEnrichment` type. JSONB storage in `enrichment` column. + +--- + +### 7B.03 ✅ `[STANDARD]` ParcelSync — Owner Search (2026-03) + +**What:** Search by owner name. DB-first (ILIKE on enrichment JSON PROPRIETARI) with eTerra API fallback. +**Status:** ✅ Done. `/api/eterra/search-owner`, `searchImmovableByOwnerName()` in client. Mode toggle in Search tab UI. + +--- + +### 7B.04 ✅ `[STANDARD]` ParcelSync — Per-UAT Analytics Dashboard (2026-03) + +**What:** Visual dashboard per UAT with KPIs, area stats, intravilan/extravilan donut, land use bars, top owners, fun facts. CSS-only (no chart libraries). +**Status:** ✅ Done. `/api/eterra/uat-dashboard` with SQL aggregates. `uat-dashboard.tsx` component. Dashboard button on each UAT card in DB tab. + +--- + +### 7B.05 ✅ `[STANDARD]` ParcelSync — Health Check + Maintenance Detection (2026-03) + +**What:** Ping eTerra every 3min, detect maintenance by keywords in HTML, block login when down, show amber "Mentenanță" state. +**Status:** ✅ Done. `eterra-health.ts` singleton, `/api/eterra/health` endpoint, session route blocks login, UI shows amber pill with message extraction. + +**Bugs found & fixed during ParcelSync development:** +- Timeout 40s too low for geometry pages → increased to 120s +- `arr[0]` access fails TS strict even after length check → assign to const +- `?? ""` on `{}` typed field produces `{}` → use `typeof` check +- Prisma `$queryRaw` result needs explicit cast + guard + +--- + ## PHASE 8 — Advanced Features > Cross-cutting features that enhance the entire platform. diff --git a/SESSION-LOG.md b/SESSION-LOG.md index 753c6ab..0ffb37a 100644 --- a/SESSION-LOG.md +++ b/SESSION-LOG.md @@ -4,6 +4,87 @@ --- +## Session — 2026-03-08 (GitHub Copilot - Claude Opus 4.6) + +### Context + +ParcelSync module development continuation — owner search, UAT dashboard, eTerra health check/maintenance detection. Documentation update. + +### Completed + +- **ParcelSync — Owner Name Search:** + - New `searchImmovableByOwnerName()` method in `eterra-client.ts` — tries personName/titularName/ownerName filter keys + - New `/api/eterra/search-owner` API route — DB search first (enrichment ILIKE with `unaccent()`), eTerra API fallback + - UI: mode toggle (Nr. Cadastral / Proprietar) in Search tab, owner results with source badges (BD/eTerra) + - Relaxed search tab guard: only requires UAT selection (not eTerra connection) + - Commit: `6558c69` + +- **ParcelSync — Per-UAT Analytics Dashboard:** + - New `/api/eterra/uat-dashboard` API route with SQL aggregates (area stats, intravilan/extravilan, land use, top owners, fun facts, median area via PERCENTILE_CONT) + - New `uat-dashboard.tsx` component — CSS-only visualizations: 6 KPI cards, donut ring (conic-gradient), horizontal bar charts (width-based), top 10 owners list, fun facts + - Dashboard button (BarChart3 icon) on each UAT card in DB tab, expands panel below + - Zero chart library dependencies + - Commit: `6557cd5` + +- **ParcelSync — eTerra Health Check + Maintenance Detection:** + - New `eterra-health.ts` service: pings eTerra every 3min, 10s timeout, detects maintenance by keywords in HTML response (includes real keywords from 2026-03-08 outage: "serviciu indisponibil", "activități de mentenanță sunt în desfășurare") + - Extracts actual maintenance message from page for display in UI + - New `/api/eterra/health` endpoint (GET cached, POST force-check) + - Session route blocks login when eTerra is in maintenance (returns 503 + `{maintenance: true}`) + - `GET /api/eterra/session` enriched with `eterraAvailable`, `eterraMaintenance`, `eterraHealthMessage` + - ConnectionPill shows amber "Mentenanță" state with AlertTriangle icon instead of red "Eroare" + - Auto-connect skips during maintenance, retries when back online (30s poll detects recovery) + - Commit: `b7a236c` + +- **eTerra Timeout Fix:** + - `DEFAULT_TIMEOUT_MS` 40000→120000 (eTerra 1000-feature geometry pages need 60-90s) + - Added `timeoutMs` option to `syncLayer()`, passed through to `EterraClient.create()` + - Commit: `8bb4a47` + +- **Documentation Update:** + - CLAUDE.md: Added ParcelSync module (#15) + Visual Copilot (#16), eTerra integration, PostGIS, new TS strict mode gotchas, eTerra/external API rules + - ROADMAP.md: Added Phase 7B with 5 completed tasks, updated module status table + - SESSION-LOG.md: This session entry + +### Bugs Found & Fixed + +| Bug | Root Cause | Fix | +|-----|-----------|-----| +| `timeout of 40000ms exceeded` on TERENURI_ACTIVE | Default 40s too short for 1000-feature geometry pages | Increased to 120s | +| `suprafata` type `{}` not assignable to `string\|number` | `?? ""` on enrichment field typed `{}` produces `{}` | `typeof x === 'number'` explicit check | +| `Object is possibly 'undefined'` on `topOwners[0]` | TS strict: `arr[0]` is `T\|undefined` even after `length > 0` | Assign to const, `if (const)` guard | +| eTerra maintenance shows as "Eroare conectare" | No health check, login attempt fails with generic error | Health check + maintenance detection + amber UI state | + +### Learnings + +- **eTerra goes down for maintenance regularly** — need pre-check before any login attempt +- **eTerra login page shows**: "Serviciu indisponibil" + "activități de mentenanță sunt în desfășurare" during outages +- **CSS-only charts work great**: conic-gradient donut rings, width-based horizontal bars — zero library overhead +- **TS strict `arr[0]`**: ALWAYS assign to const even after length check — the TS compiler doesn't narrow +- **Prisma `$queryRaw`**: always cast explicitly and guard access — it returns `unknown[]` + +### Files Changed + +- **New:** `src/modules/parcel-sync/services/eterra-health.ts` +- **New:** `src/modules/parcel-sync/components/uat-dashboard.tsx` +- **New:** `src/app/api/eterra/health/route.ts` +- **New:** `src/app/api/eterra/uat-dashboard/route.ts` +- **New:** `src/app/api/eterra/search-owner/route.ts` +- **Modified:** `src/modules/parcel-sync/services/eterra-client.ts` (timeout 120s, `searchImmovableByOwnerName()`) +- **Modified:** `src/modules/parcel-sync/services/sync-service.ts` (timeoutMs option) +- **Modified:** `src/modules/parcel-sync/services/session-store.ts` (health fields in SessionStatus) +- **Modified:** `src/modules/parcel-sync/components/parcel-sync-module.tsx` (owner search, dashboard, maintenance UI) +- **Modified:** `src/app/api/eterra/session/route.ts` (health check integration, login blocking) +- **Modified:** `CLAUDE.md`, `ROADMAP.md`, `SESSION-LOG.md` + +### Build + +- `npx next build` — zero errors +- Commits: `8bb4a47` → `6558c69` → `6557cd5` → `b7a236c` +- Pushed to main → Portainer auto-deploy + +--- + ## Session — 2026-02-28c (GitHub Copilot - Claude Opus 4.6) ### Context diff --git a/src/modules/parcel-sync/services/eterra-health.ts b/src/modules/parcel-sync/services/eterra-health.ts index 4aa3b07..e4c84c1 100644 --- a/src/modules/parcel-sync/services/eterra-health.ts +++ b/src/modules/parcel-sync/services/eterra-health.ts @@ -46,13 +46,22 @@ const MAINTENANCE_THRESHOLD = 2; /** * Keywords in eTerra HTML responses that indicate maintenance. - * Romanian maintenance pages often contain these. + * These are actual strings observed on the eTerra login page during downtime. + * Matched case-insensitively against the response body. */ const MAINTENANCE_KEYWORDS = [ - "mentenan", + // Observed on eterra.ancpi.ro/#/login during real maintenance (2026-03-08) + "serviciu indisponibil", + "activități de mentenanță", + "activitati de mentenanta", + "mentenanță", + "mentenanta", + // Generic maintenance patterns "maintenance", "indisponibil", - "temporar", + "în desfășurare", + "in desfasurare", + "temporar indisponibil", "lucrări", "lucrari", "întrerupere", @@ -90,6 +99,37 @@ function getState(): HealthState { return g.__eterraHealthState; } +/* ------------------------------------------------------------------ */ +/* Helpers */ +/* ------------------------------------------------------------------ */ + +/** + * Try to extract a human-readable maintenance message from the HTML body. + * eTerra's login page shows messages like: + * "Serviciu indisponibil" + * "activități de mentenanță sunt în desfășurare" + */ +function extractMaintenanceMessage(body: string): string | null { + // Try common patterns in the eTerra maintenance page + // Pattern: text between tags that contains maintenance keywords + const patterns = [ + /serviciu\s+indisponibil[^<]*/i, + /activit[aă][tț]i\s+de\s+mentenan[tț][aă]\s+[^<]*/i, + /mentenan[tț][aă]\s+[^<]{0,100}/i, + ]; + for (const pattern of patterns) { + const match = body.match(pattern); + if (match) { + // Capitalize first letter, trim, remove trailing HTML artifacts + const raw = match[0].replace(/<[^>]*>/g, "").trim(); + if (raw.length > 5 && raw.length < 200) { + return raw.charAt(0).toUpperCase() + raw.slice(1); + } + } + } + return null; +} + /* ------------------------------------------------------------------ */ /* Core check */ /* ------------------------------------------------------------------ */ @@ -117,13 +157,15 @@ async function performHealthCheck(): Promise { typeof response.data === "string" ? response.data.toLowerCase() : ""; // Check if the page content indicates maintenance - const isMaintenance = - status === 503 || MAINTENANCE_KEYWORDS.some((kw) => body.includes(kw)); + const matchedKeyword = MAINTENANCE_KEYWORDS.find((kw) => body.includes(kw)); + const isMaintenance = status === 503 || !!matchedKeyword; if (isMaintenance) { + // Try to extract the actual maintenance message from the page + const extractedMsg = extractMaintenanceMessage(body); state.status = { available: false, - message: "eTerra este în mentenanță programată", + message: extractedMsg || "eTerra este în mentenanță programată", lastCheckedAt: new Date().toISOString(), lastAvailableAt: state.status.lastAvailableAt, consecutiveFailures: state.status.consecutiveFailures + 1,