feat: add parcel-sync module (eTerra ANCPI integration with PostGIS)

- 31 eTerra layer catalog (terenuri, cladiri, documentatii, administrativ) - Incremental sync engine (OBJECTID comparison, only downloads new features) - PostGIS-ready Prisma schema (GisFeature, GisSyncRun, GisUat models) - 7 API routes (/api/eterra/login, count, sync, features, layers/summary, progress, sync-status) - Full UI with 3 tabs (Sincronizare, Parcele, Istoric) - Env var auth (ETERRA_USERNAME / ETERRA_PASSWORD) - Real-time sync progress tracking with polling
2026-03-06 00:36:29 +02:00
parent 51dbfcb2bd
commit 7cdea66fa2
25 changed files with 3097 additions and 12 deletions
@@ -0,0 +1,608 @@
+/**
+ * eTerra ANCPI client — server-side only.
+ *
+ * Authenticates via form-post, keeps a JSESSIONID cookie jar,
+ * and exposes typed wrappers for count / list / fetchAll / parcel-detail
+ * queries against the eTerra ArcGIS-REST API.
+ *
+ * Ported from the standalone ParcelSync project and adapted for
+ * ArchiTools module architecture.
+ */
+
+import axios, { type AxiosError, type AxiosInstance } from "axios";
+import crypto from "crypto";
+import { wrapper } from "axios-cookiejar-support";
+import { CookieJar } from "tough-cookie";
+
+/* ------------------------------------------------------------------ */
+/*  Types                                                              */
+/* ------------------------------------------------------------------ */
+
+export type EsriGeometry = { rings: number[][][] };
+
+export type EsriFeature = {
+  attributes: Record<string, unknown>;
+  geometry?: EsriGeometry;
+};
+
+export type LayerEndpoint = "aut" | "all";
+
+export type LayerConfig = {
+  id: string;
+  name: string;
+  endpoint: LayerEndpoint;
+  subLayerId?: number;
+  whereTemplate?: string;
+  spatialFilter?: boolean;
+};
+
+type EsriQueryResponse = {
+  error?: { message?: string; details?: string[] };
+  count?: number;
+  objectIds?: number[];
+  objectIdFieldName?: string;
+  features?: EsriFeature[];
+  exceededTransferLimit?: boolean;
+};
+
+type EsriLayerInfo = {
+  fields?: { name: string }[];
+  error?: { message?: string; details?: string[] };
+};
+
+type ProgressCallback = (downloaded: number, total?: number) => void;
+
+/* ------------------------------------------------------------------ */
+/*  Constants                                                          */
+/* ------------------------------------------------------------------ */
+
+const BASE_URL = "https://eterra.ancpi.ro/eterra";
+const LOGIN_URL = `${BASE_URL}/api/authentication`;
+
+const DEFAULT_TIMEOUT_MS = 40_000;
+const DEFAULT_PAGE_SIZE = 2000;
+const FALLBACK_PAGE_SIZE = 1000;
+const MAX_RETRIES = 2;
+const SESSION_TTL_MS = 9 * 60 * 1000;
+const MAX_URL_LENGTH = 1500;
+
+/* ------------------------------------------------------------------ */
+/*  Session cache (global singleton)                                   */
+/* ------------------------------------------------------------------ */
+
+type SessionEntry = {
+  jar: CookieJar;
+  client: AxiosInstance;
+  createdAt: number;
+  lastUsed: number;
+};
+
+const globalStore = globalThis as {
+  __eterraSessionStore?: Map<string, SessionEntry>;
+};
+const sessionStore =
+  globalStore.__eterraSessionStore ?? new Map<string, SessionEntry>();
+globalStore.__eterraSessionStore = sessionStore;
+
+const makeCacheKey = (u: string, p: string) =>
+  crypto.createHash("sha256").update(`${u}:${p}`).digest("hex");
+
+/* ------------------------------------------------------------------ */
+/*  Helpers                                                            */
+/* ------------------------------------------------------------------ */
+
+const isTransient = (error: unknown) => {
+  const err = error as AxiosError;
+  if (!err) return false;
+  if (
+    err.code === "ECONNRESET" ||
+    err.code === "ETIMEDOUT" ||
+    err.code === "ECONNABORTED"
+  )
+    return true;
+  const status = err.response?.status ?? 0;
+  return status >= 500;
+};
+
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+/* ------------------------------------------------------------------ */
+/*  Client                                                             */
+/* ------------------------------------------------------------------ */
+
+export class EterraClient {
+  private client: AxiosInstance;
+  private jar: CookieJar;
+  private timeoutMs: number;
+  private maxRetries: number;
+  private username: string;
+  private password: string;
+  private reloginAttempted = false;
+  private layerFieldsCache = new Map<string, string[]>();
+
+  private constructor(
+    client: AxiosInstance,
+    jar: CookieJar,
+    timeoutMs: number,
+    username: string,
+    password: string,
+    maxRetries: number,
+  ) {
+    this.client = client;
+    this.jar = jar;
+    this.timeoutMs = timeoutMs;
+    this.username = username;
+    this.password = password;
+    this.maxRetries = maxRetries;
+  }
+
+  /* ---- Factory --------------------------------------------------- */
+
+  static async create(
+    username: string,
+    password: string,
+    options?: { timeoutMs?: number; maxRetries?: number; useCache?: boolean },
+  ) {
+    const useCache = options?.useCache !== false;
+    const now = Date.now();
+    const cacheKey = makeCacheKey(username, password);
+    const cached = sessionStore.get(cacheKey);
+
+    if (useCache && cached && now - cached.lastUsed < SESSION_TTL_MS) {
+      cached.lastUsed = now;
+      return new EterraClient(
+        cached.client,
+        cached.jar,
+        options?.timeoutMs ?? DEFAULT_TIMEOUT_MS,
+        username,
+        password,
+        options?.maxRetries ?? MAX_RETRIES,
+      );
+    }
+
+    const jar = new CookieJar();
+    const client = wrapper(
+      axios.create({
+        jar,
+        withCredentials: true,
+        maxContentLength: Infinity,
+        maxBodyLength: Infinity,
+        headers: {
+          Accept: "application/json, text/plain, */*",
+          "User-Agent":
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+          Referer: `${BASE_URL}/`,
+        },
+      }),
+    );
+
+    const instance = new EterraClient(
+      client,
+      jar,
+      options?.timeoutMs ?? DEFAULT_TIMEOUT_MS,
+      username,
+      password,
+      options?.maxRetries ?? MAX_RETRIES,
+    );
+    await instance.login(username, password);
+    sessionStore.set(cacheKey, { jar, client, createdAt: now, lastUsed: now });
+    return instance;
+  }
+
+  /* ---- Auth ------------------------------------------------------ */
+
+  private async login(u: string, p: string) {
+    const body = new URLSearchParams();
+    body.set("j_username", u);
+    body.set("j_password", p);
+    body.set("j_uuid", "undefined");
+    body.set("j_isRevoked", "undefined");
+    body.set("_spring_security_remember_me", "true");
+    body.set("submit", "Login");
+
+    try {
+      await this.requestWithRetry(() =>
+        this.client.post(LOGIN_URL, body.toString(), {
+          headers: { "Content-Type": "application/x-www-form-urlencoded" },
+          timeout: this.timeoutMs,
+        }),
+      );
+    } catch (error) {
+      const err = error as AxiosError;
+      if (err?.response?.status === 401)
+        throw new Error("Login failed (invalid credentials)");
+      throw error;
+    }
+
+    const cookies = await this.jar.getCookies(LOGIN_URL);
+    if (!cookies.some((c) => c.key === "JSESSIONID"))
+      throw new Error("Login failed / session not set");
+  }
+
+  /* ---- High-level parcels --------------------------------------- */
+
+  async countParcels(siruta: string) {
+    return this.countLayer(
+      {
+        id: "TERENURI_ACTIVE",
+        name: "TERENURI_ACTIVE",
+        endpoint: "aut",
+        whereTemplate: "{{adminField}}={{siruta}} AND IS_ACTIVE=1",
+      },
+      siruta,
+    );
+  }
+
+  async fetchAllParcels(
+    siruta: string,
+    options?: {
+      pageSize?: number;
+      total?: number;
+      onProgress?: ProgressCallback;
+      delayMs?: number;
+    },
+  ) {
+    return this.fetchAllLayer(
+      {
+        id: "TERENURI_ACTIVE",
+        name: "TERENURI_ACTIVE",
+        endpoint: "aut",
+        whereTemplate: "{{adminField}}={{siruta}} AND IS_ACTIVE=1",
+      },
+      siruta,
+      options,
+    );
+  }
+
+  /* ---- Generic layer methods ------------------------------------ */
+
+  async countLayer(layer: LayerConfig, siruta: string) {
+    const where = await this.buildWhere(layer, siruta);
+    return this.countLayerByWhere(layer, where);
+  }
+
+  async countLayerByWhere(layer: LayerConfig, where: string) {
+    const params = new URLSearchParams();
+    params.set("f", "json");
+    params.set("where", where);
+    return this.countLayerWithParams(layer, params, false);
+  }
+
+  async countLayerByGeometry(layer: LayerConfig, geometry: EsriGeometry) {
+    const params = new URLSearchParams();
+    params.set("f", "json");
+    params.set("where", "1=1");
+    this.applyGeometryParams(params, geometry);
+    return this.countLayerWithParams(layer, params, true);
+  }
+
+  async listLayer(
+    layer: LayerConfig,
+    siruta: string,
+    options?: { limit?: number; outFields?: string },
+  ) {
+    const where = await this.buildWhere(layer, siruta);
+    return this.listLayerByWhere(layer, where, options);
+  }
+
+  async listLayerByWhere(
+    layer: LayerConfig,
+    where: string,
+    options?: { limit?: number; outFields?: string },
+  ) {
+    const params = new URLSearchParams();
+    params.set("f", "json");
+    params.set("where", where);
+    params.set("outFields", options?.outFields ?? "*");
+    params.set("returnGeometry", "false");
+    params.set("resultRecordCount", String(options?.limit ?? 200));
+    params.set("resultOffset", "0");
+    const data = await this.queryLayer(layer, params, false);
+    return data.features ?? [];
+  }
+
+  async fetchAllLayer(
+    layer: LayerConfig,
+    siruta: string,
+    options?: {
+      pageSize?: number;
+      total?: number;
+      onProgress?: ProgressCallback;
+      delayMs?: number;
+    },
+  ) {
+    const where = await this.buildWhere(layer, siruta);
+    return this.fetchAllLayerByWhere(layer, where, options);
+  }
+
+  async fetchAllLayerByWhere(
+    layer: LayerConfig,
+    where: string,
+    options?: {
+      pageSize?: number;
+      total?: number;
+      onProgress?: ProgressCallback;
+      delayMs?: number;
+      outFields?: string;
+      returnGeometry?: boolean;
+      geometry?: EsriGeometry;
+    },
+  ) {
+    let pageSize = options?.pageSize ?? DEFAULT_PAGE_SIZE;
+    const total = options?.total;
+    const onProgress = options?.onProgress;
+    const delayMs = options?.delayMs ?? 0;
+    const outFields = options?.outFields ?? "*";
+    const returnGeometry = options?.returnGeometry ?? true;
+    let offset = 0;
+    const all: EsriFeature[] = [];
+
+    while (true) {
+      const params = new URLSearchParams();
+      params.set("f", "json");
+      params.set("where", where);
+      if (options?.geometry) this.applyGeometryParams(params, options.geometry);
+      params.set("outFields", outFields);
+      params.set("returnGeometry", returnGeometry ? "true" : "false");
+      if (returnGeometry) params.set("outSR", "3844");
+      params.set("resultRecordCount", String(pageSize));
+      params.set("resultOffset", String(offset));
+
+      let data: EsriQueryResponse;
+      try {
+        data = await this.queryLayer(layer, params, Boolean(options?.geometry));
+      } catch {
+        if (pageSize > FALLBACK_PAGE_SIZE) {
+          pageSize = FALLBACK_PAGE_SIZE;
+          continue;
+        }
+        throw new Error(`Failed to fetch layer ${layer.name}`);
+      }
+
+      const features = data.features ?? [];
+      if (features.length === 0) {
+        if (total && all.length < total && pageSize > FALLBACK_PAGE_SIZE) {
+          pageSize = FALLBACK_PAGE_SIZE;
+          continue;
+        }
+        break;
+      }
+
+      all.push(...features);
+      offset += features.length;
+      if (onProgress) onProgress(all.length, total);
+      if (total && all.length >= total) break;
+      if (features.length < pageSize) {
+        if (total && all.length < total && pageSize > FALLBACK_PAGE_SIZE) {
+          pageSize = FALLBACK_PAGE_SIZE;
+          continue;
+        }
+        break;
+      }
+
+      if (delayMs > 0) await sleep(delayMs);
+    }
+
+    return all;
+  }
+
+  async fetchAllLayerByGeometry(
+    layer: LayerConfig,
+    geometry: EsriGeometry,
+    options?: {
+      pageSize?: number;
+      total?: number;
+      onProgress?: ProgressCallback;
+      delayMs?: number;
+      outFields?: string;
+      returnGeometry?: boolean;
+    },
+  ) {
+    return this.fetchAllLayerByWhere(layer, "1=1", { ...options, geometry });
+  }
+
+  async getLayerFieldNames(layer: LayerConfig) {
+    return this.getLayerFields(layer);
+  }
+
+  /* ---- Internals ------------------------------------------------ */
+
+  private layerQueryUrl(layer: LayerConfig) {
+    const suffix =
+      typeof layer.subLayerId === "number" ? `/${layer.subLayerId}` : "";
+    return `${BASE_URL}/api/map/rest/${layer.endpoint}/layer/${layer.name}${suffix}/query`;
+  }
+
+  private applyGeometryParams(params: URLSearchParams, geometry: EsriGeometry) {
+    params.set("geometry", JSON.stringify(geometry));
+    params.set("geometryType", "esriGeometryPolygon");
+    params.set("spatialRel", "esriSpatialRelIntersects");
+    params.set("inSR", "3844");
+  }
+
+  private async buildWhere(layer: LayerConfig, siruta: string) {
+    const fields = await this.getLayerFields(layer);
+    const adminField = this.findAdminField(fields);
+    if (!adminField)
+      throw new Error(`Layer ${layer.name} has no SIRUTA/ADMIN_UNIT field`);
+
+    if (!layer.whereTemplate) return `${adminField}=${siruta}`;
+
+    const hasIsActive = fields.some((f) => f.toUpperCase() === "IS_ACTIVE");
+    if (layer.whereTemplate.includes("IS_ACTIVE") && !hasIsActive)
+      return `${adminField}=${siruta}`;
+
+    return layer.whereTemplate
+      .replace(/\{\{adminField\}\}/g, adminField)
+      .replace(/\{\{siruta\}\}/g, siruta);
+  }
+
+  private findAdminField(fields: string[]) {
+    const preferred = [
+      "ADMIN_UNIT_ID",
+      "SIRUTA",
+      "UAT_ID",
+      "SIRUTA_UAT",
+      "UAT_SIRUTA",
+    ];
+    const upper = fields.map((f) => f.toUpperCase());
+    for (const key of preferred) {
+      const idx = upper.indexOf(key);
+      if (idx >= 0) return fields[idx];
+    }
+    return null;
+  }
+
+  private async getLayerFields(layer: LayerConfig) {
+    const cacheKey = `${layer.endpoint}:${layer.name}`;
+    const cached = this.layerFieldsCache.get(cacheKey);
+    if (cached) return cached;
+
+    const suffix =
+      typeof layer.subLayerId === "number" ? `/${layer.subLayerId}` : "";
+    const url = `${BASE_URL}/api/map/rest/${layer.endpoint}/layer/${layer.name}${suffix}?f=json`;
+
+    let response;
+    try {
+      response = await this.requestWithRetry(() =>
+        this.client.get(url, { timeout: this.timeoutMs }),
+      );
+    } catch (error) {
+      const err = error as AxiosError;
+      if (err?.response?.status === 401 && !this.reloginAttempted) {
+        this.reloginAttempted = true;
+        await this.login(this.username, this.password);
+        response = await this.requestWithRetry(() =>
+          this.client.get(url, { timeout: this.timeoutMs }),
+        );
+      } else throw error;
+    }
+
+    const data = response.data as EsriLayerInfo | string;
+    if (typeof data === "string")
+      throw new Error(`Layer info invalid for ${layer.name}`);
+    if (data.error) {
+      const details = data.error.details?.join(" ") ?? "";
+      throw new Error(
+        `${data.error.message ?? "Layer error"}${details ? ` (${details})` : ""}`,
+      );
+    }
+
+    let fields = data.fields?.map((f) => f.name) ?? [];
+    if (!fields.length) {
+      const params = new URLSearchParams();
+      params.set("f", "json");
+      params.set("where", "1=1");
+      params.set("outFields", "*");
+      params.set("returnGeometry", "false");
+      params.set("resultRecordCount", "1");
+      params.set("resultOffset", "0");
+      const sample = await this.getJson(
+        `${this.layerQueryUrl(layer)}?${params}`,
+      );
+      fields = Object.keys(sample.features?.[0]?.attributes ?? {});
+    }
+    if (!fields.length)
+      throw new Error(`Layer ${layer.name} returned no fields`);
+
+    this.layerFieldsCache.set(cacheKey, fields);
+    return fields;
+  }
+
+  private async getJson(url: string): Promise<EsriQueryResponse> {
+    return this.requestJson(() =>
+      this.client.get(url, { timeout: this.timeoutMs }),
+    );
+  }
+
+  private async postJson(
+    url: string,
+    body: URLSearchParams,
+  ): Promise<EsriQueryResponse> {
+    return this.requestJson(() =>
+      this.client.post(url, body.toString(), {
+        headers: { "Content-Type": "application/x-www-form-urlencoded" },
+        timeout: this.timeoutMs,
+      }),
+    );
+  }
+
+  private async requestJson(
+    request: () => Promise<{
+      data: EsriQueryResponse | string;
+      status: number;
+    }>,
+  ): Promise<EsriQueryResponse> {
+    let response;
+    try {
+      response = await this.requestWithRetry(request);
+    } catch (error) {
+      const err = error as AxiosError;
+      if (err?.response?.status === 401 && !this.reloginAttempted) {
+        this.reloginAttempted = true;
+        await this.login(this.username, this.password);
+        response = await this.requestWithRetry(request);
+      } else if (err?.response?.status === 401) {
+        throw new Error("Session expired (401)");
+      } else throw error;
+    }
+    const data = response.data as EsriQueryResponse | string;
+    if (typeof data === "string")
+      throw new Error("Session expired or invalid response from eTerra");
+    if (data.error) {
+      const details = data.error.details?.join(" ") ?? "";
+      throw new Error(
+        `${data.error.message ?? "eTerra query error"}${details ? ` (${details})` : ""}`,
+      );
+    }
+    return data;
+  }
+
+  private async queryLayer(
+    layer: LayerConfig,
+    params: URLSearchParams,
+    usePost: boolean,
+  ): Promise<EsriQueryResponse> {
+    const url = this.layerQueryUrl(layer);
+    const qs = params.toString();
+    const shouldPost = usePost || qs.length > MAX_URL_LENGTH;
+    if (shouldPost) return this.postJson(url, params);
+    try {
+      return await this.getJson(`${url}?${qs}`);
+    } catch (error) {
+      const err = error as AxiosError;
+      if (err?.response?.status === 414) return this.postJson(url, params);
+      throw error;
+    }
+  }
+
+  private async countLayerWithParams(
+    layer: LayerConfig,
+    params: URLSearchParams,
+    usePost: boolean,
+  ) {
+    const cp = new URLSearchParams(params);
+    cp.set("returnCountOnly", "true");
+    const data = await this.queryLayer(layer, cp, usePost);
+    if (typeof data.count === "number") return data.count;
+    const message = data.error?.message;
+    throw new Error(
+      message
+        ? `Count unavailable (${layer.name}): ${message}`
+        : `Count unavailable (${layer.name})`,
+    );
+  }
+
+  private async requestWithRetry<T>(request: () => Promise<T>) {
+    let attempt = 0;
+    while (true) {
+      try {
+        return await request();
+      } catch (error) {
+        if (attempt >= this.maxRetries || !isTransient(error)) throw error;
+        attempt += 1;
+        await sleep(300 * attempt);
+      }
+    }
+  }
+}