Files
ArchiTools/src/modules/parcel-sync/services/enrich-service.ts
T
AI Assistant bcc7a54325 perf: reverse enrichment order — direct parcel details first, skip immApps
- fetchImmovableParcelDetails called FIRST (1 call, no applicationId needed)
- app-based fetchParcelFolosinte only as fallback when direct returns nothing
- SOLICITANT skipped entirely (was always '-' for old CF records)
- Remove unused pickApplication helper
- Net savings: ~500+ API calls per UAT enrichment (50-65% reduction)
- copycf/get returns same data as list (no enrichment value, kept as utility)
2026-03-08 01:15:28 +02:00

700 lines
23 KiB
TypeScript

/* eslint-disable @typescript-eslint/no-explicit-any */
/**
* Enrich service — fetches CF/owner/address/building data from eTerra
* and stores it in GisFeature.enrichment JSON column.
*
* Called after sync to add "magic" data to parcels.
* Idempotent: re-running overwrites previous enrichment.
*/
import { Prisma } from "@prisma/client";
import { prisma } from "@/core/storage/prisma";
import { EterraClient } from "./eterra-client";
import {
setProgress,
clearProgress,
type SyncProgress,
} from "./progress-store";
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
export type EnrichResult = {
siruta: string;
enrichedCount: number;
totalFeatures?: number;
unenrichedCount?: number;
buildingCrossRefs: number;
status: "done" | "error";
error?: string;
};
/* ── Helpers (extracted from export-bundle) ──────────────────── */
const formatNumber = (value: number) =>
Number.isFinite(value) ? value.toFixed(2).replace(/\.00$/, "") : "";
const normalizeId = (value: unknown) => {
if (value === null || value === undefined) return "";
const text = String(value).trim();
if (!text) return "";
return text.replace(/\.0$/, "");
};
const normalizeCadRef = (value: unknown) =>
normalizeId(value).replace(/\s+/g, "").toUpperCase();
const baseCadRef = (value: unknown) => {
const ref = normalizeCadRef(value);
if (!ref) return "";
return ref.includes("-") ? ref.split("-")[0]! : ref;
};
const makeWorkspaceKey = (workspaceId: unknown, immovableId: unknown) => {
const ws = normalizeId(workspaceId);
const im = normalizeId(immovableId);
if (!ws || !im) return "";
return `${ws}:${im}`;
};
const isRetryable = (error: unknown) => {
const err = error as { response?: { status?: number }; code?: string };
const status = err?.response?.status ?? 0;
if ([429, 500, 502, 503, 504].includes(status)) return true;
return err?.code === "ECONNRESET" || err?.code === "ETIMEDOUT";
};
const normalizeIntravilan = (values: string[]) => {
const normalized = values
.map((v) =>
String(v ?? "")
.trim()
.toLowerCase(),
)
.filter(Boolean);
const unique = new Set(normalized);
if (!unique.size) return "-";
if (unique.size === 1)
return unique.has("da") ? "Da" : unique.has("nu") ? "Nu" : "Mixt";
return "Mixt";
};
const formatCategories = (entries: any[]) => {
const map = new Map<string, number>();
for (const entry of entries) {
// Support both API formats:
// fetchParcelFolosinte (via app): categorieFolosinta + suprafata
// fetchImmovableParcelDetails (direct): useCategory (no area)
const key = String(
entry?.categorieFolosinta ?? entry?.useCategory ?? "",
).trim();
if (!key) continue;
const area = Number(entry?.suprafata ?? 0);
map.set(key, (map.get(key) ?? 0) + (Number.isFinite(area) ? area : 0));
}
return Array.from(map.entries())
.map(([k, a]) => (a > 0 ? `${k}:${formatNumber(a)}` : k))
.join("; ");
};
const formatAddress = (item?: any) => {
const address = item?.immovableAddresses?.[0]?.address ?? null;
if (!address) return "-";
const parts: string[] = [];
if (address.addressDescription) parts.push(address.addressDescription);
if (address.street) parts.push(`Str. ${address.street}`);
if (address.buildingNo) parts.push(`Nr. ${address.buildingNo}`);
if (address.locality?.name) parts.push(address.locality.name);
return parts.length ? parts.join(", ") : "-";
};
/**
* Enrichment data stored per-feature in the `enrichment` JSON column.
*/
export type FeatureEnrichment = {
NR_CAD: string;
NR_CF: string;
NR_CF_VECHI: string;
NR_TOPO: string;
ADRESA: string;
PROPRIETARI: string;
PROPRIETARI_VECHI: string;
SUPRAFATA_2D: number | string;
SUPRAFATA_R: number | string;
SOLICITANT: string;
INTRAVILAN: string;
CATEGORIE_FOLOSINTA: string;
HAS_BUILDING: number;
BUILD_LEGAL: number;
};
/**
* Enrich all TERENURI_ACTIVE features for a given UAT.
*
* Reads features from DB, fetches extra data from eTerra (immovable list,
* documentation, owners, folosinte), cross-references with CLADIRI_ACTIVE,
* and stores the enrichment JSON on each GisFeature.
*/
export async function enrichFeatures(
client: EterraClient,
siruta: string,
options?: {
jobId?: string;
onProgress?: (done: number, total: number, phase: string) => void;
},
): Promise<EnrichResult> {
const jobId = options?.jobId;
const push = (partial: Partial<SyncProgress>) => {
if (!jobId) return;
setProgress({
jobId,
downloaded: 0,
status: "running",
...partial,
} as SyncProgress);
};
try {
// Load terenuri and cladiri from DB
const terenuri = await prisma.gisFeature.findMany({
where: { layerId: "TERENURI_ACTIVE", siruta },
select: {
id: true,
objectId: true,
attributes: true,
cadastralRef: true,
enrichedAt: true,
enrichment: true,
},
});
const cladiri = await prisma.gisFeature.findMany({
where: { layerId: "CLADIRI_ACTIVE", siruta },
select: { attributes: true },
});
if (terenuri.length === 0) {
return {
siruta,
enrichedCount: 0,
buildingCrossRefs: 0,
status: "done",
};
}
// Resolve workspace PK from feature attributes or GisUat DB
let resolvedWsPk: number | null = null;
for (const f of terenuri) {
const ws = (f.attributes as Record<string, unknown>).WORKSPACE_ID;
if (ws != null) {
const n = Number(ws);
if (Number.isFinite(n) && n > 0) {
resolvedWsPk = n;
break;
}
}
}
if (!resolvedWsPk) {
try {
const row = await prisma.gisUat.findUnique({
where: { siruta },
select: { workspacePk: true },
});
if (row?.workspacePk && row.workspacePk > 0)
resolvedWsPk = row.workspacePk;
} catch {
/* ignore */
}
}
if (!resolvedWsPk) {
// Last resort: try ArcGIS layer query for 1 feature
try {
const features = await client.listLayer(
{
id: "TERENURI_ACTIVE",
name: "TERENURI_ACTIVE",
endpoint: "aut",
whereTemplate: "{{adminField}}={{siruta}} AND IS_ACTIVE=1",
},
siruta,
{ limit: 1, outFields: "WORKSPACE_ID" },
);
const wsId = features?.[0]?.attributes?.WORKSPACE_ID;
if (wsId != null) {
const n = Number(wsId);
if (Number.isFinite(n) && n > 0) resolvedWsPk = n;
}
} catch {
/* ignore */
}
}
// If still null, enrichment will fail gracefully with empty lists
const workspacePkForApi = resolvedWsPk ?? 65;
console.log(`[enrich] siruta=${siruta} workspacePk=${workspacePkForApi}`);
push({
phase: "Pregătire îmbogățire",
downloaded: 0,
total: terenuri.length,
});
// ── Throttled request helper ──
let lastRequest = 0;
const minInterval = 250;
const throttled = async <T>(fn: () => Promise<T>) => {
let attempt = 0;
while (true) {
const now = Date.now();
const wait = Math.max(0, lastRequest + minInterval - now);
if (wait > 0) await sleep(wait);
try {
const result = await fn();
lastRequest = Date.now();
return result;
} catch (error) {
if (!isRetryable(error) || attempt >= 2) throw error;
attempt += 1;
const backoff = Math.min(5000, 1000 * attempt);
await sleep(backoff);
}
}
};
// ── Building cross-ref map (from local DB cladiri) ──
const buildingMap = new Map<string, { has: boolean; legal: boolean }>();
for (const feature of cladiri) {
const attrs = feature.attributes as Record<string, unknown>;
const immovableId = attrs.IMMOVABLE_ID ?? attrs.IMOVABLE_ID ?? null;
const workspaceId = attrs.WORKSPACE_ID ?? null;
const baseRef = baseCadRef(attrs.NATIONAL_CADASTRAL_REFERENCE ?? "");
const isLegal =
Number(attrs.IS_LEGAL ?? 0) === 1 ||
String(attrs.IS_LEGAL ?? "").toLowerCase() === "true";
const add = (key: string) => {
if (!key) return;
const existing = buildingMap.get(key) ?? { has: false, legal: false };
existing.has = true;
if (isLegal) existing.legal = true;
buildingMap.set(key, existing);
};
const immKey = normalizeId(immovableId);
const wKey = makeWorkspaceKey(workspaceId, immovableId);
if (immKey) add(immKey);
if (wKey) add(wKey);
if (baseRef) add(baseRef);
}
// ── Fetch immovable list from eTerra ──
push({ phase: "Descărcare listă imobile", downloaded: 0 });
const immovableListById = new Map<string, any>();
const immovableListByCad = new Map<string, any>();
const ownersByLandbook = new Map<string, Set<string>>();
const cancelledOwnersByLandbook = new Map<string, Set<string>>();
const addOwner = (landbook: string, name: string, radiated = false) => {
if (!landbook || !name) return;
const targetMap = radiated ? cancelledOwnersByLandbook : ownersByLandbook;
const existing = targetMap.get(landbook) ?? new Set<string>();
existing.add(name);
targetMap.set(landbook, existing);
};
let listPage = 0;
let listTotalPages = 1;
let includeInscrisCF = true;
while (listPage < listTotalPages) {
const listResponse = await throttled(() =>
client.fetchImmovableListByAdminUnit(
workspacePkForApi,
siruta,
listPage,
200,
includeInscrisCF,
),
);
if (
listPage === 0 &&
!(listResponse?.content ?? []).length &&
includeInscrisCF
) {
includeInscrisCF = false;
listPage = 0;
listTotalPages = 1;
continue;
}
listTotalPages =
typeof listResponse?.totalPages === "number"
? listResponse.totalPages
: listTotalPages;
(listResponse?.content ?? []).forEach((item: any) => {
const idKey = normalizeId(item?.immovablePk);
if (idKey) immovableListById.set(idKey, item);
const cadKey = normalizeCadRef(item?.identifierDetails ?? "");
if (cadKey) immovableListByCad.set(cadKey, item);
});
listPage += 1;
}
// ── Fetch documentation/owner data ──
push({ phase: "Descărcare documentații CF" });
const docByImmovable = new Map<string, any>();
const immovableIds = Array.from(immovableListById.keys());
const docBatchSize = 50;
for (let i = 0; i < immovableIds.length; i += docBatchSize) {
const batch = immovableIds.slice(i, i + docBatchSize);
const docResponse = await throttled(() =>
client.fetchDocumentationData(workspacePkForApi, batch),
);
(docResponse?.immovables ?? []).forEach((item: any) => {
const idKey = normalizeId(item?.immovablePk);
if (idKey) docByImmovable.set(idKey, item);
});
// Build nodeId → entry map for radiated detection
const regs: any[] = docResponse?.partTwoRegs ?? [];
const nodeMap = new Map<number, any>();
for (const reg of regs) {
if (reg?.nodeId != null) nodeMap.set(Number(reg.nodeId), reg);
}
// Check if an entry or any ancestor "I" inscription is radiated
const isRadiated = (entry: any, depth = 0): boolean => {
if (!entry || depth > 10) return false;
if (entry?.nodeStatus === -1) return true;
const pid = entry?.parentId;
if (pid != null) {
const parent = nodeMap.get(Number(pid));
if (parent) return isRadiated(parent, depth + 1);
}
return false;
};
for (const reg of regs) {
if (
String(reg?.nodeType ?? "").toUpperCase() !== "P" ||
!reg?.landbookIE
)
continue;
const name = String(reg?.nodeName ?? "").trim();
if (name) addOwner(String(reg.landbookIE), name, isRadiated(reg));
}
}
// ── Enrich each teren feature ──
push({
phase: "Îmbogățire parcele",
downloaded: 0,
total: terenuri.length,
});
const folCache = new Map<string, any[]>();
let enrichedCount = 0;
let buildingCrossRefs = 0;
const now = new Date();
for (let index = 0; index < terenuri.length; index += 1) {
const feature = terenuri[index]!;
const attrs = feature.attributes as Record<string, unknown>;
// Skip features with complete enrichment (resume after crash/interruption).
// Re-enrich if enrichment schema is incomplete (e.g., missing PROPRIETARI_VECHI
// added in a later version).
if (feature.enrichedAt != null) {
const enrichJson = feature.enrichment as Record<string, unknown> | null;
const isComplete =
enrichJson != null &&
[
"NR_CAD",
"NR_CF",
"PROPRIETARI",
"PROPRIETARI_VECHI",
"ADRESA",
"CATEGORIE_FOLOSINTA",
"HAS_BUILDING",
].every((k) => k in enrichJson && enrichJson[k] !== undefined);
if (isComplete) {
enrichedCount += 1;
if (index % 50 === 0) {
options?.onProgress?.(
index + 1,
terenuri.length,
"Îmbogățire parcele (skip enriched)",
);
}
continue;
}
// Stale enrichment — will be re-enriched below
}
const immovableId = attrs.IMMOVABLE_ID ?? "";
const workspaceId = attrs.WORKSPACE_ID ?? "";
const applicationId = (attrs.APPLICATION_ID as number) ?? null;
// SOLICITANT skipped — saves ~500+ API calls; value was always "-"
// for old CF records and rarely useful for modern ones.
const solicitant = "-";
let intravilan = "-";
let categorie = "-";
let proprietari = "-";
let nrCF = "-";
let nrCFVechi = "-";
let nrTopo = "-";
let addressText = "-";
if (immovableId && workspaceId) {
// ── Strategy: direct parcel details FIRST (1 call, no applicationId needed) ──
// This endpoint works for both GIS features and no-geom imports.
// Saves ~50-65% of API calls vs the old app-based flow.
const immPkForDetails =
immovableListById.get(normalizeId(immovableId))?.immovablePk ??
immovableId;
const detKey = `${workspaceId}:${immPkForDetails}:details`;
let details = folCache.get(detKey);
if (!details) {
try {
details = await throttled(() =>
client.fetchImmovableParcelDetails(
workspaceId as string | number,
immPkForDetails as string | number,
),
);
} catch {
details = [];
}
folCache.set(detKey, details);
}
if (details && details.length > 0) {
intravilan = normalizeIntravilan(
details.map((d: any) => d?.intravilan ?? ""),
);
categorie = formatCategories(details);
}
// ── Fallback: app-based flow (only if direct details returned nothing) ──
// Uses applicationId from GIS feature → fetchParcelFolosinte.
// This path adds 1 extra API call.
if (categorie === "-" && applicationId) {
const appId = Number(applicationId);
if (appId > 0) {
const folKey = `${workspaceId}:${immovableId}:${appId}`;
let fol = folCache.get(folKey);
if (!fol) {
fol = await throttled(() =>
client.fetchParcelFolosinte(
workspaceId as string | number,
immovableId as string | number,
appId,
),
);
folCache.set(folKey, fol);
}
if (fol && fol.length > 0) {
const folIntravilan = normalizeIntravilan(
fol.map((item: any) => item?.intravilan ?? ""),
);
const folCategorie = formatCategories(fol);
if (folCategorie && folCategorie !== "-")
categorie = folCategorie;
if (folIntravilan && folIntravilan !== "-" && intravilan === "-")
intravilan = folIntravilan;
}
}
}
}
const cadRefRaw = (attrs.NATIONAL_CADASTRAL_REFERENCE ?? "") as string;
const cadRef = normalizeCadRef(cadRefRaw);
const immKey = normalizeId(immovableId);
const listItem =
(immKey ? immovableListById.get(immKey) : undefined) ??
(cadRef ? immovableListByCad.get(cadRef) : undefined);
const docKey = listItem?.immovablePk
? normalizeId(listItem.immovablePk)
: "";
const docItem = docKey ? docByImmovable.get(docKey) : undefined;
const landbookIE = docItem?.landbookIE ?? "";
const owners =
landbookIE && ownersByLandbook.get(String(landbookIE))
? Array.from(ownersByLandbook.get(String(landbookIE)) ?? [])
: [];
const ownersByCad =
cadRefRaw && ownersByLandbook.get(String(cadRefRaw))
? Array.from(ownersByLandbook.get(String(cadRefRaw)) ?? [])
: [];
proprietari =
Array.from(new Set([...owners, ...ownersByCad])).join("; ") ||
proprietari;
// Cancelled/old owners
const cancelledOwners =
landbookIE && cancelledOwnersByLandbook.get(String(landbookIE))
? Array.from(cancelledOwnersByLandbook.get(String(landbookIE)) ?? [])
: [];
const cancelledByCad =
cadRefRaw && cancelledOwnersByLandbook.get(String(cadRefRaw))
? Array.from(cancelledOwnersByLandbook.get(String(cadRefRaw)) ?? [])
: [];
const activeSet = new Set([...owners, ...ownersByCad]);
const proprietariVechi = Array.from(
new Set([...cancelledOwners, ...cancelledByCad]),
)
.filter((n) => !activeSet.has(n))
.join("; ");
nrCF =
docItem?.landbookIE ||
listItem?.paperLbNo ||
listItem?.paperCadNo ||
nrCF;
const nrCFVechiRaw = listItem?.paperLbNo || listItem?.paperCadNo || "";
nrCFVechi =
docItem?.landbookIE && nrCFVechiRaw !== nrCF ? nrCFVechiRaw : nrCFVechi;
nrTopo =
listItem?.topNo || docItem?.topNo || listItem?.paperCadNo || nrTopo;
addressText = listItem ? formatAddress(listItem) : addressText;
const parcelRef = baseCadRef(cadRefRaw);
const wKey = makeWorkspaceKey(workspaceId, immovableId);
const build = (immKey ? buildingMap.get(immKey) : undefined) ??
(wKey ? buildingMap.get(wKey) : undefined) ??
(parcelRef ? buildingMap.get(parcelRef) : undefined) ?? {
has: false,
legal: false,
};
const hasBuilding = build.has ? 1 : 0;
const buildLegal = build.has ? (build.legal ? 1 : 0) : 0;
if (hasBuilding) buildingCrossRefs += 1;
// Area: prefer GIS AREA_VALUE, fall back to measuredArea/legalArea from
// immovable list (important for no-geometry features where AREA_VALUE
// may have been stored from measuredArea at import time, or may be null).
let areaValue =
typeof attrs.AREA_VALUE === "number" ? attrs.AREA_VALUE : null;
if (areaValue == null && listItem) {
areaValue =
(typeof listItem.measuredArea === "number" &&
listItem.measuredArea > 0
? listItem.measuredArea
: null) ??
(typeof listItem.legalArea === "number" && listItem.legalArea > 0
? listItem.legalArea
: null);
}
const enrichment: FeatureEnrichment = {
NR_CAD: cadRefRaw,
NR_CF: nrCF,
NR_CF_VECHI: nrCFVechi,
NR_TOPO: nrTopo,
ADRESA: addressText,
PROPRIETARI: proprietari,
PROPRIETARI_VECHI: proprietariVechi,
SUPRAFATA_2D: areaValue !== null ? Number(areaValue.toFixed(2)) : "",
SUPRAFATA_R: areaValue !== null ? Math.round(areaValue) : "",
SOLICITANT: solicitant,
INTRAVILAN: intravilan,
CATEGORIE_FOLOSINTA: categorie,
HAS_BUILDING: hasBuilding,
BUILD_LEGAL: buildLegal,
};
// Store enrichment in DB
await prisma.gisFeature.update({
where: { id: feature.id },
data: {
enrichment: enrichment as unknown as Prisma.InputJsonValue,
enrichedAt: now,
},
});
enrichedCount += 1;
if (index % 10 === 0) {
push({
phase: "Îmbogățire parcele",
downloaded: index + 1,
total: terenuri.length,
});
options?.onProgress?.(index + 1, terenuri.length, "Îmbogățire parcele");
}
}
// ── Post-enrichment verification ──
// Check that ALL features now have enrichment (no gaps)
const unenriched = terenuri.length - enrichedCount;
if (unenriched > 0) {
console.warn(
`[enrich] ${unenriched}/${terenuri.length} features remain unenriched for siruta=${siruta}`,
);
} else {
console.log(
`[enrich] ✓ 100% enrichment: ${enrichedCount}/${terenuri.length} features for siruta=${siruta}`,
);
}
push({
phase: "Îmbogățire completă",
status: "done",
downloaded: terenuri.length,
total: terenuri.length,
});
if (jobId) setTimeout(() => clearProgress(jobId), 60_000);
return {
siruta,
enrichedCount,
totalFeatures: terenuri.length,
unenrichedCount: unenriched,
buildingCrossRefs,
status: "done",
};
} catch (error) {
const msg = error instanceof Error ? error.message : "Unknown error";
push({ phase: "Eroare îmbogățire", status: "error", message: msg });
if (jobId) setTimeout(() => clearProgress(jobId), 60_000);
return {
siruta,
enrichedCount: 0,
buildingCrossRefs: 0,
status: "error",
error: msg,
};
}
}
/**
* Check data freshness for a UAT + layer.
* Returns the most recent sync run's completedAt, or null if never synced.
*/
export async function getLayerFreshness(
siruta: string,
layerId: string,
): Promise<{
lastSynced: Date | null;
featureCount: number;
enrichedCount: number;
}> {
const lastRun = await prisma.gisSyncRun.findFirst({
where: { siruta, layerId, status: "done" },
orderBy: { completedAt: "desc" },
select: { completedAt: true },
});
const featureCount = await prisma.gisFeature.count({
where: { siruta, layerId },
});
const enrichedCount = await prisma.gisFeature.count({
where: { siruta, layerId, enrichedAt: { not: null } },
});
return {
lastSynced: lastRun?.completedAt ?? null,
featureCount,
enrichedCount,
};
}
/**
* Check if layer data is "fresh enough" (synced within maxAgeHours).
*/
export function isFresh(lastSynced: Date | null, maxAgeHours = 168): boolean {
if (!lastSynced) return false;
const ageMs = Date.now() - lastSynced.getTime();
return ageMs < maxAgeHours * 60 * 60 * 1000;
}