fix: remove all hardcoded workspaceId=65 + add robustness for large UATs

- enrich-service: resolve workspacePk from feature attrs / GisUat DB / ArcGIS
  (was hardcoded 65, broke enrichment for non-BN counties)
- enrich-service: skip already-enriched features (resume after crash)
- no-geom-sync: use resolved wsPk in synthetic attributes
- no-geom-sync: batched DB inserts (50/batch) with retry + exponential backoff
- Fixes: Magic export for Cluj/other counties getting empty enrichment
This commit is contained in:
AI Assistant
2026-03-07 17:17:55 +02:00
parent db6ac5d3a3
commit 40b9522e12
2 changed files with 158 additions and 61 deletions
@@ -170,6 +170,7 @@ export async function enrichFeatures(
objectId: true, objectId: true,
attributes: true, attributes: true,
cadastralRef: true, cadastralRef: true,
enrichedAt: true,
}, },
}); });
@@ -187,6 +188,56 @@ export async function enrichFeatures(
}; };
} }
// Resolve workspace PK from feature attributes or GisUat DB
let resolvedWsPk: number | null = null;
for (const f of terenuri) {
const ws = (f.attributes as Record<string, unknown>).WORKSPACE_ID;
if (ws != null) {
const n = Number(ws);
if (Number.isFinite(n) && n > 0) {
resolvedWsPk = n;
break;
}
}
}
if (!resolvedWsPk) {
try {
const row = await prisma.gisUat.findUnique({
where: { siruta },
select: { workspacePk: true },
});
if (row?.workspacePk && row.workspacePk > 0)
resolvedWsPk = row.workspacePk;
} catch {
/* ignore */
}
}
if (!resolvedWsPk) {
// Last resort: try ArcGIS layer query for 1 feature
try {
const features = await client.listLayer(
{
id: "TERENURI_ACTIVE",
name: "TERENURI_ACTIVE",
endpoint: "aut",
whereTemplate: "{{adminField}}={{siruta}} AND IS_ACTIVE=1",
},
siruta,
{ limit: 1, outFields: "WORKSPACE_ID" },
);
const wsId = features?.[0]?.attributes?.WORKSPACE_ID;
if (wsId != null) {
const n = Number(wsId);
if (Number.isFinite(n) && n > 0) resolvedWsPk = n;
}
} catch {
/* ignore */
}
}
// If still null, enrichment will fail gracefully with empty lists
const workspacePkForApi = resolvedWsPk ?? 65;
console.log(`[enrich] siruta=${siruta} workspacePk=${workspacePkForApi}`);
push({ push({
phase: "Pregătire îmbogățire", phase: "Pregătire îmbogățire",
downloaded: 0, downloaded: 0,
@@ -261,7 +312,7 @@ export async function enrichFeatures(
while (listPage < listTotalPages) { while (listPage < listTotalPages) {
const listResponse = await throttled(() => const listResponse = await throttled(() =>
client.fetchImmovableListByAdminUnit( client.fetchImmovableListByAdminUnit(
65, workspacePkForApi,
siruta, siruta,
listPage, listPage,
200, 200,
@@ -299,7 +350,7 @@ export async function enrichFeatures(
for (let i = 0; i < immovableIds.length; i += docBatchSize) { for (let i = 0; i < immovableIds.length; i += docBatchSize) {
const batch = immovableIds.slice(i, i + docBatchSize); const batch = immovableIds.slice(i, i + docBatchSize);
const docResponse = await throttled(() => const docResponse = await throttled(() =>
client.fetchDocumentationData(65, batch), client.fetchDocumentationData(workspacePkForApi, batch),
); );
(docResponse?.immovables ?? []).forEach((item: any) => { (docResponse?.immovables ?? []).forEach((item: any) => {
const idKey = normalizeId(item?.immovablePk); const idKey = normalizeId(item?.immovablePk);
@@ -348,6 +399,20 @@ export async function enrichFeatures(
for (let index = 0; index < terenuri.length; index += 1) { for (let index = 0; index < terenuri.length; index += 1) {
const feature = terenuri[index]!; const feature = terenuri[index]!;
const attrs = feature.attributes as Record<string, unknown>; const attrs = feature.attributes as Record<string, unknown>;
// Skip features already enriched (resume after crash/interruption)
if (feature.enrichedAt != null) {
enrichedCount += 1;
if (index % 50 === 0) {
options?.onProgress?.(
index + 1,
terenuri.length,
"Îmbogățire parcele (skip enriched)",
);
}
continue;
}
const immovableId = attrs.IMMOVABLE_ID ?? ""; const immovableId = attrs.IMMOVABLE_ID ?? "";
const workspaceId = attrs.WORKSPACE_ID ?? ""; const workspaceId = attrs.WORKSPACE_ID ?? "";
const applicationId = (attrs.APPLICATION_ID as number) ?? null; const applicationId = (attrs.APPLICATION_ID as number) ?? null;
@@ -252,14 +252,23 @@ export async function syncNoGeometryParcels(
return { imported: 0, skipped: 0, errors: 0, status: "done" }; return { imported: 0, skipped: 0, errors: 0, status: "done" };
} }
// 4. Import candidates // 4. Import candidates in batches with retry
let imported = 0; let imported = 0;
let skipped = 0; let skipped = 0;
let errors = 0; let errors = 0;
const total = candidates.length; const total = candidates.length;
const BATCH_SIZE = 50;
const MAX_RETRIES = 3;
for (let i = 0; i < candidates.length; i++) { for (
const item = candidates[i]!; let batchStart = 0;
batchStart < candidates.length;
batchStart += BATCH_SIZE
) {
const batch = candidates.slice(batchStart, batchStart + BATCH_SIZE);
const ops: Array<ReturnType<typeof prisma.gisFeature.upsert>> = [];
for (const item of batch) {
const immPk = Number(item.immovablePk ?? 0); const immPk = Number(item.immovablePk ?? 0);
if (immPk <= 0) { if (immPk <= 0) {
skipped++; skipped++;
@@ -269,17 +278,15 @@ export async function syncNoGeometryParcels(
const cadRef = String(item.identifierDetails ?? "").trim(); const cadRef = String(item.identifierDetails ?? "").trim();
const areaValue = typeof item.area === "number" ? item.area : null; const areaValue = typeof item.area === "number" ? item.area : null;
// Build synthetic attributes to match the eTerra GIS layer format
const attributes: Record<string, unknown> = { const attributes: Record<string, unknown> = {
OBJECTID: -immPk, // synthetic negative OBJECTID: -immPk,
IMMOVABLE_ID: immPk, IMMOVABLE_ID: immPk,
WORKSPACE_ID: item.workspacePk ?? 65, WORKSPACE_ID: item.workspacePk ?? wsPk,
APPLICATION_ID: item.applicationId ?? null, APPLICATION_ID: item.applicationId ?? null,
NATIONAL_CADASTRAL_REFERENCE: cadRef, NATIONAL_CADASTRAL_REFERENCE: cadRef,
AREA_VALUE: areaValue, AREA_VALUE: areaValue,
IS_ACTIVE: 1, IS_ACTIVE: 1,
ADMIN_UNIT_ID: Number(siruta), ADMIN_UNIT_ID: Number(siruta),
// Metadata from immovable list
PAPER_CAD_NO: item.paperCadNo ?? null, PAPER_CAD_NO: item.paperCadNo ?? null,
PAPER_CF_NO: item.paperCfNo ?? null, PAPER_CF_NO: item.paperCfNo ?? null,
PAPER_LB_NO: item.paperLbNo ?? null, PAPER_LB_NO: item.paperLbNo ?? null,
@@ -288,8 +295,8 @@ export async function syncNoGeometryParcels(
NO_GEOMETRY_SOURCE: "ETERRA_IMMOVABLE_LIST", NO_GEOMETRY_SOURCE: "ETERRA_IMMOVABLE_LIST",
}; };
try { ops.push(
await prisma.gisFeature.upsert({ prisma.gisFeature.upsert({
where: { where: {
layerId_objectId: { layerId_objectId: {
layerId: "TERENURI_ACTIVE", layerId: "TERENURI_ACTIVE",
@@ -314,15 +321,40 @@ export async function syncNoGeometryParcels(
geometrySource: "NO_GEOMETRY", geometrySource: "NO_GEOMETRY",
updatedAt: new Date(), updatedAt: new Date(),
}, },
}); }),
);
}
// Execute batch with retry
if (ops.length > 0) {
let attempt = 0;
while (attempt < MAX_RETRIES) {
try {
await prisma.$transaction(ops);
imported += ops.length;
break;
} catch (err) {
attempt++;
if (attempt >= MAX_RETRIES) {
// Fall back to individual upserts for this batch
for (const op of ops) {
try {
await op;
imported++; imported++;
} catch { } catch {
errors++; errors++;
} }
if (i % 20 === 0 || i === total - 1) {
options?.onProgress?.(i + 1, total, "Import parcele fără geometrie");
} }
} else {
// Wait before retry (exponential backoff)
await new Promise((r) => setTimeout(r, 500 * attempt));
}
}
}
}
const done = Math.min(batchStart + BATCH_SIZE, total);
options?.onProgress?.(done, total, "Import parcele fără geometrie");
} }
return { imported, skipped, errors, status: "done" }; return { imported, skipped, errors, status: "done" };