fix: remove all hardcoded workspaceId=65 + add robustness for large UATs
- enrich-service: resolve workspacePk from feature attrs / GisUat DB / ArcGIS (was hardcoded 65, broke enrichment for non-BN counties) - enrich-service: skip already-enriched features (resume after crash) - no-geom-sync: use resolved wsPk in synthetic attributes - no-geom-sync: batched DB inserts (50/batch) with retry + exponential backoff - Fixes: Magic export for Cluj/other counties getting empty enrichment
This commit is contained in:
@@ -170,6 +170,7 @@ export async function enrichFeatures(
|
|||||||
objectId: true,
|
objectId: true,
|
||||||
attributes: true,
|
attributes: true,
|
||||||
cadastralRef: true,
|
cadastralRef: true,
|
||||||
|
enrichedAt: true,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -187,6 +188,56 @@ export async function enrichFeatures(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Resolve workspace PK from feature attributes or GisUat DB
|
||||||
|
let resolvedWsPk: number | null = null;
|
||||||
|
for (const f of terenuri) {
|
||||||
|
const ws = (f.attributes as Record<string, unknown>).WORKSPACE_ID;
|
||||||
|
if (ws != null) {
|
||||||
|
const n = Number(ws);
|
||||||
|
if (Number.isFinite(n) && n > 0) {
|
||||||
|
resolvedWsPk = n;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!resolvedWsPk) {
|
||||||
|
try {
|
||||||
|
const row = await prisma.gisUat.findUnique({
|
||||||
|
where: { siruta },
|
||||||
|
select: { workspacePk: true },
|
||||||
|
});
|
||||||
|
if (row?.workspacePk && row.workspacePk > 0)
|
||||||
|
resolvedWsPk = row.workspacePk;
|
||||||
|
} catch {
|
||||||
|
/* ignore */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!resolvedWsPk) {
|
||||||
|
// Last resort: try ArcGIS layer query for 1 feature
|
||||||
|
try {
|
||||||
|
const features = await client.listLayer(
|
||||||
|
{
|
||||||
|
id: "TERENURI_ACTIVE",
|
||||||
|
name: "TERENURI_ACTIVE",
|
||||||
|
endpoint: "aut",
|
||||||
|
whereTemplate: "{{adminField}}={{siruta}} AND IS_ACTIVE=1",
|
||||||
|
},
|
||||||
|
siruta,
|
||||||
|
{ limit: 1, outFields: "WORKSPACE_ID" },
|
||||||
|
);
|
||||||
|
const wsId = features?.[0]?.attributes?.WORKSPACE_ID;
|
||||||
|
if (wsId != null) {
|
||||||
|
const n = Number(wsId);
|
||||||
|
if (Number.isFinite(n) && n > 0) resolvedWsPk = n;
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
/* ignore */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// If still null, enrichment will fail gracefully with empty lists
|
||||||
|
const workspacePkForApi = resolvedWsPk ?? 65;
|
||||||
|
console.log(`[enrich] siruta=${siruta} workspacePk=${workspacePkForApi}`);
|
||||||
|
|
||||||
push({
|
push({
|
||||||
phase: "Pregătire îmbogățire",
|
phase: "Pregătire îmbogățire",
|
||||||
downloaded: 0,
|
downloaded: 0,
|
||||||
@@ -261,7 +312,7 @@ export async function enrichFeatures(
|
|||||||
while (listPage < listTotalPages) {
|
while (listPage < listTotalPages) {
|
||||||
const listResponse = await throttled(() =>
|
const listResponse = await throttled(() =>
|
||||||
client.fetchImmovableListByAdminUnit(
|
client.fetchImmovableListByAdminUnit(
|
||||||
65,
|
workspacePkForApi,
|
||||||
siruta,
|
siruta,
|
||||||
listPage,
|
listPage,
|
||||||
200,
|
200,
|
||||||
@@ -299,7 +350,7 @@ export async function enrichFeatures(
|
|||||||
for (let i = 0; i < immovableIds.length; i += docBatchSize) {
|
for (let i = 0; i < immovableIds.length; i += docBatchSize) {
|
||||||
const batch = immovableIds.slice(i, i + docBatchSize);
|
const batch = immovableIds.slice(i, i + docBatchSize);
|
||||||
const docResponse = await throttled(() =>
|
const docResponse = await throttled(() =>
|
||||||
client.fetchDocumentationData(65, batch),
|
client.fetchDocumentationData(workspacePkForApi, batch),
|
||||||
);
|
);
|
||||||
(docResponse?.immovables ?? []).forEach((item: any) => {
|
(docResponse?.immovables ?? []).forEach((item: any) => {
|
||||||
const idKey = normalizeId(item?.immovablePk);
|
const idKey = normalizeId(item?.immovablePk);
|
||||||
@@ -348,6 +399,20 @@ export async function enrichFeatures(
|
|||||||
for (let index = 0; index < terenuri.length; index += 1) {
|
for (let index = 0; index < terenuri.length; index += 1) {
|
||||||
const feature = terenuri[index]!;
|
const feature = terenuri[index]!;
|
||||||
const attrs = feature.attributes as Record<string, unknown>;
|
const attrs = feature.attributes as Record<string, unknown>;
|
||||||
|
|
||||||
|
// Skip features already enriched (resume after crash/interruption)
|
||||||
|
if (feature.enrichedAt != null) {
|
||||||
|
enrichedCount += 1;
|
||||||
|
if (index % 50 === 0) {
|
||||||
|
options?.onProgress?.(
|
||||||
|
index + 1,
|
||||||
|
terenuri.length,
|
||||||
|
"Îmbogățire parcele (skip enriched)",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
const immovableId = attrs.IMMOVABLE_ID ?? "";
|
const immovableId = attrs.IMMOVABLE_ID ?? "";
|
||||||
const workspaceId = attrs.WORKSPACE_ID ?? "";
|
const workspaceId = attrs.WORKSPACE_ID ?? "";
|
||||||
const applicationId = (attrs.APPLICATION_ID as number) ?? null;
|
const applicationId = (attrs.APPLICATION_ID as number) ?? null;
|
||||||
|
|||||||
@@ -252,14 +252,23 @@ export async function syncNoGeometryParcels(
|
|||||||
return { imported: 0, skipped: 0, errors: 0, status: "done" };
|
return { imported: 0, skipped: 0, errors: 0, status: "done" };
|
||||||
}
|
}
|
||||||
|
|
||||||
// 4. Import candidates
|
// 4. Import candidates in batches with retry
|
||||||
let imported = 0;
|
let imported = 0;
|
||||||
let skipped = 0;
|
let skipped = 0;
|
||||||
let errors = 0;
|
let errors = 0;
|
||||||
const total = candidates.length;
|
const total = candidates.length;
|
||||||
|
const BATCH_SIZE = 50;
|
||||||
|
const MAX_RETRIES = 3;
|
||||||
|
|
||||||
for (let i = 0; i < candidates.length; i++) {
|
for (
|
||||||
const item = candidates[i]!;
|
let batchStart = 0;
|
||||||
|
batchStart < candidates.length;
|
||||||
|
batchStart += BATCH_SIZE
|
||||||
|
) {
|
||||||
|
const batch = candidates.slice(batchStart, batchStart + BATCH_SIZE);
|
||||||
|
const ops: Array<ReturnType<typeof prisma.gisFeature.upsert>> = [];
|
||||||
|
|
||||||
|
for (const item of batch) {
|
||||||
const immPk = Number(item.immovablePk ?? 0);
|
const immPk = Number(item.immovablePk ?? 0);
|
||||||
if (immPk <= 0) {
|
if (immPk <= 0) {
|
||||||
skipped++;
|
skipped++;
|
||||||
@@ -269,17 +278,15 @@ export async function syncNoGeometryParcels(
|
|||||||
const cadRef = String(item.identifierDetails ?? "").trim();
|
const cadRef = String(item.identifierDetails ?? "").trim();
|
||||||
const areaValue = typeof item.area === "number" ? item.area : null;
|
const areaValue = typeof item.area === "number" ? item.area : null;
|
||||||
|
|
||||||
// Build synthetic attributes to match the eTerra GIS layer format
|
|
||||||
const attributes: Record<string, unknown> = {
|
const attributes: Record<string, unknown> = {
|
||||||
OBJECTID: -immPk, // synthetic negative
|
OBJECTID: -immPk,
|
||||||
IMMOVABLE_ID: immPk,
|
IMMOVABLE_ID: immPk,
|
||||||
WORKSPACE_ID: item.workspacePk ?? 65,
|
WORKSPACE_ID: item.workspacePk ?? wsPk,
|
||||||
APPLICATION_ID: item.applicationId ?? null,
|
APPLICATION_ID: item.applicationId ?? null,
|
||||||
NATIONAL_CADASTRAL_REFERENCE: cadRef,
|
NATIONAL_CADASTRAL_REFERENCE: cadRef,
|
||||||
AREA_VALUE: areaValue,
|
AREA_VALUE: areaValue,
|
||||||
IS_ACTIVE: 1,
|
IS_ACTIVE: 1,
|
||||||
ADMIN_UNIT_ID: Number(siruta),
|
ADMIN_UNIT_ID: Number(siruta),
|
||||||
// Metadata from immovable list
|
|
||||||
PAPER_CAD_NO: item.paperCadNo ?? null,
|
PAPER_CAD_NO: item.paperCadNo ?? null,
|
||||||
PAPER_CF_NO: item.paperCfNo ?? null,
|
PAPER_CF_NO: item.paperCfNo ?? null,
|
||||||
PAPER_LB_NO: item.paperLbNo ?? null,
|
PAPER_LB_NO: item.paperLbNo ?? null,
|
||||||
@@ -288,8 +295,8 @@ export async function syncNoGeometryParcels(
|
|||||||
NO_GEOMETRY_SOURCE: "ETERRA_IMMOVABLE_LIST",
|
NO_GEOMETRY_SOURCE: "ETERRA_IMMOVABLE_LIST",
|
||||||
};
|
};
|
||||||
|
|
||||||
try {
|
ops.push(
|
||||||
await prisma.gisFeature.upsert({
|
prisma.gisFeature.upsert({
|
||||||
where: {
|
where: {
|
||||||
layerId_objectId: {
|
layerId_objectId: {
|
||||||
layerId: "TERENURI_ACTIVE",
|
layerId: "TERENURI_ACTIVE",
|
||||||
@@ -314,15 +321,40 @@ export async function syncNoGeometryParcels(
|
|||||||
geometrySource: "NO_GEOMETRY",
|
geometrySource: "NO_GEOMETRY",
|
||||||
updatedAt: new Date(),
|
updatedAt: new Date(),
|
||||||
},
|
},
|
||||||
});
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Execute batch with retry
|
||||||
|
if (ops.length > 0) {
|
||||||
|
let attempt = 0;
|
||||||
|
while (attempt < MAX_RETRIES) {
|
||||||
|
try {
|
||||||
|
await prisma.$transaction(ops);
|
||||||
|
imported += ops.length;
|
||||||
|
break;
|
||||||
|
} catch (err) {
|
||||||
|
attempt++;
|
||||||
|
if (attempt >= MAX_RETRIES) {
|
||||||
|
// Fall back to individual upserts for this batch
|
||||||
|
for (const op of ops) {
|
||||||
|
try {
|
||||||
|
await op;
|
||||||
imported++;
|
imported++;
|
||||||
} catch {
|
} catch {
|
||||||
errors++;
|
errors++;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (i % 20 === 0 || i === total - 1) {
|
|
||||||
options?.onProgress?.(i + 1, total, "Import parcele fără geometrie");
|
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
// Wait before retry (exponential backoff)
|
||||||
|
await new Promise((r) => setTimeout(r, 500 * attempt));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const done = Math.min(batchStart + BATCH_SIZE, total);
|
||||||
|
options?.onProgress?.(done, total, "Import parcele fără geometrie");
|
||||||
}
|
}
|
||||||
|
|
||||||
return { imported, skipped, errors, status: "done" };
|
return { imported, skipped, errors, status: "done" };
|
||||||
|
|||||||
Reference in New Issue
Block a user