perf(enrich): rolling doc check resolves changes in-place, always returns early
Instead of marking features enrichedAt=null and falling through to the full enrichment flow (which downloads the entire immovable list ~5min), the rolling doc check now merges updated PROPRIETARI/DATA_CERERE directly into existing enrichment and returns immediately. Also touches enrichedAt on all checked features to rotate the batch, ensuring different features are checked on each daily run. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -217,6 +217,7 @@ export async function enrichFeatures(
|
|||||||
attributes: true,
|
attributes: true,
|
||||||
cadastralRef: true,
|
cadastralRef: true,
|
||||||
enrichedAt: true,
|
enrichedAt: true,
|
||||||
|
enrichment: true,
|
||||||
},
|
},
|
||||||
orderBy: { enrichedAt: "asc" },
|
orderBy: { enrichedAt: "asc" },
|
||||||
take: ROLLING_BATCH,
|
take: ROLLING_BATCH,
|
||||||
@@ -245,25 +246,32 @@ export async function enrichFeatures(
|
|||||||
} catch { /* ignore */ }
|
} catch { /* ignore */ }
|
||||||
}
|
}
|
||||||
|
|
||||||
let rollingMarked = 0;
|
let rollingUpdated = 0;
|
||||||
if (rollingWsPk) {
|
if (rollingWsPk) {
|
||||||
// Collect immovable PKs for the batch + map immPk → feature IDs
|
// Collect immovable PKs for the batch + map immPk → feature data
|
||||||
const rollingPks: string[] = [];
|
const rollingPks: string[] = [];
|
||||||
const enrichedAtMap = new Map<string, Date>();
|
const enrichedAtMap = new Map<string, Date>();
|
||||||
const immPkToFeatureIds = new Map<string, string[]>();
|
const immPkToFeatures = new Map<
|
||||||
|
string,
|
||||||
|
Array<{ id: string; enrichment: Record<string, unknown> | null }>
|
||||||
|
>();
|
||||||
for (const f of oldestEnriched) {
|
for (const f of oldestEnriched) {
|
||||||
const a = f.attributes as Record<string, unknown>;
|
const a = f.attributes as Record<string, unknown>;
|
||||||
const immId = normalizeId(a.IMMOVABLE_ID);
|
const immId = normalizeId(a.IMMOVABLE_ID);
|
||||||
if (immId && f.enrichedAt) {
|
if (immId && f.enrichedAt) {
|
||||||
rollingPks.push(immId);
|
rollingPks.push(immId);
|
||||||
enrichedAtMap.set(immId, f.enrichedAt);
|
enrichedAtMap.set(immId, f.enrichedAt);
|
||||||
const existing = immPkToFeatureIds.get(immId) ?? [];
|
const existing = immPkToFeatures.get(immId) ?? [];
|
||||||
existing.push(f.id);
|
existing.push({
|
||||||
immPkToFeatureIds.set(immId, existing);
|
id: f.id,
|
||||||
|
enrichment: (f as { enrichment?: Record<string, unknown> | null })
|
||||||
|
.enrichment ?? null,
|
||||||
|
});
|
||||||
|
immPkToFeatures.set(immId, existing);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fetch documentation in batches of 50
|
// Fetch documentation in batches of 50 — detect AND resolve changes in-place
|
||||||
const DOC_BATCH = 50;
|
const DOC_BATCH = 50;
|
||||||
for (let i = 0; i < rollingPks.length; i += DOC_BATCH) {
|
for (let i = 0; i < rollingPks.length; i += DOC_BATCH) {
|
||||||
const batch = rollingPks.slice(i, i + DOC_BATCH);
|
const batch = rollingPks.slice(i, i + DOC_BATCH);
|
||||||
@@ -272,52 +280,93 @@ export async function enrichFeatures(
|
|||||||
rollingWsPk,
|
rollingWsPk,
|
||||||
batch,
|
batch,
|
||||||
);
|
);
|
||||||
// Check each registration's appDate against enrichedAt
|
|
||||||
const regs: Array<{
|
const regs: Array<{
|
||||||
landbookIE?: number;
|
landbookIE?: number;
|
||||||
|
nodeType?: string;
|
||||||
|
nodeName?: string;
|
||||||
|
nodeStatus?: number;
|
||||||
application?: { appDate?: number };
|
application?: { appDate?: number };
|
||||||
immovablePk?: number;
|
|
||||||
}> = docResp?.partTwoRegs ?? [];
|
}> = docResp?.partTwoRegs ?? [];
|
||||||
|
|
||||||
// Map immovablePk → latest appDate from registrations
|
|
||||||
const immToMaxApp = new Map<string, number>();
|
|
||||||
// Build immovablePk from doc response immovables
|
|
||||||
const docImmovables: Array<{
|
const docImmovables: Array<{
|
||||||
immovablePk?: number;
|
immovablePk?: number;
|
||||||
landbookIE?: number;
|
landbookIE?: number;
|
||||||
}> = docResp?.immovables ?? [];
|
}> = docResp?.immovables ?? [];
|
||||||
|
|
||||||
|
// Map landbookIE → immovablePk
|
||||||
const lbToImm = new Map<string, string>();
|
const lbToImm = new Map<string, string>();
|
||||||
for (const di of docImmovables) {
|
for (const di of docImmovables) {
|
||||||
if (di.landbookIE && di.immovablePk) {
|
if (di.landbookIE && di.immovablePk)
|
||||||
lbToImm.set(String(di.landbookIE), normalizeId(di.immovablePk));
|
lbToImm.set(
|
||||||
}
|
String(di.landbookIE),
|
||||||
|
normalizeId(di.immovablePk),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Collect max appDate + owner names per immovablePk
|
||||||
|
const immToMaxApp = new Map<string, number>();
|
||||||
|
const ownersByImm = new Map<string, string[]>();
|
||||||
for (const reg of regs) {
|
for (const reg of regs) {
|
||||||
const appDate = reg.application?.appDate;
|
|
||||||
if (typeof appDate !== "number" || appDate <= 0) continue;
|
|
||||||
// Resolve to immovablePk via landbookIE
|
|
||||||
const lb = reg.landbookIE ? String(reg.landbookIE) : "";
|
const lb = reg.landbookIE ? String(reg.landbookIE) : "";
|
||||||
const immPk = lb ? lbToImm.get(lb) : undefined;
|
const immPk = lb ? lbToImm.get(lb) : undefined;
|
||||||
if (!immPk) continue;
|
if (!immPk) continue;
|
||||||
const current = immToMaxApp.get(immPk) ?? 0;
|
const appDate = reg.application?.appDate;
|
||||||
if (appDate > current) immToMaxApp.set(immPk, appDate);
|
if (typeof appDate === "number" && appDate > 0) {
|
||||||
|
const c = immToMaxApp.get(immPk) ?? 0;
|
||||||
|
if (appDate > c) immToMaxApp.set(immPk, appDate);
|
||||||
|
}
|
||||||
|
// Collect current owner names (nodeType=P, not radiated)
|
||||||
|
if (
|
||||||
|
String(reg.nodeType ?? "").toUpperCase() === "P" &&
|
||||||
|
reg.nodeName &&
|
||||||
|
(reg.nodeStatus ?? 0) >= 0
|
||||||
|
) {
|
||||||
|
const owners = ownersByImm.get(immPk) ?? [];
|
||||||
|
const name = String(reg.nodeName).trim();
|
||||||
|
if (name && !owners.includes(name)) owners.push(name);
|
||||||
|
ownersByImm.set(immPk, owners);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Mark features where latest appDate > enrichedAt
|
// Update features where appDate > enrichedAt — merge into existing enrichment
|
||||||
|
const now = new Date();
|
||||||
for (const [immPk, maxApp] of immToMaxApp) {
|
for (const [immPk, maxApp] of immToMaxApp) {
|
||||||
const enrichedAt = enrichedAtMap.get(immPk);
|
const enrichedAt = enrichedAtMap.get(immPk);
|
||||||
if (enrichedAt && maxApp > enrichedAt.getTime()) {
|
if (!enrichedAt || maxApp <= enrichedAt.getTime()) continue;
|
||||||
const featureIds = immPkToFeatureIds.get(immPk) ?? [];
|
const features = immPkToFeatures.get(immPk) ?? [];
|
||||||
if (featureIds.length > 0) {
|
const owners = ownersByImm.get(immPk) ?? [];
|
||||||
await prisma.gisFeature.updateMany({
|
const ownerStr = owners.join("; ") || "-";
|
||||||
where: { id: { in: featureIds } },
|
const appDateIso = new Date(maxApp)
|
||||||
data: { enrichedAt: null },
|
.toISOString()
|
||||||
});
|
.slice(0, 10);
|
||||||
rollingMarked += featureIds.length;
|
for (const feat of features) {
|
||||||
}
|
// Merge: keep existing enrichment, update doc-based fields
|
||||||
|
const existing = feat.enrichment ?? {};
|
||||||
|
const merged = {
|
||||||
|
...existing,
|
||||||
|
PROPRIETARI: ownerStr,
|
||||||
|
DATA_CERERE: appDateIso,
|
||||||
|
};
|
||||||
|
await prisma.gisFeature.update({
|
||||||
|
where: { id: feat.id },
|
||||||
|
data: {
|
||||||
|
enrichment:
|
||||||
|
merged as unknown as Prisma.InputJsonValue,
|
||||||
|
enrichedAt: now,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
rollingUpdated++;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Touch enrichedAt on checked features (even if unchanged) to rotate the batch
|
||||||
|
const checkedIds = batch
|
||||||
|
.flatMap((pk) => (immPkToFeatures.get(pk) ?? []).map((f) => f.id));
|
||||||
|
if (checkedIds.length > 0) {
|
||||||
|
await prisma.gisFeature.updateMany({
|
||||||
|
where: { id: { in: checkedIds }, enrichedAt: { not: null } },
|
||||||
|
data: { enrichedAt: now },
|
||||||
|
});
|
||||||
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.warn(
|
console.warn(
|
||||||
`[enrich] Rolling doc check batch failed:`,
|
`[enrich] Rolling doc check batch failed:`,
|
||||||
@@ -327,29 +376,26 @@ export async function enrichFeatures(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (rollingMarked > 0) {
|
// Always return early — rolling check is self-contained
|
||||||
console.log(
|
const rollingNote = rollingUpdated > 0
|
||||||
`[enrich] siruta=${siruta}: rolling check found ${rollingMarked} features with new documentation — will re-enrich`,
|
? `Rolling: ${rollingUpdated} parcele actualizate`
|
||||||
);
|
: "Date deja complete";
|
||||||
// Don't return early — fall through to normal enrichment
|
console.log(
|
||||||
} else {
|
`[enrich] siruta=${siruta}: ${rollingNote} (checked ${oldestEnriched.length})`,
|
||||||
console.log(
|
);
|
||||||
`[enrich] siruta=${siruta}: rolling check OK — all ${_totalCount} features up to date`,
|
options?.onProgress?.(
|
||||||
);
|
_totalCount,
|
||||||
options?.onProgress?.(
|
_totalCount,
|
||||||
_totalCount,
|
`Îmbogățire — ${rollingNote}`,
|
||||||
_totalCount,
|
);
|
||||||
"Îmbogățire — date deja complete",
|
return {
|
||||||
);
|
siruta,
|
||||||
return {
|
enrichedCount: _totalCount,
|
||||||
siruta,
|
totalFeatures: _totalCount,
|
||||||
enrichedCount: _totalCount,
|
unenrichedCount: 0,
|
||||||
totalFeatures: _totalCount,
|
buildingCrossRefs: rollingUpdated,
|
||||||
unenrichedCount: 0,
|
status: "done",
|
||||||
buildingCrossRefs: 0,
|
};
|
||||||
status: "done",
|
|
||||||
};
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
// No enriched features to check — early bailout
|
// No enriched features to check — early bailout
|
||||||
options?.onProgress?.(
|
options?.onProgress?.(
|
||||||
|
|||||||
Reference in New Issue
Block a user