feat(parcel-sync): incremental sync, smart export, auto-refresh + weekend deep sync

Sync Incremental:
- Add fetchObjectIds (returnIdsOnly) to eterra-client — fetches only OBJECTIDs in 1 request
- Add fetchFeaturesByObjectIds — downloads only delta features by OBJECTID IN (...)
- Rewrite syncLayer: compare remote IDs vs local, download only new features
- Fallback to full sync for first sync, forceFullSync, or delta > 50%
- Reduces sync time from ~10 min to ~5-10s for typical updates

Smart Export Tab:
- Hero buttons detect DB freshness — use export-local (instant) when data is fresh
- Dynamic subtitles: "Din DB (sync acum Xh)" / "Sync incremental" / "Sync complet"
- Re-sync link when data is fresh but user wants forced refresh
- Removed duplicate "Descarca din DB" buttons from background section

Auto-Refresh Scheduler:
- Self-contained timer via instrumentation.ts (Next.js startup hook)
- Weekday 1-5 AM: incremental refresh for existing UATs in DB
- Staggered processing with random delays between UATs
- Health check before processing, respects eTerra maintenance

Weekend Deep Sync:
- Full Magic processing for 9 large municipalities (Cluj, Bistrita, TgMures, etc.)
- Runs Fri/Sat/Sun 23:00-04:00, round-robin intercalated between cities
- 4 steps per city: sync terenuri, sync cladiri, import no-geom, enrichment
- State persisted in KeyValueStore — survives restarts, continues across nights
- Email status report at end of each session via Brevo SMTP
- Admin page at /wds: add/remove cities, view progress, reset
- Hint link on export tab pointing to /wds

API endpoints:
- POST /api/eterra/auto-refresh — N8N-compatible cron endpoint (Bearer token auth)
- GET/POST /api/eterra/weekend-sync — queue management for /wds page

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
AI Assistant
2026-03-26 20:50:34 +02:00
parent 8f65efd5d1
commit 3b456eb481
11 changed files with 1929 additions and 128 deletions
@@ -0,0 +1,522 @@
/**
* Weekend Deep Sync — full Magic processing for large cities.
*
* Runs Fri/Sat/Sun nights 23:0004:00. Processes cities in round-robin
* (one step per city, then rotate) so progress is spread across cities.
* State is persisted in KeyValueStore — survives restarts and continues
* across multiple nights/weekends.
*
* Steps per city (each is resumable):
* 1. sync_terenuri — syncLayer TERENURI_ACTIVE
* 2. sync_cladiri — syncLayer CLADIRI_ACTIVE
* 3. import_nogeom — import parcels without geometry
* 4. enrich — enrichFeatures (slowest, naturally resumable)
*/
import { PrismaClient, Prisma } from "@prisma/client";
import { syncLayer } from "./sync-service";
import { EterraClient } from "./eterra-client";
import { isEterraAvailable } from "./eterra-health";
import { sendEmail } from "@/core/notifications/email-service";
const prisma = new PrismaClient();
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
/* ------------------------------------------------------------------ */
/* City queue configuration */
/* ------------------------------------------------------------------ */
export type CityConfig = {
siruta: string;
name: string;
county: string;
priority: number; // lower = higher priority
};
/** Initial queue — priority 1 = first processed */
const DEFAULT_CITIES: CityConfig[] = [
{ siruta: "54975", name: "Cluj-Napoca", county: "Cluj", priority: 1 },
{ siruta: "32394", name: "Bistri\u021Ba", county: "Bistri\u021Ba-N\u0103s\u0103ud", priority: 1 },
{ siruta: "114319", name: "T\u00E2rgu Mure\u0219", county: "Mure\u0219", priority: 2 },
{ siruta: "139704", name: "Zal\u0103u", county: "S\u0103laj", priority: 2 },
{ siruta: "26564", name: "Oradea", county: "Bihor", priority: 2 },
{ siruta: "9262", name: "Arad", county: "Arad", priority: 2 },
{ siruta: "155243", name: "Timi\u0219oara", county: "Timi\u0219", priority: 2 },
{ siruta: "143450", name: "Sibiu", county: "Sibiu", priority: 2 },
{ siruta: "40198", name: "Bra\u0219ov", county: "Bra\u0219ov", priority: 2 },
];
/* ------------------------------------------------------------------ */
/* Step definitions */
/* ------------------------------------------------------------------ */
const STEPS = [
"sync_terenuri",
"sync_cladiri",
"import_nogeom",
"enrich",
] as const;
type StepName = (typeof STEPS)[number];
type StepStatus = "pending" | "done" | "error";
/* ------------------------------------------------------------------ */
/* Persisted state */
/* ------------------------------------------------------------------ */
type CityState = {
siruta: string;
name: string;
county: string;
priority: number;
steps: Record<StepName, StepStatus>;
lastActivity?: string;
errorMessage?: string;
};
type WeekendSyncState = {
cities: CityState[];
lastSessionDate?: string;
totalSessions: number;
completedCycles: number; // how many full cycles (all cities done)
};
const KV_NAMESPACE = "parcel-sync-weekend";
const KV_KEY = "queue-state";
async function loadState(): Promise<WeekendSyncState> {
const row = await prisma.keyValueStore.findUnique({
where: { namespace_key: { namespace: KV_NAMESPACE, key: KV_KEY } },
});
if (row?.value && typeof row.value === "object") {
return row.value as unknown as WeekendSyncState;
}
// Initialize with default cities
return {
cities: DEFAULT_CITIES.map((c) => ({
...c,
steps: {
sync_terenuri: "pending",
sync_cladiri: "pending",
import_nogeom: "pending",
enrich: "pending",
},
})),
totalSessions: 0,
completedCycles: 0,
};
}
async function saveState(state: WeekendSyncState): Promise<void> {
// Retry once on failure — state persistence is critical for resume
for (let attempt = 0; attempt < 2; attempt++) {
try {
await prisma.keyValueStore.upsert({
where: { namespace_key: { namespace: KV_NAMESPACE, key: KV_KEY } },
update: { value: state as unknown as Prisma.InputJsonValue },
create: {
namespace: KV_NAMESPACE,
key: KV_KEY,
value: state as unknown as Prisma.InputJsonValue,
},
});
return;
} catch (err) {
if (attempt === 0) {
console.warn("[weekend-sync] saveState retry...");
await sleep(2000);
} else {
const msg = err instanceof Error ? err.message : String(err);
console.error(`[weekend-sync] saveState failed: ${msg}`);
}
}
}
}
/* ------------------------------------------------------------------ */
/* Time window */
/* ------------------------------------------------------------------ */
const WEEKEND_START_HOUR = 23;
const WEEKEND_END_HOUR = 4;
const PAUSE_BETWEEN_STEPS_MS = 60_000 + Math.random() * 60_000; // 60-120s
/** Check if current time is within the weekend sync window */
export function isWeekendWindow(): boolean {
const now = new Date();
const day = now.getDay(); // 0=Sun, 5=Fri, 6=Sat
const hour = now.getHours();
// Fri 23:00+ or Sat 23:00+ or Sun 23:00+
if ((day === 5 || day === 6 || day === 0) && hour >= WEEKEND_START_HOUR) {
return true;
}
// Sat 00-04 (continuation of Friday night) or Sun 00-04 or Mon 00-04
if ((day === 6 || day === 0 || day === 1) && hour < WEEKEND_END_HOUR) {
return true;
}
return false;
}
/** Check if still within the window (called during processing) */
function stillInWindow(): boolean {
const hour = new Date().getHours();
// We can be in 23,0,1,2,3 — stop at 4
if (hour >= WEEKEND_END_HOUR && hour < WEEKEND_START_HOUR) return false;
return isWeekendWindow();
}
/* ------------------------------------------------------------------ */
/* Step executors */
/* ------------------------------------------------------------------ */
async function executeStep(
city: CityState,
step: StepName,
client: EterraClient,
): Promise<{ success: boolean; message: string }> {
const start = Date.now();
switch (step) {
case "sync_terenuri": {
const res = await syncLayer(
process.env.ETERRA_USERNAME!,
process.env.ETERRA_PASSWORD!,
city.siruta,
"TERENURI_ACTIVE",
{ uatName: city.name, forceFullSync: true },
);
const dur = ((Date.now() - start) / 1000).toFixed(1);
return {
success: res.status === "done",
message: `Terenuri: ${res.totalLocal} local (+${res.newFeatures}/-${res.removedFeatures}) [${dur}s]`,
};
}
case "sync_cladiri": {
const res = await syncLayer(
process.env.ETERRA_USERNAME!,
process.env.ETERRA_PASSWORD!,
city.siruta,
"CLADIRI_ACTIVE",
{ uatName: city.name, forceFullSync: true },
);
const dur = ((Date.now() - start) / 1000).toFixed(1);
return {
success: res.status === "done",
message: `Cl\u0103diri: ${res.totalLocal} local (+${res.newFeatures}/-${res.removedFeatures}) [${dur}s]`,
};
}
case "import_nogeom": {
const { syncNoGeometryParcels } = await import("./no-geom-sync");
const res = await syncNoGeometryParcels(client, city.siruta);
const dur = ((Date.now() - start) / 1000).toFixed(1);
return {
success: res.status !== "error",
message: `No-geom: ${res.imported} importate, ${res.skipped} skip [${dur}s]`,
};
}
case "enrich": {
const { enrichFeatures } = await import("./enrich-service");
const res = await enrichFeatures(client, city.siruta);
const dur = ((Date.now() - start) / 1000).toFixed(1);
return {
success: res.status === "done",
message: res.status === "done"
? `Enrichment: ${res.enrichedCount}/${res.totalFeatures ?? "?"} (${dur}s)`
: `Enrichment eroare: ${res.error ?? "necunoscuta"} (${dur}s)`,
};
}
}
}
/* ------------------------------------------------------------------ */
/* Main runner */
/* ------------------------------------------------------------------ */
type SessionLog = {
city: string;
step: string;
success: boolean;
message: string;
};
export async function runWeekendDeepSync(): Promise<void> {
const username = process.env.ETERRA_USERNAME;
const password = process.env.ETERRA_PASSWORD;
if (!username || !password) return;
if (!isEterraAvailable()) {
console.log("[weekend-sync] eTerra indisponibil, skip.");
return;
}
const state = await loadState();
const today = new Date().toISOString().slice(0, 10);
// Prevent running twice in the same session
if (state.lastSessionDate === today) return;
state.totalSessions++;
state.lastSessionDate = today;
// Ensure new default cities are added if config expanded
for (const dc of DEFAULT_CITIES) {
if (!state.cities.some((c) => c.siruta === dc.siruta)) {
state.cities.push({
...dc,
steps: {
sync_terenuri: "pending",
sync_cladiri: "pending",
import_nogeom: "pending",
enrich: "pending",
},
});
}
}
const sessionStart = Date.now();
const log: SessionLog[] = [];
let stepsCompleted = 0;
console.log(
`[weekend-sync] Sesiune #${state.totalSessions} pornita. ${state.cities.length} orase in coada.`,
);
// Create eTerra client (shared across steps)
let client: EterraClient;
try {
client = await EterraClient.create(username, password);
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
console.error(`[weekend-sync] Nu se poate conecta la eTerra: ${msg}`);
await saveState(state);
return;
}
// Sort cities: priority first, then shuffle within same priority
const sorted = [...state.cities].sort((a, b) => {
if (a.priority !== b.priority) return a.priority - b.priority;
return Math.random() - 0.5; // random within same priority
});
// Round-robin: iterate through steps, for each step iterate through cities
for (const stepName of STEPS) {
// Find cities that still need this step
const needsStep = sorted.filter((c) => c.steps[stepName] === "pending");
if (needsStep.length === 0) continue;
for (const city of needsStep) {
// Check time window
if (!stillInWindow()) {
console.log("[weekend-sync] Fereastra s-a inchis, opresc.");
await saveState(state);
await sendStatusEmail(state, log, sessionStart);
return;
}
// Check eTerra health
if (!isEterraAvailable()) {
console.log("[weekend-sync] eTerra indisponibil, opresc.");
await saveState(state);
await sendStatusEmail(state, log, sessionStart);
return;
}
// Pause between steps
if (stepsCompleted > 0) {
const pause = 60_000 + Math.random() * 60_000;
console.log(
`[weekend-sync] Pauza ${Math.round(pause / 1000)}s inainte de ${city.name} / ${stepName}`,
);
await sleep(pause);
}
// Execute step
console.log(`[weekend-sync] ${city.name}: ${stepName}...`);
try {
const result = await executeStep(city, stepName, client);
city.steps[stepName] = result.success ? "done" : "error";
if (!result.success) city.errorMessage = result.message;
city.lastActivity = new Date().toISOString();
log.push({
city: city.name,
step: stepName,
success: result.success,
message: result.message,
});
console.log(
`[weekend-sync] ${city.name}: ${stepName}${result.success ? "OK" : "EROARE"}${result.message}`,
);
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
city.steps[stepName] = "error";
city.errorMessage = msg;
city.lastActivity = new Date().toISOString();
log.push({
city: city.name,
step: stepName,
success: false,
message: msg,
});
console.error(
`[weekend-sync] ${city.name}: ${stepName} EROARE: ${msg}`,
);
}
stepsCompleted++;
// Save state after each step (crash safety)
await saveState(state);
}
}
// Check if all cities completed all steps → new cycle
const allDone = state.cities.every((c) =>
STEPS.every((s) => c.steps[s] === "done"),
);
if (allDone) {
state.completedCycles++;
// Reset for next cycle
for (const city of state.cities) {
for (const step of STEPS) {
city.steps[step] = "pending";
}
}
console.log(
`[weekend-sync] Ciclu complet #${state.completedCycles}! Reset pentru urmatorul ciclu.`,
);
}
await saveState(state);
await sendStatusEmail(state, log, sessionStart);
console.log(`[weekend-sync] Sesiune finalizata. ${stepsCompleted} pasi executati.`);
}
/* ------------------------------------------------------------------ */
/* Email status report */
/* ------------------------------------------------------------------ */
async function sendStatusEmail(
state: WeekendSyncState,
log: SessionLog[],
sessionStart: number,
): Promise<void> {
const emailTo = process.env.WEEKEND_SYNC_EMAIL;
if (!emailTo) return;
try {
const duration = Date.now() - sessionStart;
const durMin = Math.round(duration / 60_000);
const durStr =
durMin >= 60
? `${Math.floor(durMin / 60)}h ${durMin % 60}m`
: `${durMin}m`;
const now = new Date();
const dayNames = [
"Duminic\u0103",
"Luni",
"Mar\u021Bi",
"Miercuri",
"Joi",
"Vineri",
"S\u00E2mb\u0103t\u0103",
];
const dayName = dayNames[now.getDay()] ?? "";
const dateStr = now.toLocaleDateString("ro-RO", {
day: "2-digit",
month: "2-digit",
year: "numeric",
});
// Build city progress table
const cityRows = state.cities
.sort((a, b) => a.priority - b.priority)
.map((c) => {
const doneCount = STEPS.filter((s) => c.steps[s] === "done").length;
const errorCount = STEPS.filter((s) => c.steps[s] === "error").length;
const icon =
doneCount === STEPS.length
? "\u2713"
: doneCount > 0
? "\u25D0"
: "\u25CB";
const color =
doneCount === STEPS.length
? "#22c55e"
: errorCount > 0
? "#ef4444"
: doneCount > 0
? "#f59e0b"
: "#9ca3af";
const stepDetail = STEPS.map(
(s) =>
`<span style="color:${c.steps[s] === "done" ? "#22c55e" : c.steps[s] === "error" ? "#ef4444" : "#9ca3af"}">${s.replace("_", " ")}</span>`,
).join(" \u2192 ");
return `<tr>
<td style="padding:4px 8px;color:${color};font-size:16px">${icon}</td>
<td style="padding:4px 8px;font-weight:600">${c.name}</td>
<td style="padding:4px 8px;color:#6b7280;font-size:12px">${c.county}</td>
<td style="padding:4px 8px">${doneCount}/${STEPS.length}</td>
<td style="padding:4px 8px;font-size:11px">${stepDetail}</td>
</tr>`;
})
.join("\n");
// Build session log
const logRows =
log.length > 0
? log
.map(
(l) =>
`<tr>
<td style="padding:2px 6px;font-size:12px">${l.success ? "\u2713" : "\u2717"}</td>
<td style="padding:2px 6px;font-size:12px">${l.city}</td>
<td style="padding:2px 6px;font-size:12px;color:#6b7280">${l.step}</td>
<td style="padding:2px 6px;font-size:11px;color:#6b7280">${l.message}</td>
</tr>`,
)
.join("\n")
: '<tr><td colspan="4" style="padding:8px;color:#9ca3af;font-size:12px">Niciun pas executat in aceasta sesiune</td></tr>';
const html = `
<div style="font-family:system-ui,sans-serif;max-width:700px;margin:0 auto">
<h2 style="color:#1f2937;margin-bottom:4px">Weekend Sync — ${dayName} ${dateStr}</h2>
<p style="color:#6b7280;margin-top:0">Durata sesiune: ${durStr} | Sesiunea #${state.totalSessions} | Cicluri complete: ${state.completedCycles}</p>
<h3 style="color:#374151;margin-bottom:8px">Progres per ora\u0219</h3>
<table style="border-collapse:collapse;width:100%;border:1px solid #e5e7eb;border-radius:6px">
<thead><tr style="background:#f9fafb">
<th style="padding:6px 8px;text-align:left;font-size:12px"></th>
<th style="padding:6px 8px;text-align:left;font-size:12px">Ora\u0219</th>
<th style="padding:6px 8px;text-align:left;font-size:12px">Jude\u021B</th>
<th style="padding:6px 8px;text-align:left;font-size:12px">Pa\u0219i</th>
<th style="padding:6px 8px;text-align:left;font-size:12px">Detaliu</th>
</tr></thead>
<tbody>${cityRows}</tbody>
</table>
<h3 style="color:#374151;margin-top:16px;margin-bottom:8px">Activitate sesiune curent\u0103</h3>
<table style="border-collapse:collapse;width:100%;border:1px solid #e5e7eb">
<tbody>${logRows}</tbody>
</table>
<p style="color:#9ca3af;font-size:11px;margin-top:16px">
Generat automat de ArchiTools Weekend Sync
</p>
</div>
`;
await sendEmail({
to: emailTo,
subject: `[ArchiTools] Weekend Sync — ${dayName} ${dateStr}`,
html,
});
console.log(`[weekend-sync] Email status trimis la ${emailTo}`);
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
console.warn(`[weekend-sync] Nu s-a putut trimite email: ${msg}`);
}
}