/** * Weekend Deep Sync — full Magic processing for large cities. * * Runs Fri/Sat/Sun nights 23:00–04:00. Processes cities in round-robin * (one step per city, then rotate) so progress is spread across cities. * State is persisted in KeyValueStore — survives restarts and continues * across multiple nights/weekends. * * Steps per city (each is resumable): * 1. sync_terenuri — syncLayer TERENURI_ACTIVE * 2. sync_cladiri — syncLayer CLADIRI_ACTIVE * 3. import_nogeom — import parcels without geometry * 4. enrich — enrichFeatures (slowest, naturally resumable) */ import { PrismaClient, Prisma } from "@prisma/client"; import { syncLayer } from "./sync-service"; import { EterraClient } from "./eterra-client"; import { isEterraAvailable } from "./eterra-health"; import { sendEmail } from "@/core/notifications/email-service"; const prisma = new PrismaClient(); const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); /* ------------------------------------------------------------------ */ /* Live activity tracking (globalThis — same process) */ /* ------------------------------------------------------------------ */ const g = globalThis as { __weekendSyncActivity?: { city: string; step: string; startedAt: string; } | null; __parcelSyncRunning?: boolean; }; export function getWeekendSyncActivity() { return g.__weekendSyncActivity ?? null; } /* ------------------------------------------------------------------ */ /* City queue configuration */ /* ------------------------------------------------------------------ */ export type CityConfig = { siruta: string; name: string; county: string; priority: number; // lower = higher priority }; /** Initial queue — priority 1 = first processed */ const DEFAULT_CITIES: CityConfig[] = [ { siruta: "54975", name: "Cluj-Napoca", county: "Cluj", priority: 1 }, { siruta: "32394", name: "Bistri\u021Ba", county: "Bistri\u021Ba-N\u0103s\u0103ud", priority: 1 }, { siruta: "114319", name: "T\u00E2rgu Mure\u0219", county: "Mure\u0219", priority: 2 }, { siruta: "139704", name: "Zal\u0103u", county: "S\u0103laj", priority: 2 }, { siruta: "26564", name: "Oradea", county: "Bihor", priority: 2 }, { siruta: "9262", name: "Arad", county: "Arad", priority: 2 }, { siruta: "155243", name: "Timi\u0219oara", county: "Timi\u0219", priority: 2 }, { siruta: "143450", name: "Sibiu", county: "Sibiu", priority: 2 }, { siruta: "40198", name: "Bra\u0219ov", county: "Bra\u0219ov", priority: 2 }, ]; /* ------------------------------------------------------------------ */ /* Step definitions */ /* ------------------------------------------------------------------ */ const STEPS = [ "sync_terenuri", "sync_cladiri", "import_nogeom", "enrich", ] as const; type StepName = (typeof STEPS)[number]; type StepStatus = "pending" | "done" | "error"; /* ------------------------------------------------------------------ */ /* Persisted state */ /* ------------------------------------------------------------------ */ type CityState = { siruta: string; name: string; county: string; priority: number; steps: Record; lastActivity?: string; errorMessage?: string; }; type WeekendSyncState = { cities: CityState[]; lastSessionDate?: string; totalSessions: number; completedCycles: number; // how many full cycles (all cities done) }; const KV_NAMESPACE = "parcel-sync-weekend"; const KV_KEY = "queue-state"; async function loadState(): Promise { const row = await prisma.keyValueStore.findUnique({ where: { namespace_key: { namespace: KV_NAMESPACE, key: KV_KEY } }, }); if (row?.value && typeof row.value === "object") { return row.value as unknown as WeekendSyncState; } // Initialize with default cities return { cities: DEFAULT_CITIES.map((c) => ({ ...c, steps: { sync_terenuri: "pending", sync_cladiri: "pending", import_nogeom: "pending", enrich: "pending", }, })), totalSessions: 0, completedCycles: 0, }; } async function saveState(state: WeekendSyncState): Promise { // Retry once on failure — state persistence is critical for resume for (let attempt = 0; attempt < 2; attempt++) { try { await prisma.keyValueStore.upsert({ where: { namespace_key: { namespace: KV_NAMESPACE, key: KV_KEY } }, update: { value: state as unknown as Prisma.InputJsonValue }, create: { namespace: KV_NAMESPACE, key: KV_KEY, value: state as unknown as Prisma.InputJsonValue, }, }); return; } catch (err) { if (attempt === 0) { console.warn("[weekend-sync] saveState retry..."); await sleep(2000); } else { const msg = err instanceof Error ? err.message : String(err); console.error(`[weekend-sync] saveState failed: ${msg}`); } } } } /* ------------------------------------------------------------------ */ /* Time window */ /* ------------------------------------------------------------------ */ const WEEKEND_START_HOUR = 23; const WEEKEND_END_HOUR = 4; const PAUSE_BETWEEN_STEPS_MS = 60_000 + Math.random() * 60_000; // 60-120s /** Check if current time is within the weekend sync window */ export function isWeekendWindow(): boolean { const now = new Date(); const day = now.getDay(); // 0=Sun, 5=Fri, 6=Sat const hour = now.getHours(); // Fri 23:00+ or Sat 23:00+ or Sun 23:00+ if ((day === 5 || day === 6 || day === 0) && hour >= WEEKEND_START_HOUR) { return true; } // Sat 00-04 (continuation of Friday night) or Sun 00-04 or Mon 00-04 if ((day === 6 || day === 0 || day === 1) && hour < WEEKEND_END_HOUR) { return true; } return false; } /** Check if still within the window (called during processing) */ function stillInWindow(force?: boolean): boolean { if (force) { // Force mode: any night 22:00–05:00 const hour = new Date().getHours(); return hour >= 22 || hour < 5; } const hour = new Date().getHours(); // We can be in 23,0,1,2,3 — stop at 4 if (hour >= WEEKEND_END_HOUR && hour < WEEKEND_START_HOUR) return false; return isWeekendWindow(); } /* ------------------------------------------------------------------ */ /* Step executors */ /* ------------------------------------------------------------------ */ async function executeStep( city: CityState, step: StepName, client: EterraClient, ): Promise<{ success: boolean; message: string }> { const start = Date.now(); switch (step) { case "sync_terenuri": { const res = await syncLayer( process.env.ETERRA_USERNAME!, process.env.ETERRA_PASSWORD!, city.siruta, "TERENURI_ACTIVE", { uatName: city.name, forceFullSync: true }, ); // Also sync admin layers (lightweight, non-fatal) for (const adminLayer of ["LIMITE_INTRAV_DYNAMIC", "LIMITE_UAT"]) { try { await syncLayer( process.env.ETERRA_USERNAME!, process.env.ETERRA_PASSWORD!, city.siruta, adminLayer, { uatName: city.name }, ); } catch { // admin layers are best-effort } } const dur = ((Date.now() - start) / 1000).toFixed(1); return { success: res.status === "done", message: `Terenuri: ${res.totalLocal} local (+${res.newFeatures}/-${res.removedFeatures}) + intravilan [${dur}s]`, }; } case "sync_cladiri": { const res = await syncLayer( process.env.ETERRA_USERNAME!, process.env.ETERRA_PASSWORD!, city.siruta, "CLADIRI_ACTIVE", { uatName: city.name, forceFullSync: true }, ); const dur = ((Date.now() - start) / 1000).toFixed(1); return { success: res.status === "done", message: `Cl\u0103diri: ${res.totalLocal} local (+${res.newFeatures}/-${res.removedFeatures}) [${dur}s]`, }; } case "import_nogeom": { const { syncNoGeometryParcels } = await import("./no-geom-sync"); const res = await syncNoGeometryParcels(client, city.siruta); const dur = ((Date.now() - start) / 1000).toFixed(1); return { success: res.status !== "error", message: `No-geom: ${res.imported} importate, ${res.skipped} skip [${dur}s]`, }; } case "enrich": { const { enrichFeatures } = await import("./enrich-service"); const res = await enrichFeatures(client, city.siruta); const dur = ((Date.now() - start) / 1000).toFixed(1); return { success: res.status === "done", message: res.status === "done" ? `Enrichment: ${res.enrichedCount}/${res.totalFeatures ?? "?"} (${dur}s)` : `Enrichment eroare: ${res.error ?? "necunoscuta"} (${dur}s)`, }; } } } /* ------------------------------------------------------------------ */ /* Main runner */ /* ------------------------------------------------------------------ */ type SessionLog = { city: string; step: string; success: boolean; message: string; }; export async function runWeekendDeepSync(options?: { force?: boolean; }): Promise { const force = options?.force ?? false; const username = process.env.ETERRA_USERNAME; const password = process.env.ETERRA_PASSWORD; if (!username || !password) return; if (!isEterraAvailable()) { console.log("[weekend-sync] eTerra indisponibil, skip."); return; } const state = await loadState(); const today = new Date().toISOString().slice(0, 10); // Prevent running twice in the same session (force bypasses) if (!force && state.lastSessionDate === today) return; state.totalSessions++; state.lastSessionDate = today; // Ensure new default cities are added if config expanded for (const dc of DEFAULT_CITIES) { if (!state.cities.some((c) => c.siruta === dc.siruta)) { state.cities.push({ ...dc, steps: { sync_terenuri: "pending", sync_cladiri: "pending", import_nogeom: "pending", enrich: "pending", }, }); } } const sessionStart = Date.now(); const log: SessionLog[] = []; let stepsCompleted = 0; console.log( `[weekend-sync] Sesiune #${state.totalSessions} pornita. ${state.cities.length} orase in coada.`, ); // Sort cities: priority first, then shuffle within same priority const sorted = [...state.cities].sort((a, b) => { if (a.priority !== b.priority) return a.priority - b.priority; return Math.random() - 0.5; // random within same priority }); // Round-robin: iterate through steps, for each step iterate through cities for (const stepName of STEPS) { // Find cities that still need this step const needsStep = sorted.filter((c) => c.steps[stepName] === "pending"); if (needsStep.length === 0) continue; for (const city of needsStep) { // Check time window if (!stillInWindow(force)) { console.log("[weekend-sync] Fereastra s-a inchis, opresc."); g.__weekendSyncActivity = null; await saveState(state); await sendStatusEmail(state, log, sessionStart); return; } // Check eTerra health if (!isEterraAvailable()) { console.log("[weekend-sync] eTerra indisponibil, opresc."); g.__weekendSyncActivity = null; await saveState(state); await sendStatusEmail(state, log, sessionStart); return; } // Pause between steps if (stepsCompleted > 0) { const pause = 60_000 + Math.random() * 60_000; console.log( `[weekend-sync] Pauza ${Math.round(pause / 1000)}s inainte de ${city.name} / ${stepName}`, ); await sleep(pause); } // Execute step — fresh client per step (sessions expire after ~10 min) console.log(`[weekend-sync] ${city.name}: ${stepName}...`); g.__weekendSyncActivity = { city: city.name, step: stepName, startedAt: new Date().toISOString(), }; try { const client = await EterraClient.create(username, password); const result = await executeStep(city, stepName, client); city.steps[stepName] = result.success ? "done" : "error"; if (!result.success) { city.errorMessage = result.message; await sendStepErrorEmail(city, stepName, result.message); } city.lastActivity = new Date().toISOString(); log.push({ city: city.name, step: stepName, success: result.success, message: result.message, }); console.log( `[weekend-sync] ${city.name}: ${stepName} → ${result.success ? "OK" : "EROARE"} — ${result.message}`, ); } catch (err) { const msg = err instanceof Error ? err.message : String(err); city.steps[stepName] = "error"; city.errorMessage = msg; city.lastActivity = new Date().toISOString(); log.push({ city: city.name, step: stepName, success: false, message: msg, }); console.error( `[weekend-sync] ${city.name}: ${stepName} EROARE: ${msg}`, ); await sendStepErrorEmail(city, stepName, msg); } g.__weekendSyncActivity = null; stepsCompleted++; // Save state after each step (crash safety) await saveState(state); } } // Check if all cities completed all steps → new cycle const allDone = state.cities.every((c) => STEPS.every((s) => c.steps[s] === "done"), ); if (allDone) { state.completedCycles++; // Reset for next cycle for (const city of state.cities) { for (const step of STEPS) { city.steps[step] = "pending"; } } console.log( `[weekend-sync] Ciclu complet #${state.completedCycles}! Reset pentru urmatorul ciclu.`, ); // Notify N8N to rebuild PMTiles (overview tiles for geoportal) await fireSyncWebhook(state.completedCycles); } await saveState(state); await sendStatusEmail(state, log, sessionStart); console.log(`[weekend-sync] Sesiune finalizata. ${stepsCompleted} pasi executati.`); } /* ------------------------------------------------------------------ */ /* Immediate error email */ /* ------------------------------------------------------------------ */ async function sendStepErrorEmail( city: CityState, step: StepName, errorMsg: string, ): Promise { const emailTo = process.env.WEEKEND_SYNC_EMAIL; if (!emailTo) return; try { const now = new Date(); const timeStr = now.toLocaleString("ro-RO", { day: "2-digit", month: "2-digit", year: "numeric", hour: "2-digit", minute: "2-digit", }); const stepLabel: Record = { sync_terenuri: "Sync Terenuri", sync_cladiri: "Sync Cladiri", import_nogeom: "Import No-Geom", enrich: "Enrichment", }; const html = `

Weekend Sync — Eroare

${timeStr}

Oras ${city.name} (${city.county})
Pas ${stepLabel[step]}
Eroare ${errorMsg}

Generat automat de ArchiTools Weekend Sync

`; await sendEmail({ to: emailTo, subject: `[ArchiTools] WDS Eroare: ${city.name} — ${stepLabel[step]}`, html, }); console.log(`[weekend-sync] Email eroare trimis: ${city.name}/${step}`); } catch (err) { const msg = err instanceof Error ? err.message : String(err); console.warn(`[weekend-sync] Nu s-a putut trimite email eroare: ${msg}`); } } /* ------------------------------------------------------------------ */ /* Email status report */ /* ------------------------------------------------------------------ */ async function sendStatusEmail( state: WeekendSyncState, log: SessionLog[], sessionStart: number, ): Promise { const emailTo = process.env.WEEKEND_SYNC_EMAIL; if (!emailTo) return; try { const duration = Date.now() - sessionStart; const durMin = Math.round(duration / 60_000); const durStr = durMin >= 60 ? `${Math.floor(durMin / 60)}h ${durMin % 60}m` : `${durMin}m`; const now = new Date(); const dayNames = [ "Duminic\u0103", "Luni", "Mar\u021Bi", "Miercuri", "Joi", "Vineri", "S\u00E2mb\u0103t\u0103", ]; const dayName = dayNames[now.getDay()] ?? ""; const dateStr = now.toLocaleDateString("ro-RO", { day: "2-digit", month: "2-digit", year: "numeric", }); // Build city progress table const cityRows = state.cities .sort((a, b) => a.priority - b.priority) .map((c) => { const doneCount = STEPS.filter((s) => c.steps[s] === "done").length; const errorCount = STEPS.filter((s) => c.steps[s] === "error").length; const icon = doneCount === STEPS.length ? "\u2713" : doneCount > 0 ? "\u25D0" : "\u25CB"; const color = doneCount === STEPS.length ? "#22c55e" : errorCount > 0 ? "#ef4444" : doneCount > 0 ? "#f59e0b" : "#9ca3af"; const stepDetail = STEPS.map( (s) => `${s.replace("_", " ")}`, ).join(" \u2192 "); return ` ${icon} ${c.name} ${c.county} ${doneCount}/${STEPS.length} ${stepDetail} `; }) .join("\n"); // Build session log const logRows = log.length > 0 ? log .map( (l) => ` ${l.success ? "\u2713" : "\u2717"} ${l.city} ${l.step} ${l.message} `, ) .join("\n") : 'Niciun pas executat in aceasta sesiune'; const html = `

Weekend Sync — ${dayName} ${dateStr}

Durata sesiune: ${durStr} | Sesiunea #${state.totalSessions} | Cicluri complete: ${state.completedCycles}

Progres per ora\u0219

${cityRows}
Ora\u0219 Jude\u021B Pa\u0219i Detaliu

Activitate sesiune curent\u0103

${logRows}

Generat automat de ArchiTools Weekend Sync

`; await sendEmail({ to: emailTo, subject: `[ArchiTools] Weekend Sync — ${dayName} ${dateStr}`, html, }); console.log(`[weekend-sync] Email status trimis la ${emailTo}`); } catch (err) { const msg = err instanceof Error ? err.message : String(err); console.warn(`[weekend-sync] Nu s-a putut trimite email: ${msg}`); } } /* ------------------------------------------------------------------ */ /* Manual force trigger */ /* ------------------------------------------------------------------ */ /** * Trigger a sync run outside the weekend window. * Resets error steps, clears lastSessionDate, and starts immediately. * Uses an extended night window (22:00–05:00) for the stillInWindow check. */ export async function triggerForceSync(): Promise<{ started: boolean; reason?: string }> { if (g.__parcelSyncRunning) { return { started: false, reason: "O sincronizare ruleaza deja" }; } const username = process.env.ETERRA_USERNAME; const password = process.env.ETERRA_PASSWORD; if (!username || !password) { return { started: false, reason: "ETERRA credentials lipsesc" }; } if (!isEterraAvailable()) { return { started: false, reason: "eTerra indisponibil" }; } // Reset error steps + lastSessionDate in DB so the run proceeds const state = await loadState(); for (const city of state.cities) { for (const step of STEPS) { if (city.steps[step] === "error") { city.steps[step] = "pending"; city.errorMessage = undefined; } } } state.lastSessionDate = undefined; await saveState(state); // Start in background — don't block the API response g.__parcelSyncRunning = true; void (async () => { try { console.log("[weekend-sync] Force sync declansat manual."); await runWeekendDeepSync({ force: true }); } catch (err) { const msg = err instanceof Error ? err.message : String(err); console.error(`[weekend-sync] Force sync eroare: ${msg}`); } finally { g.__parcelSyncRunning = false; } })(); return { started: true }; } /* ------------------------------------------------------------------ */ /* N8N Webhook — trigger PMTiles rebuild after sync cycle */ /* ------------------------------------------------------------------ */ async function fireSyncWebhook(cycle: number): Promise { const url = process.env.N8N_WEBHOOK_URL; if (!url) return; try { await fetch(url, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ event: "weekend-sync-cycle-complete", cycle, timestamp: new Date().toISOString(), }), }); console.log(`[weekend-sync] Webhook trimis la N8N (ciclu #${cycle})`); } catch (err) { const msg = err instanceof Error ? err.message : String(err); console.warn(`[weekend-sync] Webhook N8N esuat: ${msg}`); } }