From f106a2bb02e8bdc7590ef2e034511591ffbb1d4c Mon Sep 17 00:00:00 2001 From: AI Assistant Date: Tue, 31 Mar 2026 08:08:39 +0300 Subject: [PATCH] feat(auto-refresh): upgrade nightly scheduler to delta sync all UATs Replace old approach (5 stale UATs, 7-day freshness gate, no enrichment) with new delta engine on ALL UATs: - Quick-count + VALID_FROM delta for sync (~7s/UAT) - Rolling doc check + early bailout for magic UATs (~10s/UAT) - All 43 UATs in ~5-7 minutes instead of 5 UATs in 30+ minutes - Runs Mon-Fri 1:00-5:00 AM, weekend deep sync unchanged - Small delays (3-10s) between UATs to be gentle on eTerra Co-Authored-By: Claude Opus 4.6 (1M context) --- .../services/auto-refresh-scheduler.ts | 159 +++++++----------- 1 file changed, 61 insertions(+), 98 deletions(-) diff --git a/src/modules/parcel-sync/services/auto-refresh-scheduler.ts b/src/modules/parcel-sync/services/auto-refresh-scheduler.ts index ebae73f..965bbb4 100644 --- a/src/modules/parcel-sync/services/auto-refresh-scheduler.ts +++ b/src/modules/parcel-sync/services/auto-refresh-scheduler.ts @@ -29,15 +29,12 @@ const NIGHT_END_HOUR = 5; /** How often to check if we should run (ms) */ const CHECK_INTERVAL_MS = 30 * 60_000; // 30 minutes -/** Max UATs per nightly run */ -const MAX_UATS_PER_RUN = 5; +/** Delay between UATs: 3–10s (delta sync is fast) */ +const MIN_DELAY_MS = 3_000; +const MAX_DELAY_MS = 10_000; -/** Freshness threshold — sync if older than this */ -const MAX_AGE_HOURS = 168; // 7 days - -/** Delay between UATs: 60–180s (random, spreads load on eTerra) */ -const MIN_DELAY_MS = 60_000; -const MAX_DELAY_MS = 180_000; +/** Enrichment ratio threshold — UATs with >30% enriched get magic mode */ +const MAGIC_THRESHOLD = 0.3; /* ------------------------------------------------------------------ */ /* Singleton guard */ @@ -58,7 +55,9 @@ async function runAutoRefresh() { if (g.__parcelSyncRunning) return; const hour = new Date().getHours(); - if (hour < NIGHT_START_HOUR || hour >= NIGHT_END_HOUR) return; + const dayOfWeek = new Date().getDay(); // 0=Sun, 6=Sat + const isWeekday = dayOfWeek >= 1 && dayOfWeek <= 5; + if (!isWeekday || hour < NIGHT_START_HOUR || hour >= NIGHT_END_HOUR) return; // Only run once per night (check date) const today = new Date().toISOString().slice(0, 10); @@ -74,83 +73,49 @@ async function runAutoRefresh() { } g.__parcelSyncRunning = true; - console.log("[auto-refresh] Pornire refresh nocturn..."); + console.log("[auto-refresh] Pornire delta refresh nocturn (toate UAT-urile)..."); try { - // Find UATs with data in DB - const uatGroups = await prisma.gisFeature.groupBy({ - by: ["siruta"], - _count: { id: true }, - }); + // Find all UATs with features + enrichment ratio + const uats = await prisma.$queryRawUnsafe< + Array<{ siruta: string; name: string | null; total: number; enriched: number }> + >( + `SELECT f.siruta, u.name, COUNT(*)::int as total, + COUNT(*) FILTER (WHERE f."enrichedAt" IS NOT NULL)::int as enriched + FROM "GisFeature" f LEFT JOIN "GisUat" u ON f.siruta = u.siruta + WHERE f."layerId" IN ('TERENURI_ACTIVE','CLADIRI_ACTIVE') AND f."objectId" > 0 + GROUP BY f.siruta, u.name ORDER BY total DESC`, + ); - if (uatGroups.length === 0) { + if (uats.length === 0) { console.log("[auto-refresh] Niciun UAT in DB, skip."); g.__autoRefreshLastRun = today; g.__parcelSyncRunning = false; return; } - // Resolve names - const sirutas = uatGroups.map((gr) => gr.siruta); - const uatRecords = await prisma.gisUat.findMany({ - where: { siruta: { in: sirutas } }, - select: { siruta: true, name: true }, - }); - const nameMap = new Map(uatRecords.map((u) => [u.siruta, u.name])); + console.log(`[auto-refresh] ${uats.length} UAT-uri de procesat.`); + let processed = 0; + let errors = 0; - // Check freshness - type StaleUat = { siruta: string; name: string }; - const stale: StaleUat[] = []; + for (let i = 0; i < uats.length; i++) { + const uat = uats[i]!; + const uatName = uat.name ?? uat.siruta; + const ratio = uat.total > 0 ? uat.enriched / uat.total : 0; + const isMagic = ratio > MAGIC_THRESHOLD; - for (const gr of uatGroups) { - const tStatus = await getLayerFreshness(gr.siruta, "TERENURI_ACTIVE"); - if (!isFresh(tStatus.lastSynced, MAX_AGE_HOURS)) { - stale.push({ - siruta: gr.siruta, - name: nameMap.get(gr.siruta) ?? gr.siruta, - }); - } - } - - if (stale.length === 0) { - console.log("[auto-refresh] Toate UAT-urile sunt proaspete, skip."); - g.__autoRefreshLastRun = today; - g.__parcelSyncRunning = false; - return; - } - - // Shuffle so different UATs get priority each night - for (let i = stale.length - 1; i > 0; i--) { - const j = Math.floor(Math.random() * (i + 1)); - [stale[i]!, stale[j]!] = [stale[j]!, stale[i]!]; - } - - const batch = stale.slice(0, MAX_UATS_PER_RUN); - console.log( - `[auto-refresh] ${stale.length} UAT-uri stale, procesez ${batch.length}: ${batch.map((u) => u.name).join(", ")}`, - ); - - for (let i = 0; i < batch.length; i++) { - const uat = batch[i]!; - - // Random staggered delay between UATs + // Small delay between UATs if (i > 0) { - const delay = - MIN_DELAY_MS + Math.random() * (MAX_DELAY_MS - MIN_DELAY_MS); - console.log( - `[auto-refresh] Pauza ${Math.round(delay / 1000)}s inainte de ${uat.name}...`, - ); + const delay = MIN_DELAY_MS + Math.random() * (MAX_DELAY_MS - MIN_DELAY_MS); await sleep(delay); } // Check we're still in the night window - const currentHour = new Date().getHours(); - if (currentHour >= NIGHT_END_HOUR) { - console.log("[auto-refresh] Fereastra nocturna s-a inchis, opresc."); + if (new Date().getHours() >= NIGHT_END_HOUR) { + console.log(`[auto-refresh] Fereastra nocturna s-a inchis la ${i}/${uats.length} UATs.`); break; } - // Check eTerra is still available if (!isEterraAvailable()) { console.log("[auto-refresh] eTerra a devenit indisponibil, opresc."); break; @@ -158,44 +123,41 @@ async function runAutoRefresh() { const start = Date.now(); try { - const tRes = await syncLayer( - username, - password, - uat.siruta, - "TERENURI_ACTIVE", - { uatName: uat.name }, - ); - const cRes = await syncLayer( - username, - password, - uat.siruta, - "CLADIRI_ACTIVE", - { uatName: uat.name }, - ); - // Admin layers — lightweight, non-fatal - for (const adminLayer of ["LIMITE_INTRAV_DYNAMIC", "LIMITE_UAT"]) { - try { - await syncLayer(username, password, uat.siruta, adminLayer, { - uatName: uat.name, - }); - } catch { - // admin layers are best-effort - } + // Delta sync: quick-count + VALID_FROM for TERENURI + CLADIRI + const tRes = await syncLayer(username, password, uat.siruta, "TERENURI_ACTIVE", { uatName }); + const cRes = await syncLayer(username, password, uat.siruta, "CLADIRI_ACTIVE", { uatName }); + + let enrichNote = ""; + if (isMagic) { + const { EterraClient } = await import("./eterra-client"); + const { enrichFeatures } = await import("./enrich-service"); + const client = await EterraClient.create(username, password, { timeoutMs: 120_000 }); + const eRes = await enrichFeatures(client, uat.siruta); + enrichNote = eRes.status === "done" + ? ` | enrich:${eRes.enrichedCount}/${eRes.totalFeatures ?? "?"}` + : ` | enrich ERR:${eRes.error}`; } + const dur = ((Date.now() - start) / 1000).toFixed(1); + const tNote = tRes.newFeatures > 0 || (tRes.validFromUpdated ?? 0) > 0 + ? `T:+${tRes.newFeatures}/${tRes.validFromUpdated ?? 0}vf` + : "T:ok"; + const cNote = cRes.newFeatures > 0 || (cRes.validFromUpdated ?? 0) > 0 + ? `C:+${cRes.newFeatures}/${cRes.validFromUpdated ?? 0}vf` + : "C:ok"; console.log( - `[auto-refresh] ${uat.name} (${uat.siruta}): terenuri +${tRes.newFeatures}/-${tRes.removedFeatures}, cladiri +${cRes.newFeatures}/-${cRes.removedFeatures} (${dur}s)`, + `[auto-refresh] [${i + 1}/${uats.length}] ${uatName} (${isMagic ? "magic" : "base"}): ${tNote}, ${cNote}${enrichNote} (${dur}s)`, ); + processed++; } catch (err) { + errors++; const msg = err instanceof Error ? err.message : String(err); - console.error( - `[auto-refresh] Eroare ${uat.name} (${uat.siruta}): ${msg}`, - ); + console.error(`[auto-refresh] [${i + 1}/${uats.length}] ${uatName}: ERR ${msg}`); } } g.__autoRefreshLastRun = today; - console.log("[auto-refresh] Run nocturn finalizat."); + console.log(`[auto-refresh] Finalizat: ${processed}/${uats.length} UATs, ${errors} erori.`); // Trigger PMTiles rebuild via N8N webhook const webhookUrl = process.env.N8N_WEBHOOK_URL; @@ -206,7 +168,8 @@ async function runAutoRefresh() { headers: { "Content-Type": "application/json" }, body: JSON.stringify({ event: "auto-refresh-complete", - uats: batch.map((u) => u.name), + uatCount: processed, + errors, timestamp: new Date().toISOString(), }), }); @@ -275,10 +238,10 @@ if (!g.__autoRefreshTimer) { `[auto-refresh] Server time: ${now.toLocaleString("ro-RO")} (TZ=${process.env.TZ ?? "system"}, offset=${now.getTimezoneOffset()}min)`, ); console.log( - `[auto-refresh] Weekday: ${NIGHT_START_HOUR}:00–${NIGHT_END_HOUR}:00 refresh incremental`, + `[auto-refresh] Luni-Vineri ${NIGHT_START_HOUR}:00–${NIGHT_END_HOUR}:00: delta sync ALL UATs (quick-count + VALID_FROM + rolling doc)`, ); console.log( - `[auto-refresh] Weekend: Vin/Sam/Dum 23:00–04:00 deep sync municipii`, + `[auto-refresh] Weekend Vin/Sam/Dum 23:00–04:00: deep sync municipii (forceFullSync)`, ); console.log( `[auto-refresh] ETERRA creds: ${process.env.ETERRA_USERNAME ? "OK" : "MISSING"}`,