From 377b88c48d429c50a9d3ca9fc8cee690fdc26bbb Mon Sep 17 00:00:00 2001 From: Claude VM Date: Thu, 9 Apr 2026 14:59:18 +0300 Subject: [PATCH] feat(sync): auto-trigger PMTiles rebuild after sync + fix progress display - Add pmtiles-webhook.ts shared helper for triggering PMTiles rebuild - sync-county: trigger rebuild when new features synced, pass jobId to syncLayer for sub-progress, update % after UAT completion (not before) - sync-all-counties: same progress fix + rebuild trigger at end - geoportal monitor: use shared helper instead of raw fetch - weekend-deep-sync + auto-refresh: consolidate webhook code via helper - docker-compose: default N8N_WEBHOOK_URL to pmtiles-webhook on satra:9876 Co-Authored-By: Claude Opus 4.6 (1M context) --- .env.example | 4 +- docker-compose.yml | 4 +- src/app/api/eterra/sync-all-counties/route.ts | 26 ++++++++++-- src/app/api/eterra/sync-county/route.ts | 40 +++++++++++++++++-- src/app/api/geoportal/monitor/route.ts | 35 ++++++---------- .../services/auto-refresh-scheduler.ts | 23 ++--------- .../parcel-sync/services/pmtiles-webhook.ts | 39 ++++++++++++++++++ .../parcel-sync/services/weekend-deep-sync.ts | 22 ++-------- 8 files changed, 120 insertions(+), 73 deletions(-) create mode 100644 src/modules/parcel-sync/services/pmtiles-webhook.ts diff --git a/.env.example b/.env.example index cb46448..66d7528 100644 --- a/.env.example +++ b/.env.example @@ -49,8 +49,8 @@ AUTHENTIK_CLIENT_ID=your-authentik-client-id AUTHENTIK_CLIENT_SECRET=your-authentik-client-secret AUTHENTIK_ISSUER=https://auth.beletage.ro/application/o/architools/ -# N8N automation (future) -# N8N_WEBHOOK_URL=http://10.10.10.166:5678/webhook +# PMTiles rebuild webhook (pmtiles-webhook systemd service on satra) +N8N_WEBHOOK_URL=http://10.10.10.166:9876 # External tool URLs (displayed in dashboard) NEXT_PUBLIC_GITEA_URL=http://10.10.10.166:3002 diff --git a/docker-compose.yml b/docker-compose.yml index c2a5c18..f5a95ee 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -72,8 +72,8 @@ services: - NOTIFICATION_CRON_SECRET=1547a198feca43af6c05622588c6d3b820bad5163b8c20175b2b5bbf8fc1a987 # Weekend Deep Sync email reports (comma-separated for multiple recipients) - WEEKEND_SYNC_EMAIL=${WEEKEND_SYNC_EMAIL:-} - # N8N webhook — triggers PMTiles rebuild after sync cycle - - N8N_WEBHOOK_URL=${N8N_WEBHOOK_URL:-} + # PMTiles rebuild webhook (pmtiles-webhook systemd service on host) + - N8N_WEBHOOK_URL=${N8N_WEBHOOK_URL:-http://10.10.10.166:9876} # Portal-only users (comma-separated, redirected to /portal) - PORTAL_ONLY_USERS=dtiurbe,d.tiurbe # Address Book API (inter-service auth for external tools) diff --git a/src/app/api/eterra/sync-all-counties/route.ts b/src/app/api/eterra/sync-all-counties/route.ts index a796bdf..62f8465 100644 --- a/src/app/api/eterra/sync-all-counties/route.ts +++ b/src/app/api/eterra/sync-all-counties/route.ts @@ -20,6 +20,7 @@ import { EterraClient } from "@/modules/parcel-sync/services/eterra-client"; import { checkEterraHealthNow } from "@/modules/parcel-sync/services/eterra-health"; import { createAppNotification } from "@/core/notifications/app-notifications"; import { getSessionCredentials } from "@/modules/parcel-sync/services/session-store"; +import { firePmtilesRebuild } from "@/modules/parcel-sync/services/pmtiles-webhook"; export const runtime = "nodejs"; export const dynamic = "force-dynamic"; @@ -185,7 +186,7 @@ async function runAllCountiesSync( const isMagic = ratio > 0.3; const mode = isMagic ? "magic" : "base"; - // Progress: county level + UAT level + // Progress: county level + UAT level — update before starting UAT const countyPct = ci / counties.length; const uatPct = i / uats.length; const overallPct = Math.round((countyPct + uatPct / counties.length) * 100); @@ -200,12 +201,12 @@ async function runAllCountiesSync( }); try { - await syncLayer(username, password, uat.siruta, "TERENURI_ACTIVE", { uatName }); - await syncLayer(username, password, uat.siruta, "CLADIRI_ACTIVE", { uatName }); + await syncLayer(username, password, uat.siruta, "TERENURI_ACTIVE", { uatName, jobId, isSubStep: true }); + await syncLayer(username, password, uat.siruta, "CLADIRI_ACTIVE", { uatName, jobId, isSubStep: true }); // LIMITE_INTRAV_DYNAMIC — best effort try { - await syncLayer(username, password, uat.siruta, "LIMITE_INTRAV_DYNAMIC", { uatName }); + await syncLayer(username, password, uat.siruta, "LIMITE_INTRAV_DYNAMIC", { uatName, jobId, isSubStep: true }); } catch { /* skip */ } // Enrichment for magic mode @@ -222,6 +223,15 @@ async function runAllCountiesSync( const msg = err instanceof Error ? err.message : "Unknown"; console.error(`[sync-all] ${county}/${uatName}: ${msg}`); } + + // Update progress AFTER UAT completion + const completedUatPct = (i + 1) / uats.length; + const completedOverallPct = Math.round((countyPct + completedUatPct / counties.length) * 100); + push({ + downloaded: completedOverallPct, + total: 100, + phase: `[${ci + 1}/${counties.length}] ${county} — [${i + 1}/${uats.length}] ${uatName} finalizat`, + }); } const dur = Math.round((Date.now() - countyStart) / 1000); @@ -256,6 +266,14 @@ async function runAllCountiesSync( }); console.log(`[sync-all] Done: ${summary}`); + + // Trigger PMTiles rebuild after full Romania sync + await firePmtilesRebuild("all-counties-sync-complete", { + counties: counties.length, + totalUats, + totalErrors, + }); + setTimeout(() => clearProgress(jobId), 12 * 3_600_000); } catch (err) { const msg = err instanceof Error ? err.message : "Unknown"; diff --git a/src/app/api/eterra/sync-county/route.ts b/src/app/api/eterra/sync-county/route.ts index 6ce6be7..18cb2d9 100644 --- a/src/app/api/eterra/sync-county/route.ts +++ b/src/app/api/eterra/sync-county/route.ts @@ -21,6 +21,7 @@ import { EterraClient } from "@/modules/parcel-sync/services/eterra-client"; import { checkEterraHealthNow } from "@/modules/parcel-sync/services/eterra-health"; import { createAppNotification } from "@/core/notifications/app-notifications"; import { getSessionCredentials } from "@/modules/parcel-sync/services/session-store"; +import { firePmtilesRebuild } from "@/modules/parcel-sync/services/pmtiles-webhook"; export const runtime = "nodejs"; export const dynamic = "force-dynamic"; @@ -169,6 +170,8 @@ async function runCountySync( }> = []; let errors = 0; + let totalNewFeatures = 0; + for (let i = 0; i < uats.length; i++) { const uat = uats[i]!; const uatName = uat.name ?? uat.siruta; @@ -189,12 +192,12 @@ async function runCountySync( const uatStart = Date.now(); try { - // Sync TERENURI + CLADIRI + // Sync TERENURI + CLADIRI — pass jobId for sub-progress const tRes = await syncLayer(username, password, uat.siruta, "TERENURI_ACTIVE", { - uatName, + uatName, jobId, isSubStep: true, }); const cRes = await syncLayer(username, password, uat.siruta, "CLADIRI_ACTIVE", { - uatName, + uatName, jobId, isSubStep: true, }); // Sync ADMINISTRATIV (intravilan) — wrapped in try/catch since it needs UAT geometry @@ -205,7 +208,7 @@ async function runCountySync( password, uat.siruta, "LIMITE_INTRAV_DYNAMIC", - { uatName }, + { uatName, jobId, isSubStep: true }, ); if (aRes.newFeatures > 0) { adminNote = ` | A:+${aRes.newFeatures}`; @@ -236,8 +239,19 @@ async function runCountySync( ? `C:+${cRes.newFeatures}/${cRes.validFromUpdated ?? 0}vf` : "C:ok", ]; + totalNewFeatures += tRes.newFeatures + cRes.newFeatures; const note = `${parts.join(", ")}${adminNote}${enrichNote} (${dur}s)`; results.push({ siruta: uat.siruta, name: uatName, mode, duration: dur, note }); + + // Update progress AFTER UAT completion (so % reflects completed work) + const completedPct = Math.round(((i + 1) / uats.length) * 100); + push({ + downloaded: completedPct, + total: 100, + phase: `[${i + 1}/${uats.length}] ${uatName} finalizat`, + note: `${note}`, + }); + console.log(`[sync-county:${county}] ${i + 1}/${uats.length} ${uatName}: ${note}`); } catch (err) { errors++; @@ -250,6 +264,13 @@ async function runCountySync( duration: dur, note: `ERR: ${msg}`, }); + // Still update progress after error + const completedPct = Math.round(((i + 1) / uats.length) * 100); + push({ + downloaded: completedPct, + total: 100, + phase: `[${i + 1}/${uats.length}] ${uatName} — eroare`, + }); console.error(`[sync-county:${county}] ${uatName}: ${msg}`); } } @@ -278,6 +299,17 @@ async function runCountySync( }); console.log(`[sync-county:${county}] Done: ${summary}`); + + // Trigger PMTiles rebuild if new features were synced + if (totalNewFeatures > 0) { + await firePmtilesRebuild("county-sync-complete", { + county, + uatCount: uats.length, + newFeatures: totalNewFeatures, + errors, + }); + } + setTimeout(() => clearProgress(jobId), 6 * 3_600_000); } catch (err) { const msg = err instanceof Error ? err.message : "Unknown"; diff --git a/src/app/api/geoportal/monitor/route.ts b/src/app/api/geoportal/monitor/route.ts index 392259f..508fcd3 100644 --- a/src/app/api/geoportal/monitor/route.ts +++ b/src/app/api/geoportal/monitor/route.ts @@ -3,6 +3,7 @@ * POST /api/geoportal/monitor — trigger actions (rebuild, warm-cache) */ import { NextRequest, NextResponse } from "next/server"; +import { firePmtilesRebuild } from "@/modules/parcel-sync/services/pmtiles-webhook"; export const runtime = "nodejs"; export const dynamic = "force-dynamic"; @@ -159,31 +160,21 @@ export async function POST(request: NextRequest) { const action = body.action; if (action === "rebuild") { - if (!N8N_WEBHOOK_URL) { - return NextResponse.json({ error: "N8N_WEBHOOK_URL not configured" }, { status: 400 }); - } // Get current PMTiles state before rebuild const before = await getPmtilesInfo(); - try { - const webhookRes = await fetch(N8N_WEBHOOK_URL, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - event: "manual-rebuild", - timestamp: new Date().toISOString(), - }), - }); - return NextResponse.json({ - ok: true, - action: "rebuild", - webhookStatus: webhookRes.status, - previousPmtiles: before, - message: `Webhook trimis la N8N (HTTP ${webhookRes.status}). Rebuild-ul ruleaza ~8 min. Urmareste PMTiles last-modified.`, - }); - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - return NextResponse.json({ error: `Webhook esuat: ${msg}` }, { status: 500 }); + const ok = await firePmtilesRebuild("manual-rebuild"); + if (!ok) { + return NextResponse.json( + { error: "Webhook PMTiles indisponibil — verifica N8N_WEBHOOK_URL si serviciul pmtiles-webhook" }, + { status: 500 }, + ); } + return NextResponse.json({ + ok: true, + action: "rebuild", + previousPmtiles: before, + message: "Rebuild PMTiles pornit. Dureaza ~8 min. Urmareste PMTiles last-modified.", + }); } if (action === "check-rebuild") { diff --git a/src/modules/parcel-sync/services/auto-refresh-scheduler.ts b/src/modules/parcel-sync/services/auto-refresh-scheduler.ts index 965bbb4..736a8dc 100644 --- a/src/modules/parcel-sync/services/auto-refresh-scheduler.ts +++ b/src/modules/parcel-sync/services/auto-refresh-scheduler.ts @@ -159,26 +159,9 @@ async function runAutoRefresh() { g.__autoRefreshLastRun = today; console.log(`[auto-refresh] Finalizat: ${processed}/${uats.length} UATs, ${errors} erori.`); - // Trigger PMTiles rebuild via N8N webhook - const webhookUrl = process.env.N8N_WEBHOOK_URL; - if (webhookUrl) { - try { - await fetch(webhookUrl, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - event: "auto-refresh-complete", - uatCount: processed, - errors, - timestamp: new Date().toISOString(), - }), - }); - console.log("[auto-refresh] Webhook PMTiles rebuild trimis la N8N."); - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - console.warn(`[auto-refresh] Webhook N8N esuat: ${msg}`); - } - } + // Trigger PMTiles rebuild + const { firePmtilesRebuild } = await import("./pmtiles-webhook"); + await firePmtilesRebuild("auto-refresh-complete", { uatCount: processed, errors }); } catch (err) { const msg = err instanceof Error ? err.message : String(err); console.error(`[auto-refresh] Eroare generala: ${msg}`); diff --git a/src/modules/parcel-sync/services/pmtiles-webhook.ts b/src/modules/parcel-sync/services/pmtiles-webhook.ts new file mode 100644 index 0000000..f0a1bc1 --- /dev/null +++ b/src/modules/parcel-sync/services/pmtiles-webhook.ts @@ -0,0 +1,39 @@ +/** + * Shared helper — triggers PMTiles rebuild via webhook after sync operations. + * The webhook server (pmtiles-webhook systemd service on satra) runs + * `docker run architools-tippecanoe` to regenerate overview tiles. + */ + +const WEBHOOK_URL = process.env.N8N_WEBHOOK_URL || ""; + +export async function firePmtilesRebuild( + event: string, + metadata?: Record, +): Promise { + if (!WEBHOOK_URL) { + console.warn("[pmtiles-webhook] N8N_WEBHOOK_URL not configured — skipping rebuild trigger"); + return false; + } + + try { + const res = await fetch(WEBHOOK_URL, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + event, + timestamp: new Date().toISOString(), + ...metadata, + }), + }); + if (res.ok) { + console.log(`[pmtiles-webhook] Rebuild triggered (event: ${event}, HTTP ${res.status})`); + return true; + } + console.warn(`[pmtiles-webhook] Webhook returned HTTP ${res.status}`); + return false; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + console.warn(`[pmtiles-webhook] Failed: ${msg}`); + return false; + } +} diff --git a/src/modules/parcel-sync/services/weekend-deep-sync.ts b/src/modules/parcel-sync/services/weekend-deep-sync.ts index 191f8d4..50c4450 100644 --- a/src/modules/parcel-sync/services/weekend-deep-sync.ts +++ b/src/modules/parcel-sync/services/weekend-deep-sync.ts @@ -686,26 +686,10 @@ export async function triggerForceSync(options?: { } /* ------------------------------------------------------------------ */ -/* N8N Webhook — trigger PMTiles rebuild after sync cycle */ +/* PMTiles Webhook — trigger rebuild after sync cycle */ /* ------------------------------------------------------------------ */ async function fireSyncWebhook(cycle: number): Promise { - const url = process.env.N8N_WEBHOOK_URL; - if (!url) return; - - try { - await fetch(url, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - event: "weekend-sync-cycle-complete", - cycle, - timestamp: new Date().toISOString(), - }), - }); - console.log(`[weekend-sync] Webhook trimis la N8N (ciclu #${cycle})`); - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - console.warn(`[weekend-sync] Webhook N8N esuat: ${msg}`); - } + const { firePmtilesRebuild } = await import("./pmtiles-webhook"); + await firePmtilesRebuild("weekend-sync-cycle-complete", { cycle }); }