feat(parcel-sync): incremental sync, smart export, auto-refresh + weekend deep sync

Sync Incremental: - Add fetchObjectIds (returnIdsOnly) to eterra-client — fetches only OBJECTIDs in 1 request - Add fetchFeaturesByObjectIds — downloads only delta features by OBJECTID IN (...) - Rewrite syncLayer: compare remote IDs vs local, download only new features - Fallback to full sync for first sync, forceFullSync, or delta > 50% - Reduces sync time from ~10 min to ~5-10s for typical updates Smart Export Tab: - Hero buttons detect DB freshness — use export-local (instant) when data is fresh - Dynamic subtitles: "Din DB (sync acum Xh)" / "Sync incremental" / "Sync complet" - Re-sync link when data is fresh but user wants forced refresh - Removed duplicate "Descarca din DB" buttons from background section Auto-Refresh Scheduler: - Self-contained timer via instrumentation.ts (Next.js startup hook) - Weekday 1-5 AM: incremental refresh for existing UATs in DB - Staggered processing with random delays between UATs - Health check before processing, respects eTerra maintenance Weekend Deep Sync: - Full Magic processing for 9 large municipalities (Cluj, Bistrita, TgMures, etc.) - Runs Fri/Sat/Sun 23:00-04:00, round-robin intercalated between cities - 4 steps per city: sync terenuri, sync cladiri, import no-geom, enrichment - State persisted in KeyValueStore — survives restarts, continues across nights - Email status report at end of each session via Brevo SMTP - Admin page at /wds: add/remove cities, view progress, reset - Hint link on export tab pointing to /wds API endpoints: - POST /api/eterra/auto-refresh — N8N-compatible cron endpoint (Bearer token auth) - GET/POST /api/eterra/weekend-sync — queue management for /wds page Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-26 20:50:34 +02:00
parent 8f65efd5d1
commit 3b456eb481
11 changed files with 1929 additions and 128 deletions
@@ -0,0 +1,252 @@
+import { NextResponse } from "next/server";
+import { PrismaClient } from "@prisma/client";
+import { syncLayer } from "@/modules/parcel-sync/services/sync-service";
+import {
+  getLayerFreshness,
+  isFresh,
+} from "@/modules/parcel-sync/services/enrich-service";
+import { checkEterraHealthNow } from "@/modules/parcel-sync/services/eterra-health";
+
+export const runtime = "nodejs";
+export const dynamic = "force-dynamic";
+export const maxDuration = 300; // 5 min max — N8N handles overall timeout
+
+const prisma = new PrismaClient();
+
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+type UatRefreshResult = {
+  siruta: string;
+  uatName: string;
+  action: "synced" | "fresh" | "error";
+  reason?: string;
+  terenuri?: { new: number; removed: number };
+  cladiri?: { new: number; removed: number };
+  durationMs?: number;
+};
+
+/**
+ * POST /api/eterra/auto-refresh
+ *
+ * Server-to-server endpoint called by N8N cron to keep DB data fresh.
+ * Auth: Authorization: Bearer <NOTIFICATION_CRON_SECRET>
+ *
+ * Query params:
+ *   ?maxUats=5          — max UATs to process per run (default 5, max 10)
+ *   ?maxAgeHours=168    — freshness threshold in hours (default 168 = 7 days)
+ *   ?forceFullSync=true — force full re-download (for weekly deep sync)
+ *   ?includeEnrichment=true — re-enrich UATs with partial enrichment
+ */
+export async function POST(request: Request) {
+  // ── Auth ──
+  const secret = process.env.NOTIFICATION_CRON_SECRET;
+  if (!secret) {
+    return NextResponse.json(
+      { error: "NOTIFICATION_CRON_SECRET not configured" },
+      { status: 500 },
+    );
+  }
+  const authHeader = request.headers.get("Authorization");
+  const token = authHeader?.replace("Bearer ", "");
+  if (token !== secret) {
+    return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
+  }
+
+  // ── Parse params ──
+  const url = new URL(request.url);
+  const maxUats = Math.min(
+    Number(url.searchParams.get("maxUats") ?? "5") || 5,
+    10,
+  );
+  const maxAgeHours =
+    Number(url.searchParams.get("maxAgeHours") ?? "168") || 168;
+  const forceFullSync = url.searchParams.get("forceFullSync") === "true";
+  const includeEnrichment =
+    url.searchParams.get("includeEnrichment") === "true";
+
+  // ── Credentials ──
+  const username = process.env.ETERRA_USERNAME;
+  const password = process.env.ETERRA_PASSWORD;
+  if (!username || !password) {
+    return NextResponse.json(
+      { error: "ETERRA_USERNAME / ETERRA_PASSWORD not configured" },
+      { status: 500 },
+    );
+  }
+
+  // ── Health check ──
+  const health = await checkEterraHealthNow();
+  if (!health.available) {
+    return NextResponse.json({
+      processed: 0,
+      skipped: 0,
+      errors: 0,
+      duration: "0s",
+      message: `eTerra indisponibil: ${health.message ?? "maintenance"}`,
+      details: [],
+    });
+  }
+
+  // ── Find UATs with data in DB ──
+  const uatGroups = await prisma.gisFeature.groupBy({
+    by: ["siruta"],
+    _count: { id: true },
+  });
+
+  // Resolve UAT names
+  const sirutas = uatGroups.map((g) => g.siruta);
+  const uatRecords = await prisma.gisUat.findMany({
+    where: { siruta: { in: sirutas } },
+    select: { siruta: true, name: true },
+  });
+  const nameMap = new Map(uatRecords.map((u) => [u.siruta, u.name]));
+
+  // ── Check freshness per UAT ──
+  type UatCandidate = {
+    siruta: string;
+    uatName: string;
+    featureCount: number;
+    terenuriStale: boolean;
+    cladiriStale: boolean;
+    enrichedCount: number;
+    totalCount: number;
+  };
+
+  const stale: UatCandidate[] = [];
+  const fresh: string[] = [];
+
+  for (const group of uatGroups) {
+    const sir = group.siruta;
+    const [tStatus, cStatus] = await Promise.all([
+      getLayerFreshness(sir, "TERENURI_ACTIVE"),
+      getLayerFreshness(sir, "CLADIRI_ACTIVE"),
+    ]);
+    const tFresh = isFresh(tStatus.lastSynced, maxAgeHours);
+    const cFresh = isFresh(cStatus.lastSynced, maxAgeHours);
+
+    if (forceFullSync || !tFresh || !cFresh) {
+      stale.push({
+        siruta: sir,
+        uatName: nameMap.get(sir) ?? sir,
+        featureCount: group._count.id,
+        terenuriStale: !tFresh || forceFullSync,
+        cladiriStale: !cFresh || forceFullSync,
+        enrichedCount: tStatus.enrichedCount,
+        totalCount: tStatus.featureCount + cStatus.featureCount,
+      });
+    } else {
+      fresh.push(sir);
+    }
+  }
+
+  // Shuffle stale UATs so we don't always process the same ones first
+  for (let i = stale.length - 1; i > 0; i--) {
+    const j = Math.floor(Math.random() * (i + 1));
+    [stale[i]!, stale[j]!] = [stale[j]!, stale[i]!];
+  }
+
+  const toProcess = stale.slice(0, maxUats);
+  const startTime = Date.now();
+  const details: UatRefreshResult[] = [];
+  let errorCount = 0;
+
+  // ── Process stale UATs ──
+  for (let idx = 0; idx < toProcess.length; idx++) {
+    const uat = toProcess[idx]!;
+
+    // Random delay between UATs (30-120s) to spread load
+    if (idx > 0) {
+      const delay = 30_000 + Math.random() * 90_000;
+      await sleep(delay);
+    }
+
+    const uatStart = Date.now();
+    console.log(
+      `[auto-refresh] Processing UAT ${uat.siruta} (${uat.uatName})...`,
+    );
+
+    try {
+      let terenuriResult = { newFeatures: 0, removedFeatures: 0 };
+      let cladiriResult = { newFeatures: 0, removedFeatures: 0 };
+
+      if (uat.terenuriStale) {
+        const res = await syncLayer(username, password, uat.siruta, "TERENURI_ACTIVE", {
+          uatName: uat.uatName,
+          forceFullSync,
+        });
+        terenuriResult = { newFeatures: res.newFeatures, removedFeatures: res.removedFeatures };
+      }
+
+      if (uat.cladiriStale) {
+        const res = await syncLayer(username, password, uat.siruta, "CLADIRI_ACTIVE", {
+          uatName: uat.uatName,
+          forceFullSync,
+        });
+        cladiriResult = { newFeatures: res.newFeatures, removedFeatures: res.removedFeatures };
+      }
+
+      // Optional: re-enrich if partial enrichment
+      if (includeEnrichment && uat.enrichedCount < uat.totalCount) {
+        try {
+          const { EterraClient } = await import(
+            "@/modules/parcel-sync/services/eterra-client"
+          );
+          const { enrichFeatures } = await import(
+            "@/modules/parcel-sync/services/enrich-service"
+          );
+          const enrichClient = await EterraClient.create(username, password);
+          await enrichFeatures(enrichClient, uat.siruta);
+        } catch (enrichErr) {
+          console.warn(
+            `[auto-refresh] Enrichment failed for ${uat.siruta}:`,
+            enrichErr instanceof Error ? enrichErr.message : enrichErr,
+          );
+        }
+      }
+
+      const durationMs = Date.now() - uatStart;
+      console.log(
+        `[auto-refresh] UAT ${uat.siruta}: terenuri +${terenuriResult.newFeatures}/-${terenuriResult.removedFeatures}, cladiri +${cladiriResult.newFeatures}/-${cladiriResult.removedFeatures} (${(durationMs / 1000).toFixed(1)}s)`,
+      );
+
+      details.push({
+        siruta: uat.siruta,
+        uatName: uat.uatName,
+        action: "synced",
+        terenuri: { new: terenuriResult.newFeatures, removed: terenuriResult.removedFeatures },
+        cladiri: { new: cladiriResult.newFeatures, removed: cladiriResult.removedFeatures },
+        durationMs,
+      });
+    } catch (error) {
+      errorCount++;
+      const msg = error instanceof Error ? error.message : "Unknown error";
+      console.error(`[auto-refresh] Error on UAT ${uat.siruta}: ${msg}`);
+      details.push({
+        siruta: uat.siruta,
+        uatName: uat.uatName,
+        action: "error",
+        reason: msg,
+        durationMs: Date.now() - uatStart,
+      });
+    }
+  }
+
+  const totalDuration = Date.now() - startTime;
+  const durationStr =
+    totalDuration > 60_000
+      ? `${Math.floor(totalDuration / 60_000)}m ${Math.round((totalDuration % 60_000) / 1000)}s`
+      : `${Math.round(totalDuration / 1000)}s`;
+
+  console.log(
+    `[auto-refresh] Completed ${toProcess.length}/${stale.length} UATs, ${errorCount} errors (${durationStr})`,
+  );
+
+  return NextResponse.json({
+    processed: toProcess.length,
+    skipped: fresh.length,
+    staleTotal: stale.length,
+    errors: errorCount,
+    duration: durationStr,
+    details,
+  });
+}
@@ -0,0 +1,181 @@
+import { NextResponse } from "next/server";
+import { PrismaClient, Prisma } from "@prisma/client";
+
+const prisma = new PrismaClient();
+
+const KV_NAMESPACE = "parcel-sync-weekend";
+const KV_KEY = "queue-state";
+
+type StepName = "sync_terenuri" | "sync_cladiri" | "import_nogeom" | "enrich";
+type StepStatus = "pending" | "done" | "error";
+
+type CityState = {
+  siruta: string;
+  name: string;
+  county: string;
+  priority: number;
+  steps: Record<StepName, StepStatus>;
+  lastActivity?: string;
+  errorMessage?: string;
+};
+
+type WeekendSyncState = {
+  cities: CityState[];
+  lastSessionDate?: string;
+  totalSessions: number;
+  completedCycles: number;
+};
+
+/**
+ * GET /api/eterra/weekend-sync
+ * Returns the current queue state.
+ */
+export async function GET() {
+  // Auth handled by middleware (route is not excluded)
+  const row = await prisma.keyValueStore.findUnique({
+    where: { namespace_key: { namespace: KV_NAMESPACE, key: KV_KEY } },
+  });
+
+  if (!row?.value) {
+    return NextResponse.json({ state: null });
+  }
+
+  // Enrich with DB feature counts per city
+  const state = row.value as unknown as WeekendSyncState;
+  const sirutas = state.cities.map((c) => c.siruta);
+
+  const counts = await prisma.gisFeature.groupBy({
+    by: ["siruta", "layerId"],
+    where: { siruta: { in: sirutas } },
+    _count: { id: true },
+  });
+
+  const enrichedCounts = await prisma.gisFeature.groupBy({
+    by: ["siruta"],
+    where: { siruta: { in: sirutas }, enrichedAt: { not: null } },
+    _count: { id: true },
+  });
+
+  const enrichedMap = new Map(enrichedCounts.map((e) => [e.siruta, e._count.id]));
+
+  type CityStats = {
+    terenuri: number;
+    cladiri: number;
+    total: number;
+    enriched: number;
+  };
+  const statsMap = new Map<string, CityStats>();
+
+  for (const c of counts) {
+    const existing = statsMap.get(c.siruta) ?? { terenuri: 0, cladiri: 0, total: 0, enriched: 0 };
+    existing.total += c._count.id;
+    if (c.layerId === "TERENURI_ACTIVE") existing.terenuri = c._count.id;
+    if (c.layerId === "CLADIRI_ACTIVE") existing.cladiri = c._count.id;
+    existing.enriched = enrichedMap.get(c.siruta) ?? 0;
+    statsMap.set(c.siruta, existing);
+  }
+
+  const citiesWithStats = state.cities.map((c) => ({
+    ...c,
+    dbStats: statsMap.get(c.siruta) ?? { terenuri: 0, cladiri: 0, total: 0, enriched: 0 },
+  }));
+
+  return NextResponse.json({
+    state: { ...state, cities: citiesWithStats },
+  });
+}
+
+/**
+ * POST /api/eterra/weekend-sync
+ * Modify the queue: add/remove cities, reset steps, change priority.
+ */
+export async function POST(request: Request) {
+  // Auth handled by middleware (route is not excluded)
+  const body = (await request.json()) as {
+    action: "add" | "remove" | "reset" | "reset_all" | "set_priority";
+    siruta?: string;
+    name?: string;
+    county?: string;
+    priority?: number;
+  };
+
+  // Load current state
+  const row = await prisma.keyValueStore.findUnique({
+    where: { namespace_key: { namespace: KV_NAMESPACE, key: KV_KEY } },
+  });
+
+  const state: WeekendSyncState = row?.value
+    ? (row.value as unknown as WeekendSyncState)
+    : { cities: [], totalSessions: 0, completedCycles: 0 };
+
+  const freshSteps: Record<StepName, StepStatus> = {
+    sync_terenuri: "pending",
+    sync_cladiri: "pending",
+    import_nogeom: "pending",
+    enrich: "pending",
+  };
+
+  switch (body.action) {
+    case "add": {
+      if (!body.siruta || !body.name) {
+        return NextResponse.json(
+          { error: "siruta si name sunt obligatorii" },
+          { status: 400 },
+        );
+      }
+      if (state.cities.some((c) => c.siruta === body.siruta)) {
+        return NextResponse.json(
+          { error: `${body.name} (${body.siruta}) e deja in coada` },
+          { status: 409 },
+        );
+      }
+      state.cities.push({
+        siruta: body.siruta,
+        name: body.name,
+        county: body.county ?? "",
+        priority: body.priority ?? 3,
+        steps: { ...freshSteps },
+      });
+      break;
+    }
+    case "remove": {
+      state.cities = state.cities.filter((c) => c.siruta !== body.siruta);
+      break;
+    }
+    case "reset": {
+      const city = state.cities.find((c) => c.siruta === body.siruta);
+      if (city) {
+        city.steps = { ...freshSteps };
+        city.errorMessage = undefined;
+      }
+      break;
+    }
+    case "reset_all": {
+      for (const city of state.cities) {
+        city.steps = { ...freshSteps };
+        city.errorMessage = undefined;
+      }
+      state.completedCycles = 0;
+      break;
+    }
+    case "set_priority": {
+      const city = state.cities.find((c) => c.siruta === body.siruta);
+      if (city && body.priority != null) {
+        city.priority = body.priority;
+      }
+      break;
+    }
+  }
+
+  await prisma.keyValueStore.upsert({
+    where: { namespace_key: { namespace: KV_NAMESPACE, key: KV_KEY } },
+    update: { value: state as unknown as Prisma.InputJsonValue },
+    create: {
+      namespace: KV_NAMESPACE,
+      key: KV_KEY,
+      value: state as unknown as Prisma.InputJsonValue,
+    },
+  });
+
+  return NextResponse.json({ ok: true, cities: state.cities.length });
+}