feat(monitor): add Sync All Romania + live GIS stats
- /api/eterra/stats: lightweight polling endpoint (8 parallel Prisma queries, 30s poll) - /api/eterra/sync-all-counties: iterates all counties in DB sequentially, syncs TERENURI + CLADIRI + INTRAVILAN + enrichment (magic mode) per UAT - Monitor page: live stat cards (UATs, parcels, buildings, DB size), Sync All Romania button with progress tracking at county+UAT level - Concurrency guard: blocks county sync while all-Romania sync runs Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude VM
@@ -0,0 +1,86 @@
|
||||
/**
|
||||
* GET /api/eterra/stats
|
||||
*
|
||||
* Lightweight endpoint for the monitor page — returns aggregate counts
|
||||
* suitable for polling every 30s without heavy DB load.
|
||||
*
|
||||
* Response:
|
||||
* {
|
||||
* totalUats: number,
|
||||
* totalFeatures: number,
|
||||
* totalTerenuri: number,
|
||||
* totalCladiri: number,
|
||||
* totalEnriched: number,
|
||||
* totalNoGeom: number,
|
||||
* countiesWithData: number,
|
||||
* lastSyncAt: string | null,
|
||||
* dbSizeMb: number | null,
|
||||
* }
|
||||
*/
|
||||
|
||||
import { prisma } from "@/core/storage/prisma";
|
||||
import { NextResponse } from "next/server";
|
||||
|
||||
export const runtime = "nodejs";
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
export async function GET() {
|
||||
try {
|
||||
const [
|
||||
totalUats,
|
||||
totalFeatures,
|
||||
totalTerenuri,
|
||||
totalCladiri,
|
||||
totalEnriched,
|
||||
totalNoGeom,
|
||||
countyAgg,
|
||||
lastSync,
|
||||
dbSize,
|
||||
] = await Promise.all([
|
||||
prisma.gisUat.count(),
|
||||
prisma.gisFeature.count({ where: { objectId: { gt: 0 } } }),
|
||||
prisma.gisFeature.count({ where: { layerId: "TERENURI_ACTIVE", objectId: { gt: 0 } } }),
|
||||
prisma.gisFeature.count({ where: { layerId: "CLADIRI_ACTIVE", objectId: { gt: 0 } } }),
|
||||
prisma.gisFeature.count({ where: { enrichedAt: { not: null } } }),
|
||||
prisma.gisFeature.count({ where: { geometrySource: "NO_GEOMETRY" } }),
|
||||
prisma.gisUat.groupBy({
|
||||
by: ["county"],
|
||||
where: { county: { not: null } },
|
||||
_count: true,
|
||||
}),
|
||||
prisma.gisSyncRun.findFirst({
|
||||
where: { status: "done" },
|
||||
orderBy: { completedAt: "desc" },
|
||||
select: { completedAt: true },
|
||||
}),
|
||||
prisma.$queryRaw<Array<{ size: string }>>`
|
||||
SELECT pg_size_pretty(pg_database_size(current_database())) as size
|
||||
`,
|
||||
]);
|
||||
|
||||
// Parse DB size to MB
|
||||
const sizeStr = dbSize[0]?.size ?? "";
|
||||
let dbSizeMb: number | null = null;
|
||||
const mbMatch = sizeStr.match(/([\d.]+)\s*(MB|GB|TB)/i);
|
||||
if (mbMatch) {
|
||||
const val = parseFloat(mbMatch[1]!);
|
||||
const unit = mbMatch[2]!.toUpperCase();
|
||||
dbSizeMb = unit === "GB" ? val * 1024 : unit === "TB" ? val * 1024 * 1024 : val;
|
||||
}
|
||||
|
||||
return NextResponse.json({
|
||||
totalUats,
|
||||
totalFeatures,
|
||||
totalTerenuri,
|
||||
totalCladiri,
|
||||
totalEnriched,
|
||||
totalNoGeom,
|
||||
countiesWithData: countyAgg.length,
|
||||
lastSyncAt: lastSync?.completedAt?.toISOString() ?? null,
|
||||
dbSizeMb: dbSizeMb ? Math.round(dbSizeMb) : null,
|
||||
});
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "Eroare server";
|
||||
return NextResponse.json({ error: message }, { status: 500 });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,289 @@
|
||||
/**
|
||||
* POST /api/eterra/sync-all-counties
|
||||
*
|
||||
* Starts a background sync for ALL counties in the database (entire Romania).
|
||||
* Iterates counties sequentially, running county-sync logic for each.
|
||||
* Returns immediately with jobId — progress via /api/eterra/progress.
|
||||
*
|
||||
* Body: {} (no params needed)
|
||||
*/
|
||||
|
||||
import { prisma } from "@/core/storage/prisma";
|
||||
import {
|
||||
setProgress,
|
||||
clearProgress,
|
||||
type SyncProgress,
|
||||
} from "@/modules/parcel-sync/services/progress-store";
|
||||
import { syncLayer } from "@/modules/parcel-sync/services/sync-service";
|
||||
import { enrichFeatures } from "@/modules/parcel-sync/services/enrich-service";
|
||||
import { EterraClient } from "@/modules/parcel-sync/services/eterra-client";
|
||||
import { checkEterraHealthNow } from "@/modules/parcel-sync/services/eterra-health";
|
||||
import { createAppNotification } from "@/core/notifications/app-notifications";
|
||||
import { getSessionCredentials } from "@/modules/parcel-sync/services/session-store";
|
||||
|
||||
export const runtime = "nodejs";
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
/* Concurrency guard — blocks both this and single county sync */
|
||||
const g = globalThis as {
|
||||
__countySyncRunning?: string;
|
||||
__allCountiesSyncRunning?: boolean;
|
||||
};
|
||||
|
||||
export async function POST() {
|
||||
const session = getSessionCredentials();
|
||||
const username = String(session?.username || process.env.ETERRA_USERNAME || "").trim();
|
||||
const password = String(session?.password || process.env.ETERRA_PASSWORD || "").trim();
|
||||
if (!username || !password) {
|
||||
return Response.json(
|
||||
{ error: "Credentiale lipsa — conecteaza-te la eTerra mai intai." },
|
||||
{ status: 401 },
|
||||
);
|
||||
}
|
||||
|
||||
if (g.__allCountiesSyncRunning) {
|
||||
return Response.json(
|
||||
{ error: "Sync All Romania deja in curs" },
|
||||
{ status: 409 },
|
||||
);
|
||||
}
|
||||
|
||||
if (g.__countySyncRunning) {
|
||||
return Response.json(
|
||||
{ error: `Sync judet deja in curs: ${g.__countySyncRunning}` },
|
||||
{ status: 409 },
|
||||
);
|
||||
}
|
||||
|
||||
const jobId = crypto.randomUUID();
|
||||
g.__allCountiesSyncRunning = true;
|
||||
|
||||
setProgress({
|
||||
jobId,
|
||||
downloaded: 0,
|
||||
total: 100,
|
||||
status: "running",
|
||||
phase: "Pregatire sync Romania...",
|
||||
});
|
||||
|
||||
void runAllCountiesSync(jobId, username, password);
|
||||
|
||||
return Response.json(
|
||||
{ jobId, message: "Sync All Romania pornit" },
|
||||
{ status: 202 },
|
||||
);
|
||||
}
|
||||
|
||||
async function runAllCountiesSync(
|
||||
jobId: string,
|
||||
username: string,
|
||||
password: string,
|
||||
) {
|
||||
const push = (p: Partial<SyncProgress>) =>
|
||||
setProgress({
|
||||
jobId,
|
||||
downloaded: 0,
|
||||
total: 100,
|
||||
status: "running",
|
||||
...p,
|
||||
} as SyncProgress);
|
||||
|
||||
try {
|
||||
// Health check
|
||||
const health = await checkEterraHealthNow();
|
||||
if (!health.available) {
|
||||
setProgress({
|
||||
jobId,
|
||||
downloaded: 0,
|
||||
total: 100,
|
||||
status: "error",
|
||||
phase: "eTerra indisponibil",
|
||||
message: health.message ?? "maintenance",
|
||||
});
|
||||
g.__allCountiesSyncRunning = false;
|
||||
setTimeout(() => clearProgress(jobId), 3_600_000);
|
||||
return;
|
||||
}
|
||||
|
||||
// Get all distinct counties, ordered alphabetically
|
||||
const countyRows = await prisma.gisUat.groupBy({
|
||||
by: ["county"],
|
||||
where: { county: { not: null } },
|
||||
_count: true,
|
||||
orderBy: { county: "asc" },
|
||||
});
|
||||
|
||||
const counties = countyRows
|
||||
.map((r) => r.county)
|
||||
.filter((c): c is string => c != null);
|
||||
|
||||
if (counties.length === 0) {
|
||||
setProgress({
|
||||
jobId,
|
||||
downloaded: 100,
|
||||
total: 100,
|
||||
status: "done",
|
||||
phase: "Niciun judet gasit in DB",
|
||||
});
|
||||
g.__allCountiesSyncRunning = false;
|
||||
setTimeout(() => clearProgress(jobId), 3_600_000);
|
||||
return;
|
||||
}
|
||||
|
||||
push({ phase: `0/${counties.length} judete — pornire...` });
|
||||
|
||||
const countyResults: Array<{
|
||||
county: string;
|
||||
uatCount: number;
|
||||
errors: number;
|
||||
duration: number;
|
||||
}> = [];
|
||||
let totalErrors = 0;
|
||||
let totalUats = 0;
|
||||
|
||||
for (let ci = 0; ci < counties.length; ci++) {
|
||||
const county = counties[ci]!;
|
||||
g.__countySyncRunning = county;
|
||||
|
||||
// Get UATs for this county
|
||||
const uats = await prisma.$queryRawUnsafe<
|
||||
Array<{
|
||||
siruta: string;
|
||||
name: string | null;
|
||||
total: number;
|
||||
enriched: number;
|
||||
}>
|
||||
>(
|
||||
`SELECT u.siruta, u.name,
|
||||
COALESCE(f.total, 0)::int as total,
|
||||
COALESCE(f.enriched, 0)::int as enriched
|
||||
FROM "GisUat" u
|
||||
LEFT JOIN (
|
||||
SELECT siruta, COUNT(*)::int as total,
|
||||
COUNT(*) FILTER (WHERE "enrichedAt" IS NOT NULL)::int as enriched
|
||||
FROM "GisFeature"
|
||||
WHERE "layerId" IN ('TERENURI_ACTIVE','CLADIRI_ACTIVE') AND "objectId" > 0
|
||||
GROUP BY siruta
|
||||
) f ON u.siruta = f.siruta
|
||||
WHERE u.county = $1
|
||||
ORDER BY COALESCE(f.total, 0) DESC`,
|
||||
county,
|
||||
);
|
||||
|
||||
if (uats.length === 0) {
|
||||
countyResults.push({ county, uatCount: 0, errors: 0, duration: 0 });
|
||||
continue;
|
||||
}
|
||||
|
||||
const countyStart = Date.now();
|
||||
let countyErrors = 0;
|
||||
|
||||
for (let i = 0; i < uats.length; i++) {
|
||||
const uat = uats[i]!;
|
||||
const uatName = uat.name ?? uat.siruta;
|
||||
const ratio = uat.total > 0 ? uat.enriched / uat.total : 0;
|
||||
const isMagic = ratio > 0.3;
|
||||
const mode = isMagic ? "magic" : "base";
|
||||
|
||||
// Progress: county level + UAT level
|
||||
const countyPct = ci / counties.length;
|
||||
const uatPct = i / uats.length;
|
||||
const overallPct = Math.round((countyPct + uatPct / counties.length) * 100);
|
||||
|
||||
push({
|
||||
downloaded: overallPct,
|
||||
total: 100,
|
||||
phase: `[${ci + 1}/${counties.length}] ${county} — [${i + 1}/${uats.length}] ${uatName} (${mode})`,
|
||||
note: countyResults.length > 0
|
||||
? `Ultimul judet: ${countyResults[countyResults.length - 1]!.county} (${countyResults[countyResults.length - 1]!.uatCount} UAT, ${countyResults[countyResults.length - 1]!.errors} err)`
|
||||
: undefined,
|
||||
});
|
||||
|
||||
try {
|
||||
await syncLayer(username, password, uat.siruta, "TERENURI_ACTIVE", { uatName });
|
||||
await syncLayer(username, password, uat.siruta, "CLADIRI_ACTIVE", { uatName });
|
||||
|
||||
// LIMITE_INTRAV_DYNAMIC — best effort
|
||||
try {
|
||||
await syncLayer(username, password, uat.siruta, "LIMITE_INTRAV_DYNAMIC", { uatName });
|
||||
} catch { /* skip */ }
|
||||
|
||||
// Enrichment for magic mode
|
||||
if (isMagic) {
|
||||
try {
|
||||
const client = await EterraClient.create(username, password, { timeoutMs: 120_000 });
|
||||
await enrichFeatures(client, uat.siruta);
|
||||
} catch {
|
||||
// Enrichment failure is non-fatal
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
countyErrors++;
|
||||
const msg = err instanceof Error ? err.message : "Unknown";
|
||||
console.error(`[sync-all] ${county}/${uatName}: ${msg}`);
|
||||
}
|
||||
}
|
||||
|
||||
const dur = Math.round((Date.now() - countyStart) / 1000);
|
||||
countyResults.push({ county, uatCount: uats.length, errors: countyErrors, duration: dur });
|
||||
totalErrors += countyErrors;
|
||||
totalUats += uats.length;
|
||||
|
||||
console.log(
|
||||
`[sync-all] ${ci + 1}/${counties.length} ${county}: ${uats.length} UAT, ${countyErrors} err, ${dur}s`,
|
||||
);
|
||||
}
|
||||
|
||||
const totalDur = countyResults.reduce((s, r) => s + r.duration, 0);
|
||||
const summary = `${counties.length} judete, ${totalUats} UAT-uri, ${totalErrors} erori, ${formatDuration(totalDur)}`;
|
||||
|
||||
setProgress({
|
||||
jobId,
|
||||
downloaded: 100,
|
||||
total: 100,
|
||||
status: totalErrors > 0 && totalErrors === totalUats ? "error" : "done",
|
||||
phase: "Sync Romania finalizat",
|
||||
message: summary,
|
||||
});
|
||||
|
||||
await createAppNotification({
|
||||
type: totalErrors > 0 ? "sync-error" : "sync-complete",
|
||||
title: totalErrors > 0
|
||||
? `Sync Romania: ${totalErrors} erori din ${totalUats} UAT-uri`
|
||||
: `Sync Romania: ${totalUats} UAT-uri in ${counties.length} judete`,
|
||||
message: summary,
|
||||
metadata: { jobId, counties: counties.length, totalUats, totalErrors, totalDuration: totalDur },
|
||||
});
|
||||
|
||||
console.log(`[sync-all] Done: ${summary}`);
|
||||
setTimeout(() => clearProgress(jobId), 12 * 3_600_000);
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : "Unknown";
|
||||
setProgress({
|
||||
jobId,
|
||||
downloaded: 0,
|
||||
total: 100,
|
||||
status: "error",
|
||||
phase: "Eroare",
|
||||
message: msg,
|
||||
});
|
||||
await createAppNotification({
|
||||
type: "sync-error",
|
||||
title: "Sync Romania: eroare generala",
|
||||
message: msg,
|
||||
metadata: { jobId },
|
||||
});
|
||||
setTimeout(() => clearProgress(jobId), 3_600_000);
|
||||
} finally {
|
||||
g.__allCountiesSyncRunning = false;
|
||||
g.__countySyncRunning = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function formatDuration(seconds: number): string {
|
||||
if (seconds < 60) return `${seconds}s`;
|
||||
if (seconds < 3600) return `${Math.floor(seconds / 60)}m${String(seconds % 60).padStart(2, "0")}s`;
|
||||
const h = Math.floor(seconds / 3600);
|
||||
const m = Math.floor((seconds % 3600) / 60);
|
||||
return `${h}h${String(m).padStart(2, "0")}m`;
|
||||
}
|
||||
@@ -26,7 +26,7 @@ export const runtime = "nodejs";
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
/* Concurrency guard */
|
||||
const g = globalThis as { __countySyncRunning?: string };
|
||||
const g = globalThis as { __countySyncRunning?: string; __allCountiesSyncRunning?: boolean };
|
||||
|
||||
export async function POST(req: Request) {
|
||||
let body: { county?: string };
|
||||
@@ -51,6 +51,13 @@ export async function POST(req: Request) {
|
||||
return Response.json({ error: "Judetul lipseste" }, { status: 400 });
|
||||
}
|
||||
|
||||
if (g.__allCountiesSyncRunning) {
|
||||
return Response.json(
|
||||
{ error: "Sync All Romania in curs — asteapta sa se termine" },
|
||||
{ status: 409 },
|
||||
);
|
||||
}
|
||||
|
||||
if (g.__countySyncRunning) {
|
||||
return Response.json(
|
||||
{ error: `Sync judet deja in curs: ${g.__countySyncRunning}` },
|
||||
|
||||
Reference in New Issue
Block a user