feat(parcel-sync): incremental sync, smart export, auto-refresh + weekend deep sync
Sync Incremental: - Add fetchObjectIds (returnIdsOnly) to eterra-client — fetches only OBJECTIDs in 1 request - Add fetchFeaturesByObjectIds — downloads only delta features by OBJECTID IN (...) - Rewrite syncLayer: compare remote IDs vs local, download only new features - Fallback to full sync for first sync, forceFullSync, or delta > 50% - Reduces sync time from ~10 min to ~5-10s for typical updates Smart Export Tab: - Hero buttons detect DB freshness — use export-local (instant) when data is fresh - Dynamic subtitles: "Din DB (sync acum Xh)" / "Sync incremental" / "Sync complet" - Re-sync link when data is fresh but user wants forced refresh - Removed duplicate "Descarca din DB" buttons from background section Auto-Refresh Scheduler: - Self-contained timer via instrumentation.ts (Next.js startup hook) - Weekday 1-5 AM: incremental refresh for existing UATs in DB - Staggered processing with random delays between UATs - Health check before processing, respects eTerra maintenance Weekend Deep Sync: - Full Magic processing for 9 large municipalities (Cluj, Bistrita, TgMures, etc.) - Runs Fri/Sat/Sun 23:00-04:00, round-robin intercalated between cities - 4 steps per city: sync terenuri, sync cladiri, import no-geom, enrichment - State persisted in KeyValueStore — survives restarts, continues across nights - Email status report at end of each session via Brevo SMTP - Admin page at /wds: add/remove cities, view progress, reset - Hint link on export tab pointing to /wds API endpoints: - POST /api/eterra/auto-refresh — N8N-compatible cron endpoint (Bearer token auth) - GET/POST /api/eterra/weekend-sync — queue management for /wds page Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
AI Assistant
@@ -0,0 +1,181 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { PrismaClient, Prisma } from "@prisma/client";
|
||||
|
||||
const prisma = new PrismaClient();
|
||||
|
||||
const KV_NAMESPACE = "parcel-sync-weekend";
|
||||
const KV_KEY = "queue-state";
|
||||
|
||||
type StepName = "sync_terenuri" | "sync_cladiri" | "import_nogeom" | "enrich";
|
||||
type StepStatus = "pending" | "done" | "error";
|
||||
|
||||
type CityState = {
|
||||
siruta: string;
|
||||
name: string;
|
||||
county: string;
|
||||
priority: number;
|
||||
steps: Record<StepName, StepStatus>;
|
||||
lastActivity?: string;
|
||||
errorMessage?: string;
|
||||
};
|
||||
|
||||
type WeekendSyncState = {
|
||||
cities: CityState[];
|
||||
lastSessionDate?: string;
|
||||
totalSessions: number;
|
||||
completedCycles: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* GET /api/eterra/weekend-sync
|
||||
* Returns the current queue state.
|
||||
*/
|
||||
export async function GET() {
|
||||
// Auth handled by middleware (route is not excluded)
|
||||
const row = await prisma.keyValueStore.findUnique({
|
||||
where: { namespace_key: { namespace: KV_NAMESPACE, key: KV_KEY } },
|
||||
});
|
||||
|
||||
if (!row?.value) {
|
||||
return NextResponse.json({ state: null });
|
||||
}
|
||||
|
||||
// Enrich with DB feature counts per city
|
||||
const state = row.value as unknown as WeekendSyncState;
|
||||
const sirutas = state.cities.map((c) => c.siruta);
|
||||
|
||||
const counts = await prisma.gisFeature.groupBy({
|
||||
by: ["siruta", "layerId"],
|
||||
where: { siruta: { in: sirutas } },
|
||||
_count: { id: true },
|
||||
});
|
||||
|
||||
const enrichedCounts = await prisma.gisFeature.groupBy({
|
||||
by: ["siruta"],
|
||||
where: { siruta: { in: sirutas }, enrichedAt: { not: null } },
|
||||
_count: { id: true },
|
||||
});
|
||||
|
||||
const enrichedMap = new Map(enrichedCounts.map((e) => [e.siruta, e._count.id]));
|
||||
|
||||
type CityStats = {
|
||||
terenuri: number;
|
||||
cladiri: number;
|
||||
total: number;
|
||||
enriched: number;
|
||||
};
|
||||
const statsMap = new Map<string, CityStats>();
|
||||
|
||||
for (const c of counts) {
|
||||
const existing = statsMap.get(c.siruta) ?? { terenuri: 0, cladiri: 0, total: 0, enriched: 0 };
|
||||
existing.total += c._count.id;
|
||||
if (c.layerId === "TERENURI_ACTIVE") existing.terenuri = c._count.id;
|
||||
if (c.layerId === "CLADIRI_ACTIVE") existing.cladiri = c._count.id;
|
||||
existing.enriched = enrichedMap.get(c.siruta) ?? 0;
|
||||
statsMap.set(c.siruta, existing);
|
||||
}
|
||||
|
||||
const citiesWithStats = state.cities.map((c) => ({
|
||||
...c,
|
||||
dbStats: statsMap.get(c.siruta) ?? { terenuri: 0, cladiri: 0, total: 0, enriched: 0 },
|
||||
}));
|
||||
|
||||
return NextResponse.json({
|
||||
state: { ...state, cities: citiesWithStats },
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /api/eterra/weekend-sync
|
||||
* Modify the queue: add/remove cities, reset steps, change priority.
|
||||
*/
|
||||
export async function POST(request: Request) {
|
||||
// Auth handled by middleware (route is not excluded)
|
||||
const body = (await request.json()) as {
|
||||
action: "add" | "remove" | "reset" | "reset_all" | "set_priority";
|
||||
siruta?: string;
|
||||
name?: string;
|
||||
county?: string;
|
||||
priority?: number;
|
||||
};
|
||||
|
||||
// Load current state
|
||||
const row = await prisma.keyValueStore.findUnique({
|
||||
where: { namespace_key: { namespace: KV_NAMESPACE, key: KV_KEY } },
|
||||
});
|
||||
|
||||
const state: WeekendSyncState = row?.value
|
||||
? (row.value as unknown as WeekendSyncState)
|
||||
: { cities: [], totalSessions: 0, completedCycles: 0 };
|
||||
|
||||
const freshSteps: Record<StepName, StepStatus> = {
|
||||
sync_terenuri: "pending",
|
||||
sync_cladiri: "pending",
|
||||
import_nogeom: "pending",
|
||||
enrich: "pending",
|
||||
};
|
||||
|
||||
switch (body.action) {
|
||||
case "add": {
|
||||
if (!body.siruta || !body.name) {
|
||||
return NextResponse.json(
|
||||
{ error: "siruta si name sunt obligatorii" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
if (state.cities.some((c) => c.siruta === body.siruta)) {
|
||||
return NextResponse.json(
|
||||
{ error: `${body.name} (${body.siruta}) e deja in coada` },
|
||||
{ status: 409 },
|
||||
);
|
||||
}
|
||||
state.cities.push({
|
||||
siruta: body.siruta,
|
||||
name: body.name,
|
||||
county: body.county ?? "",
|
||||
priority: body.priority ?? 3,
|
||||
steps: { ...freshSteps },
|
||||
});
|
||||
break;
|
||||
}
|
||||
case "remove": {
|
||||
state.cities = state.cities.filter((c) => c.siruta !== body.siruta);
|
||||
break;
|
||||
}
|
||||
case "reset": {
|
||||
const city = state.cities.find((c) => c.siruta === body.siruta);
|
||||
if (city) {
|
||||
city.steps = { ...freshSteps };
|
||||
city.errorMessage = undefined;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "reset_all": {
|
||||
for (const city of state.cities) {
|
||||
city.steps = { ...freshSteps };
|
||||
city.errorMessage = undefined;
|
||||
}
|
||||
state.completedCycles = 0;
|
||||
break;
|
||||
}
|
||||
case "set_priority": {
|
||||
const city = state.cities.find((c) => c.siruta === body.siruta);
|
||||
if (city && body.priority != null) {
|
||||
city.priority = body.priority;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
await prisma.keyValueStore.upsert({
|
||||
where: { namespace_key: { namespace: KV_NAMESPACE, key: KV_KEY } },
|
||||
update: { value: state as unknown as Prisma.InputJsonValue },
|
||||
create: {
|
||||
namespace: KV_NAMESPACE,
|
||||
key: KV_KEY,
|
||||
value: state as unknown as Prisma.InputJsonValue,
|
||||
},
|
||||
});
|
||||
|
||||
return NextResponse.json({ ok: true, cities: state.cities.length });
|
||||
}
|
||||
Reference in New Issue
Block a user