feat(parcel-sync): sync-first architecture — DB as ground truth
- Rewrite export-bundle to sync-first: check freshness -> sync layers -> enrich (magic) -> build GPKG/CSV from local DB - Rewrite export-layer-gpkg to sync-first: sync if stale -> export from DB - Create enrich-service.ts: extracted magic enrichment logic (CF, owners, addresses) with DB storage - Add enrichment + enrichedAt columns to GisFeature schema - Update PostGIS views to include enrichment data - UI: update button labels for sync-first semantics, refresh sync status after exports - Smart caching: skip sync if data is fresh (168h / 1 week default)
This commit is contained in:
AI Assistant
@@ -1,9 +1,23 @@
|
||||
import { EterraClient } from "@/modules/parcel-sync/services/eterra-client";
|
||||
/**
|
||||
* POST /api/eterra/export-layer-gpkg (v2 — sync-first)
|
||||
*
|
||||
* Flow:
|
||||
* 1. Check local DB freshness for the requested layer
|
||||
* 2. If stale/empty → sync from eTerra (stores in DB)
|
||||
* 3. Build GPKG from local DB
|
||||
* 4. Return GPKG
|
||||
*
|
||||
* Body: { username?, password?, siruta, layerId, jobId?, forceSync? }
|
||||
*/
|
||||
import { prisma } from "@/core/storage/prisma";
|
||||
import { findLayerById } from "@/modules/parcel-sync/services/eterra-layers";
|
||||
import { esriToGeojson } from "@/modules/parcel-sync/services/esri-geojson";
|
||||
import { getEpsg3844Wkt } from "@/modules/parcel-sync/services/reproject";
|
||||
import { buildGpkg } from "@/modules/parcel-sync/services/gpkg-export";
|
||||
import { fetchUatGeometry } from "@/modules/parcel-sync/services/uat-geometry";
|
||||
import { syncLayer } from "@/modules/parcel-sync/services/sync-service";
|
||||
import {
|
||||
getLayerFreshness,
|
||||
isFresh,
|
||||
} from "@/modules/parcel-sync/services/enrich-service";
|
||||
import {
|
||||
clearProgress,
|
||||
setProgress,
|
||||
@@ -13,6 +27,7 @@ import {
|
||||
registerJob,
|
||||
unregisterJob,
|
||||
} from "@/modules/parcel-sync/services/session-store";
|
||||
import type { GeoJsonFeature } from "@/modules/parcel-sync/services/esri-geojson";
|
||||
|
||||
export const runtime = "nodejs";
|
||||
export const dynamic = "force-dynamic";
|
||||
@@ -23,10 +38,10 @@ type ExportLayerRequest = {
|
||||
siruta?: string | number;
|
||||
layerId?: string;
|
||||
jobId?: string;
|
||||
forceSync?: boolean;
|
||||
};
|
||||
|
||||
const validate = (body: ExportLayerRequest) => {
|
||||
// Priority: request body > session store > env vars
|
||||
const session = getSessionCredentials();
|
||||
const username = String(
|
||||
body.username || session?.username || process.env.ETERRA_USERNAME || "",
|
||||
@@ -37,13 +52,14 @@ const validate = (body: ExportLayerRequest) => {
|
||||
const siruta = String(body.siruta ?? "").trim();
|
||||
const layerId = String(body.layerId ?? "").trim();
|
||||
const jobId = body.jobId ? String(body.jobId).trim() : undefined;
|
||||
const forceSync = body.forceSync === true;
|
||||
|
||||
if (!username) throw new Error("Email is required");
|
||||
if (!password) throw new Error("Password is required");
|
||||
if (!/^\d+$/.test(siruta)) throw new Error("SIRUTA must be numeric");
|
||||
if (!layerId) throw new Error("Layer ID missing");
|
||||
|
||||
return { username, password, siruta, layerId, jobId };
|
||||
return { username, password, siruta, layerId, jobId, forceSync };
|
||||
};
|
||||
|
||||
const scheduleClear = (jobId?: string) => {
|
||||
@@ -54,7 +70,7 @@ const scheduleClear = (jobId?: string) => {
|
||||
export async function POST(req: Request) {
|
||||
let jobId: string | undefined;
|
||||
let message: string | undefined;
|
||||
let phase = "Initializare";
|
||||
let phase = "Inițializare";
|
||||
let note: string | undefined;
|
||||
let status: "running" | "done" | "error" = "running";
|
||||
let downloaded = 0;
|
||||
@@ -139,121 +155,88 @@ export async function POST(req: Request) {
|
||||
const layer = findLayerById(validated.layerId);
|
||||
if (!layer) throw new Error("Layer not configured");
|
||||
|
||||
const weights = {
|
||||
auth: 5,
|
||||
count: 5,
|
||||
download: 70,
|
||||
gpkg: 15,
|
||||
finalize: 5,
|
||||
};
|
||||
const weights = { sync: 60, gpkg: 30, finalize: 10 };
|
||||
|
||||
/* Auth */
|
||||
setPhaseState("Autentificare", weights.auth, 1);
|
||||
const client = await EterraClient.create(
|
||||
validated.username,
|
||||
validated.password,
|
||||
{ timeoutMs: 120_000 },
|
||||
/* ── Phase 1: Check freshness & sync if needed ── */
|
||||
setPhaseState("Verificare date locale", weights.sync, 1);
|
||||
|
||||
const freshness = await getLayerFreshness(
|
||||
validated.siruta,
|
||||
validated.layerId,
|
||||
);
|
||||
const needsSync =
|
||||
validated.forceSync ||
|
||||
!isFresh(freshness.lastSynced) ||
|
||||
freshness.featureCount === 0;
|
||||
|
||||
let syncedFromCache = true;
|
||||
|
||||
if (needsSync) {
|
||||
syncedFromCache = false;
|
||||
phase = `Sincronizare ${layer.name}`;
|
||||
note =
|
||||
freshness.featureCount > 0
|
||||
? "Re-sync (date expirate)"
|
||||
: "Sync inițial de la eTerra";
|
||||
pushProgress();
|
||||
|
||||
await syncLayer(
|
||||
validated.username,
|
||||
validated.password,
|
||||
validated.siruta,
|
||||
validated.layerId,
|
||||
{ forceFullSync: validated.forceSync },
|
||||
);
|
||||
} else {
|
||||
note = "Date proaspete în baza de date — skip sync";
|
||||
pushProgress();
|
||||
}
|
||||
updatePhaseProgress(1, 1);
|
||||
finishPhase();
|
||||
|
||||
/* Count */
|
||||
let geometry;
|
||||
setPhaseState("Numarare", weights.count, 2);
|
||||
let count: number | undefined;
|
||||
try {
|
||||
if (layer.spatialFilter) {
|
||||
geometry = await fetchUatGeometry(client, validated.siruta);
|
||||
count = await client.countLayerByGeometry(layer, geometry);
|
||||
} else {
|
||||
count = await client.countLayer(layer, validated.siruta);
|
||||
}
|
||||
} catch (error) {
|
||||
const msg = error instanceof Error ? error.message : "Count error";
|
||||
if (!msg.toLowerCase().includes("count unavailable")) throw error;
|
||||
}
|
||||
updatePhaseProgress(2, 2);
|
||||
finishPhase();
|
||||
/* ── Phase 2: Build GPKG from local DB ── */
|
||||
setPhaseState("Generare GPKG din baza de date", weights.gpkg, 1);
|
||||
|
||||
if (layer.spatialFilter && !geometry) {
|
||||
geometry = await fetchUatGeometry(client, validated.siruta);
|
||||
const features = await prisma.gisFeature.findMany({
|
||||
where: { layerId: validated.layerId, siruta: validated.siruta },
|
||||
select: { attributes: true, geometry: true },
|
||||
});
|
||||
|
||||
if (features.length === 0) {
|
||||
throw new Error(
|
||||
`Niciun feature în DB pentru ${layer.name} / SIRUTA ${validated.siruta}`,
|
||||
);
|
||||
}
|
||||
|
||||
const pageSize =
|
||||
typeof count === "number"
|
||||
? Math.min(1000, Math.max(200, Math.ceil(count / 8)))
|
||||
: 500;
|
||||
const geoFeatures: GeoJsonFeature[] = features
|
||||
.filter((f) => f.geometry != null)
|
||||
.map((f) => ({
|
||||
type: "Feature" as const,
|
||||
geometry: f.geometry as GeoJsonFeature["geometry"],
|
||||
properties: f.attributes as Record<string, unknown>,
|
||||
}));
|
||||
|
||||
/* Download */
|
||||
setPhaseState("Descarcare", weights.download, count);
|
||||
const features = layer.spatialFilter
|
||||
? await client.fetchAllLayerByGeometry(layer, geometry!, {
|
||||
total: count,
|
||||
pageSize,
|
||||
delayMs: 250,
|
||||
onProgress: (value, totalCount) => {
|
||||
updatePhaseProgress(
|
||||
value,
|
||||
typeof totalCount === "number" ? totalCount : value + pageSize,
|
||||
);
|
||||
},
|
||||
})
|
||||
: await client.fetchAllLayer(layer, validated.siruta, {
|
||||
total: count,
|
||||
pageSize,
|
||||
delayMs: 250,
|
||||
onProgress: (value, totalCount) => {
|
||||
updatePhaseProgress(
|
||||
value,
|
||||
typeof totalCount === "number" ? totalCount : value + pageSize,
|
||||
);
|
||||
},
|
||||
});
|
||||
updatePhaseProgress(features.length, count ?? features.length);
|
||||
finishPhase();
|
||||
const fields = Object.keys(geoFeatures[0]?.properties ?? {});
|
||||
|
||||
/* Fields */
|
||||
let fields: string[] = [];
|
||||
try {
|
||||
fields = await client.getLayerFieldNames(layer);
|
||||
} catch (error) {
|
||||
const msg = error instanceof Error ? error.message : "";
|
||||
if (!msg.toLowerCase().includes("returned no fields")) throw error;
|
||||
}
|
||||
if (!fields.length) {
|
||||
fields = Object.keys(features[0]?.attributes ?? {});
|
||||
}
|
||||
|
||||
/* GPKG */
|
||||
setPhaseState("GPKG", weights.gpkg, 1);
|
||||
const geo = esriToGeojson(features);
|
||||
const gpkg = await withHeartbeat(() =>
|
||||
buildGpkg({
|
||||
srsId: 3844,
|
||||
srsWkt: getEpsg3844Wkt(),
|
||||
layers: [
|
||||
{
|
||||
name: layer.name,
|
||||
fields,
|
||||
features: geo.features,
|
||||
},
|
||||
],
|
||||
layers: [{ name: layer.name, fields, features: geoFeatures }],
|
||||
}),
|
||||
);
|
||||
updatePhaseProgress(1, 1);
|
||||
finishPhase();
|
||||
|
||||
/* Finalize */
|
||||
/* ── Phase 3: Finalize ── */
|
||||
setPhaseState("Finalizare", weights.finalize, 1);
|
||||
updatePhaseProgress(1, 1);
|
||||
finishPhase();
|
||||
|
||||
const suffix = syncedFromCache ? " (din cache local)" : "";
|
||||
status = "done";
|
||||
phase = "Finalizat";
|
||||
message =
|
||||
typeof count === "number"
|
||||
? `Finalizat 100% · ${features.length}/${count} elemente`
|
||||
: `Finalizat 100% · ${features.length} elemente`;
|
||||
message = `Finalizat 100% · ${geoFeatures.length} elemente${suffix}`;
|
||||
pushProgress();
|
||||
scheduleClear(jobId);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user