feat(parcel-sync): incremental sync, smart export, auto-refresh + weekend deep sync
Sync Incremental: - Add fetchObjectIds (returnIdsOnly) to eterra-client — fetches only OBJECTIDs in 1 request - Add fetchFeaturesByObjectIds — downloads only delta features by OBJECTID IN (...) - Rewrite syncLayer: compare remote IDs vs local, download only new features - Fallback to full sync for first sync, forceFullSync, or delta > 50% - Reduces sync time from ~10 min to ~5-10s for typical updates Smart Export Tab: - Hero buttons detect DB freshness — use export-local (instant) when data is fresh - Dynamic subtitles: "Din DB (sync acum Xh)" / "Sync incremental" / "Sync complet" - Re-sync link when data is fresh but user wants forced refresh - Removed duplicate "Descarca din DB" buttons from background section Auto-Refresh Scheduler: - Self-contained timer via instrumentation.ts (Next.js startup hook) - Weekday 1-5 AM: incremental refresh for existing UATs in DB - Staggered processing with random delays between UATs - Health check before processing, respects eTerra maintenance Weekend Deep Sync: - Full Magic processing for 9 large municipalities (Cluj, Bistrita, TgMures, etc.) - Runs Fri/Sat/Sun 23:00-04:00, round-robin intercalated between cities - 4 steps per city: sync terenuri, sync cladiri, import no-geom, enrichment - State persisted in KeyValueStore — survives restarts, continues across nights - Email status report at end of each session via Brevo SMTP - Admin page at /wds: add/remove cities, view progress, reset - Hint link on export tab pointing to /wds API endpoints: - POST /api/eterra/auto-refresh — N8N-compatible cron endpoint (Bearer token auth) - GET/POST /api/eterra/weekend-sync — queue management for /wds page Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -297,6 +297,81 @@ export class EterraClient {
|
||||
return this.countLayerWithParams(layer, params, true);
|
||||
}
|
||||
|
||||
/* ---- Incremental sync: fetch only OBJECTIDs -------------------- */
|
||||
|
||||
async fetchObjectIds(layer: LayerConfig, siruta: string): Promise<number[]> {
|
||||
const where = await this.buildWhere(layer, siruta);
|
||||
return this.fetchObjectIdsByWhere(layer, where);
|
||||
}
|
||||
|
||||
async fetchObjectIdsByWhere(
|
||||
layer: LayerConfig,
|
||||
where: string,
|
||||
): Promise<number[]> {
|
||||
const params = new URLSearchParams();
|
||||
params.set("f", "json");
|
||||
params.set("where", where);
|
||||
params.set("returnIdsOnly", "true");
|
||||
const data = await this.queryLayer(layer, params, false);
|
||||
return data.objectIds ?? [];
|
||||
}
|
||||
|
||||
async fetchObjectIdsByGeometry(
|
||||
layer: LayerConfig,
|
||||
geometry: EsriGeometry,
|
||||
): Promise<number[]> {
|
||||
const params = new URLSearchParams();
|
||||
params.set("f", "json");
|
||||
params.set("where", "1=1");
|
||||
params.set("returnIdsOnly", "true");
|
||||
this.applyGeometryParams(params, geometry);
|
||||
const data = await this.queryLayer(layer, params, true);
|
||||
return data.objectIds ?? [];
|
||||
}
|
||||
|
||||
/* ---- Fetch specific features by OBJECTID list ------------------- */
|
||||
|
||||
async fetchFeaturesByObjectIds(
|
||||
layer: LayerConfig,
|
||||
objectIds: number[],
|
||||
options?: {
|
||||
baseWhere?: string;
|
||||
outFields?: string;
|
||||
returnGeometry?: boolean;
|
||||
onProgress?: ProgressCallback;
|
||||
delayMs?: number;
|
||||
},
|
||||
): Promise<EsriFeature[]> {
|
||||
if (objectIds.length === 0) return [];
|
||||
const chunkSize = 500;
|
||||
const all: EsriFeature[] = [];
|
||||
const total = objectIds.length;
|
||||
for (let i = 0; i < objectIds.length; i += chunkSize) {
|
||||
const chunk = objectIds.slice(i, i + chunkSize);
|
||||
const idList = chunk.join(",");
|
||||
const idWhere = `OBJECTID IN (${idList})`;
|
||||
const where = options?.baseWhere
|
||||
? `(${options.baseWhere}) AND ${idWhere}`
|
||||
: idWhere;
|
||||
try {
|
||||
const features = await this.fetchAllLayerByWhere(layer, where, {
|
||||
outFields: options?.outFields ?? "*",
|
||||
returnGeometry: options?.returnGeometry ?? true,
|
||||
delayMs: options?.delayMs ?? 200,
|
||||
});
|
||||
all.push(...features);
|
||||
} catch (err) {
|
||||
// Log but continue with remaining chunks — partial results better than none
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
console.warn(
|
||||
`[fetchFeaturesByObjectIds] Chunk ${Math.floor(i / chunkSize) + 1} failed (${chunk.length} IDs): ${msg}`,
|
||||
);
|
||||
}
|
||||
options?.onProgress?.(all.length, total);
|
||||
}
|
||||
return all;
|
||||
}
|
||||
|
||||
async listLayer(
|
||||
layer: LayerConfig,
|
||||
siruta: string,
|
||||
|
||||
Reference in New Issue
Block a user