feat(epay): auto-delete ePay CF extracts 45 days after issuance
An ePay extract is valid 30 days after issuance; at 45 days it's worthless, so
delete the DB row + its MinIO object to declutter the list and free storage.
Only type='epay' rows are touched — the free cf-intern extracts are kept.
- cleanupExpiredEpayExtracts({olderThanDays=45, dryRun}): COALESCE(documentDate,
createdAt) < cutoff; deletes MinIO objects (batched, best-effort) then the
rows. Idempotent.
- Self-contained scheduler (epay-cleanup.ts, same pattern as
auto-refresh-scheduler): boot run (+90s) then every 24h, started from
instrumentation.ts. Works with zero external config; idempotent so a
redeploy/interrupt is harmless.
- GET/POST /api/ancpi/cleanup for manual preview (dry-run) / on-demand run —
staff session OR cron Bearer (EPAY_CLEANUP_CRON_SECRET /
NOTIFICATION_CRON_SECRET); excluded from the auth middleware (fail-closed
in-route). ?days overrides the window.
- deleteCfExtractObjects() helper in epay-storage.
Verified on prod: 0 epay rows currently qualify (all recent); the 8 old intern
rows are correctly left untouched.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,64 @@
|
|||||||
|
// GET /api/ancpi/cleanup?dryRun=1 — preview what would be deleted
|
||||||
|
// POST /api/ancpi/cleanup — run the cleanup now
|
||||||
|
//
|
||||||
|
// On-demand control over the 45-day ePay extract auto-cleanup (the scheduler
|
||||||
|
// in epay-cleanup.ts runs it automatically on boot + every 24h). Useful to
|
||||||
|
// preview (dryRun) before trusting the automatic run, or to trigger it now.
|
||||||
|
//
|
||||||
|
// Auth: a staff session (requireCfAccess), OR a cron Bearer token
|
||||||
|
// (EPAY_CLEANUP_CRON_SECRET / NOTIFICATION_CRON_SECRET) so an external
|
||||||
|
// scheduler can call it. ?days overrides the retention window.
|
||||||
|
|
||||||
|
import { NextResponse } from "next/server";
|
||||||
|
import {
|
||||||
|
cleanupExpiredEpayExtracts,
|
||||||
|
EPAY_RETENTION_DAYS,
|
||||||
|
} from "@/modules/parcel-sync/services/epay-cleanup";
|
||||||
|
import { requireCfAccess } from "@/core/auth/cf-access";
|
||||||
|
|
||||||
|
export const runtime = "nodejs";
|
||||||
|
export const dynamic = "force-dynamic";
|
||||||
|
|
||||||
|
function cronAuthorized(req: Request): boolean {
|
||||||
|
const secret =
|
||||||
|
process.env.EPAY_CLEANUP_CRON_SECRET ?? process.env.NOTIFICATION_CRON_SECRET;
|
||||||
|
if (!secret) return false;
|
||||||
|
const auth = req.headers.get("authorization") ?? "";
|
||||||
|
return auth === `Bearer ${secret}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function authorize(req: Request): Promise<boolean> {
|
||||||
|
if (cronAuthorized(req)) return true;
|
||||||
|
const access = await requireCfAccess();
|
||||||
|
return access.ok;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseDays(req: Request): number {
|
||||||
|
const raw = new URL(req.url).searchParams.get("days");
|
||||||
|
const n = raw ? parseInt(raw, 10) : NaN;
|
||||||
|
return Number.isFinite(n) && n > 0 ? n : EPAY_RETENTION_DAYS;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function GET(req: Request) {
|
||||||
|
if (!(await authorize(req))) {
|
||||||
|
return NextResponse.json({ error: "Neautorizat." }, { status: 401 });
|
||||||
|
}
|
||||||
|
// GET is always a dry-run (no side effects) — safe to preview from a browser.
|
||||||
|
const result = await cleanupExpiredEpayExtracts({
|
||||||
|
olderThanDays: parseDays(req),
|
||||||
|
dryRun: true,
|
||||||
|
});
|
||||||
|
return NextResponse.json(result);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function POST(req: Request) {
|
||||||
|
if (!(await authorize(req))) {
|
||||||
|
return NextResponse.json({ error: "Neautorizat." }, { status: 401 });
|
||||||
|
}
|
||||||
|
const dryRun = new URL(req.url).searchParams.get("dryRun") === "1";
|
||||||
|
const result = await cleanupExpiredEpayExtracts({
|
||||||
|
olderThanDays: parseDays(req),
|
||||||
|
dryRun,
|
||||||
|
});
|
||||||
|
return NextResponse.json(result);
|
||||||
|
}
|
||||||
@@ -8,5 +8,9 @@ export async function register() {
|
|||||||
// ParcelSync auto-refresh scheduler DISABLED during GIS DB overhaul.
|
// ParcelSync auto-refresh scheduler DISABLED during GIS DB overhaul.
|
||||||
// Re-enable by uncommenting the import below once the new schema is stable.
|
// Re-enable by uncommenting the import below once the new schema is stable.
|
||||||
// await import("@/modules/parcel-sync/services/auto-refresh-scheduler");
|
// await import("@/modules/parcel-sync/services/auto-refresh-scheduler");
|
||||||
|
|
||||||
|
// ePay CF extract auto-cleanup (deletes rows + MinIO objects 45 days
|
||||||
|
// after issuance). Self-contained scheduler; safe to run every deploy.
|
||||||
|
await import("@/modules/parcel-sync/services/epay-cleanup");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-1
@@ -58,6 +58,6 @@ export const config = {
|
|||||||
* - /favicon.ico, /robots.txt, /sitemap.xml
|
* - /favicon.ico, /robots.txt, /sitemap.xml
|
||||||
* - Files with extensions (images, fonts, etc.)
|
* - Files with extensions (images, fonts, etc.)
|
||||||
*/
|
*/
|
||||||
"/((?!api/auth|api/version|api/basemap-|api/notifications/digest|api/eterra/auto-refresh|api/compress-pdf|api/address-book|api/projects|auth/signin|_next|favicon\\.ico|robots\\.txt|sitemap\\.xml|.*\\..*).*)",
|
"/((?!api/auth|api/version|api/basemap-|api/notifications/digest|api/eterra/auto-refresh|api/ancpi/cleanup|api/compress-pdf|api/address-book|api/projects|auth/signin|_next|favicon\\.ico|robots\\.txt|sitemap\\.xml|.*\\..*).*)",
|
||||||
],
|
],
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -0,0 +1,114 @@
|
|||||||
|
/**
|
||||||
|
* Auto-cleanup of old ePay CF extracts.
|
||||||
|
*
|
||||||
|
* An ePay extract is valid 30 days after issuance (documentDate); after that
|
||||||
|
* it's worthless (you'd re-order). At 45 days we delete the row + its MinIO
|
||||||
|
* object to declutter the list and free storage. Only `type='epay'` rows are
|
||||||
|
* touched — the free `cf-intern` (copycf) extracts are kept.
|
||||||
|
*
|
||||||
|
* Self-contained scheduler (same pattern as auto-refresh-scheduler): started
|
||||||
|
* by importing this module from instrumentation.ts. Runs once shortly after
|
||||||
|
* boot (so it happens at least once per deploy, since redeploys reset the
|
||||||
|
* interval) and then every 24h. The cleanup is idempotent — a partial run
|
||||||
|
* (e.g. interrupted by a restart) is simply finished by the next run.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { PrismaClient } from "@prisma/client";
|
||||||
|
import { deleteCfExtractObjects } from "./epay-storage";
|
||||||
|
|
||||||
|
const prisma = new PrismaClient();
|
||||||
|
|
||||||
|
/** Delete ePay extracts this many days after issuance. */
|
||||||
|
export const EPAY_RETENTION_DAYS = 45;
|
||||||
|
|
||||||
|
const g = globalThis as {
|
||||||
|
__epayCleanupTimer?: ReturnType<typeof setInterval>;
|
||||||
|
__epayCleanupRunning?: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type CleanupResult = {
|
||||||
|
candidates: number;
|
||||||
|
rowsDeleted: number;
|
||||||
|
objectsDeleted: number;
|
||||||
|
cutoff: string;
|
||||||
|
dryRun: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Find and (unless dryRun) delete ePay extracts older than `olderThanDays`
|
||||||
|
* from issuance. Issuance = documentDate, falling back to createdAt for rows
|
||||||
|
* that never got a document (old failed/cancelled). Deletes the MinIO object
|
||||||
|
* first, then the DB rows.
|
||||||
|
*/
|
||||||
|
export async function cleanupExpiredEpayExtracts(opts?: {
|
||||||
|
olderThanDays?: number;
|
||||||
|
dryRun?: boolean;
|
||||||
|
}): Promise<CleanupResult> {
|
||||||
|
const olderThanDays = opts?.olderThanDays ?? EPAY_RETENTION_DAYS;
|
||||||
|
const dryRun = opts?.dryRun ?? false;
|
||||||
|
const cutoff = new Date(Date.now() - olderThanDays * 24 * 60 * 60 * 1000);
|
||||||
|
|
||||||
|
// COALESCE(documentDate, createdAt) < cutoff — raw query so the date logic
|
||||||
|
// runs in Postgres and uses the createdAt index where possible.
|
||||||
|
const rows = (await prisma.$queryRaw`
|
||||||
|
SELECT id, "minioPath"
|
||||||
|
FROM "CfExtract"
|
||||||
|
WHERE type = 'epay'
|
||||||
|
AND COALESCE("documentDate", "createdAt") < ${cutoff}
|
||||||
|
`) as Array<{ id: string; minioPath: string | null }>;
|
||||||
|
|
||||||
|
const result: CleanupResult = {
|
||||||
|
candidates: rows.length,
|
||||||
|
rowsDeleted: 0,
|
||||||
|
objectsDeleted: 0,
|
||||||
|
cutoff: cutoff.toISOString(),
|
||||||
|
dryRun,
|
||||||
|
};
|
||||||
|
|
||||||
|
if (rows.length === 0 || dryRun) {
|
||||||
|
console.log(
|
||||||
|
`[epay-cleanup] ${dryRun ? "(dry-run) " : ""}${rows.length} ePay extract(s) older than ${olderThanDays}d (cutoff ${cutoff.toISOString().slice(0, 10)})`,
|
||||||
|
);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete MinIO objects first (best-effort) so a deleted DB row never leaves
|
||||||
|
// an orphan file.
|
||||||
|
const paths = rows.map((r) => r.minioPath).filter((p): p is string => !!p);
|
||||||
|
result.objectsDeleted = await deleteCfExtractObjects(paths);
|
||||||
|
|
||||||
|
const del = await prisma.cfExtract.deleteMany({
|
||||||
|
where: { id: { in: rows.map((r) => r.id) } },
|
||||||
|
});
|
||||||
|
result.rowsDeleted = del.count;
|
||||||
|
|
||||||
|
console.log(
|
||||||
|
`[epay-cleanup] deleted ${result.rowsDeleted} row(s) + ${result.objectsDeleted} object(s) older than ${olderThanDays}d`,
|
||||||
|
);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Run the cleanup once, guarded against overlap. Never throws. */
|
||||||
|
async function runCleanupSafely(): Promise<void> {
|
||||||
|
if (g.__epayCleanupRunning) return;
|
||||||
|
g.__epayCleanupRunning = true;
|
||||||
|
try {
|
||||||
|
await cleanupExpiredEpayExtracts();
|
||||||
|
} catch (error) {
|
||||||
|
console.error("[epay-cleanup] run failed:", error);
|
||||||
|
} finally {
|
||||||
|
g.__epayCleanupRunning = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const ONE_DAY_MS = 24 * 60 * 60 * 1000;
|
||||||
|
const BOOT_DELAY_MS = 90_000; // let the app finish starting first
|
||||||
|
|
||||||
|
// Start the scheduler (idempotent — one per process).
|
||||||
|
if (!g.__epayCleanupTimer) {
|
||||||
|
setTimeout(() => void runCleanupSafely(), BOOT_DELAY_MS);
|
||||||
|
g.__epayCleanupTimer = setInterval(() => void runCleanupSafely(), ONE_DAY_MS);
|
||||||
|
console.log(
|
||||||
|
`[epay-cleanup] scheduler armed (retention ${EPAY_RETENTION_DAYS}d, boot run in ${BOOT_DELAY_MS / 1000}s, then every 24h)`,
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -134,6 +134,35 @@ export async function getCfExtractStream(
|
|||||||
return minioClient.getObject(BUCKET, minioPath);
|
return minioClient.getObject(BUCKET, minioPath);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delete stored CF extract objects from MinIO (best-effort, batched).
|
||||||
|
* Returns how many were removed. Used by the 45-day auto-cleanup.
|
||||||
|
*/
|
||||||
|
export async function deleteCfExtractObjects(
|
||||||
|
minioPaths: string[],
|
||||||
|
): Promise<number> {
|
||||||
|
const paths = minioPaths.filter(Boolean);
|
||||||
|
if (paths.length === 0) return 0;
|
||||||
|
try {
|
||||||
|
await minioClient.removeObjects(BUCKET, paths);
|
||||||
|
return paths.length;
|
||||||
|
} catch (error) {
|
||||||
|
// removeObjects can fail wholesale on a transport error — fall back to
|
||||||
|
// per-object deletes so one bad key doesn't block the rest.
|
||||||
|
console.warn("[epay-storage] batch delete failed, retrying per-object:", error);
|
||||||
|
let removed = 0;
|
||||||
|
for (const p of paths) {
|
||||||
|
try {
|
||||||
|
await minioClient.removeObject(BUCKET, p);
|
||||||
|
removed++;
|
||||||
|
} catch (err) {
|
||||||
|
console.warn(`[epay-storage] could not delete ${p}:`, err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return removed;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* List all stored CF extracts for a cadastral number.
|
* List all stored CF extracts for a cadastral number.
|
||||||
*/
|
*/
|
||||||
|
|||||||
Reference in New Issue
Block a user