feat(epay): auto-delete ePay CF extracts 45 days after issuance
An ePay extract is valid 30 days after issuance; at 45 days it's worthless, so
delete the DB row + its MinIO object to declutter the list and free storage.
Only type='epay' rows are touched — the free cf-intern extracts are kept.
- cleanupExpiredEpayExtracts({olderThanDays=45, dryRun}): COALESCE(documentDate,
createdAt) < cutoff; deletes MinIO objects (batched, best-effort) then the
rows. Idempotent.
- Self-contained scheduler (epay-cleanup.ts, same pattern as
auto-refresh-scheduler): boot run (+90s) then every 24h, started from
instrumentation.ts. Works with zero external config; idempotent so a
redeploy/interrupt is harmless.
- GET/POST /api/ancpi/cleanup for manual preview (dry-run) / on-demand run —
staff session OR cron Bearer (EPAY_CLEANUP_CRON_SECRET /
NOTIFICATION_CRON_SECRET); excluded from the auth middleware (fail-closed
in-route). ?days overrides the window.
- deleteCfExtractObjects() helper in epay-storage.
Verified on prod: 0 epay rows currently qualify (all recent); the 8 old intern
rows are correctly left untouched.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,64 @@
|
||||
// GET /api/ancpi/cleanup?dryRun=1 — preview what would be deleted
|
||||
// POST /api/ancpi/cleanup — run the cleanup now
|
||||
//
|
||||
// On-demand control over the 45-day ePay extract auto-cleanup (the scheduler
|
||||
// in epay-cleanup.ts runs it automatically on boot + every 24h). Useful to
|
||||
// preview (dryRun) before trusting the automatic run, or to trigger it now.
|
||||
//
|
||||
// Auth: a staff session (requireCfAccess), OR a cron Bearer token
|
||||
// (EPAY_CLEANUP_CRON_SECRET / NOTIFICATION_CRON_SECRET) so an external
|
||||
// scheduler can call it. ?days overrides the retention window.
|
||||
|
||||
import { NextResponse } from "next/server";
|
||||
import {
|
||||
cleanupExpiredEpayExtracts,
|
||||
EPAY_RETENTION_DAYS,
|
||||
} from "@/modules/parcel-sync/services/epay-cleanup";
|
||||
import { requireCfAccess } from "@/core/auth/cf-access";
|
||||
|
||||
export const runtime = "nodejs";
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
function cronAuthorized(req: Request): boolean {
|
||||
const secret =
|
||||
process.env.EPAY_CLEANUP_CRON_SECRET ?? process.env.NOTIFICATION_CRON_SECRET;
|
||||
if (!secret) return false;
|
||||
const auth = req.headers.get("authorization") ?? "";
|
||||
return auth === `Bearer ${secret}`;
|
||||
}
|
||||
|
||||
async function authorize(req: Request): Promise<boolean> {
|
||||
if (cronAuthorized(req)) return true;
|
||||
const access = await requireCfAccess();
|
||||
return access.ok;
|
||||
}
|
||||
|
||||
function parseDays(req: Request): number {
|
||||
const raw = new URL(req.url).searchParams.get("days");
|
||||
const n = raw ? parseInt(raw, 10) : NaN;
|
||||
return Number.isFinite(n) && n > 0 ? n : EPAY_RETENTION_DAYS;
|
||||
}
|
||||
|
||||
export async function GET(req: Request) {
|
||||
if (!(await authorize(req))) {
|
||||
return NextResponse.json({ error: "Neautorizat." }, { status: 401 });
|
||||
}
|
||||
// GET is always a dry-run (no side effects) — safe to preview from a browser.
|
||||
const result = await cleanupExpiredEpayExtracts({
|
||||
olderThanDays: parseDays(req),
|
||||
dryRun: true,
|
||||
});
|
||||
return NextResponse.json(result);
|
||||
}
|
||||
|
||||
export async function POST(req: Request) {
|
||||
if (!(await authorize(req))) {
|
||||
return NextResponse.json({ error: "Neautorizat." }, { status: 401 });
|
||||
}
|
||||
const dryRun = new URL(req.url).searchParams.get("dryRun") === "1";
|
||||
const result = await cleanupExpiredEpayExtracts({
|
||||
olderThanDays: parseDays(req),
|
||||
dryRun,
|
||||
});
|
||||
return NextResponse.json(result);
|
||||
}
|
||||
@@ -8,5 +8,9 @@ export async function register() {
|
||||
// ParcelSync auto-refresh scheduler DISABLED during GIS DB overhaul.
|
||||
// Re-enable by uncommenting the import below once the new schema is stable.
|
||||
// await import("@/modules/parcel-sync/services/auto-refresh-scheduler");
|
||||
|
||||
// ePay CF extract auto-cleanup (deletes rows + MinIO objects 45 days
|
||||
// after issuance). Self-contained scheduler; safe to run every deploy.
|
||||
await import("@/modules/parcel-sync/services/epay-cleanup");
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -58,6 +58,6 @@ export const config = {
|
||||
* - /favicon.ico, /robots.txt, /sitemap.xml
|
||||
* - Files with extensions (images, fonts, etc.)
|
||||
*/
|
||||
"/((?!api/auth|api/version|api/basemap-|api/notifications/digest|api/eterra/auto-refresh|api/compress-pdf|api/address-book|api/projects|auth/signin|_next|favicon\\.ico|robots\\.txt|sitemap\\.xml|.*\\..*).*)",
|
||||
"/((?!api/auth|api/version|api/basemap-|api/notifications/digest|api/eterra/auto-refresh|api/ancpi/cleanup|api/compress-pdf|api/address-book|api/projects|auth/signin|_next|favicon\\.ico|robots\\.txt|sitemap\\.xml|.*\\..*).*)",
|
||||
],
|
||||
};
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
/**
|
||||
* Auto-cleanup of old ePay CF extracts.
|
||||
*
|
||||
* An ePay extract is valid 30 days after issuance (documentDate); after that
|
||||
* it's worthless (you'd re-order). At 45 days we delete the row + its MinIO
|
||||
* object to declutter the list and free storage. Only `type='epay'` rows are
|
||||
* touched — the free `cf-intern` (copycf) extracts are kept.
|
||||
*
|
||||
* Self-contained scheduler (same pattern as auto-refresh-scheduler): started
|
||||
* by importing this module from instrumentation.ts. Runs once shortly after
|
||||
* boot (so it happens at least once per deploy, since redeploys reset the
|
||||
* interval) and then every 24h. The cleanup is idempotent — a partial run
|
||||
* (e.g. interrupted by a restart) is simply finished by the next run.
|
||||
*/
|
||||
|
||||
import { PrismaClient } from "@prisma/client";
|
||||
import { deleteCfExtractObjects } from "./epay-storage";
|
||||
|
||||
const prisma = new PrismaClient();
|
||||
|
||||
/** Delete ePay extracts this many days after issuance. */
|
||||
export const EPAY_RETENTION_DAYS = 45;
|
||||
|
||||
const g = globalThis as {
|
||||
__epayCleanupTimer?: ReturnType<typeof setInterval>;
|
||||
__epayCleanupRunning?: boolean;
|
||||
};
|
||||
|
||||
export type CleanupResult = {
|
||||
candidates: number;
|
||||
rowsDeleted: number;
|
||||
objectsDeleted: number;
|
||||
cutoff: string;
|
||||
dryRun: boolean;
|
||||
};
|
||||
|
||||
/**
|
||||
* Find and (unless dryRun) delete ePay extracts older than `olderThanDays`
|
||||
* from issuance. Issuance = documentDate, falling back to createdAt for rows
|
||||
* that never got a document (old failed/cancelled). Deletes the MinIO object
|
||||
* first, then the DB rows.
|
||||
*/
|
||||
export async function cleanupExpiredEpayExtracts(opts?: {
|
||||
olderThanDays?: number;
|
||||
dryRun?: boolean;
|
||||
}): Promise<CleanupResult> {
|
||||
const olderThanDays = opts?.olderThanDays ?? EPAY_RETENTION_DAYS;
|
||||
const dryRun = opts?.dryRun ?? false;
|
||||
const cutoff = new Date(Date.now() - olderThanDays * 24 * 60 * 60 * 1000);
|
||||
|
||||
// COALESCE(documentDate, createdAt) < cutoff — raw query so the date logic
|
||||
// runs in Postgres and uses the createdAt index where possible.
|
||||
const rows = (await prisma.$queryRaw`
|
||||
SELECT id, "minioPath"
|
||||
FROM "CfExtract"
|
||||
WHERE type = 'epay'
|
||||
AND COALESCE("documentDate", "createdAt") < ${cutoff}
|
||||
`) as Array<{ id: string; minioPath: string | null }>;
|
||||
|
||||
const result: CleanupResult = {
|
||||
candidates: rows.length,
|
||||
rowsDeleted: 0,
|
||||
objectsDeleted: 0,
|
||||
cutoff: cutoff.toISOString(),
|
||||
dryRun,
|
||||
};
|
||||
|
||||
if (rows.length === 0 || dryRun) {
|
||||
console.log(
|
||||
`[epay-cleanup] ${dryRun ? "(dry-run) " : ""}${rows.length} ePay extract(s) older than ${olderThanDays}d (cutoff ${cutoff.toISOString().slice(0, 10)})`,
|
||||
);
|
||||
return result;
|
||||
}
|
||||
|
||||
// Delete MinIO objects first (best-effort) so a deleted DB row never leaves
|
||||
// an orphan file.
|
||||
const paths = rows.map((r) => r.minioPath).filter((p): p is string => !!p);
|
||||
result.objectsDeleted = await deleteCfExtractObjects(paths);
|
||||
|
||||
const del = await prisma.cfExtract.deleteMany({
|
||||
where: { id: { in: rows.map((r) => r.id) } },
|
||||
});
|
||||
result.rowsDeleted = del.count;
|
||||
|
||||
console.log(
|
||||
`[epay-cleanup] deleted ${result.rowsDeleted} row(s) + ${result.objectsDeleted} object(s) older than ${olderThanDays}d`,
|
||||
);
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Run the cleanup once, guarded against overlap. Never throws. */
|
||||
async function runCleanupSafely(): Promise<void> {
|
||||
if (g.__epayCleanupRunning) return;
|
||||
g.__epayCleanupRunning = true;
|
||||
try {
|
||||
await cleanupExpiredEpayExtracts();
|
||||
} catch (error) {
|
||||
console.error("[epay-cleanup] run failed:", error);
|
||||
} finally {
|
||||
g.__epayCleanupRunning = false;
|
||||
}
|
||||
}
|
||||
|
||||
const ONE_DAY_MS = 24 * 60 * 60 * 1000;
|
||||
const BOOT_DELAY_MS = 90_000; // let the app finish starting first
|
||||
|
||||
// Start the scheduler (idempotent — one per process).
|
||||
if (!g.__epayCleanupTimer) {
|
||||
setTimeout(() => void runCleanupSafely(), BOOT_DELAY_MS);
|
||||
g.__epayCleanupTimer = setInterval(() => void runCleanupSafely(), ONE_DAY_MS);
|
||||
console.log(
|
||||
`[epay-cleanup] scheduler armed (retention ${EPAY_RETENTION_DAYS}d, boot run in ${BOOT_DELAY_MS / 1000}s, then every 24h)`,
|
||||
);
|
||||
}
|
||||
@@ -134,6 +134,35 @@ export async function getCfExtractStream(
|
||||
return minioClient.getObject(BUCKET, minioPath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete stored CF extract objects from MinIO (best-effort, batched).
|
||||
* Returns how many were removed. Used by the 45-day auto-cleanup.
|
||||
*/
|
||||
export async function deleteCfExtractObjects(
|
||||
minioPaths: string[],
|
||||
): Promise<number> {
|
||||
const paths = minioPaths.filter(Boolean);
|
||||
if (paths.length === 0) return 0;
|
||||
try {
|
||||
await minioClient.removeObjects(BUCKET, paths);
|
||||
return paths.length;
|
||||
} catch (error) {
|
||||
// removeObjects can fail wholesale on a transport error — fall back to
|
||||
// per-object deletes so one bad key doesn't block the rest.
|
||||
console.warn("[epay-storage] batch delete failed, retrying per-object:", error);
|
||||
let removed = 0;
|
||||
for (const p of paths) {
|
||||
try {
|
||||
await minioClient.removeObject(BUCKET, p);
|
||||
removed++;
|
||||
} catch (err) {
|
||||
console.warn(`[epay-storage] could not delete ${p}:`, err);
|
||||
}
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* List all stored CF extracts for a cadastral number.
|
||||
*/
|
||||
|
||||
Reference in New Issue
Block a user