fix: move blob migration server-side, restore lightweight list loading

The client-side migration was downloading 25-50MB of base64 data to the
browser before showing anything. getAllEntries also lost its lightweight flag.

Fix:
- New POST /api/storage/migrate-blobs endpoint runs entirely server-side
  (loads entries one-at-a-time from PostgreSQL, never sends heavy data to browser)
- Restore lightweight:true on getAllEntries (strips remaining base64 in API)
- Migration fires on mount (fire-and-forget) while list loads independently
- Remove client-side migrateEntryBlobs function
This commit is contained in:
AI Assistant
2026-02-28 00:03:26 +02:00
parent 578f6580a4
commit a0ec4aed3f
3 changed files with 147 additions and 58 deletions
+139
View File
@@ -0,0 +1,139 @@
import { NextResponse } from "next/server";
import { prisma } from "@/core/storage/prisma";
/**
* Server-side migration: move base64 attachment data from registry entries
* to a separate "registratura-blobs" namespace. This runs entirely on the
* server — heavy data never reaches the browser.
*
* POST /api/storage/migrate-blobs
*/
export async function POST() {
try {
// 1. Check migration flag
const flag = await prisma.keyValueStore.findUnique({
where: {
namespace_key: {
namespace: "registratura",
key: "__blobs_migrated__",
},
},
});
if (flag) {
return NextResponse.json({ migrated: 0, alreadyDone: true });
}
// 2. Get ONLY keys (no heavy value column) for registratura entries
const keyRows = await prisma.keyValueStore.findMany({
where: { namespace: "registratura", key: { startsWith: "entry:" } },
select: { key: true },
});
let migrated = 0;
// 3. Process entries ONE AT A TIME server-side
for (const { key } of keyRows) {
const row = await prisma.keyValueStore.findUnique({
where: { namespace_key: { namespace: "registratura", key } },
});
if (!row?.value) continue;
const entry = row.value as Record<string, unknown>;
const entryId = (entry.id as string) || key.replace("entry:", "");
const blobs: Record<string, unknown> = {};
let hasBlobs = false;
// Strip attachment data
const attachments =
(entry.attachments as Array<Record<string, unknown>>) ?? [];
const strippedAttachments = attachments.map((att) => {
const data = att.data as string | undefined;
if (
data &&
typeof data === "string" &&
data.length > 1024 &&
data !== "__stripped__"
) {
if (!blobs.attachments) blobs.attachments = {};
(blobs.attachments as Record<string, string>)[att.id as string] =
data;
hasBlobs = true;
return { ...att, data: "" };
}
return att;
});
// Strip closure attachment data
let strippedEntry: Record<string, unknown> = {
...entry,
attachments: strippedAttachments,
};
const closureInfo = entry.closureInfo as
| Record<string, unknown>
| undefined;
if (closureInfo?.attachment) {
const closureAtt = closureInfo.attachment as Record<string, unknown>;
const data = closureAtt.data as string | undefined;
if (
data &&
typeof data === "string" &&
data.length > 1024 &&
data !== "__stripped__"
) {
blobs.closureAttachment = data;
hasBlobs = true;
strippedEntry = {
...strippedEntry,
closureInfo: {
...closureInfo,
attachment: { ...closureAtt, data: "" },
},
};
}
}
if (hasBlobs) {
// Save stripped entry + blobs in parallel
await Promise.all([
prisma.keyValueStore.update({
where: { namespace_key: { namespace: "registratura", key } },
data: { value: strippedEntry as object },
}),
prisma.keyValueStore.upsert({
where: {
namespace_key: { namespace: "registratura-blobs", key: entryId },
},
update: { value: blobs as object },
create: {
namespace: "registratura-blobs",
key: entryId,
value: blobs as object,
},
}),
]);
migrated++;
}
}
// 4. Set migration flag
await prisma.keyValueStore.upsert({
where: {
namespace_key: {
namespace: "registratura",
key: "__blobs_migrated__",
},
},
update: { value: true },
create: {
namespace: "registratura",
key: "__blobs_migrated__",
value: true,
},
});
return NextResponse.json({ migrated, alreadyDone: false });
} catch (error) {
console.error("Blob migration error:", error);
return NextResponse.json({ error: "Migration failed" }, { status: 500 });
}
}
@@ -17,7 +17,6 @@ import {
saveEntry, saveEntry,
deleteEntry, deleteEntry,
generateRegistryNumber, generateRegistryNumber,
migrateEntryBlobs,
} from "../services/registry-service"; } from "../services/registry-service";
import { import {
createTrackedDeadline, createTrackedDeadline,
@@ -54,18 +53,19 @@ export function useRegistry() {
setLoading(false); setLoading(false);
}, [storage]); }, [storage]);
// On mount: run migration (once), then load entries // On mount: trigger server-side blob migration (fire-and-forget), then load list
// eslint-disable-next-line react-hooks/set-state-in-effect // eslint-disable-next-line react-hooks/set-state-in-effect
useEffect(() => { useEffect(() => {
const init = async () => { const init = async () => {
// Trigger server-side migration (runs inside Node.js, not browser)
if (!migrationRan.current) { if (!migrationRan.current) {
migrationRan.current = true; migrationRan.current = true;
await migrateEntryBlobs(storage, blobStorage); fetch("/api/storage/migrate-blobs", { method: "POST" }).catch(() => {});
} }
await refresh(); await refresh();
}; };
init(); init();
}, [refresh, storage, blobStorage]); }, [refresh]);
const addEntry = useCallback( const addEntry = useCallback(
async ( async (
@@ -111,13 +111,14 @@ function mergeBlobs(
} }
/** /**
* Load all registry entries. Entries are inherently lightweight because * Load all registry entries in a SINGLE lightweight request.
* base64 blobs are stored in a separate namespace. No SQL stripping needed. * Uses exportAll({ lightweight: true }) to strip any remaining base64
* data server-side, ensuring the browser receives only small JSON.
*/ */
export async function getAllEntries( export async function getAllEntries(
storage: RegistryStorage, storage: RegistryStorage,
): Promise<RegistryEntry[]> { ): Promise<RegistryEntry[]> {
const all = await storage.exportAll(); const all = await storage.exportAll({ lightweight: true });
const entries: RegistryEntry[] = []; const entries: RegistryEntry[] = [];
for (const [key, value] of Object.entries(all)) { for (const [key, value] of Object.entries(all)) {
if (key.startsWith(STORAGE_PREFIX) && value) { if (key.startsWith(STORAGE_PREFIX) && value) {
@@ -172,57 +173,6 @@ export async function deleteEntry(
await blobStorage.delete(id).catch(() => {}); await blobStorage.delete(id).catch(() => {});
} }
/**
* One-time migration: move base64 data from entries to blob namespace.
* Runs on first load if unmigrated entries exist. After migration,
* entries are inherently lightweight.
*/
export async function migrateEntryBlobs(
storage: RegistryStorage,
blobStorage: RegistryStorage,
): Promise<number> {
// Check migration flag FIRST to avoid any heavy loading
const migrated = await storage.get<boolean>("__blobs_migrated__");
if (migrated) return 0;
// Load entries (may be heavy on first migration — runs only once)
const all = await storage.exportAll();
// Load full data for entries that need migration
let count = 0;
for (const [key, value] of Object.entries(all)) {
if (!key.startsWith(STORAGE_PREFIX) || !value) continue;
const entry = value as RegistryEntry;
// Check if this entry might have attachments that need migrating
const hasAttachments =
(entry.attachments ?? []).length > 0 || entry.closureInfo?.attachment;
if (!hasAttachments) continue;
// Check if blobs already exist for this entry
const existingBlobs = await blobStorage.get<EntryBlobs>(entry.id);
if (existingBlobs) continue;
// Load the full entry (with base64 data) from DB
const full = await storage.get<RegistryEntry>(
`${STORAGE_PREFIX}${entry.id}`,
);
if (!full) continue;
// Check if there's actually heavy data to extract
const { stripped, blobs } = extractBlobs(full);
if (blobs) {
await storage.set(`${STORAGE_PREFIX}${entry.id}`, stripped);
await blobStorage.set(entry.id, blobs);
count++;
}
}
// Mark migration done
await storage.set("__blobs_migrated__", true);
return count;
}
const COMPANY_PREFIXES: Record<CompanyId, string> = { const COMPANY_PREFIXES: Record<CompanyId, string> = {
beletage: "B", beletage: "B",
"urban-switch": "US", "urban-switch": "US",