fix: move blob migration server-side, restore lightweight list loading
The client-side migration was downloading 25-50MB of base64 data to the browser before showing anything. getAllEntries also lost its lightweight flag. Fix: - New POST /api/storage/migrate-blobs endpoint runs entirely server-side (loads entries one-at-a-time from PostgreSQL, never sends heavy data to browser) - Restore lightweight:true on getAllEntries (strips remaining base64 in API) - Migration fires on mount (fire-and-forget) while list loads independently - Remove client-side migrateEntryBlobs function
This commit is contained in:
AI Assistant
@@ -0,0 +1,139 @@
|
||||
import { NextResponse } from "next/server";
|
||||
import { prisma } from "@/core/storage/prisma";
|
||||
|
||||
/**
|
||||
* Server-side migration: move base64 attachment data from registry entries
|
||||
* to a separate "registratura-blobs" namespace. This runs entirely on the
|
||||
* server — heavy data never reaches the browser.
|
||||
*
|
||||
* POST /api/storage/migrate-blobs
|
||||
*/
|
||||
export async function POST() {
|
||||
try {
|
||||
// 1. Check migration flag
|
||||
const flag = await prisma.keyValueStore.findUnique({
|
||||
where: {
|
||||
namespace_key: {
|
||||
namespace: "registratura",
|
||||
key: "__blobs_migrated__",
|
||||
},
|
||||
},
|
||||
});
|
||||
if (flag) {
|
||||
return NextResponse.json({ migrated: 0, alreadyDone: true });
|
||||
}
|
||||
|
||||
// 2. Get ONLY keys (no heavy value column) for registratura entries
|
||||
const keyRows = await prisma.keyValueStore.findMany({
|
||||
where: { namespace: "registratura", key: { startsWith: "entry:" } },
|
||||
select: { key: true },
|
||||
});
|
||||
|
||||
let migrated = 0;
|
||||
|
||||
// 3. Process entries ONE AT A TIME server-side
|
||||
for (const { key } of keyRows) {
|
||||
const row = await prisma.keyValueStore.findUnique({
|
||||
where: { namespace_key: { namespace: "registratura", key } },
|
||||
});
|
||||
if (!row?.value) continue;
|
||||
|
||||
const entry = row.value as Record<string, unknown>;
|
||||
const entryId = (entry.id as string) || key.replace("entry:", "");
|
||||
const blobs: Record<string, unknown> = {};
|
||||
let hasBlobs = false;
|
||||
|
||||
// Strip attachment data
|
||||
const attachments =
|
||||
(entry.attachments as Array<Record<string, unknown>>) ?? [];
|
||||
const strippedAttachments = attachments.map((att) => {
|
||||
const data = att.data as string | undefined;
|
||||
if (
|
||||
data &&
|
||||
typeof data === "string" &&
|
||||
data.length > 1024 &&
|
||||
data !== "__stripped__"
|
||||
) {
|
||||
if (!blobs.attachments) blobs.attachments = {};
|
||||
(blobs.attachments as Record<string, string>)[att.id as string] =
|
||||
data;
|
||||
hasBlobs = true;
|
||||
return { ...att, data: "" };
|
||||
}
|
||||
return att;
|
||||
});
|
||||
|
||||
// Strip closure attachment data
|
||||
let strippedEntry: Record<string, unknown> = {
|
||||
...entry,
|
||||
attachments: strippedAttachments,
|
||||
};
|
||||
const closureInfo = entry.closureInfo as
|
||||
| Record<string, unknown>
|
||||
| undefined;
|
||||
if (closureInfo?.attachment) {
|
||||
const closureAtt = closureInfo.attachment as Record<string, unknown>;
|
||||
const data = closureAtt.data as string | undefined;
|
||||
if (
|
||||
data &&
|
||||
typeof data === "string" &&
|
||||
data.length > 1024 &&
|
||||
data !== "__stripped__"
|
||||
) {
|
||||
blobs.closureAttachment = data;
|
||||
hasBlobs = true;
|
||||
strippedEntry = {
|
||||
...strippedEntry,
|
||||
closureInfo: {
|
||||
...closureInfo,
|
||||
attachment: { ...closureAtt, data: "" },
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (hasBlobs) {
|
||||
// Save stripped entry + blobs in parallel
|
||||
await Promise.all([
|
||||
prisma.keyValueStore.update({
|
||||
where: { namespace_key: { namespace: "registratura", key } },
|
||||
data: { value: strippedEntry as object },
|
||||
}),
|
||||
prisma.keyValueStore.upsert({
|
||||
where: {
|
||||
namespace_key: { namespace: "registratura-blobs", key: entryId },
|
||||
},
|
||||
update: { value: blobs as object },
|
||||
create: {
|
||||
namespace: "registratura-blobs",
|
||||
key: entryId,
|
||||
value: blobs as object,
|
||||
},
|
||||
}),
|
||||
]);
|
||||
migrated++;
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Set migration flag
|
||||
await prisma.keyValueStore.upsert({
|
||||
where: {
|
||||
namespace_key: {
|
||||
namespace: "registratura",
|
||||
key: "__blobs_migrated__",
|
||||
},
|
||||
},
|
||||
update: { value: true },
|
||||
create: {
|
||||
namespace: "registratura",
|
||||
key: "__blobs_migrated__",
|
||||
value: true,
|
||||
},
|
||||
});
|
||||
|
||||
return NextResponse.json({ migrated, alreadyDone: false });
|
||||
} catch (error) {
|
||||
console.error("Blob migration error:", error);
|
||||
return NextResponse.json({ error: "Migration failed" }, { status: 500 });
|
||||
}
|
||||
}
|
||||
@@ -17,7 +17,6 @@ import {
|
||||
saveEntry,
|
||||
deleteEntry,
|
||||
generateRegistryNumber,
|
||||
migrateEntryBlobs,
|
||||
} from "../services/registry-service";
|
||||
import {
|
||||
createTrackedDeadline,
|
||||
@@ -54,18 +53,19 @@ export function useRegistry() {
|
||||
setLoading(false);
|
||||
}, [storage]);
|
||||
|
||||
// On mount: run migration (once), then load entries
|
||||
// On mount: trigger server-side blob migration (fire-and-forget), then load list
|
||||
// eslint-disable-next-line react-hooks/set-state-in-effect
|
||||
useEffect(() => {
|
||||
const init = async () => {
|
||||
// Trigger server-side migration (runs inside Node.js, not browser)
|
||||
if (!migrationRan.current) {
|
||||
migrationRan.current = true;
|
||||
await migrateEntryBlobs(storage, blobStorage);
|
||||
fetch("/api/storage/migrate-blobs", { method: "POST" }).catch(() => {});
|
||||
}
|
||||
await refresh();
|
||||
};
|
||||
init();
|
||||
}, [refresh, storage, blobStorage]);
|
||||
}, [refresh]);
|
||||
|
||||
const addEntry = useCallback(
|
||||
async (
|
||||
|
||||
@@ -111,13 +111,14 @@ function mergeBlobs(
|
||||
}
|
||||
|
||||
/**
|
||||
* Load all registry entries. Entries are inherently lightweight because
|
||||
* base64 blobs are stored in a separate namespace. No SQL stripping needed.
|
||||
* Load all registry entries in a SINGLE lightweight request.
|
||||
* Uses exportAll({ lightweight: true }) to strip any remaining base64
|
||||
* data server-side, ensuring the browser receives only small JSON.
|
||||
*/
|
||||
export async function getAllEntries(
|
||||
storage: RegistryStorage,
|
||||
): Promise<RegistryEntry[]> {
|
||||
const all = await storage.exportAll();
|
||||
const all = await storage.exportAll({ lightweight: true });
|
||||
const entries: RegistryEntry[] = [];
|
||||
for (const [key, value] of Object.entries(all)) {
|
||||
if (key.startsWith(STORAGE_PREFIX) && value) {
|
||||
@@ -172,57 +173,6 @@ export async function deleteEntry(
|
||||
await blobStorage.delete(id).catch(() => {});
|
||||
}
|
||||
|
||||
/**
|
||||
* One-time migration: move base64 data from entries to blob namespace.
|
||||
* Runs on first load if unmigrated entries exist. After migration,
|
||||
* entries are inherently lightweight.
|
||||
*/
|
||||
export async function migrateEntryBlobs(
|
||||
storage: RegistryStorage,
|
||||
blobStorage: RegistryStorage,
|
||||
): Promise<number> {
|
||||
// Check migration flag FIRST to avoid any heavy loading
|
||||
const migrated = await storage.get<boolean>("__blobs_migrated__");
|
||||
if (migrated) return 0;
|
||||
|
||||
// Load entries (may be heavy on first migration — runs only once)
|
||||
const all = await storage.exportAll();
|
||||
|
||||
// Load full data for entries that need migration
|
||||
let count = 0;
|
||||
for (const [key, value] of Object.entries(all)) {
|
||||
if (!key.startsWith(STORAGE_PREFIX) || !value) continue;
|
||||
const entry = value as RegistryEntry;
|
||||
|
||||
// Check if this entry might have attachments that need migrating
|
||||
const hasAttachments =
|
||||
(entry.attachments ?? []).length > 0 || entry.closureInfo?.attachment;
|
||||
if (!hasAttachments) continue;
|
||||
|
||||
// Check if blobs already exist for this entry
|
||||
const existingBlobs = await blobStorage.get<EntryBlobs>(entry.id);
|
||||
if (existingBlobs) continue;
|
||||
|
||||
// Load the full entry (with base64 data) from DB
|
||||
const full = await storage.get<RegistryEntry>(
|
||||
`${STORAGE_PREFIX}${entry.id}`,
|
||||
);
|
||||
if (!full) continue;
|
||||
|
||||
// Check if there's actually heavy data to extract
|
||||
const { stripped, blobs } = extractBlobs(full);
|
||||
if (blobs) {
|
||||
await storage.set(`${STORAGE_PREFIX}${entry.id}`, stripped);
|
||||
await blobStorage.set(entry.id, blobs);
|
||||
count++;
|
||||
}
|
||||
}
|
||||
|
||||
// Mark migration done
|
||||
await storage.set("__blobs_migrated__", true);
|
||||
return count;
|
||||
}
|
||||
|
||||
const COMPANY_PREFIXES: Record<CompanyId, string> = {
|
||||
beletage: "B",
|
||||
"urban-switch": "US",
|
||||
|
||||
Reference in New Issue
Block a user