fix: move blob migration server-side, restore lightweight list loading
The client-side migration was downloading 25-50MB of base64 data to the browser before showing anything. getAllEntries also lost its lightweight flag. Fix: - New POST /api/storage/migrate-blobs endpoint runs entirely server-side (loads entries one-at-a-time from PostgreSQL, never sends heavy data to browser) - Restore lightweight:true on getAllEntries (strips remaining base64 in API) - Migration fires on mount (fire-and-forget) while list loads independently - Remove client-side migrateEntryBlobs function
This commit is contained in:
@@ -0,0 +1,139 @@
|
|||||||
|
import { NextResponse } from "next/server";
|
||||||
|
import { prisma } from "@/core/storage/prisma";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Server-side migration: move base64 attachment data from registry entries
|
||||||
|
* to a separate "registratura-blobs" namespace. This runs entirely on the
|
||||||
|
* server — heavy data never reaches the browser.
|
||||||
|
*
|
||||||
|
* POST /api/storage/migrate-blobs
|
||||||
|
*/
|
||||||
|
export async function POST() {
|
||||||
|
try {
|
||||||
|
// 1. Check migration flag
|
||||||
|
const flag = await prisma.keyValueStore.findUnique({
|
||||||
|
where: {
|
||||||
|
namespace_key: {
|
||||||
|
namespace: "registratura",
|
||||||
|
key: "__blobs_migrated__",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
if (flag) {
|
||||||
|
return NextResponse.json({ migrated: 0, alreadyDone: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Get ONLY keys (no heavy value column) for registratura entries
|
||||||
|
const keyRows = await prisma.keyValueStore.findMany({
|
||||||
|
where: { namespace: "registratura", key: { startsWith: "entry:" } },
|
||||||
|
select: { key: true },
|
||||||
|
});
|
||||||
|
|
||||||
|
let migrated = 0;
|
||||||
|
|
||||||
|
// 3. Process entries ONE AT A TIME server-side
|
||||||
|
for (const { key } of keyRows) {
|
||||||
|
const row = await prisma.keyValueStore.findUnique({
|
||||||
|
where: { namespace_key: { namespace: "registratura", key } },
|
||||||
|
});
|
||||||
|
if (!row?.value) continue;
|
||||||
|
|
||||||
|
const entry = row.value as Record<string, unknown>;
|
||||||
|
const entryId = (entry.id as string) || key.replace("entry:", "");
|
||||||
|
const blobs: Record<string, unknown> = {};
|
||||||
|
let hasBlobs = false;
|
||||||
|
|
||||||
|
// Strip attachment data
|
||||||
|
const attachments =
|
||||||
|
(entry.attachments as Array<Record<string, unknown>>) ?? [];
|
||||||
|
const strippedAttachments = attachments.map((att) => {
|
||||||
|
const data = att.data as string | undefined;
|
||||||
|
if (
|
||||||
|
data &&
|
||||||
|
typeof data === "string" &&
|
||||||
|
data.length > 1024 &&
|
||||||
|
data !== "__stripped__"
|
||||||
|
) {
|
||||||
|
if (!blobs.attachments) blobs.attachments = {};
|
||||||
|
(blobs.attachments as Record<string, string>)[att.id as string] =
|
||||||
|
data;
|
||||||
|
hasBlobs = true;
|
||||||
|
return { ...att, data: "" };
|
||||||
|
}
|
||||||
|
return att;
|
||||||
|
});
|
||||||
|
|
||||||
|
// Strip closure attachment data
|
||||||
|
let strippedEntry: Record<string, unknown> = {
|
||||||
|
...entry,
|
||||||
|
attachments: strippedAttachments,
|
||||||
|
};
|
||||||
|
const closureInfo = entry.closureInfo as
|
||||||
|
| Record<string, unknown>
|
||||||
|
| undefined;
|
||||||
|
if (closureInfo?.attachment) {
|
||||||
|
const closureAtt = closureInfo.attachment as Record<string, unknown>;
|
||||||
|
const data = closureAtt.data as string | undefined;
|
||||||
|
if (
|
||||||
|
data &&
|
||||||
|
typeof data === "string" &&
|
||||||
|
data.length > 1024 &&
|
||||||
|
data !== "__stripped__"
|
||||||
|
) {
|
||||||
|
blobs.closureAttachment = data;
|
||||||
|
hasBlobs = true;
|
||||||
|
strippedEntry = {
|
||||||
|
...strippedEntry,
|
||||||
|
closureInfo: {
|
||||||
|
...closureInfo,
|
||||||
|
attachment: { ...closureAtt, data: "" },
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (hasBlobs) {
|
||||||
|
// Save stripped entry + blobs in parallel
|
||||||
|
await Promise.all([
|
||||||
|
prisma.keyValueStore.update({
|
||||||
|
where: { namespace_key: { namespace: "registratura", key } },
|
||||||
|
data: { value: strippedEntry as object },
|
||||||
|
}),
|
||||||
|
prisma.keyValueStore.upsert({
|
||||||
|
where: {
|
||||||
|
namespace_key: { namespace: "registratura-blobs", key: entryId },
|
||||||
|
},
|
||||||
|
update: { value: blobs as object },
|
||||||
|
create: {
|
||||||
|
namespace: "registratura-blobs",
|
||||||
|
key: entryId,
|
||||||
|
value: blobs as object,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
migrated++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. Set migration flag
|
||||||
|
await prisma.keyValueStore.upsert({
|
||||||
|
where: {
|
||||||
|
namespace_key: {
|
||||||
|
namespace: "registratura",
|
||||||
|
key: "__blobs_migrated__",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
update: { value: true },
|
||||||
|
create: {
|
||||||
|
namespace: "registratura",
|
||||||
|
key: "__blobs_migrated__",
|
||||||
|
value: true,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
return NextResponse.json({ migrated, alreadyDone: false });
|
||||||
|
} catch (error) {
|
||||||
|
console.error("Blob migration error:", error);
|
||||||
|
return NextResponse.json({ error: "Migration failed" }, { status: 500 });
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -17,7 +17,6 @@ import {
|
|||||||
saveEntry,
|
saveEntry,
|
||||||
deleteEntry,
|
deleteEntry,
|
||||||
generateRegistryNumber,
|
generateRegistryNumber,
|
||||||
migrateEntryBlobs,
|
|
||||||
} from "../services/registry-service";
|
} from "../services/registry-service";
|
||||||
import {
|
import {
|
||||||
createTrackedDeadline,
|
createTrackedDeadline,
|
||||||
@@ -54,18 +53,19 @@ export function useRegistry() {
|
|||||||
setLoading(false);
|
setLoading(false);
|
||||||
}, [storage]);
|
}, [storage]);
|
||||||
|
|
||||||
// On mount: run migration (once), then load entries
|
// On mount: trigger server-side blob migration (fire-and-forget), then load list
|
||||||
// eslint-disable-next-line react-hooks/set-state-in-effect
|
// eslint-disable-next-line react-hooks/set-state-in-effect
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
const init = async () => {
|
const init = async () => {
|
||||||
|
// Trigger server-side migration (runs inside Node.js, not browser)
|
||||||
if (!migrationRan.current) {
|
if (!migrationRan.current) {
|
||||||
migrationRan.current = true;
|
migrationRan.current = true;
|
||||||
await migrateEntryBlobs(storage, blobStorage);
|
fetch("/api/storage/migrate-blobs", { method: "POST" }).catch(() => {});
|
||||||
}
|
}
|
||||||
await refresh();
|
await refresh();
|
||||||
};
|
};
|
||||||
init();
|
init();
|
||||||
}, [refresh, storage, blobStorage]);
|
}, [refresh]);
|
||||||
|
|
||||||
const addEntry = useCallback(
|
const addEntry = useCallback(
|
||||||
async (
|
async (
|
||||||
|
|||||||
@@ -111,13 +111,14 @@ function mergeBlobs(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Load all registry entries. Entries are inherently lightweight because
|
* Load all registry entries in a SINGLE lightweight request.
|
||||||
* base64 blobs are stored in a separate namespace. No SQL stripping needed.
|
* Uses exportAll({ lightweight: true }) to strip any remaining base64
|
||||||
|
* data server-side, ensuring the browser receives only small JSON.
|
||||||
*/
|
*/
|
||||||
export async function getAllEntries(
|
export async function getAllEntries(
|
||||||
storage: RegistryStorage,
|
storage: RegistryStorage,
|
||||||
): Promise<RegistryEntry[]> {
|
): Promise<RegistryEntry[]> {
|
||||||
const all = await storage.exportAll();
|
const all = await storage.exportAll({ lightweight: true });
|
||||||
const entries: RegistryEntry[] = [];
|
const entries: RegistryEntry[] = [];
|
||||||
for (const [key, value] of Object.entries(all)) {
|
for (const [key, value] of Object.entries(all)) {
|
||||||
if (key.startsWith(STORAGE_PREFIX) && value) {
|
if (key.startsWith(STORAGE_PREFIX) && value) {
|
||||||
@@ -172,57 +173,6 @@ export async function deleteEntry(
|
|||||||
await blobStorage.delete(id).catch(() => {});
|
await blobStorage.delete(id).catch(() => {});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* One-time migration: move base64 data from entries to blob namespace.
|
|
||||||
* Runs on first load if unmigrated entries exist. After migration,
|
|
||||||
* entries are inherently lightweight.
|
|
||||||
*/
|
|
||||||
export async function migrateEntryBlobs(
|
|
||||||
storage: RegistryStorage,
|
|
||||||
blobStorage: RegistryStorage,
|
|
||||||
): Promise<number> {
|
|
||||||
// Check migration flag FIRST to avoid any heavy loading
|
|
||||||
const migrated = await storage.get<boolean>("__blobs_migrated__");
|
|
||||||
if (migrated) return 0;
|
|
||||||
|
|
||||||
// Load entries (may be heavy on first migration — runs only once)
|
|
||||||
const all = await storage.exportAll();
|
|
||||||
|
|
||||||
// Load full data for entries that need migration
|
|
||||||
let count = 0;
|
|
||||||
for (const [key, value] of Object.entries(all)) {
|
|
||||||
if (!key.startsWith(STORAGE_PREFIX) || !value) continue;
|
|
||||||
const entry = value as RegistryEntry;
|
|
||||||
|
|
||||||
// Check if this entry might have attachments that need migrating
|
|
||||||
const hasAttachments =
|
|
||||||
(entry.attachments ?? []).length > 0 || entry.closureInfo?.attachment;
|
|
||||||
if (!hasAttachments) continue;
|
|
||||||
|
|
||||||
// Check if blobs already exist for this entry
|
|
||||||
const existingBlobs = await blobStorage.get<EntryBlobs>(entry.id);
|
|
||||||
if (existingBlobs) continue;
|
|
||||||
|
|
||||||
// Load the full entry (with base64 data) from DB
|
|
||||||
const full = await storage.get<RegistryEntry>(
|
|
||||||
`${STORAGE_PREFIX}${entry.id}`,
|
|
||||||
);
|
|
||||||
if (!full) continue;
|
|
||||||
|
|
||||||
// Check if there's actually heavy data to extract
|
|
||||||
const { stripped, blobs } = extractBlobs(full);
|
|
||||||
if (blobs) {
|
|
||||||
await storage.set(`${STORAGE_PREFIX}${entry.id}`, stripped);
|
|
||||||
await blobStorage.set(entry.id, blobs);
|
|
||||||
count++;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Mark migration done
|
|
||||||
await storage.set("__blobs_migrated__", true);
|
|
||||||
return count;
|
|
||||||
}
|
|
||||||
|
|
||||||
const COMPANY_PREFIXES: Record<CompanyId, string> = {
|
const COMPANY_PREFIXES: Record<CompanyId, string> = {
|
||||||
beletage: "B",
|
beletage: "B",
|
||||||
"urban-switch": "US",
|
"urban-switch": "US",
|
||||||
|
|||||||
Reference in New Issue
Block a user