fix: move blob migration server-side, restore lightweight list loading
The client-side migration was downloading 25-50MB of base64 data to the browser before showing anything. getAllEntries also lost its lightweight flag. Fix: - New POST /api/storage/migrate-blobs endpoint runs entirely server-side (loads entries one-at-a-time from PostgreSQL, never sends heavy data to browser) - Restore lightweight:true on getAllEntries (strips remaining base64 in API) - Migration fires on mount (fire-and-forget) while list loads independently - Remove client-side migrateEntryBlobs function
This commit is contained in:
@@ -17,7 +17,6 @@ import {
|
||||
saveEntry,
|
||||
deleteEntry,
|
||||
generateRegistryNumber,
|
||||
migrateEntryBlobs,
|
||||
} from "../services/registry-service";
|
||||
import {
|
||||
createTrackedDeadline,
|
||||
@@ -54,18 +53,19 @@ export function useRegistry() {
|
||||
setLoading(false);
|
||||
}, [storage]);
|
||||
|
||||
// On mount: run migration (once), then load entries
|
||||
// On mount: trigger server-side blob migration (fire-and-forget), then load list
|
||||
// eslint-disable-next-line react-hooks/set-state-in-effect
|
||||
useEffect(() => {
|
||||
const init = async () => {
|
||||
// Trigger server-side migration (runs inside Node.js, not browser)
|
||||
if (!migrationRan.current) {
|
||||
migrationRan.current = true;
|
||||
await migrateEntryBlobs(storage, blobStorage);
|
||||
fetch("/api/storage/migrate-blobs", { method: "POST" }).catch(() => {});
|
||||
}
|
||||
await refresh();
|
||||
};
|
||||
init();
|
||||
}, [refresh, storage, blobStorage]);
|
||||
}, [refresh]);
|
||||
|
||||
const addEntry = useCallback(
|
||||
async (
|
||||
|
||||
@@ -111,13 +111,14 @@ function mergeBlobs(
|
||||
}
|
||||
|
||||
/**
|
||||
* Load all registry entries. Entries are inherently lightweight because
|
||||
* base64 blobs are stored in a separate namespace. No SQL stripping needed.
|
||||
* Load all registry entries in a SINGLE lightweight request.
|
||||
* Uses exportAll({ lightweight: true }) to strip any remaining base64
|
||||
* data server-side, ensuring the browser receives only small JSON.
|
||||
*/
|
||||
export async function getAllEntries(
|
||||
storage: RegistryStorage,
|
||||
): Promise<RegistryEntry[]> {
|
||||
const all = await storage.exportAll();
|
||||
const all = await storage.exportAll({ lightweight: true });
|
||||
const entries: RegistryEntry[] = [];
|
||||
for (const [key, value] of Object.entries(all)) {
|
||||
if (key.startsWith(STORAGE_PREFIX) && value) {
|
||||
@@ -172,57 +173,6 @@ export async function deleteEntry(
|
||||
await blobStorage.delete(id).catch(() => {});
|
||||
}
|
||||
|
||||
/**
|
||||
* One-time migration: move base64 data from entries to blob namespace.
|
||||
* Runs on first load if unmigrated entries exist. After migration,
|
||||
* entries are inherently lightweight.
|
||||
*/
|
||||
export async function migrateEntryBlobs(
|
||||
storage: RegistryStorage,
|
||||
blobStorage: RegistryStorage,
|
||||
): Promise<number> {
|
||||
// Check migration flag FIRST to avoid any heavy loading
|
||||
const migrated = await storage.get<boolean>("__blobs_migrated__");
|
||||
if (migrated) return 0;
|
||||
|
||||
// Load entries (may be heavy on first migration — runs only once)
|
||||
const all = await storage.exportAll();
|
||||
|
||||
// Load full data for entries that need migration
|
||||
let count = 0;
|
||||
for (const [key, value] of Object.entries(all)) {
|
||||
if (!key.startsWith(STORAGE_PREFIX) || !value) continue;
|
||||
const entry = value as RegistryEntry;
|
||||
|
||||
// Check if this entry might have attachments that need migrating
|
||||
const hasAttachments =
|
||||
(entry.attachments ?? []).length > 0 || entry.closureInfo?.attachment;
|
||||
if (!hasAttachments) continue;
|
||||
|
||||
// Check if blobs already exist for this entry
|
||||
const existingBlobs = await blobStorage.get<EntryBlobs>(entry.id);
|
||||
if (existingBlobs) continue;
|
||||
|
||||
// Load the full entry (with base64 data) from DB
|
||||
const full = await storage.get<RegistryEntry>(
|
||||
`${STORAGE_PREFIX}${entry.id}`,
|
||||
);
|
||||
if (!full) continue;
|
||||
|
||||
// Check if there's actually heavy data to extract
|
||||
const { stripped, blobs } = extractBlobs(full);
|
||||
if (blobs) {
|
||||
await storage.set(`${STORAGE_PREFIX}${entry.id}`, stripped);
|
||||
await blobStorage.set(entry.id, blobs);
|
||||
count++;
|
||||
}
|
||||
}
|
||||
|
||||
// Mark migration done
|
||||
await storage.set("__blobs_migrated__", true);
|
||||
return count;
|
||||
}
|
||||
|
||||
const COMPANY_PREFIXES: Record<CompanyId, string> = {
|
||||
beletage: "B",
|
||||
"urban-switch": "US",
|
||||
|
||||
Reference in New Issue
Block a user