perf: separate blob storage for registratura attachments
Root cause: even with SQL-level stripping, PostgreSQL must TOAST-decompress entire multi-MB JSONB values from disk before any processing. For 5 entries with PDF attachments (25-50MB total), this takes several seconds. Fix: store base64 attachment data in separate namespace 'registratura-blobs'. Main entries are inherently small (~1-2KB). List queries never touch heavy data. Changes: - registry-service.ts: extractBlobs/mergeBlobs split base64 on save/load, migrateEntryBlobs() one-time migration for existing entries - use-registry.ts: dual namespace (registratura + registratura-blobs), migration runs on first mount - registratura-module.tsx: removed useContacts/useTags hooks that triggered 2 unnecessary API fetches on page load (write-only ops use direct storage) Before: 3 API calls on mount, one reading 25-50MB from PostgreSQL After: 1 API call on mount, reading ~5-10KB total
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
import type { CompanyId } from "@/core/auth/types";
|
||||
import type { RegistryEntry } from "../types";
|
||||
import type { RegistryEntry, RegistryAttachment } from "../types";
|
||||
|
||||
const STORAGE_PREFIX = "entry:";
|
||||
|
||||
@@ -13,15 +13,108 @@ export interface RegistryStorage {
|
||||
}): Promise<Record<string, unknown>>;
|
||||
}
|
||||
|
||||
// ── Blob separation ──
|
||||
// Base64 attachment data is stored in a SEPARATE namespace ("registratura-blobs")
|
||||
// so the main entries are always small (~1-2KB each). PostgreSQL never needs to
|
||||
// decompress multi-MB TOAST chunks for list queries.
|
||||
|
||||
/** Shape of the blob record for one entry */
|
||||
interface EntryBlobs {
|
||||
/** attachmentId → base64 data */
|
||||
attachments?: Record<string, string>;
|
||||
/** closure attachment base64 */
|
||||
closureAttachment?: string;
|
||||
}
|
||||
|
||||
/** Strip base64 from attachments, return stripped entry + extracted blobs */
|
||||
function extractBlobs(entry: RegistryEntry): {
|
||||
stripped: RegistryEntry;
|
||||
blobs: EntryBlobs | null;
|
||||
} {
|
||||
const blobs: EntryBlobs = {};
|
||||
let hasBlobs = false;
|
||||
|
||||
// Strip attachment data (keep metadata: id, name, type, size, addedAt)
|
||||
const strippedAttachments: RegistryAttachment[] = (
|
||||
entry.attachments ?? []
|
||||
).map((att) => {
|
||||
if (att.data && att.data.length > 1024 && att.data !== "__stripped__") {
|
||||
if (!blobs.attachments) blobs.attachments = {};
|
||||
blobs.attachments[att.id] = att.data;
|
||||
hasBlobs = true;
|
||||
return { ...att, data: "" };
|
||||
}
|
||||
return att;
|
||||
});
|
||||
|
||||
let stripped: RegistryEntry = { ...entry, attachments: strippedAttachments };
|
||||
|
||||
// Strip closure attachment data
|
||||
if (
|
||||
entry.closureInfo?.attachment?.data &&
|
||||
entry.closureInfo.attachment.data.length > 1024 &&
|
||||
entry.closureInfo.attachment.data !== "__stripped__"
|
||||
) {
|
||||
blobs.closureAttachment = entry.closureInfo.attachment.data;
|
||||
hasBlobs = true;
|
||||
stripped = {
|
||||
...stripped,
|
||||
closureInfo: {
|
||||
...entry.closureInfo,
|
||||
attachment: { ...entry.closureInfo.attachment, data: "" },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return { stripped, blobs: hasBlobs ? blobs : null };
|
||||
}
|
||||
|
||||
/** Merge blob data back into a stripped entry */
|
||||
function mergeBlobs(
|
||||
entry: RegistryEntry,
|
||||
blobs: EntryBlobs | null,
|
||||
): RegistryEntry {
|
||||
if (!blobs) return entry;
|
||||
|
||||
let merged = entry;
|
||||
|
||||
if (blobs.attachments) {
|
||||
merged = {
|
||||
...merged,
|
||||
attachments: (merged.attachments ?? []).map((att) => {
|
||||
const data = blobs.attachments?.[att.id];
|
||||
if (data && (!att.data || att.data === "" || att.data === "__stripped__")) {
|
||||
return { ...att, data };
|
||||
}
|
||||
return att;
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
if (blobs.closureAttachment && merged.closureInfo?.attachment) {
|
||||
merged = {
|
||||
...merged,
|
||||
closureInfo: {
|
||||
...merged.closureInfo,
|
||||
attachment: {
|
||||
...merged.closureInfo.attachment,
|
||||
data: blobs.closureAttachment,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return merged;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load all registry entries in a SINGLE lightweight request.
|
||||
* Uses exportAll({ lightweight: true }) which strips base64 attachment data
|
||||
* server-side, reducing payload from potentially 30-60MB to <100KB.
|
||||
* Load all registry entries. Entries are inherently lightweight because
|
||||
* base64 blobs are stored in a separate namespace. No SQL stripping needed.
|
||||
*/
|
||||
export async function getAllEntries(
|
||||
storage: RegistryStorage,
|
||||
): Promise<RegistryEntry[]> {
|
||||
const all = await storage.exportAll({ lightweight: true });
|
||||
const all = await storage.exportAll();
|
||||
const entries: RegistryEntry[] = [];
|
||||
for (const [key, value] of Object.entries(all)) {
|
||||
if (key.startsWith(STORAGE_PREFIX) && value) {
|
||||
@@ -33,27 +126,98 @@ export async function getAllEntries(
|
||||
}
|
||||
|
||||
/**
|
||||
* Load a single full entry (with attachment data) for editing.
|
||||
* Load a single full entry WITH attachment data (for editing).
|
||||
* Loads the lightweight entry + its blob record and merges them.
|
||||
*/
|
||||
export async function getFullEntry(
|
||||
storage: RegistryStorage,
|
||||
blobStorage: RegistryStorage,
|
||||
id: string,
|
||||
): Promise<RegistryEntry | null> {
|
||||
return storage.get<RegistryEntry>(`${STORAGE_PREFIX}${id}`);
|
||||
const entry = await storage.get<RegistryEntry>(`${STORAGE_PREFIX}${id}`);
|
||||
if (!entry) return null;
|
||||
const blobs = await blobStorage.get<EntryBlobs>(id);
|
||||
return mergeBlobs(entry, blobs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Save an entry. Strips base64 from the main entry and stores blobs
|
||||
* in the separate blob namespace so list queries stay fast.
|
||||
*/
|
||||
export async function saveEntry(
|
||||
storage: RegistryStorage,
|
||||
blobStorage: RegistryStorage,
|
||||
entry: RegistryEntry,
|
||||
): Promise<void> {
|
||||
await storage.set(`${STORAGE_PREFIX}${entry.id}`, entry);
|
||||
const { stripped, blobs } = extractBlobs(entry);
|
||||
await storage.set(`${STORAGE_PREFIX}${entry.id}`, stripped);
|
||||
if (blobs) {
|
||||
await blobStorage.set(entry.id, blobs);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete an entry and its blob data.
|
||||
*/
|
||||
export async function deleteEntry(
|
||||
storage: RegistryStorage,
|
||||
blobStorage: RegistryStorage,
|
||||
id: string,
|
||||
): Promise<void> {
|
||||
await storage.delete(`${STORAGE_PREFIX}${id}`);
|
||||
// Clean up blob data (ignore errors if no blobs exist)
|
||||
await blobStorage.delete(id).catch(() => {});
|
||||
}
|
||||
|
||||
/**
|
||||
* One-time migration: move base64 data from entries to blob namespace.
|
||||
* Runs on first load if unmigrated entries exist. After migration,
|
||||
* entries are inherently lightweight.
|
||||
*/
|
||||
export async function migrateEntryBlobs(
|
||||
storage: RegistryStorage,
|
||||
blobStorage: RegistryStorage,
|
||||
): Promise<number> {
|
||||
// Use lightweight=true so PostgreSQL SQL strips data (works for old entries)
|
||||
const all = await storage.exportAll({ lightweight: true });
|
||||
|
||||
// Check if we already migrated (marker key)
|
||||
const migrated = await storage.get<boolean>("__blobs_migrated__");
|
||||
if (migrated) return 0;
|
||||
|
||||
// Load full data for entries that need migration
|
||||
let count = 0;
|
||||
for (const [key, value] of Object.entries(all)) {
|
||||
if (!key.startsWith(STORAGE_PREFIX) || !value) continue;
|
||||
const entry = value as RegistryEntry;
|
||||
|
||||
// Check if this entry might have attachments that need migrating
|
||||
const hasAttachments =
|
||||
(entry.attachments ?? []).length > 0 || entry.closureInfo?.attachment;
|
||||
if (!hasAttachments) continue;
|
||||
|
||||
// Check if blobs already exist for this entry
|
||||
const existingBlobs = await blobStorage.get<EntryBlobs>(entry.id);
|
||||
if (existingBlobs) continue;
|
||||
|
||||
// Load the full entry (with base64 data) from DB
|
||||
const full = await storage.get<RegistryEntry>(
|
||||
`${STORAGE_PREFIX}${entry.id}`,
|
||||
);
|
||||
if (!full) continue;
|
||||
|
||||
// Check if there's actually heavy data to extract
|
||||
const { stripped, blobs } = extractBlobs(full);
|
||||
if (blobs) {
|
||||
await storage.set(`${STORAGE_PREFIX}${entry.id}`, stripped);
|
||||
await blobStorage.set(entry.id, blobs);
|
||||
count++;
|
||||
}
|
||||
}
|
||||
|
||||
// Mark migration done
|
||||
await storage.set("__blobs_migrated__", true);
|
||||
return count;
|
||||
}
|
||||
|
||||
const COMPANY_PREFIXES: Record<CompanyId, string> = {
|
||||
|
||||
Reference in New Issue
Block a user