perf: separate blob storage for registratura attachments

Root cause: even with SQL-level stripping, PostgreSQL must TOAST-decompress
entire multi-MB JSONB values from disk before any processing. For 5 entries
with PDF attachments (25-50MB total), this takes several seconds.

Fix: store base64 attachment data in separate namespace 'registratura-blobs'.
Main entries are inherently small (~1-2KB). List queries never touch heavy data.

Changes:
- registry-service.ts: extractBlobs/mergeBlobs split base64 on save/load,
  migrateEntryBlobs() one-time migration for existing entries
- use-registry.ts: dual namespace (registratura + registratura-blobs),
  migration runs on first mount
- registratura-module.tsx: removed useContacts/useTags hooks that triggered
  2 unnecessary API fetches on page load (write-only ops use direct storage)

Before: 3 API calls on mount, one reading 25-50MB from PostgreSQL
After: 1 API call on mount, reading ~5-10KB total
This commit is contained in:
AI Assistant
2026-02-27 23:35:04 +02:00
parent 8385041bb0
commit f8c19bb5b4
3 changed files with 227 additions and 53 deletions
@@ -1,5 +1,5 @@
import type { CompanyId } from "@/core/auth/types";
import type { RegistryEntry } from "../types";
import type { RegistryEntry, RegistryAttachment } from "../types";
const STORAGE_PREFIX = "entry:";
@@ -13,15 +13,108 @@ export interface RegistryStorage {
}): Promise<Record<string, unknown>>;
}
// ── Blob separation ──
// Base64 attachment data is stored in a SEPARATE namespace ("registratura-blobs")
// so the main entries are always small (~1-2KB each). PostgreSQL never needs to
// decompress multi-MB TOAST chunks for list queries.
/** Shape of the blob record for one entry */
interface EntryBlobs {
/** attachmentId → base64 data */
attachments?: Record<string, string>;
/** closure attachment base64 */
closureAttachment?: string;
}
/** Strip base64 from attachments, return stripped entry + extracted blobs */
function extractBlobs(entry: RegistryEntry): {
stripped: RegistryEntry;
blobs: EntryBlobs | null;
} {
const blobs: EntryBlobs = {};
let hasBlobs = false;
// Strip attachment data (keep metadata: id, name, type, size, addedAt)
const strippedAttachments: RegistryAttachment[] = (
entry.attachments ?? []
).map((att) => {
if (att.data && att.data.length > 1024 && att.data !== "__stripped__") {
if (!blobs.attachments) blobs.attachments = {};
blobs.attachments[att.id] = att.data;
hasBlobs = true;
return { ...att, data: "" };
}
return att;
});
let stripped: RegistryEntry = { ...entry, attachments: strippedAttachments };
// Strip closure attachment data
if (
entry.closureInfo?.attachment?.data &&
entry.closureInfo.attachment.data.length > 1024 &&
entry.closureInfo.attachment.data !== "__stripped__"
) {
blobs.closureAttachment = entry.closureInfo.attachment.data;
hasBlobs = true;
stripped = {
...stripped,
closureInfo: {
...entry.closureInfo,
attachment: { ...entry.closureInfo.attachment, data: "" },
},
};
}
return { stripped, blobs: hasBlobs ? blobs : null };
}
/** Merge blob data back into a stripped entry */
function mergeBlobs(
entry: RegistryEntry,
blobs: EntryBlobs | null,
): RegistryEntry {
if (!blobs) return entry;
let merged = entry;
if (blobs.attachments) {
merged = {
...merged,
attachments: (merged.attachments ?? []).map((att) => {
const data = blobs.attachments?.[att.id];
if (data && (!att.data || att.data === "" || att.data === "__stripped__")) {
return { ...att, data };
}
return att;
}),
};
}
if (blobs.closureAttachment && merged.closureInfo?.attachment) {
merged = {
...merged,
closureInfo: {
...merged.closureInfo,
attachment: {
...merged.closureInfo.attachment,
data: blobs.closureAttachment,
},
},
};
}
return merged;
}
/**
* Load all registry entries in a SINGLE lightweight request.
* Uses exportAll({ lightweight: true }) which strips base64 attachment data
* server-side, reducing payload from potentially 30-60MB to <100KB.
* Load all registry entries. Entries are inherently lightweight because
* base64 blobs are stored in a separate namespace. No SQL stripping needed.
*/
export async function getAllEntries(
storage: RegistryStorage,
): Promise<RegistryEntry[]> {
const all = await storage.exportAll({ lightweight: true });
const all = await storage.exportAll();
const entries: RegistryEntry[] = [];
for (const [key, value] of Object.entries(all)) {
if (key.startsWith(STORAGE_PREFIX) && value) {
@@ -33,27 +126,98 @@ export async function getAllEntries(
}
/**
* Load a single full entry (with attachment data) for editing.
* Load a single full entry WITH attachment data (for editing).
* Loads the lightweight entry + its blob record and merges them.
*/
export async function getFullEntry(
storage: RegistryStorage,
blobStorage: RegistryStorage,
id: string,
): Promise<RegistryEntry | null> {
return storage.get<RegistryEntry>(`${STORAGE_PREFIX}${id}`);
const entry = await storage.get<RegistryEntry>(`${STORAGE_PREFIX}${id}`);
if (!entry) return null;
const blobs = await blobStorage.get<EntryBlobs>(id);
return mergeBlobs(entry, blobs);
}
/**
* Save an entry. Strips base64 from the main entry and stores blobs
* in the separate blob namespace so list queries stay fast.
*/
export async function saveEntry(
storage: RegistryStorage,
blobStorage: RegistryStorage,
entry: RegistryEntry,
): Promise<void> {
await storage.set(`${STORAGE_PREFIX}${entry.id}`, entry);
const { stripped, blobs } = extractBlobs(entry);
await storage.set(`${STORAGE_PREFIX}${entry.id}`, stripped);
if (blobs) {
await blobStorage.set(entry.id, blobs);
}
}
/**
* Delete an entry and its blob data.
*/
export async function deleteEntry(
storage: RegistryStorage,
blobStorage: RegistryStorage,
id: string,
): Promise<void> {
await storage.delete(`${STORAGE_PREFIX}${id}`);
// Clean up blob data (ignore errors if no blobs exist)
await blobStorage.delete(id).catch(() => {});
}
/**
* One-time migration: move base64 data from entries to blob namespace.
* Runs on first load if unmigrated entries exist. After migration,
* entries are inherently lightweight.
*/
export async function migrateEntryBlobs(
storage: RegistryStorage,
blobStorage: RegistryStorage,
): Promise<number> {
// Use lightweight=true so PostgreSQL SQL strips data (works for old entries)
const all = await storage.exportAll({ lightweight: true });
// Check if we already migrated (marker key)
const migrated = await storage.get<boolean>("__blobs_migrated__");
if (migrated) return 0;
// Load full data for entries that need migration
let count = 0;
for (const [key, value] of Object.entries(all)) {
if (!key.startsWith(STORAGE_PREFIX) || !value) continue;
const entry = value as RegistryEntry;
// Check if this entry might have attachments that need migrating
const hasAttachments =
(entry.attachments ?? []).length > 0 || entry.closureInfo?.attachment;
if (!hasAttachments) continue;
// Check if blobs already exist for this entry
const existingBlobs = await blobStorage.get<EntryBlobs>(entry.id);
if (existingBlobs) continue;
// Load the full entry (with base64 data) from DB
const full = await storage.get<RegistryEntry>(
`${STORAGE_PREFIX}${entry.id}`,
);
if (!full) continue;
// Check if there's actually heavy data to extract
const { stripped, blobs } = extractBlobs(full);
if (blobs) {
await storage.set(`${STORAGE_PREFIX}${entry.id}`, stripped);
await blobStorage.set(entry.id, blobs);
count++;
}
}
// Mark migration done
await storage.set("__blobs_migrated__", true);
return count;
}
const COMPANY_PREFIXES: Record<CompanyId, string> = {