perf(registratura): lightweight API mode strips base64 attachments from list
ROOT CAUSE: RegistryEntry stores file attachments as base64 strings in JSON.
A single 5MB PDF becomes ~6.7MB of base64. With 6 entries, the exportAll()
endpoint was sending 30-60MB of JSON on every page load taking 2+ minutes.
Fix: Added ?lightweight=true parameter to /api/storage GET endpoint.
When enabled, stripHeavyFields() recursively removes large 'data' and
'fileData' string fields (>1KB) from JSON values, replacing with '__stripped__'.
Changes:
- /api/storage route.ts: stripHeavyFields() + lightweight query param
- StorageService.export(): accepts { lightweight?: boolean } option
- DatabaseStorageAdapter.export(): passes lightweight flag to API
- LocalStorageAdapter.export(): accepts option (no-op, localStorage is fast)
- useStorage.exportAll(): passes options through
- registry-service.ts: getAllEntries() uses lightweight=true by default
- registry-service.ts: new getFullEntry() loads single entry with full data
- use-registry.ts: exports loadFullEntry() for on-demand full loading
- registratura-module.tsx: handleEdit/handleNavigateEntry load full entry
Result: List loading transfers ~100KB instead of 30-60MB. Editing loads
full data for a single entry on demand (~5-10MB for one entry vs all).
This commit is contained in:
@@ -1,10 +1,40 @@
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { prisma } from "@/core/storage/prisma";
|
||||
|
||||
/**
|
||||
* Strip heavy base64 data from JSON values to enable fast list loading.
|
||||
* Targets: attachments[].data, versions[].fileData, and any top-level `data`
|
||||
* field that looks like base64 (>1KB). Replaces with "__stripped__" marker.
|
||||
*/
|
||||
function stripHeavyFields(value: unknown): unknown {
|
||||
if (!value || typeof value !== "object") return value;
|
||||
if (Array.isArray(value)) return value.map(stripHeavyFields);
|
||||
|
||||
const obj = value as Record<string, unknown>;
|
||||
const result: Record<string, unknown> = {};
|
||||
|
||||
for (const [k, v] of Object.entries(obj)) {
|
||||
if (k === "data" && typeof v === "string" && v.length > 1024) {
|
||||
// Strip large base64 data fields, keep a marker
|
||||
result[k] = "__stripped__";
|
||||
} else if (k === "fileData" && typeof v === "string" && v.length > 1024) {
|
||||
result[k] = "__stripped__";
|
||||
} else if (Array.isArray(v)) {
|
||||
result[k] = v.map(stripHeavyFields);
|
||||
} else if (v && typeof v === "object") {
|
||||
result[k] = stripHeavyFields(v);
|
||||
} else {
|
||||
result[k] = v;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
const searchParams = request.nextUrl.searchParams;
|
||||
const namespace = searchParams.get("namespace");
|
||||
const key = searchParams.get("key");
|
||||
const lightweight = searchParams.get("lightweight") === "true";
|
||||
|
||||
if (!namespace) {
|
||||
return NextResponse.json(
|
||||
@@ -34,8 +64,12 @@ export async function GET(request: NextRequest) {
|
||||
// Return as a record { [key]: value }
|
||||
const result: Record<string, any> = {};
|
||||
for (const item of items) {
|
||||
if (lightweight) {
|
||||
result[item.key] = stripHeavyFields(item.value);
|
||||
} else {
|
||||
result[item.key] = item.value;
|
||||
}
|
||||
}
|
||||
return NextResponse.json({ items: result });
|
||||
}
|
||||
} catch (error) {
|
||||
|
||||
@@ -82,11 +82,16 @@ export class DatabaseStorageAdapter implements StorageService {
|
||||
}
|
||||
}
|
||||
|
||||
async export(namespace: string): Promise<Record<string, unknown>> {
|
||||
async export(
|
||||
namespace: string,
|
||||
options?: { lightweight?: boolean },
|
||||
): Promise<Record<string, unknown>> {
|
||||
try {
|
||||
const res = await fetch(
|
||||
`/api/storage?namespace=${encodeURIComponent(namespace)}`,
|
||||
);
|
||||
let url = `/api/storage?namespace=${encodeURIComponent(namespace)}`;
|
||||
if (options?.lightweight) {
|
||||
url += "&lightweight=true";
|
||||
}
|
||||
const res = await fetch(url);
|
||||
if (!res.ok) return {};
|
||||
const data = await res.json();
|
||||
return data.items || {};
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import type { StorageService } from '../types';
|
||||
import type { StorageService } from "../types";
|
||||
|
||||
function nsKey(namespace: string, key: string): string {
|
||||
return `architools:${namespace}:${key}`;
|
||||
@@ -10,7 +10,7 @@ function nsPrefix(namespace: string): string {
|
||||
|
||||
export class LocalStorageAdapter implements StorageService {
|
||||
async get<T>(namespace: string, key: string): Promise<T | null> {
|
||||
if (typeof window === 'undefined') return null;
|
||||
if (typeof window === "undefined") return null;
|
||||
try {
|
||||
const raw = window.localStorage.getItem(nsKey(namespace, key));
|
||||
if (raw === null) return null;
|
||||
@@ -21,17 +21,17 @@ export class LocalStorageAdapter implements StorageService {
|
||||
}
|
||||
|
||||
async set<T>(namespace: string, key: string, value: T): Promise<void> {
|
||||
if (typeof window === 'undefined') return;
|
||||
if (typeof window === "undefined") return;
|
||||
window.localStorage.setItem(nsKey(namespace, key), JSON.stringify(value));
|
||||
}
|
||||
|
||||
async delete(namespace: string, key: string): Promise<void> {
|
||||
if (typeof window === 'undefined') return;
|
||||
if (typeof window === "undefined") return;
|
||||
window.localStorage.removeItem(nsKey(namespace, key));
|
||||
}
|
||||
|
||||
async list(namespace: string): Promise<string[]> {
|
||||
if (typeof window === 'undefined') return [];
|
||||
if (typeof window === "undefined") return [];
|
||||
const prefix = nsPrefix(namespace);
|
||||
const keys: string[] = [];
|
||||
for (let i = 0; i < window.localStorage.length; i++) {
|
||||
@@ -43,7 +43,10 @@ export class LocalStorageAdapter implements StorageService {
|
||||
return keys;
|
||||
}
|
||||
|
||||
async query<T>(namespace: string, predicate: (item: T) => boolean): Promise<T[]> {
|
||||
async query<T>(
|
||||
namespace: string,
|
||||
predicate: (item: T) => boolean,
|
||||
): Promise<T[]> {
|
||||
const keys = await this.list(namespace);
|
||||
const results: T[] = [];
|
||||
for (const key of keys) {
|
||||
@@ -56,14 +59,17 @@ export class LocalStorageAdapter implements StorageService {
|
||||
}
|
||||
|
||||
async clear(namespace: string): Promise<void> {
|
||||
if (typeof window === 'undefined') return;
|
||||
if (typeof window === "undefined") return;
|
||||
const keys = await this.list(namespace);
|
||||
for (const key of keys) {
|
||||
window.localStorage.removeItem(nsKey(namespace, key));
|
||||
}
|
||||
}
|
||||
|
||||
async export(namespace: string): Promise<Record<string, unknown>> {
|
||||
async export(
|
||||
namespace: string,
|
||||
_options?: { lightweight?: boolean },
|
||||
): Promise<Record<string, unknown>> {
|
||||
const keys = await this.list(namespace);
|
||||
const data: Record<string, unknown> = {};
|
||||
for (const key of keys) {
|
||||
@@ -72,7 +78,10 @@ export class LocalStorageAdapter implements StorageService {
|
||||
return data;
|
||||
}
|
||||
|
||||
async import(namespace: string, data: Record<string, unknown>): Promise<void> {
|
||||
async import(
|
||||
namespace: string,
|
||||
data: Record<string, unknown>,
|
||||
): Promise<void> {
|
||||
for (const [key, value] of Object.entries(data)) {
|
||||
await this.set(namespace, key, value);
|
||||
}
|
||||
|
||||
@@ -5,6 +5,9 @@ export interface StorageService {
|
||||
list(namespace: string): Promise<string[]>;
|
||||
query<T>(namespace: string, predicate: (item: T) => boolean): Promise<T[]>;
|
||||
clear(namespace: string): Promise<void>;
|
||||
export(namespace: string): Promise<Record<string, unknown>>;
|
||||
export(
|
||||
namespace: string,
|
||||
options?: { lightweight?: boolean },
|
||||
): Promise<Record<string, unknown>>;
|
||||
import(namespace: string, data: Record<string, unknown>): Promise<void>;
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
'use client';
|
||||
"use client";
|
||||
|
||||
import { useCallback, useMemo } from 'react';
|
||||
import { useStorageService } from './storage-provider';
|
||||
import { useCallback, useMemo } from "react";
|
||||
import { useStorageService } from "./storage-provider";
|
||||
|
||||
export interface NamespacedStorage {
|
||||
get: <T>(key: string) => Promise<T | null>;
|
||||
@@ -10,7 +10,9 @@ export interface NamespacedStorage {
|
||||
list: () => Promise<string[]>;
|
||||
query: <T>(predicate: (item: T) => boolean) => Promise<T[]>;
|
||||
clear: () => Promise<void>;
|
||||
exportAll: () => Promise<Record<string, unknown>>;
|
||||
exportAll: (options?: {
|
||||
lightweight?: boolean;
|
||||
}) => Promise<Record<string, unknown>>;
|
||||
importAll: (data: Record<string, unknown>) => Promise<void>;
|
||||
}
|
||||
|
||||
@@ -18,38 +20,45 @@ export function useStorage(namespace: string): NamespacedStorage {
|
||||
const service = useStorageService();
|
||||
|
||||
const get = useCallback(
|
||||
<T,>(key: string) => service.get<T>(namespace, key),
|
||||
[service, namespace]
|
||||
<T>(key: string) => service.get<T>(namespace, key),
|
||||
[service, namespace],
|
||||
);
|
||||
|
||||
const set = useCallback(
|
||||
<T,>(key: string, value: T) => service.set<T>(namespace, key, value),
|
||||
[service, namespace]
|
||||
<T>(key: string, value: T) => service.set<T>(namespace, key, value),
|
||||
[service, namespace],
|
||||
);
|
||||
|
||||
const del = useCallback(
|
||||
(key: string) => service.delete(namespace, key),
|
||||
[service, namespace]
|
||||
[service, namespace],
|
||||
);
|
||||
|
||||
const list = useCallback(() => service.list(namespace), [service, namespace]);
|
||||
|
||||
const query = useCallback(
|
||||
<T,>(predicate: (item: T) => boolean) => service.query<T>(namespace, predicate),
|
||||
[service, namespace]
|
||||
<T>(predicate: (item: T) => boolean) =>
|
||||
service.query<T>(namespace, predicate),
|
||||
[service, namespace],
|
||||
);
|
||||
|
||||
const clear = useCallback(() => service.clear(namespace), [service, namespace]);
|
||||
const clear = useCallback(
|
||||
() => service.clear(namespace),
|
||||
[service, namespace],
|
||||
);
|
||||
|
||||
const exportAll = useCallback(() => service.export(namespace), [service, namespace]);
|
||||
const exportAll = useCallback(
|
||||
(options?: { lightweight?: boolean }) => service.export(namespace, options),
|
||||
[service, namespace],
|
||||
);
|
||||
|
||||
const importAll = useCallback(
|
||||
(data: Record<string, unknown>) => service.import(namespace, data),
|
||||
[service, namespace]
|
||||
[service, namespace],
|
||||
);
|
||||
|
||||
return useMemo(
|
||||
() => ({ get, set, delete: del, list, query, clear, exportAll, importAll }),
|
||||
[get, set, del, list, query, clear, exportAll, importAll]
|
||||
[get, set, del, list, query, clear, exportAll, importAll],
|
||||
);
|
||||
}
|
||||
|
||||
@@ -49,6 +49,7 @@ export function RegistraturaModule() {
|
||||
updateEntry,
|
||||
removeEntry,
|
||||
closeEntry,
|
||||
loadFullEntry,
|
||||
addDeadline,
|
||||
resolveDeadline,
|
||||
removeDeadline,
|
||||
@@ -120,13 +121,16 @@ export function RegistraturaModule() {
|
||||
setViewMode("list");
|
||||
};
|
||||
|
||||
const handleEdit = (entry: RegistryEntry) => {
|
||||
setEditingEntry(entry);
|
||||
const handleEdit = async (entry: RegistryEntry) => {
|
||||
// Load full entry with attachment data (list mode strips base64)
|
||||
const full = await loadFullEntry(entry.id);
|
||||
setEditingEntry(full ?? entry);
|
||||
setViewMode("edit");
|
||||
};
|
||||
|
||||
const handleNavigateEntry = (entry: RegistryEntry) => {
|
||||
setEditingEntry(entry);
|
||||
const handleNavigateEntry = async (entry: RegistryEntry) => {
|
||||
const full = await loadFullEntry(entry.id);
|
||||
setEditingEntry(full ?? entry);
|
||||
setViewMode("edit");
|
||||
};
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ import type {
|
||||
} from "../types";
|
||||
import {
|
||||
getAllEntries,
|
||||
getFullEntry,
|
||||
saveEntry,
|
||||
deleteEntry,
|
||||
generateRegistryNumber,
|
||||
@@ -255,6 +256,17 @@ export function useRegistry() {
|
||||
return true;
|
||||
});
|
||||
|
||||
/**
|
||||
* Load a single entry WITH full attachment data (for editing).
|
||||
* The list uses lightweight mode that strips base64 data.
|
||||
*/
|
||||
const loadFullEntry = useCallback(
|
||||
async (id: string): Promise<RegistryEntry | null> => {
|
||||
return getFullEntry(storage, id);
|
||||
},
|
||||
[storage],
|
||||
);
|
||||
|
||||
return {
|
||||
entries: filteredEntries,
|
||||
allEntries: entries,
|
||||
@@ -265,6 +277,7 @@ export function useRegistry() {
|
||||
updateEntry,
|
||||
removeEntry,
|
||||
closeEntry,
|
||||
loadFullEntry,
|
||||
addDeadline,
|
||||
resolveDeadline,
|
||||
removeDeadline,
|
||||
|
||||
@@ -8,18 +8,20 @@ export interface RegistryStorage {
|
||||
set<T>(key: string, value: T): Promise<void>;
|
||||
delete(key: string): Promise<void>;
|
||||
list(): Promise<string[]>;
|
||||
exportAll(): Promise<Record<string, unknown>>;
|
||||
exportAll(options?: {
|
||||
lightweight?: boolean;
|
||||
}): Promise<Record<string, unknown>>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load all registry entries in a SINGLE request.
|
||||
* Uses exportAll() which fetches namespace data in one HTTP call,
|
||||
* avoiding the N+1 pattern (list keys → get each one individually).
|
||||
* Load all registry entries in a SINGLE lightweight request.
|
||||
* Uses exportAll({ lightweight: true }) which strips base64 attachment data
|
||||
* server-side, reducing payload from potentially 30-60MB to <100KB.
|
||||
*/
|
||||
export async function getAllEntries(
|
||||
storage: RegistryStorage,
|
||||
): Promise<RegistryEntry[]> {
|
||||
const all = await storage.exportAll();
|
||||
const all = await storage.exportAll({ lightweight: true });
|
||||
const entries: RegistryEntry[] = [];
|
||||
for (const [key, value] of Object.entries(all)) {
|
||||
if (key.startsWith(STORAGE_PREFIX) && value) {
|
||||
@@ -30,6 +32,16 @@ export async function getAllEntries(
|
||||
return entries;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load a single full entry (with attachment data) for editing.
|
||||
*/
|
||||
export async function getFullEntry(
|
||||
storage: RegistryStorage,
|
||||
id: string,
|
||||
): Promise<RegistryEntry | null> {
|
||||
return storage.get<RegistryEntry>(`${STORAGE_PREFIX}${id}`);
|
||||
}
|
||||
|
||||
export async function saveEntry(
|
||||
storage: RegistryStorage,
|
||||
entry: RegistryEntry,
|
||||
|
||||
Reference in New Issue
Block a user