perf: strip heavy base64 data at PostgreSQL level using raw SQL
Previous fix stripped data in Node.js AFTER Prisma loaded the full JSON from PostgreSQL. For 5 entries with PDF attachments, this still meant 25-50MB transferring from DB to Node.js on every page load. Now uses prisma.\ with nested jsonb_each/jsonb_object_agg to strip data/fileData/imageUrl strings >1KB inside the database itself. Heavy base64 never leaves PostgreSQL when lightweight=true.
This commit is contained in:
@@ -57,19 +57,67 @@ export async function GET(request: NextRequest) {
|
||||
return NextResponse.json({ value: item ? item.value : null });
|
||||
} else {
|
||||
// Get all items in namespace
|
||||
const items = await prisma.keyValueStore.findMany({
|
||||
where: { namespace },
|
||||
});
|
||||
const result: Record<string, unknown> = {};
|
||||
|
||||
// Return as a record { [key]: value }
|
||||
const result: Record<string, any> = {};
|
||||
for (const item of items) {
|
||||
if (lightweight) {
|
||||
result[item.key] = stripHeavyFields(item.value);
|
||||
} else {
|
||||
if (lightweight) {
|
||||
// Strip heavy base64 fields AT THE DATABASE LEVEL using raw SQL.
|
||||
// This prevents PostgreSQL from transferring megabytes of base64
|
||||
// data to Node.js — the heavy strings never leave the DB.
|
||||
// Handles: top-level data/fileData/imageUrl + same keys inside array elements.
|
||||
const rows = await prisma.$queryRaw<
|
||||
Array<{ key: string; value: unknown }>
|
||||
>`
|
||||
SELECT key,
|
||||
CASE WHEN jsonb_typeof(value) = 'object' THEN (
|
||||
SELECT COALESCE(jsonb_object_agg(
|
||||
k,
|
||||
CASE
|
||||
WHEN k IN ('data', 'fileData', 'imageUrl')
|
||||
AND jsonb_typeof(v) = 'string'
|
||||
AND length(v #>> '{}') > 1024
|
||||
THEN '"__stripped__"'::jsonb
|
||||
WHEN jsonb_typeof(v) = 'array' THEN (
|
||||
SELECT COALESCE(jsonb_agg(
|
||||
CASE
|
||||
WHEN jsonb_typeof(el) = 'object' THEN (
|
||||
SELECT COALESCE(jsonb_object_agg(
|
||||
ek,
|
||||
CASE
|
||||
WHEN ek IN ('data', 'fileData', 'imageUrl')
|
||||
AND jsonb_typeof(ev) = 'string'
|
||||
AND length(ev #>> '{}') > 1024
|
||||
THEN '"__stripped__"'::jsonb
|
||||
ELSE ev
|
||||
END
|
||||
), '{}'::jsonb) FROM jsonb_each(el) AS ie(ek, ev)
|
||||
)
|
||||
ELSE el
|
||||
END
|
||||
), '[]'::jsonb)
|
||||
FROM jsonb_array_elements(v) AS ae(el)
|
||||
)
|
||||
ELSE v
|
||||
END
|
||||
), '{}'::jsonb)
|
||||
FROM jsonb_each(value) AS oe(k, v)
|
||||
)
|
||||
ELSE value
|
||||
END AS value
|
||||
FROM "KeyValueStore"
|
||||
WHERE namespace = ${namespace}
|
||||
`;
|
||||
for (const row of rows) {
|
||||
result[row.key] = row.value;
|
||||
}
|
||||
} else {
|
||||
const items = await prisma.keyValueStore.findMany({
|
||||
where: { namespace },
|
||||
});
|
||||
for (const item of items) {
|
||||
result[item.key] = item.value;
|
||||
}
|
||||
}
|
||||
|
||||
return NextResponse.json({ items: result });
|
||||
}
|
||||
} catch (error) {
|
||||
|
||||
Reference in New Issue
Block a user