perf: strip heavy base64 data at PostgreSQL level using raw SQL
Previous fix stripped data in Node.js AFTER Prisma loaded the full JSON from PostgreSQL. For 5 entries with PDF attachments, this still meant 25-50MB transferring from DB to Node.js on every page load. Now uses prisma.\ with nested jsonb_each/jsonb_object_agg to strip data/fileData/imageUrl strings >1KB inside the database itself. Heavy base64 never leaves PostgreSQL when lightweight=true.
This commit is contained in:
@@ -57,19 +57,67 @@ export async function GET(request: NextRequest) {
|
|||||||
return NextResponse.json({ value: item ? item.value : null });
|
return NextResponse.json({ value: item ? item.value : null });
|
||||||
} else {
|
} else {
|
||||||
// Get all items in namespace
|
// Get all items in namespace
|
||||||
|
const result: Record<string, unknown> = {};
|
||||||
|
|
||||||
|
if (lightweight) {
|
||||||
|
// Strip heavy base64 fields AT THE DATABASE LEVEL using raw SQL.
|
||||||
|
// This prevents PostgreSQL from transferring megabytes of base64
|
||||||
|
// data to Node.js — the heavy strings never leave the DB.
|
||||||
|
// Handles: top-level data/fileData/imageUrl + same keys inside array elements.
|
||||||
|
const rows = await prisma.$queryRaw<
|
||||||
|
Array<{ key: string; value: unknown }>
|
||||||
|
>`
|
||||||
|
SELECT key,
|
||||||
|
CASE WHEN jsonb_typeof(value) = 'object' THEN (
|
||||||
|
SELECT COALESCE(jsonb_object_agg(
|
||||||
|
k,
|
||||||
|
CASE
|
||||||
|
WHEN k IN ('data', 'fileData', 'imageUrl')
|
||||||
|
AND jsonb_typeof(v) = 'string'
|
||||||
|
AND length(v #>> '{}') > 1024
|
||||||
|
THEN '"__stripped__"'::jsonb
|
||||||
|
WHEN jsonb_typeof(v) = 'array' THEN (
|
||||||
|
SELECT COALESCE(jsonb_agg(
|
||||||
|
CASE
|
||||||
|
WHEN jsonb_typeof(el) = 'object' THEN (
|
||||||
|
SELECT COALESCE(jsonb_object_agg(
|
||||||
|
ek,
|
||||||
|
CASE
|
||||||
|
WHEN ek IN ('data', 'fileData', 'imageUrl')
|
||||||
|
AND jsonb_typeof(ev) = 'string'
|
||||||
|
AND length(ev #>> '{}') > 1024
|
||||||
|
THEN '"__stripped__"'::jsonb
|
||||||
|
ELSE ev
|
||||||
|
END
|
||||||
|
), '{}'::jsonb) FROM jsonb_each(el) AS ie(ek, ev)
|
||||||
|
)
|
||||||
|
ELSE el
|
||||||
|
END
|
||||||
|
), '[]'::jsonb)
|
||||||
|
FROM jsonb_array_elements(v) AS ae(el)
|
||||||
|
)
|
||||||
|
ELSE v
|
||||||
|
END
|
||||||
|
), '{}'::jsonb)
|
||||||
|
FROM jsonb_each(value) AS oe(k, v)
|
||||||
|
)
|
||||||
|
ELSE value
|
||||||
|
END AS value
|
||||||
|
FROM "KeyValueStore"
|
||||||
|
WHERE namespace = ${namespace}
|
||||||
|
`;
|
||||||
|
for (const row of rows) {
|
||||||
|
result[row.key] = row.value;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
const items = await prisma.keyValueStore.findMany({
|
const items = await prisma.keyValueStore.findMany({
|
||||||
where: { namespace },
|
where: { namespace },
|
||||||
});
|
});
|
||||||
|
|
||||||
// Return as a record { [key]: value }
|
|
||||||
const result: Record<string, any> = {};
|
|
||||||
for (const item of items) {
|
for (const item of items) {
|
||||||
if (lightweight) {
|
|
||||||
result[item.key] = stripHeavyFields(item.value);
|
|
||||||
} else {
|
|
||||||
result[item.key] = item.value;
|
result[item.key] = item.value;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return NextResponse.json({ items: result });
|
return NextResponse.json({ items: result });
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|||||||
Reference in New Issue
Block a user