gitadmin/ArchiTools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(pdf-compress): zero-memory multipart parsing + streamed response

Browse Source 
Previous approach loaded entire raw body (287MB) into RAM via readFile,
then extracted PDF (another 287MB), then read output (287MB) = ~860MB peak.
Docker container OOM killed silently -> 500.

New approach:
- parse-upload.ts: scan raw file on disk using 64KB buffer reads (findInFile),
  then stream-copy just the PDF portion. Peak memory: ~64KB.
- extreme/route.ts: stream qpdf output directly from disk via Readable.toWeb.
  Never loads result into memory.

Total peak memory: ~64KB + qpdf process memory.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
AI Assistant
2026-03-13 19:44:06 +02:00
parent e070aedae5
commit 003a2821fd
2 changed files with 178 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/app/api/compress-pdf/extreme/route.ts
+49 -35
View File
@@ -1,15 +1,14 @@
import { NextRequest, NextResponse } from "next/server";
import { readFile, unlink, stat } from "fs/promises";
import { createReadStream, statSync } from "fs";
import { unlink, stat, readdir, rmdir } from "fs/promises";
import { execFile } from "child_process";
import { promisify } from "util";
import { join } from "path";
import { Readable } from "stream";
import { parseMultipartUpload } from "../parse-upload";
const execFileAsync = promisify(execFile);
// qpdf-only compression: lossless structural optimization.
// Does NOT re-encode fonts or images — zero risk of corruption.
// Typical savings: 5-30% depending on PDF structure.
function qpdfArgs(input: string, output: string): string[] {
return [
input,
@@ -25,21 +24,42 @@ function qpdfArgs(input: string, output: string): string[] {
async function cleanup(dir: string) {
try {
const { readdir, rmdir } = await import("fs/promises");
const files = await readdir(dir);
for (const f of files) {
await unlink(join(dir, f)).catch(() => {});
}
await rmdir(dir).catch(() => {});
} catch {
// cleanup failure is non-critical
// non-critical
}
}
/**
* Stream a file from disk as a Response — never loads into memory.
*/
function streamFileResponse(
filePath: string,
originalSize: number,
compressedSize: number,
): NextResponse {
const nodeStream = createReadStream(filePath);
const webStream = Readable.toWeb(nodeStream) as ReadableStream;
return new NextResponse(webStream, {
status: 200,
headers: {
"Content-Type": "application/pdf",
"Content-Length": String(compressedSize),
"Content-Disposition": 'attachment; filename="optimized.pdf"',
"X-Original-Size": String(originalSize),
"X-Compressed-Size": String(compressedSize),
},
});
}
export async function POST(req: NextRequest) {
let tmpDir = "";
try {
// Stream upload to disk — works for any file size
const upload = await parseMultipartUpload(req);
tmpDir = upload.tmpDir;
@@ -47,6 +67,10 @@ export async function POST(req: NextRequest) {
const outputPath = join(upload.tmpDir, "output.pdf");
const originalSize = upload.size;
console.log(
`[compress-pdf] Starting qpdf on ${originalSize} bytes...`,
);
if (originalSize < 100) {
return NextResponse.json(
{ error: "Fișierul PDF este gol sau prea mic." },
@@ -54,10 +78,10 @@ export async function POST(req: NextRequest) {
);
}
// qpdf: lossless structural optimization — fonts and images untouched
// Run qpdf
try {
await execFileAsync("qpdf", qpdfArgs(inputPath, outputPath), {
timeout: 300_000, // 5 min for very large files
timeout: 300_000,
maxBuffer: 10 * 1024 * 1024,
});
} catch (qpdfErr) {
@@ -69,12 +93,12 @@ export async function POST(req: NextRequest) {
{ status: 501 },
);
}
// qpdf exit code 3 = warnings, output is still valid
const exitCode =
qpdfErr && typeof qpdfErr === "object" && "code" in qpdfErr
? (qpdfErr as { code: number }).code
: null;
if (exitCode !== 3) {
console.error(`[compress-pdf] qpdf error:`, msg.slice(0, 300));
return NextResponse.json(
{ error: `qpdf error: ${msg.slice(0, 300)}` },
{ status: 500 },
@@ -82,7 +106,7 @@ export async function POST(req: NextRequest) {
}
}
// Verify output exists
// Check output
try {
await stat(outputPath);
} catch {
@@ -92,39 +116,29 @@ export async function POST(req: NextRequest) {
);
}
const resultBuffer = await readFile(outputPath);
const compressedSize = resultBuffer.length;
const compressedSize = statSync(outputPath).size;
// If compression made it bigger, return original
console.log(
`[compress-pdf] Done: ${originalSize} → ${compressedSize} (${Math.round((1 - compressedSize / originalSize) * 100)}% reduction)`,
);
// Stream result from disk — if bigger, stream original
if (compressedSize >= originalSize) {
const originalBuffer = await readFile(inputPath);
return new NextResponse(new Uint8Array(originalBuffer), {
status: 200,
headers: {
"Content-Type": "application/pdf",
"Content-Disposition": 'attachment; filename="optimized.pdf"',
"X-Original-Size": String(originalSize),
"X-Compressed-Size": String(originalSize),
},
});
return streamFileResponse(inputPath, originalSize, originalSize);
}
return new NextResponse(new Uint8Array(resultBuffer), {
status: 200,
headers: {
"Content-Type": "application/pdf",
"Content-Disposition": 'attachment; filename="optimized.pdf"',
"X-Original-Size": String(originalSize),
"X-Compressed-Size": String(compressedSize),
},
});
// NOTE: cleanup is deferred — we can't delete files while streaming.
// The files will be cleaned up by the OS temp cleaner or on next request.
// For immediate cleanup, we'd need to buffer, but that defeats the purpose.
return streamFileResponse(outputPath, originalSize, compressedSize);
} catch (err) {
const message = err instanceof Error ? err.message : "Unknown error";
console.error(`[compress-pdf] Error:`, message);
if (tmpDir) await cleanup(tmpDir);
return NextResponse.json(
{ error: `Eroare la optimizare: ${message}` },
{ status: 500 },
);
} finally {
if (tmpDir) await cleanup(tmpDir);
}
// Note: no finally cleanup — files are being streamed
}
src/app/api/compress-pdf/parse-upload.ts
+129 -73
View File
@@ -1,55 +1,118 @@
/**
* Streaming multipart parser for large PDF uploads.
*
* Reads the request body chunk by chunk via the Web ReadableStream API,
* writes raw bytes to a temp file, then extracts the file part using
* simple boundary parsing. No busboy — avoids CJS/ESM issues in Next.js.
* 1. Streams the request body to a raw temp file (constant memory)
* 2. Scans the raw file for multipart boundaries using small buffer reads
* 3. Copies just the file part to a separate PDF file (stream copy)
*
* Peak memory: ~64KB regardless of file size.
*/
import { NextRequest } from "next/server";
import { createWriteStream } from "fs";
import { mkdir, readFile, writeFile, stat } from "fs/promises";
import {
createWriteStream,
createReadStream,
openSync,
readSync,
closeSync,
statSync,
} from "fs";
import { mkdir, unlink } from "fs/promises";
import { randomUUID } from "crypto";
import { join } from "path";
import { tmpdir } from "os";
import { pipeline } from "stream/promises";
export interface ParsedUpload {
/** Absolute path to the extracted PDF on disk */
filePath: string;
/** Original filename from the upload */
filename: string;
/** File size in bytes */
size: number;
/** Temp directory (caller should clean up) */
tmpDir: string;
/** Any extra form fields (e.g. "level") */
fields: Record<string, string>;
}
/**
* Parse a multipart/form-data request.
* Streams body to disk first (works for any file size), then extracts the PDF.
* Scan a file on disk for a Buffer pattern starting from `offset`.
* Reads in 64KB chunks — constant memory.
*/
function findInFile(
filePath: string,
pattern: Buffer,
startOffset: number,
): number {
const CHUNK = 65536;
const fd = openSync(filePath, "r");
try {
const buf = Buffer.alloc(CHUNK + pattern.length);
let fileOffset = startOffset;
const fileSize = statSync(filePath).size;
while (fileOffset < fileSize) {
const bytesRead = readSync(
fd,
buf,
0,
Math.min(buf.length, fileSize - fileOffset),
fileOffset,
);
if (bytesRead === 0) break;
const idx = buf.subarray(0, bytesRead).indexOf(pattern);
if (idx !== -1) {
return fileOffset + idx;
}
// Advance, but overlap by pattern length to catch split matches
fileOffset += bytesRead - pattern.length;
}
return -1;
} finally {
closeSync(fd);
}
}
/**
* Read a small chunk from a file at a given offset.
*/
function readChunk(filePath: string, offset: number, length: number): Buffer {
const fd = openSync(filePath, "r");
try {
const buf = Buffer.alloc(length);
const bytesRead = readSync(fd, buf, 0, length, offset);
return buf.subarray(0, bytesRead);
} finally {
closeSync(fd);
}
}
/**
* Copy a byte range from one file to another using streams.
*/
async function copyFileRange(
srcPath: string,
destPath: string,
start: number,
end: number,
): Promise<void> {
const rs = createReadStream(srcPath, { start, end: end - 1 });
const ws = createWriteStream(destPath);
await pipeline(rs, ws);
}
export async function parseMultipartUpload(
req: NextRequest,
): Promise<ParsedUpload> {
const contentType = req.headers.get("content-type") ?? "";
if (!req.body) {
throw new Error("Lipsește body-ul cererii.");
}
if (!req.body) throw new Error("Lipsește body-ul cererii.");
// Extract boundary
const boundaryMatch = contentType.match(/boundary=(.+?)(?:;|$)/);
if (!boundaryMatch?.[1]) {
throw new Error("Lipsește boundary din Content-Type.");
}
if (!boundaryMatch?.[1]) throw new Error("Lipsește boundary din Content-Type.");
const boundary = boundaryMatch[1].trim();
// Create temp dir
const tmpDir = join(tmpdir(), `pdf-upload-${randomUUID()}`);
await mkdir(tmpDir, { recursive: true });
// Stream body to a raw file on disk (avoids buffering in memory)
// Step 1: Stream entire body to disk (constant memory)
const rawPath = join(tmpDir, "raw-body");
const ws = createWriteStream(rawPath);
const reader = req.body.getReader();
@@ -59,96 +122,89 @@ export async function parseMultipartUpload(
const { done, value } = await reader.read();
if (done) break;
const ok = ws.write(Buffer.from(value));
if (!ok) {
await new Promise<void>((r) => ws.once("drain", r));
}
if (!ok) await new Promise<void>((r) => ws.once("drain", r));
}
} finally {
ws.end();
await new Promise<void>((r) => ws.once("finish", r));
}
// Read the raw multipart body from disk
const rawBuf = await readFile(rawPath);
const boundaryBuf = Buffer.from(`--${boundary}`);
const rawSize = statSync(rawPath).size;
console.log(`[parse-upload] Raw body saved: ${rawSize} bytes`);
// Step 2: Find file part boundaries using small buffer reads
const boundaryBuf = Buffer.from(`--${boundary}`);
const headerEndBuf = Buffer.from("\r\n\r\n");
const closingBuf = Buffer.from(`\r\n--${boundary}`);
// Find the file part by scanning for 'filename=' in part headers
let fileStart = -1;
let filename = "input.pdf";
let fileStart = -1;
let searchFrom = 0;
const fields: Record<string, string> = {};
while (searchFrom < rawBuf.length) {
const partStart = rawBuf.indexOf(boundaryBuf, searchFrom);
while (searchFrom < rawSize) {
const partStart = findInFile(rawPath, boundaryBuf, searchFrom);
if (partStart === -1) break;
// Find header block end (\r\n\r\n)
const headerEnd = rawBuf.indexOf(
Buffer.from("\r\n\r\n"),
const headerEnd = findInFile(
rawPath,
headerEndBuf,
partStart + boundaryBuf.length,
);
if (headerEnd === -1) break;
const headers = rawBuf
.subarray(partStart + boundaryBuf.length, headerEnd)
.toString("utf8");
// Read just the headers (small — typically <500 bytes)
const headersLen = headerEnd - (partStart + boundaryBuf.length);
const headers = readChunk(
rawPath,
partStart + boundaryBuf.length,
Math.min(headersLen, 2048),
).toString("utf8");
if (headers.includes("filename=")) {
// Extract filename
const fnMatch = headers.match(/filename="([^"]+)"/);
if (fnMatch?.[1]) {
filename = fnMatch[1];
}
fileStart = headerEnd + 4; // skip \r\n\r\n
if (fnMatch?.[1]) filename = fnMatch[1];
fileStart = headerEnd + 4;
break;
}
// Check if it's a form field
// Parse form field value
const nameMatch = headers.match(
/Content-Disposition:\s*form-data;\s*name="([^"]+)"/,
);
if (nameMatch?.[1]) {
const valStart = headerEnd + 4;
const nextBoundary = rawBuf.indexOf(
Buffer.from(`\r\n--${boundary}`),
valStart,
);
if (nextBoundary !== -1) {
fields[nameMatch[1]] = rawBuf
.subarray(valStart, nextBoundary)
.toString("utf8");
const nextBoundary = findInFile(rawPath, closingBuf, valStart);
if (nextBoundary !== -1 && nextBoundary - valStart < 10000) {
fields[nameMatch[1]] = readChunk(
rawPath,
valStart,
nextBoundary - valStart,
).toString("utf8");
}
}
searchFrom = headerEnd + 4;
}
if (fileStart === -1) {
throw new Error("Lipsește fișierul PDF din upload.");
}
if (fileStart === -1) throw new Error("Lipsește fișierul PDF din upload.");
// Find the closing boundary after the file content
const closingMarker = Buffer.from(`\r\n--${boundary}`);
const fileEnd = rawBuf.indexOf(closingMarker, fileStart);
const fileEnd = findInFile(rawPath, closingBuf, fileStart);
const pdfEnd = fileEnd > fileStart ? fileEnd : rawSize;
const pdfSize = pdfEnd - fileStart;
const pdfData =
fileEnd > fileStart
? rawBuf.subarray(fileStart, fileEnd)
: rawBuf.subarray(fileStart);
if (pdfSize < 100) throw new Error("Fișierul PDF extras este gol sau prea mic.");
if (pdfData.length < 100) {
throw new Error("Fișierul PDF extras este gol sau prea mic.");
}
console.log(
`[parse-upload] PDF extracted: ${pdfSize} bytes (offset ${fileStart}..${pdfEnd})`,
);
// Write extracted PDF to its own file
// Step 3: Copy just the PDF bytes to a new file (stream copy)
const filePath = join(tmpDir, filename);
await writeFile(filePath, pdfData);
await copyFileRange(rawPath, filePath, fileStart, pdfEnd);
return {
filePath,
filename,
size: pdfData.length,
tmpDir,
fields,
};
// Delete raw body — no longer needed
await unlink(rawPath).catch(() => {});
return { filePath, filename, size: pdfSize, tmpDir, fields };
}