From 8c9eab621fcc74bb72b5ba241e2f50379f7d3c6e Mon Sep 17 00:00:00 2001 From: Nic Polumeyv <162764842+Nic-Polumeyv@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:31:32 -0400 Subject: [PATCH 1/2] compress each file from one read, a few files at a time --- .changeset/calm-files-compress.md | 5 +++ packages/kit/src/core/adapt/builder.js | 55 ++++++++++++-------------- 2 files changed, 31 insertions(+), 29 deletions(-) create mode 100644 .changeset/calm-files-compress.md diff --git a/.changeset/calm-files-compress.md b/.changeset/calm-files-compress.md new file mode 100644 index 000000000000..565fda9070d8 --- /dev/null +++ b/.changeset/calm-files-compress.md @@ -0,0 +1,5 @@ +--- +'@sveltejs/kit': patch +--- + +fix: keep memory flat when precompressing many files diff --git a/packages/kit/src/core/adapt/builder.js b/packages/kit/src/core/adapt/builder.js index dd96eb93e2eb..5b2775598836 100644 --- a/packages/kit/src/core/adapt/builder.js +++ b/packages/kit/src/core/adapt/builder.js @@ -5,16 +5,9 @@ /** @import { RouteData, ValidatedConfig, BuildData, ServerMetadata, ServerMetadataRoute, Prerendered, PrerenderMap, Logger, RemoteChunk } from 'types' */ import { loadEnv } from 'vite'; import * as devalue from 'devalue'; -import { - createReadStream, - createWriteStream, - existsSync, - mkdirSync, - rmSync, - statSync -} from 'node:fs'; +import { existsSync, mkdirSync, rmSync, statSync } from 'node:fs'; +import { readFile, writeFile } from 'node:fs/promises'; import path from 'node:path'; -import { pipeline } from 'node:stream'; import { promisify } from 'node:util'; import zlib from 'node:zlib'; import { copy, relative_path, walk } from '../../utils/filesystem.js'; @@ -29,7 +22,8 @@ import { handle_issues, validate } from '../../exports/internal/env.js'; import { get_mime_lookup } from '../utils.js'; import { lookup as mime_lookup } from '../../utils/mime.js'; -const pipe = promisify(pipeline); +const gzip = promisify(zlib.gzip); +const brotli = promisify(zlib.brotliCompress); const extensions = [ '.html', '.js', @@ -156,10 +150,14 @@ export function create_builder({ const files = [...walk(directory)].filter((file) => extensions.includes(path.extname(file))); + // zlib work is serialised on the threadpool and each brotli encoder is allocated up front, + // so a handful of files in flight is as fast as all of them and keeps memory flat + const queue = [...files]; await Promise.all( - files.flatMap((file) => { - const abs = path.resolve(directory, file); - return [compress_file(abs, 'gz'), compress_file(abs, 'br')]; + Array.from({ length: 16 }, async () => { + /** @type {string | undefined} */ + let file; + while ((file = queue.shift())) await compress_file(path.resolve(directory, file)); }) ); @@ -372,25 +370,24 @@ export function create_builder({ } /** + * Writes gzip and brotli variants next to `file` * @param {string} file - * @param {'gz' | 'br'} format */ -async function compress_file(file, format = 'gz') { - const compress = - format == 'br' - ? zlib.createBrotliCompress({ - params: { - [zlib.constants.BROTLI_PARAM_MODE]: zlib.constants.BROTLI_MODE_TEXT, - [zlib.constants.BROTLI_PARAM_QUALITY]: zlib.constants.BROTLI_MAX_QUALITY, - [zlib.constants.BROTLI_PARAM_SIZE_HINT]: statSync(file).size - } - }) - : zlib.createGzip({ level: zlib.constants.Z_BEST_COMPRESSION }); - - const source = createReadStream(file); - const destination = createWriteStream(`${file}.${format}`); +async function compress_file(file) { + const contents = await readFile(file); + + const [gz, br] = await Promise.all([ + gzip(contents, { level: zlib.constants.Z_BEST_COMPRESSION }), + brotli(contents, { + params: { + [zlib.constants.BROTLI_PARAM_MODE]: zlib.constants.BROTLI_MODE_TEXT, + [zlib.constants.BROTLI_PARAM_QUALITY]: zlib.constants.BROTLI_MAX_QUALITY, + [zlib.constants.BROTLI_PARAM_SIZE_HINT]: contents.length + } + }) + ]); - await pipe(source, compress, destination); + await Promise.all([writeFile(`${file}.gz`, gz), writeFile(`${file}.br`, br)]); } /** From 996da7e3e03d3d3b7ec06bf3be90539864ebba13 Mon Sep 17 00:00:00 2001 From: Nic Polumeyv <162764842+Nic-Polumeyv@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:44:07 -0400 Subject: [PATCH 2/2] use the default fs import, walk the file list by index --- packages/kit/src/core/adapt/builder.js | 38 +++++++++++++------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/packages/kit/src/core/adapt/builder.js b/packages/kit/src/core/adapt/builder.js index 5b2775598836..296ea71895ad 100644 --- a/packages/kit/src/core/adapt/builder.js +++ b/packages/kit/src/core/adapt/builder.js @@ -5,8 +5,7 @@ /** @import { RouteData, ValidatedConfig, BuildData, ServerMetadata, ServerMetadataRoute, Prerendered, PrerenderMap, Logger, RemoteChunk } from 'types' */ import { loadEnv } from 'vite'; import * as devalue from 'devalue'; -import { existsSync, mkdirSync, rmSync, statSync } from 'node:fs'; -import { readFile, writeFile } from 'node:fs/promises'; +import fs from 'node:fs'; import path from 'node:path'; import { promisify } from 'node:util'; import zlib from 'node:zlib'; @@ -109,8 +108,8 @@ export function create_builder({ return { log, - rimraf: (dir) => rmSync(dir, { force: true, recursive: true }), - mkdirp: (dir) => mkdirSync(dir, { recursive: true }), + rimraf: (dir) => fs.rmSync(dir, { force: true, recursive: true }), + mkdirp: (dir) => fs.mkdirSync(dir, { recursive: true }), copy, config, @@ -128,7 +127,7 @@ export function create_builder({ /** @type {Record} */ const files = {}; for (const file of server_assets) { - files[file] = statSync(path.resolve(build_data.out_dir, 'server', file)).size; + files[file] = fs.statSync(path.resolve(build_data.out_dir, 'server', file)).size; const ext = path.extname(file); mime_types[ext] ??= mime_lookup(ext) || ''; @@ -144,7 +143,7 @@ export function create_builder({ }, async compress(directory) { - if (!existsSync(directory)) { + if (!fs.existsSync(directory)) { return []; } @@ -152,12 +151,10 @@ export function create_builder({ // zlib work is serialised on the threadpool and each brotli encoder is allocated up front, // so a handful of files in flight is as fast as all of them and keeps memory flat - const queue = [...files]; + let i = 0; await Promise.all( Array.from({ length: 16 }, async () => { - /** @type {string | undefined} */ - let file; - while ((file = queue.shift())) await compress_file(path.resolve(directory, file)); + while (i < files.length) await compress_file(path.resolve(directory, files[i++])); }) ); @@ -184,7 +181,7 @@ export function create_builder({ assets: config.files.assets }); - if (existsSync(dest)) { + if (fs.existsSync(dest)) { log.warn( `\nOverwriting ${dest} with fallback page. Consider using a different name for the fallback.\n` ); @@ -310,7 +307,7 @@ export function create_builder({ }, hasServerInstrumentationFile() { - return existsSync(`${config.outDir}/output/server/instrumentation.server.js`); + return fs.existsSync(`${config.outDir}/output/server/instrumentation.server.js`); }, instrument({ @@ -322,24 +319,24 @@ export function create_builder({ exports: ['default'] } }) { - if (!existsSync(instrumentation)) { + if (!fs.existsSync(instrumentation)) { throw new Error( `Instrumentation file ${instrumentation} not found. This is probably a bug in your adapter.` ); } - if (!existsSync(entrypoint)) { + if (!fs.existsSync(entrypoint)) { throw new Error( `Entrypoint file ${entrypoint} not found. This is probably a bug in your adapter.` ); } - if (!existsSync(initializer)) { + if (!fs.existsSync(initializer)) { throw new Error( `Instrumentation initializer ${initializer} not found. This is probably a bug in your adapter.` ); } copy(entrypoint, start); - if (existsSync(`${entrypoint}.map`)) { + if (fs.existsSync(`${entrypoint}.map`)) { copy(`${entrypoint}.map`, `${start}.map`); } @@ -363,7 +360,7 @@ export function create_builder({ initializer: relative_initializer }); - rmSync(entrypoint, { force: true, recursive: true }); + fs.rmSync(entrypoint, { force: true, recursive: true }); write(entrypoint, facade); } }; @@ -374,7 +371,7 @@ export function create_builder({ * @param {string} file */ async function compress_file(file) { - const contents = await readFile(file); + const contents = await fs.promises.readFile(file); const [gz, br] = await Promise.all([ gzip(contents, { level: zlib.constants.Z_BEST_COMPRESSION }), @@ -387,7 +384,10 @@ async function compress_file(file) { }) ]); - await Promise.all([writeFile(`${file}.gz`, gz), writeFile(`${file}.br`, br)]); + await Promise.all([ + fs.promises.writeFile(`${file}.gz`, gz), + fs.promises.writeFile(`${file}.br`, br) + ]); } /**