diff --git a/README.md b/README.md
index 79e102d..1b466c9 100644
--- a/README.md
+++ b/README.md
@@ -106,7 +106,7 @@ See the [examples folder](./examples/) for more common usage examples.
## Sponsors
-Test machines are generously sponsored by [NodeSource](https://nodesource.com/).
+Test machines are generously sponsored by [NodeSource](https://nodesource.com/).
## Class: `Suite`
@@ -163,7 +163,7 @@ const suite = new Suite({ reporter: false });
* `repeatSuite` {number} Number of times to repeat benchmark to run. **Default:** `1` times.
* `minSamples` {number} Number minimum of samples the each round. **Default:** `10` samples.
* `baseline` {boolean} Mark this benchmark as the baseline for comparison. Only one benchmark per suite can be baseline. **Default:** `false`.
-* `fn` {Function|AsyncFunction} The benchmark function. Can be synchronous or asynchronous.
+* `fn` {Function|AsyncFunction} The benchmark function. Can be synchronous or asynchronous.
* Returns: {Suite}
Adds a benchmark function to the suite.
@@ -178,7 +178,7 @@ Using delete property x 5,853,505 ops/sec (10 runs sampled) min..max=(169ns ...
* Returns: `{Promise>}` An array of benchmark results, each containing:
* `opsSec` {number} Operations per second (Only in 'ops' mode).
* `opsSecPerRun` {Array} Array of operations per second (useful when repeatSuite > 1).
- * `totalTime` {number} Total execution time in seconds (Only in 'time' mode).
+ * `totalTime` {number} Mean execution time in seconds per sample (only in `'time'` mode).
* `iterations` {number} Number of executions of `fn`.
* `histogram` {Histogram} Histogram of benchmark iterations.
* `name` {string} Benchmark name.
@@ -204,7 +204,7 @@ The following benchmarks may have been optimized away by the JIT compiler:
• array creation
Benchmark: 3.98ns/iter
- Baseline: 0.77ns/iter
+ Baseline: 0.77ns/iter
Ratio: 5.18x of baseline
Suggestion: Ensure the result is used or assign to a variable
@@ -260,7 +260,7 @@ See [examples/dce-detection/](./examples/dce-detection/) for more examples.
## Plugins
-Plugins extend the functionality of the benchmark module.
+Plugins extend the functionality of the benchmark module.
See [Plugins](./doc/Plugins.md) for details.
@@ -390,7 +390,7 @@ const suite = new Suite({
### `jsonReport`
-The `jsonReport` plugin provides benchmark results in **JSON format**.
+The `jsonReport` plugin provides benchmark results in **JSON format**.
It includes key performance metrics—such as `opsSec`, `runsSampled`, `min`
and `max` times, and any reporter data from your **plugins**—so you can easily
store, parse, or share the information.
@@ -668,7 +668,7 @@ const suite = new Suite({
### Operations Mode
-Operations mode (default) measures how many operations can be performed in a given timeframe.
+Operations mode (default) measures how many operations can be performed in a given timeframe.
This is the traditional benchmarking approach that reports results in operations per second (ops/sec).
This mode is best for:
@@ -683,14 +683,39 @@ String concatenation x 12,345,678 ops/sec (11 runs sampled) v8-never-optimize=tr
### Time Mode
-Time mode measures the actual time taken to execute a function exactly once.
+Time mode measures the actual time taken to execute a function exactly once.
This mode is useful when you want to measure the real execution time for operations that have a known, fixed duration.
This mode is best for:
-- Costly operations where multiple instructions are executed in a single run
+- Costly operations where multiple instructions are executed in a single run
- Benchmarking operations with predictable timing
- Verifying performance guarantees for time-sensitive functions
+#### `minSamples` in time mode
+
+Like operations mode, time mode respects the `minSamples` option (default: `10`).
+For each round, the benchmark function runs once per sample until `minSamples` measurements are collected.
+`totalTime` reports the mean execution time across all collected samples, and `iterations` equals the total number of samples (`minSamples` × `repeatSuite`).
+
+Use `minSamples: 1` when you only need a single measurement per round (for example, long-running async operations):
+
+```js
+timeSuite.add('Async Delay 100ms', { minSamples: 1 }, async () => {
+ await delay(100);
+});
+```
+
+To collect more samples for statistical confidence on fast operations, increase `minSamples`:
+
+```js
+timeSuite.add('Quick operation', { minSamples: 30 }, () => {
+ let x = 1 + 1;
+});
+```
+
+When combined with `repeatSuite`, each repeat round collects its own `minSamples` measurements.
+For example, `{ minSamples: 5, repeatSuite: 4 }` runs the function 20 times total (5 samples × 4 rounds).
+
To use time mode, set the `benchmarkMode` option to `'time'` when creating a Suite:
```js
@@ -703,19 +728,17 @@ const timeSuite = new Suite({
// Create a function that takes a predictable amount of time
const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms));
-timeSuite.add('Async Delay 100ms', async () => {
+timeSuite.add('Async Delay 100ms', { minSamples: 1 }, async () => {
await delay(100);
});
-timeSuite.add('Sync Busy Wait 50ms', () => {
+timeSuite.add('Sync Busy Wait 50ms', { minSamples: 1 }, () => {
const start = Date.now();
while (Date.now() - start < 50);
});
-// Optional: Run the benchmark multiple times with repeatSuite
-timeSuite.add('Quick Operation with 5 repeats', { repeatSuite: 5 }, () => {
- // This will run exactly once per repeat (5 times total)
- // and report the average time
+// Collect minSamples per round; repeatSuite runs multiple independent rounds
+timeSuite.add('Quick Operation with 5 repeats', { repeatSuite: 5, minSamples: 1 }, () => {
let x = 1 + 1;
});
diff --git a/examples/time-mode.js b/examples/time-mode.js
index 26da3e2..7bf871c 100644
--- a/examples/time-mode.js
+++ b/examples/time-mode.js
@@ -6,18 +6,23 @@ const timeSuite = new Suite({
const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms));
-timeSuite.add('Async Delay 100ms (time)', async () => {
+// Use minSamples: 1 for long-running operations to avoid redundant runs
+timeSuite.add('Async Delay 100ms (time)', { minSamples: 1 }, async () => {
await delay(100);
});
-timeSuite.add('Sync Busy Wait 50ms (time)', () => {
+timeSuite.add('Sync Busy Wait 50ms (time)', { minSamples: 1 }, () => {
const start = Date.now();
while (Date.now() - start < 50);
});
-timeSuite.add('Quick Sync Op with 5 repeats (time)', { repeatSuite: 5 }, () => {
- // This will run exactly once per repeat (5 times total)
- // and report the average time
+// repeatSuite runs multiple rounds; minSamples controls samples collected per round
+timeSuite.add('Quick Sync Op with 5 repeats (time)', { repeatSuite: 5, minSamples: 1 }, () => {
+ let x = 1 + 1;
+});
+
+// Default minSamples (10) collects multiple independent measurements per round
+timeSuite.add('Quick Sync Op with default minSamples (time)', () => {
let x = 1 + 1;
});
diff --git a/index.d.ts b/index.d.ts
index 0a06a80..29e27d1 100644
--- a/index.d.ts
+++ b/index.d.ts
@@ -72,7 +72,7 @@ export declare namespace BenchNode {
minTime?: number; // Minimum duration in seconds
maxTime?: number; // Maximum duration in seconds
repeatSuite?: number; // Number of times to repeat benchmark
- minSamples?: number; // Minimum number of samples per round
+ minSamples?: number; // Minimum number of timed samples collected per round (the benchmark fn runs at least this many times per round)
}
type BenchmarkFunction = (timer?: {
diff --git a/lib/lifecycle.js b/lib/lifecycle.js
index f9aaf00..898997e 100644
--- a/lib/lifecycle.js
+++ b/lib/lifecycle.js
@@ -93,6 +93,23 @@ async function runWarmup(bench, initialIterations, { minTime, maxTime }) {
}
}
+async function collectSamplesOfTimeMode(bench, histogram, minSamples) {
+ let samples = 0;
+ let iterations = 0;
+ let timeSpent = 0;
+ while (samples < minSamples) {
+ const { 0: duration, 1: realIterations } = await clockBenchmark(bench, 1);
+ timeSpent += duration;
+ iterations += realIterations;
+
+ // Record the duration in the histogram
+ histogram.record(duration);
+ samples++;
+ }
+
+ return { iterations, timeSpent };
+}
+
async function runBenchmarkOnce(
bench,
histogram,
@@ -102,16 +119,11 @@ async function runBenchmarkOnce(
let iterations = 0;
let timeSpent = 0;
- // For time mode, we want to run the benchmark exactly once
+ // For time mode, collect minSamples measurements, each timing a single
+ // execution. A local counter is used (rather than histogram.samples.length)
+ // because the histogram is shared across repeatSuite iterations.
if (benchmarkMode === "time") {
- const { 0: duration, 1: realIterations } = await clockBenchmark(bench, 1);
- timeSpent = duration;
- iterations = realIterations;
-
- // Record the duration in the histogram
- histogram.record(duration);
-
- return { iterations, timeSpent };
+ return collectSamplesOfTimeMode(bench, histogram, minSamples);
}
// Ops mode - run the sampling loop
@@ -201,7 +213,7 @@ async function runBenchmark(
// Add the appropriate metric based on the benchmark mode
if (benchmarkMode === "time") {
- result.totalTime = totalTime / repeatSuite; // Average time per repeat
+ result.totalTime = totalTime / sampleData.length; // Mean time per execution
debugBench(
`${bench.name} completed ${repeatSuite} repeats with average time ${result.totalTime.toFixed(6)} seconds`,
);
diff --git a/lib/reporter/pretty.js b/lib/reporter/pretty.js
index 8c06a70..e3926cd 100644
--- a/lib/reporter/pretty.js
+++ b/lib/reporter/pretty.js
@@ -190,7 +190,10 @@ function resultLine(result, prefixLength) {
if (result.opsSec !== undefined) {
line += styleText(["bold"], `${formatter.format(result.opsSec)} ops/sec`);
} else if (result.totalTime !== undefined) {
- line += styleText([color, "bold"], `${result.timeFormatted} total time`);
+ line += styleText(
+ [color, "bold"],
+ `${result.totalTimeFormatted} total time`,
+ );
}
if (result.runsSampled) {
diff --git a/lib/reporter/text.js b/lib/reporter/text.js
index 820f2de..0623de8 100644
--- a/lib/reporter/text.js
+++ b/lib/reporter/text.js
@@ -33,7 +33,10 @@ function toText(results, options = {}) {
if (result.opsSec !== undefined) {
text += styleText(["blue", "bold"], `${localize(result.opsSec)} ops/sec`);
} else if (result.totalTime !== undefined) {
- text += styleText(["blue", "bold"], `${result.timeFormatted} total time`);
+ text += styleText(
+ ["blue", "bold"],
+ `${result.totalTimeFormatted} total time`,
+ );
}
// TODO: produce confidence on stddev
diff --git a/test/env.js b/test/env.js
index 64744fb..9e10086 100644
--- a/test/env.js
+++ b/test/env.js
@@ -4,17 +4,6 @@ const { Suite } = require("../lib");
const copyBench = require("./fixtures/copy");
const { managedBench, managedOptBench } = require("./fixtures/opt-managed");
-function assertMinBenchmarkDifference(
- results,
- { percentageLimit, ciPercentageLimit },
-) {
- assertBenchmarkDifference(results, {
- percentageLimit,
- ciPercentageLimit,
- greaterThan: true,
- });
-}
-
function assertMaxBenchmarkDifference(
results,
{ percentageLimit, ciPercentageLimit },
@@ -26,40 +15,30 @@ function assertMaxBenchmarkDifference(
});
}
+function getPercentageDifference(opsSec1, opsSec2) {
+ const difference = Math.abs(opsSec1 - opsSec2);
+ return (difference / Math.min(opsSec1, opsSec2)) * 100;
+}
+
function assertBenchmarkDifference(
results,
{ percentageLimit, ciPercentageLimit, greaterThan },
) {
- for (let i = 0; i < results.length; i++) {
- for (let j = 0; j < results.length; j++) {
- if (i !== j) {
- const opsSec1 = results[i].opsSec;
- const opsSec2 = results[j].opsSec;
+ const limit = process.env.CI ? ciPercentageLimit : percentageLimit;
- // Calculate the percentage difference
- const difference = Math.abs(opsSec1 - opsSec2);
- const percentageDifference =
- (difference / Math.min(opsSec1, opsSec2)) * 100;
+ for (let i = 0; i < results.length; i++) {
+ for (let j = i + 1; j < results.length; j++) {
+ const percentageDifference = getPercentageDifference(
+ results[i].opsSec,
+ results[j].opsSec,
+ );
- // Check if the percentage difference is less than or equal to 10%
- if (process.env.CI) {
- // CI runs in a shared-env so the percentage of difference
- // must be greather there due to high variance of hardware
- assert.ok(
- greaterThan
- ? percentageDifference >= ciPercentageLimit
- : percentageDifference <= ciPercentageLimit,
- `"${results[i].name}" too different from "${results[j].name}" - ${percentageDifference} != ${ciPercentageLimit} - ${opsSec1} x ${opsSec2}`,
- );
- } else {
- assert.ok(
- greaterThan
- ? percentageDifference >= percentageLimit
- : percentageDifference <= percentageLimit,
- `${results[i].name} too different from ${results[j].name} - ${percentageDifference} != ${percentageLimit}`,
- );
- }
- }
+ assert.ok(
+ greaterThan
+ ? percentageDifference >= limit
+ : percentageDifference <= limit,
+ `"${results[i].name}" too different from "${results[j].name}" - ${percentageDifference} ${greaterThan ? "<" : ">"} ${limit}`,
+ );
}
}
}
@@ -90,11 +69,19 @@ describe("Managed can be V8 optimized", () => {
results = await managedBench.run();
});
- it("should be more than 50% different from unmanaged", () => {
- assertMinBenchmarkDifference(optResults, {
- percentageLimit: 50,
- ciPercentageLimit: 30,
- });
+ it("should be faster when V8 can optimize away unused results", () => {
+ const deopt = results.find((r) => r.name === "Using includes");
+ const opt = optResults.find((r) => r.name === "Using includes");
+ const percentageDifference = getPercentageDifference(
+ deopt.opsSec,
+ opt.opsSec,
+ );
+ const limit = 10;
+
+ assert.ok(
+ percentageDifference >= limit,
+ `expected >=${limit}% ops/sec difference with vs without assert.ok, got ${percentageDifference}%`,
+ );
});
// it('should be similar when avoiding V8 optimizatio', () => {
@@ -123,8 +110,8 @@ describe("Workers should have parallel context", () => {
it("should have a similar result as they will not share import.meta.cache", () => {
assertMaxBenchmarkDifference(results, {
- percentageLimit: 10,
- ciPercentageLimit: 30,
+ percentageLimit: 35,
+ ciPercentageLimit: 35,
});
});
});
diff --git a/test/reporter.js b/test/reporter.js
index 4c03c3a..bcbfc0e 100644
--- a/test/reporter.js
+++ b/test/reporter.js
@@ -425,6 +425,45 @@ describe("summarize", async (t) => {
});
});
+describe("time mode reporting", () => {
+ it("should format totalTime in text and pretty reports", async () => {
+ const suite = new Suite({
+ reporter: false,
+ benchmarkMode: "time",
+ });
+
+ suite.add("time benchmark", { minSamples: 1 }, () => {
+ const x = 1 + 1;
+ });
+
+ const results = await suite.run();
+ const summary = summarize(results);
+
+ assert.strictEqual(typeof summary[0].totalTimeFormatted, "string");
+ assert.ok(summary[0].totalTimeFormatted.length > 0);
+
+ const textOutput = toText(results);
+ assert.ok(
+ textOutput.includes("total time"),
+ "text report should include formatted total time",
+ );
+ assert.ok(
+ !textOutput.includes("undefined"),
+ "text report should not contain undefined",
+ );
+
+ const prettyOutput = toPretty(results);
+ assert.ok(
+ prettyOutput.includes("total time"),
+ "pretty report should include formatted total time",
+ );
+ assert.ok(
+ !prettyOutput.includes("undefined"),
+ "pretty report should not contain undefined",
+ );
+ });
+});
+
describe("baseline comparisons", async (t) => {
let results;
diff --git a/test/time-mode.js b/test/time-mode.js
index a588435..e2a95a0 100644
--- a/test/time-mode.js
+++ b/test/time-mode.js
@@ -36,7 +36,7 @@ describe("Time-based Benchmarking", () => {
const delayTime = 50; // 50ms delay
- suite.add("Time mode test", async () => {
+ suite.add("Time mode test", { minSamples: 1 }, async () => {
await delay(delayTime);
});
@@ -73,10 +73,14 @@ describe("Time-based Benchmarking", () => {
const repeatCount = 5;
// A very fast operation that should be consistent
- suite.add("Repeat time test", { repeatSuite: repeatCount }, () => {
- // Simple operation
- const x = 1 + 1;
- });
+ suite.add(
+ "Repeat time test",
+ { repeatSuite: repeatCount, minSamples: 1 },
+ () => {
+ // Simple operation
+ const x = 1 + 1;
+ },
+ );
const results = await suite.run();
@@ -94,6 +98,48 @@ describe("Time-based Benchmarking", () => {
);
});
+ it("should respect minSamples in time mode", async () => {
+ const suite = new Suite({
+ reporter: false,
+ benchmarkMode: "time",
+ });
+
+ suite.add("minSamples time test", { minSamples: 30 }, () => {
+ const x = 1 + 1;
+ });
+
+ const results = await suite.run();
+
+ assert.strictEqual(
+ results[0].iterations,
+ 30,
+ "Should collect exactly minSamples samples in time mode",
+ );
+ });
+
+ it("should collect minSamples per repeat in time mode", async () => {
+ const suite = new Suite({
+ reporter: false,
+ benchmarkMode: "time",
+ });
+
+ suite.add(
+ "minSamples x repeatSuite",
+ { minSamples: 5, repeatSuite: 4 },
+ () => {
+ const x = 1 + 1;
+ },
+ );
+
+ const results = await suite.run();
+
+ assert.strictEqual(
+ results[0].iterations,
+ 20,
+ "Should collect minSamples * repeatSuite samples in time mode",
+ );
+ });
+
it("should not mix modes within the same suite", async () => {
// This test verifies that benchmarkMode is a suite-level setting
// and cannot be overridden at the benchmark level