Add default proxied network isolation for registry benchmarks

.github/workflows/benchmark.yaml

@@ -18,6 +18,12 @@ on:
       runs:
         description: "The number of runs on each benchmark"
         default: "5"
+      network_profile:
+        description: "Optional network isolation profile for registry benchmarks"
+        default: "registry-bandwidth"
+      network_rate_kbps:
+        description: "Bandwidth cap in KB/s for supported network profiles"
+        default: "8192"
   schedule:
     # GitHub Actions cron uses UTC. 10:17 UTC ~= 2:17 AM Pacific (PST).
     - cron: "17 10 * * *"
@@ -100,6 +106,8 @@ jobs:
       BENCH_INCLUDE: ${{ fromJson(inputs.binaries || '"npm,yarn,berry,zpm,pnpm,pnpm11,vlt,bun,deno,aube,nx,turbo,vp,node"') }}
       BENCH_WARMUP: ${{ (github.event_name == 'push' && github.ref != 'refs/heads/main') && '1' || (inputs.warmup || '2') }}
       BENCH_RUNS: ${{ (github.event_name == 'push' && github.ref != 'refs/heads/main') && '1' || (inputs.runs || '5') }}
+      BENCH_NETWORK_PROFILE: ${{ inputs.network_profile || 'registry-bandwidth' }}
+      BENCH_TOXIPROXY_RATE_KBPS: ${{ inputs.network_rate_kbps || '8192' }}
     steps:
       - uses: actions/checkout@v6
       - name: Install Node

README.md

@@ -90,12 +90,25 @@ Examples:
 
 # Run only aws registry benchmarks (requires token)
 CODEARTIFACT_AUTH_TOKEN=<token> ./bench run --variation=registry-clean --fixtures=next --registries=aws
+
+# Run isolated VSR registry benchmarks through toxiproxy with a bandwidth cap
+VLT_REGISTRY_AUTH_TOKEN=<token> ./bench run --variation=registry-clean --fixtures=next --registries=vlt --network-profile=registry-bandwidth --network-rate-kbps=8192
 ```
 
 Auth notes:
 
 - `aws` requires `CODEARTIFACT_AUTH_TOKEN`.
 
+#### Network isolation
+
+Registry benchmarks default to a local `toxiproxy` path so benchmark traffic uses a controlled network path instead of the host's default connection.
+
+- `registry-bandwidth` proxies each included registry through a dedicated local toxiproxy listener.
+- This rewrites registry hosts to local listeners, then proxies TLS traffic to the real upstream registries with symmetric bandwidth limits.
+- Use `--network-profile=none` to bypass the proxy path.
+- `--network-rate-kbps` controls both upstream and downstream bandwidth in KB/s.
+- The helper temporarily edits `/etc/hosts`, so local runs require `sudo` access.
+
 ## Testing Script Execution
 
 This suite also tests the performance of basic script execution (ex. `npm run foo`). Notably, for any given build, test or deployment task the spawning of the process is a fraction of the overall execution time. That said, this is a commonly tracked workflow by various developer tools as it involves the common set of tasks: startup, filesystem read (`package.json`) & finally, spawning the process/command.
@@ -155,6 +168,9 @@ This suite also tests the performance of basic script execution (ex. `npm run fo
   # Run multiple variations in one command
   ./bench run --fixtures=svelte --variation=lockfile,registry-lockfile,run --registries=npm,vlt
 
+  # Run an isolated VSR benchmark with a bandwidth cap
+  VLT_REGISTRY_AUTH_TOKEN=<token> ./bench run --variation=registry-clean --fixtures=next --registries=vlt --network-profile=registry-bandwidth --network-rate-kbps=8192
+
   # Script-execution benchmark
   ./bench run --variation=run --pms=vlt
 

bench

@@ -50,7 +50,7 @@ AVAILABLE_REGISTRIES=(
 usage() {
   cat <<'EOF'
 Usage:
-  ./bench run [--fixtures=<list>] [--pms=<list>] [--variation=<list>] [--runs=<n>] [--warmup=<n>] [--chart] [--process] [--date=<yyyy-mm-dd>] [--clean]
+  ./bench run [--fixtures=<list>] [--pms=<list>] [--variation=<list>] [--runs=<n>] [--warmup=<n>] [--chart] [--process] [--date=<yyyy-mm-dd>] [--clean] [--network-profile=<name>] [--network-rate-kbps=<n>]
   ./bench chart [--fixtures=<list>] [--variation=<list>] [--date=<yyyy-mm-dd>] [--clean]
   ./bench process [--dry-run]
   ./bench list
@@ -63,6 +63,8 @@ Options:
   --runs      Hyperfine runs (default: scripts/variations/common.sh default)
   --warmup    Hyperfine warmup runs (default: scripts/variations/common.sh default)
   --timeout   Per-command timeout in seconds (default: 300)
+  --network-profile  Network isolation profile for registry-* runs (default: registry-bandwidth, use none to disable)
+  --network-rate-kbps  Bandwidth cap in KB/s for supported profiles (default: 8192)
   --chart     Generate chart data and copy to app/latest
   --process   Process results after run (clean + dated/latest outputs + chart data)
   --date      Date folder for chart output (default: today)
@@ -79,6 +81,7 @@ Examples:
   ./bench run --fixtures=next --chart --process --runs=3
   ./bench run --variation=registry-clean --fixtures=next
   ./bench run --variation=registry-lockfile --registries=npm,vlt
+  ./bench run --variation=registry-clean --fixtures=next --registries=vlt --network-profile=registry-bandwidth --network-rate-kbps=8192
   CODEARTIFACT_AUTH_TOKEN=<token> ./bench run --variation=registry-clean --fixtures=next --registries=aws
   GH_REGISTRY=<url> GH_AUTH_TOKEN=<token> ./bench run --variation=registry-clean --fixtures=next --registries=github
   ./bench chart --fixtures=next --variation=clean
@@ -375,6 +378,8 @@ run_bench() {
   local registries_input="${10:-}"
   local process_results="${11:-0}"
   local timeout_input="${12:-}"
+  local network_profile_input="${13:-}"
+  local network_rate_input="${14:-}"
 
   if [[ ! -f "$BENCH_SCRIPT" ]]; then
     echo "Error: Missing benchmark script at $BENCH_SCRIPT"
@@ -396,6 +401,9 @@ run_bench() {
   if [[ -n "$timeout_input" ]]; then
     validate_number "$timeout_input" "--timeout"
   fi
+  if [[ -n "$network_rate_input" ]]; then
+    validate_number "$network_rate_input" "--network-rate-kbps"
+  fi
 
   local pms_env=""
   if [[ -n "$pms_input" ]]; then
@@ -429,6 +437,16 @@ run_bench() {
   else
     echo "  timeout: 300s (default)"
   fi
+  if [[ -n "$network_profile_input" ]]; then
+    echo "  network profile: $network_profile_input"
+  else
+    echo "  network profile: registry script default"
+  fi
+  if [[ -n "$network_rate_input" ]]; then
+    echo "  network rate: ${network_rate_input} KB/s"
+  else
+    echo "  network rate: 8192 KB/s (default)"
+  fi
   if [[ "$clean_results" -eq 1 ]]; then
     echo "  clean: enabled"
   else
@@ -485,6 +503,12 @@ run_bench() {
     if [[ -n "$timeout_input" ]]; then
       env_args+=("BENCH_TIMEOUT=$timeout_input")
     fi
+    if [[ -n "$network_profile_input" ]]; then
+      env_args+=("BENCH_NETWORK_PROFILE=$network_profile_input")
+    fi
+    if [[ -n "$network_rate_input" ]]; then
+      env_args+=("BENCH_TOXIPROXY_RATE_KBPS=$network_rate_input")
+    fi
 
     local cmd=(bash "$BENCH_SCRIPT" "$fixture" "$variation")
 
@@ -642,6 +666,8 @@ case "$command" in
     CHART_RESULTS=0
     PROCESS_RESULTS=0
     CHART_DATE=""
+    NETWORK_PROFILE_INPUT=""
+    NETWORK_RATE_INPUT=""
 
     while [[ $# -gt 0 ]]; do
       case "$1" in
@@ -701,6 +727,22 @@ case "$command" in
           TIMEOUT_INPUT="${1#*=}"
           shift
           ;;
+        --network-profile)
+          NETWORK_PROFILE_INPUT="${2:-}"
+          shift 2
+          ;;
+        --network-profile=*)
+          NETWORK_PROFILE_INPUT="${1#*=}"
+          shift
+          ;;
+        --network-rate-kbps)
+          NETWORK_RATE_INPUT="${2:-}"
+          shift 2
+          ;;
+        --network-rate-kbps=*)
+          NETWORK_RATE_INPUT="${1#*=}"
+          shift
+          ;;
         --chart)
           CHART_RESULTS=1
           shift
@@ -741,7 +783,7 @@ case "$command" in
       esac
     done
 
-    run_bench "$FIXTURES_INPUT" "$PMS_INPUT" "$VARIATION_INPUT" "$RUNS_INPUT" "$WARMUP_INPUT" "$DRY_RUN" "$CLEAN_RESULTS" "$CHART_RESULTS" "$CHART_DATE" "$REGISTRIES_INPUT" "$PROCESS_RESULTS" "$TIMEOUT_INPUT"
+    run_bench "$FIXTURES_INPUT" "$PMS_INPUT" "$VARIATION_INPUT" "$RUNS_INPUT" "$WARMUP_INPUT" "$DRY_RUN" "$CLEAN_RESULTS" "$CHART_RESULTS" "$CHART_DATE" "$REGISTRIES_INPUT" "$PROCESS_RESULTS" "$TIMEOUT_INPUT" "$NETWORK_PROFILE_INPUT" "$NETWORK_RATE_INPUT"
     ;;
   process)
     DRY_RUN=0

scripts/network-isolation.sh

@@ -0,0 +1,188 @@
+#!/usr/bin/env bash
+
+set -Eeuo pipefail
+
+BENCH_TOXIPROXY_API_HOST="${BENCH_TOXIPROXY_API_HOST:-127.0.0.1}"
+BENCH_TOXIPROXY_API_PORT="${BENCH_TOXIPROXY_API_PORT:-8474}"
+BENCH_TOXIPROXY_LISTEN_HOST="${BENCH_TOXIPROXY_LISTEN_HOST:-127.0.0.1}"
+BENCH_TOXIPROXY_LISTEN_PORT="${BENCH_TOXIPROXY_LISTEN_PORT:-7443}"
+BENCH_TOXIPROXY_RATE_KBPS="${BENCH_TOXIPROXY_RATE_KBPS:-8192}"
+BENCH_TOXIPROXY_PID_FILE="${BENCH_TOXIPROXY_PID_FILE:-/tmp/vlt-benchmarks-toxiproxy.pid}"
+BENCH_TOXIPROXY_LOG_FILE="${BENCH_TOXIPROXY_LOG_FILE:-/tmp/vlt-benchmarks-toxiproxy.log}"
+BENCH_TOXIPROXY_HOSTS_START="# >>> vlt-benchmarks toxiproxy >>>"
+BENCH_TOXIPROXY_HOSTS_END="# <<< vlt-benchmarks toxiproxy <<<"
+BENCH_TOXIPROXY_HOSTS_ENTRIES="${BENCH_TOXIPROXY_HOSTS_ENTRIES:-}"
+
+toxiproxy_api_url() {
+  echo "http://${BENCH_TOXIPROXY_API_HOST}:${BENCH_TOXIPROXY_API_PORT}"
+}
+
+ensure_toxiproxy_installed() {
+  if ! command -v toxiproxy-server >/dev/null 2>&1; then
+    echo "Error: toxiproxy-server is required for BENCH_NETWORK_PROFILE=$BENCH_NETWORK_PROFILE"
+    exit 1
+  fi
+}
+
+resolve_ipv4() {
+  local host="$1"
+
+  node -e '
+    const dns = require("node:dns").promises
+    dns.lookup(process.argv[1], { family: 4 }).then((result) => {
+      process.stdout.write(result.address)
+    }).catch((error) => {
+      console.error(error.message)
+      process.exit(1)
+    })
+  ' "$host"
+}
+
+wait_for_toxiproxy() {
+  local api
+  api="$(toxiproxy_api_url)"
+
+  for _ in {1..40}; do
+    if curl -fsS "$api/version" >/dev/null 2>&1; then
+      return 0
+    fi
+    sleep 0.25
+  done
+
+  echo "Error: toxiproxy did not start on $api"
+  exit 1
+}
+
+start_toxiproxy_server() {
+  local api
+  api="$(toxiproxy_api_url)"
+
+  if curl -fsS "$api/version" >/dev/null 2>&1; then
+    return 0
+  fi
+
+  nohup toxiproxy-server -host "$BENCH_TOXIPROXY_API_HOST" -port "$BENCH_TOXIPROXY_API_PORT" \
+    >"$BENCH_TOXIPROXY_LOG_FILE" 2>&1 &
+  echo "$!" > "$BENCH_TOXIPROXY_PID_FILE"
+  BENCH_TOXIPROXY_STARTED=1
+  wait_for_toxiproxy
+}
+
+reset_toxiproxy() {
+  curl -fsS -X POST "$(toxiproxy_api_url)/reset" >/dev/null
+}
+
+write_hosts_mapping() {
+  local entries="$1"
+
+  sudo node -e '
+    const fs = require("node:fs")
+
+    const filePath = "/etc/hosts"
+    const start = process.argv[1]
+    const end = process.argv[2]
+    const entries = process.argv[3]
+    const escapeRegex = (value) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")
+
+    let content = fs.readFileSync(filePath, "utf8")
+    const pattern = new RegExp(`${escapeRegex(start)}\\n?[\\s\\S]*?${escapeRegex(end)}\\n?`, "g")
+    content = content.replace(pattern, "")
+    content = content.replace(/\n*$/, "\n")
+    if (entries.length > 0) {
+      const block = `${start}\n${entries}\n${end}`
+      content = `${content}${block}\n`
+    }
+    fs.writeFileSync(filePath, content)
+  ' "$BENCH_TOXIPROXY_HOSTS_START" "$BENCH_TOXIPROXY_HOSTS_END" "$entries"
+}
+
+append_hosts_mapping() {
+  local host="$1"
+  local entry="${BENCH_TOXIPROXY_LISTEN_HOST} ${host}"
+
+  if ! printf '%s\n' "$BENCH_TOXIPROXY_HOSTS_ENTRIES" | grep -Fxq "$entry"; then
+    if [ -n "$BENCH_TOXIPROXY_HOSTS_ENTRIES" ]; then
+      BENCH_TOXIPROXY_HOSTS_ENTRIES="${BENCH_TOXIPROXY_HOSTS_ENTRIES}
+${entry}"
+    else
+      BENCH_TOXIPROXY_HOSTS_ENTRIES="$entry"
+    fi
+  fi
+
+  write_hosts_mapping "$BENCH_TOXIPROXY_HOSTS_ENTRIES"
+}
+
+create_toxiproxy_proxy() {
+  local proxy_name="$1"
+  local upstream_host="$2"
+  local upstream_ip="$3"
+  local upstream_port="$4"
+  local listen_port="$5"
+  local rate_kbps="$6"
+
+  curl -fsS -X POST "$(toxiproxy_api_url)/proxies" \
+    -H 'Content-Type: application/json' \
+    -d "{\"name\":\"${proxy_name}\",\"listen\":\"${BENCH_TOXIPROXY_LISTEN_HOST}:${listen_port}\",\"upstream\":\"${upstream_ip}:${upstream_port}\"}" \
+    >/dev/null
+
+  curl -fsS -X POST "$(toxiproxy_api_url)/proxies/${proxy_name}/toxics" \
+    -H 'Content-Type: application/json' \
+    -d "{\"name\":\"bandwidth_upstream\",\"type\":\"bandwidth\",\"stream\":\"upstream\",\"attributes\":{\"rate\":${rate_kbps}}}" \
+    >/dev/null
+
+  curl -fsS -X POST "$(toxiproxy_api_url)/proxies/${proxy_name}/toxics" \
+    -H 'Content-Type: application/json' \
+    -d "{\"name\":\"bandwidth_downstream\",\"type\":\"bandwidth\",\"stream\":\"downstream\",\"attributes\":{\"rate\":${rate_kbps}}}" \
+    >/dev/null
+
+  echo "Toxiproxy configured for ${upstream_host} at ${BENCH_TOXIPROXY_LISTEN_HOST}:${listen_port} (${rate_kbps} KB/s)" >&2
+}
+
+url_field() {
+  local url="$1"
+  local field="$2"
+
+  node -e '
+    const parsed = new URL(process.argv[1])
+    const field = process.argv[2]
+    const value = field === "port"
+      ? (parsed.port || (parsed.protocol === "https:" ? "443" : "80"))
+      : parsed[field]
+    process.stdout.write(value)
+  ' "$url" "$field"
+}
+
+setup_registry_bandwidth_proxy() {
+  local proxy_name="$1"
+  local registry_url="$2"
+  local listen_port="$3"
+  local upstream_host
+  local upstream_path
+  local upstream_protocol
+  local upstream_port
+  local upstream_ip
+
+  upstream_host="$(url_field "$registry_url" hostname)"
+  upstream_path="$(url_field "$registry_url" pathname)"
+  upstream_protocol="$(url_field "$registry_url" protocol)"
+  upstream_port="$(url_field "$registry_url" port)"
+  upstream_ip="$(resolve_ipv4 "$upstream_host")"
+
+  create_toxiproxy_proxy "$proxy_name" "$upstream_host" "$upstream_ip" "$upstream_port" "$listen_port" "$BENCH_TOXIPROXY_RATE_KBPS"
+  append_hosts_mapping "$upstream_host"
+
+  echo "${upstream_protocol}//${upstream_host}:${listen_port}${upstream_path}"
+}
+
+cleanup_network_isolation() {
+  write_hosts_mapping "" || true
+
+  if curl -fsS "$(toxiproxy_api_url)/version" >/dev/null 2>&1; then
+    curl -fsS -X POST "$(toxiproxy_api_url)/reset" >/dev/null || true
+  fi
+
+  if [ -n "${BENCH_TOXIPROXY_STARTED:-}" ] && [ -f "$BENCH_TOXIPROXY_PID_FILE" ]; then
+    kill "$(cat "$BENCH_TOXIPROXY_PID_FILE")" >/dev/null 2>&1 || true
+    rm -f "$BENCH_TOXIPROXY_PID_FILE"
+  fi
+}