diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index cf9adfa31b..599f438997 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -18,6 +18,12 @@ on: runs: description: "The number of runs on each benchmark" default: "5" + network_profile: + description: "Optional network isolation profile for registry benchmarks" + default: "registry-bandwidth" + network_rate_kbps: + description: "Bandwidth cap in KB/s for supported network profiles" + default: "8192" schedule: # GitHub Actions cron uses UTC. 10:17 UTC ~= 2:17 AM Pacific (PST). - cron: "17 10 * * *" @@ -100,6 +106,8 @@ jobs: BENCH_INCLUDE: ${{ fromJson(inputs.binaries || '"npm,yarn,berry,zpm,pnpm,pnpm11,vlt,bun,deno,aube,nx,turbo,vp,node"') }} BENCH_WARMUP: ${{ (github.event_name == 'push' && github.ref != 'refs/heads/main') && '1' || (inputs.warmup || '2') }} BENCH_RUNS: ${{ (github.event_name == 'push' && github.ref != 'refs/heads/main') && '1' || (inputs.runs || '5') }} + BENCH_NETWORK_PROFILE: ${{ inputs.network_profile || 'registry-bandwidth' }} + BENCH_TOXIPROXY_RATE_KBPS: ${{ inputs.network_rate_kbps || '8192' }} steps: - uses: actions/checkout@v6 - name: Install Node diff --git a/README.md b/README.md index 7ae436d815..0c5dacefcf 100644 --- a/README.md +++ b/README.md @@ -90,12 +90,25 @@ Examples: # Run only aws registry benchmarks (requires token) CODEARTIFACT_AUTH_TOKEN= ./bench run --variation=registry-clean --fixtures=next --registries=aws + +# Run isolated VSR registry benchmarks through toxiproxy with a bandwidth cap +VLT_REGISTRY_AUTH_TOKEN= ./bench run --variation=registry-clean --fixtures=next --registries=vlt --network-profile=registry-bandwidth --network-rate-kbps=8192 ``` Auth notes: - `aws` requires `CODEARTIFACT_AUTH_TOKEN`. +#### Network isolation + +Registry benchmarks default to a local `toxiproxy` path so benchmark traffic uses a controlled network path instead of the host's default connection. + +- `registry-bandwidth` proxies each included registry through a dedicated local toxiproxy listener. +- This rewrites registry hosts to local listeners, then proxies TLS traffic to the real upstream registries with symmetric bandwidth limits. +- Use `--network-profile=none` to bypass the proxy path. +- `--network-rate-kbps` controls both upstream and downstream bandwidth in KB/s. +- The helper temporarily edits `/etc/hosts`, so local runs require `sudo` access. + ## Testing Script Execution This suite also tests the performance of basic script execution (ex. `npm run foo`). Notably, for any given build, test or deployment task the spawning of the process is a fraction of the overall execution time. That said, this is a commonly tracked workflow by various developer tools as it involves the common set of tasks: startup, filesystem read (`package.json`) & finally, spawning the process/command. @@ -155,6 +168,9 @@ This suite also tests the performance of basic script execution (ex. `npm run fo # Run multiple variations in one command ./bench run --fixtures=svelte --variation=lockfile,registry-lockfile,run --registries=npm,vlt + # Run an isolated VSR benchmark with a bandwidth cap + VLT_REGISTRY_AUTH_TOKEN= ./bench run --variation=registry-clean --fixtures=next --registries=vlt --network-profile=registry-bandwidth --network-rate-kbps=8192 + # Script-execution benchmark ./bench run --variation=run --pms=vlt diff --git a/bench b/bench index 0ed2e21928..3fa0660c48 100755 --- a/bench +++ b/bench @@ -50,7 +50,7 @@ AVAILABLE_REGISTRIES=( usage() { cat <<'EOF' Usage: - ./bench run [--fixtures=] [--pms=] [--variation=] [--runs=] [--warmup=] [--chart] [--process] [--date=] [--clean] + ./bench run [--fixtures=] [--pms=] [--variation=] [--runs=] [--warmup=] [--chart] [--process] [--date=] [--clean] [--network-profile=] [--network-rate-kbps=] ./bench chart [--fixtures=] [--variation=] [--date=] [--clean] ./bench process [--dry-run] ./bench list @@ -63,6 +63,8 @@ Options: --runs Hyperfine runs (default: scripts/variations/common.sh default) --warmup Hyperfine warmup runs (default: scripts/variations/common.sh default) --timeout Per-command timeout in seconds (default: 300) + --network-profile Network isolation profile for registry-* runs (default: registry-bandwidth, use none to disable) + --network-rate-kbps Bandwidth cap in KB/s for supported profiles (default: 8192) --chart Generate chart data and copy to app/latest --process Process results after run (clean + dated/latest outputs + chart data) --date Date folder for chart output (default: today) @@ -79,6 +81,7 @@ Examples: ./bench run --fixtures=next --chart --process --runs=3 ./bench run --variation=registry-clean --fixtures=next ./bench run --variation=registry-lockfile --registries=npm,vlt + ./bench run --variation=registry-clean --fixtures=next --registries=vlt --network-profile=registry-bandwidth --network-rate-kbps=8192 CODEARTIFACT_AUTH_TOKEN= ./bench run --variation=registry-clean --fixtures=next --registries=aws GH_REGISTRY= GH_AUTH_TOKEN= ./bench run --variation=registry-clean --fixtures=next --registries=github ./bench chart --fixtures=next --variation=clean @@ -375,6 +378,8 @@ run_bench() { local registries_input="${10:-}" local process_results="${11:-0}" local timeout_input="${12:-}" + local network_profile_input="${13:-}" + local network_rate_input="${14:-}" if [[ ! -f "$BENCH_SCRIPT" ]]; then echo "Error: Missing benchmark script at $BENCH_SCRIPT" @@ -396,6 +401,9 @@ run_bench() { if [[ -n "$timeout_input" ]]; then validate_number "$timeout_input" "--timeout" fi + if [[ -n "$network_rate_input" ]]; then + validate_number "$network_rate_input" "--network-rate-kbps" + fi local pms_env="" if [[ -n "$pms_input" ]]; then @@ -429,6 +437,16 @@ run_bench() { else echo " timeout: 300s (default)" fi + if [[ -n "$network_profile_input" ]]; then + echo " network profile: $network_profile_input" + else + echo " network profile: registry script default" + fi + if [[ -n "$network_rate_input" ]]; then + echo " network rate: ${network_rate_input} KB/s" + else + echo " network rate: 8192 KB/s (default)" + fi if [[ "$clean_results" -eq 1 ]]; then echo " clean: enabled" else @@ -485,6 +503,12 @@ run_bench() { if [[ -n "$timeout_input" ]]; then env_args+=("BENCH_TIMEOUT=$timeout_input") fi + if [[ -n "$network_profile_input" ]]; then + env_args+=("BENCH_NETWORK_PROFILE=$network_profile_input") + fi + if [[ -n "$network_rate_input" ]]; then + env_args+=("BENCH_TOXIPROXY_RATE_KBPS=$network_rate_input") + fi local cmd=(bash "$BENCH_SCRIPT" "$fixture" "$variation") @@ -642,6 +666,8 @@ case "$command" in CHART_RESULTS=0 PROCESS_RESULTS=0 CHART_DATE="" + NETWORK_PROFILE_INPUT="" + NETWORK_RATE_INPUT="" while [[ $# -gt 0 ]]; do case "$1" in @@ -701,6 +727,22 @@ case "$command" in TIMEOUT_INPUT="${1#*=}" shift ;; + --network-profile) + NETWORK_PROFILE_INPUT="${2:-}" + shift 2 + ;; + --network-profile=*) + NETWORK_PROFILE_INPUT="${1#*=}" + shift + ;; + --network-rate-kbps) + NETWORK_RATE_INPUT="${2:-}" + shift 2 + ;; + --network-rate-kbps=*) + NETWORK_RATE_INPUT="${1#*=}" + shift + ;; --chart) CHART_RESULTS=1 shift @@ -741,7 +783,7 @@ case "$command" in esac done - run_bench "$FIXTURES_INPUT" "$PMS_INPUT" "$VARIATION_INPUT" "$RUNS_INPUT" "$WARMUP_INPUT" "$DRY_RUN" "$CLEAN_RESULTS" "$CHART_RESULTS" "$CHART_DATE" "$REGISTRIES_INPUT" "$PROCESS_RESULTS" "$TIMEOUT_INPUT" + run_bench "$FIXTURES_INPUT" "$PMS_INPUT" "$VARIATION_INPUT" "$RUNS_INPUT" "$WARMUP_INPUT" "$DRY_RUN" "$CLEAN_RESULTS" "$CHART_RESULTS" "$CHART_DATE" "$REGISTRIES_INPUT" "$PROCESS_RESULTS" "$TIMEOUT_INPUT" "$NETWORK_PROFILE_INPUT" "$NETWORK_RATE_INPUT" ;; process) DRY_RUN=0 diff --git a/scripts/network-isolation.sh b/scripts/network-isolation.sh new file mode 100644 index 0000000000..961f812b9e --- /dev/null +++ b/scripts/network-isolation.sh @@ -0,0 +1,188 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +BENCH_TOXIPROXY_API_HOST="${BENCH_TOXIPROXY_API_HOST:-127.0.0.1}" +BENCH_TOXIPROXY_API_PORT="${BENCH_TOXIPROXY_API_PORT:-8474}" +BENCH_TOXIPROXY_LISTEN_HOST="${BENCH_TOXIPROXY_LISTEN_HOST:-127.0.0.1}" +BENCH_TOXIPROXY_LISTEN_PORT="${BENCH_TOXIPROXY_LISTEN_PORT:-7443}" +BENCH_TOXIPROXY_RATE_KBPS="${BENCH_TOXIPROXY_RATE_KBPS:-8192}" +BENCH_TOXIPROXY_PID_FILE="${BENCH_TOXIPROXY_PID_FILE:-/tmp/vlt-benchmarks-toxiproxy.pid}" +BENCH_TOXIPROXY_LOG_FILE="${BENCH_TOXIPROXY_LOG_FILE:-/tmp/vlt-benchmarks-toxiproxy.log}" +BENCH_TOXIPROXY_HOSTS_START="# >>> vlt-benchmarks toxiproxy >>>" +BENCH_TOXIPROXY_HOSTS_END="# <<< vlt-benchmarks toxiproxy <<<" +BENCH_TOXIPROXY_HOSTS_ENTRIES="${BENCH_TOXIPROXY_HOSTS_ENTRIES:-}" + +toxiproxy_api_url() { + echo "http://${BENCH_TOXIPROXY_API_HOST}:${BENCH_TOXIPROXY_API_PORT}" +} + +ensure_toxiproxy_installed() { + if ! command -v toxiproxy-server >/dev/null 2>&1; then + echo "Error: toxiproxy-server is required for BENCH_NETWORK_PROFILE=$BENCH_NETWORK_PROFILE" + exit 1 + fi +} + +resolve_ipv4() { + local host="$1" + + node -e ' + const dns = require("node:dns").promises + dns.lookup(process.argv[1], { family: 4 }).then((result) => { + process.stdout.write(result.address) + }).catch((error) => { + console.error(error.message) + process.exit(1) + }) + ' "$host" +} + +wait_for_toxiproxy() { + local api + api="$(toxiproxy_api_url)" + + for _ in {1..40}; do + if curl -fsS "$api/version" >/dev/null 2>&1; then + return 0 + fi + sleep 0.25 + done + + echo "Error: toxiproxy did not start on $api" + exit 1 +} + +start_toxiproxy_server() { + local api + api="$(toxiproxy_api_url)" + + if curl -fsS "$api/version" >/dev/null 2>&1; then + return 0 + fi + + nohup toxiproxy-server -host "$BENCH_TOXIPROXY_API_HOST" -port "$BENCH_TOXIPROXY_API_PORT" \ + >"$BENCH_TOXIPROXY_LOG_FILE" 2>&1 & + echo "$!" > "$BENCH_TOXIPROXY_PID_FILE" + BENCH_TOXIPROXY_STARTED=1 + wait_for_toxiproxy +} + +reset_toxiproxy() { + curl -fsS -X POST "$(toxiproxy_api_url)/reset" >/dev/null +} + +write_hosts_mapping() { + local entries="$1" + + sudo node -e ' + const fs = require("node:fs") + + const filePath = "/etc/hosts" + const start = process.argv[1] + const end = process.argv[2] + const entries = process.argv[3] + const escapeRegex = (value) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") + + let content = fs.readFileSync(filePath, "utf8") + const pattern = new RegExp(`${escapeRegex(start)}\\n?[\\s\\S]*?${escapeRegex(end)}\\n?`, "g") + content = content.replace(pattern, "") + content = content.replace(/\n*$/, "\n") + if (entries.length > 0) { + const block = `${start}\n${entries}\n${end}` + content = `${content}${block}\n` + } + fs.writeFileSync(filePath, content) + ' "$BENCH_TOXIPROXY_HOSTS_START" "$BENCH_TOXIPROXY_HOSTS_END" "$entries" +} + +append_hosts_mapping() { + local host="$1" + local entry="${BENCH_TOXIPROXY_LISTEN_HOST} ${host}" + + if ! printf '%s\n' "$BENCH_TOXIPROXY_HOSTS_ENTRIES" | grep -Fxq "$entry"; then + if [ -n "$BENCH_TOXIPROXY_HOSTS_ENTRIES" ]; then + BENCH_TOXIPROXY_HOSTS_ENTRIES="${BENCH_TOXIPROXY_HOSTS_ENTRIES} +${entry}" + else + BENCH_TOXIPROXY_HOSTS_ENTRIES="$entry" + fi + fi + + write_hosts_mapping "$BENCH_TOXIPROXY_HOSTS_ENTRIES" +} + +create_toxiproxy_proxy() { + local proxy_name="$1" + local upstream_host="$2" + local upstream_ip="$3" + local upstream_port="$4" + local listen_port="$5" + local rate_kbps="$6" + + curl -fsS -X POST "$(toxiproxy_api_url)/proxies" \ + -H 'Content-Type: application/json' \ + -d "{\"name\":\"${proxy_name}\",\"listen\":\"${BENCH_TOXIPROXY_LISTEN_HOST}:${listen_port}\",\"upstream\":\"${upstream_ip}:${upstream_port}\"}" \ + >/dev/null + + curl -fsS -X POST "$(toxiproxy_api_url)/proxies/${proxy_name}/toxics" \ + -H 'Content-Type: application/json' \ + -d "{\"name\":\"bandwidth_upstream\",\"type\":\"bandwidth\",\"stream\":\"upstream\",\"attributes\":{\"rate\":${rate_kbps}}}" \ + >/dev/null + + curl -fsS -X POST "$(toxiproxy_api_url)/proxies/${proxy_name}/toxics" \ + -H 'Content-Type: application/json' \ + -d "{\"name\":\"bandwidth_downstream\",\"type\":\"bandwidth\",\"stream\":\"downstream\",\"attributes\":{\"rate\":${rate_kbps}}}" \ + >/dev/null + + echo "Toxiproxy configured for ${upstream_host} at ${BENCH_TOXIPROXY_LISTEN_HOST}:${listen_port} (${rate_kbps} KB/s)" >&2 +} + +url_field() { + local url="$1" + local field="$2" + + node -e ' + const parsed = new URL(process.argv[1]) + const field = process.argv[2] + const value = field === "port" + ? (parsed.port || (parsed.protocol === "https:" ? "443" : "80")) + : parsed[field] + process.stdout.write(value) + ' "$url" "$field" +} + +setup_registry_bandwidth_proxy() { + local proxy_name="$1" + local registry_url="$2" + local listen_port="$3" + local upstream_host + local upstream_path + local upstream_protocol + local upstream_port + local upstream_ip + + upstream_host="$(url_field "$registry_url" hostname)" + upstream_path="$(url_field "$registry_url" pathname)" + upstream_protocol="$(url_field "$registry_url" protocol)" + upstream_port="$(url_field "$registry_url" port)" + upstream_ip="$(resolve_ipv4 "$upstream_host")" + + create_toxiproxy_proxy "$proxy_name" "$upstream_host" "$upstream_ip" "$upstream_port" "$listen_port" "$BENCH_TOXIPROXY_RATE_KBPS" + append_hosts_mapping "$upstream_host" + + echo "${upstream_protocol}//${upstream_host}:${listen_port}${upstream_path}" +} + +cleanup_network_isolation() { + write_hosts_mapping "" || true + + if curl -fsS "$(toxiproxy_api_url)/version" >/dev/null 2>&1; then + curl -fsS -X POST "$(toxiproxy_api_url)/reset" >/dev/null || true + fi + + if [ -n "${BENCH_TOXIPROXY_STARTED:-}" ] && [ -f "$BENCH_TOXIPROXY_PID_FILE" ]; then + kill "$(cat "$BENCH_TOXIPROXY_PID_FILE")" >/dev/null 2>&1 || true + rm -f "$BENCH_TOXIPROXY_PID_FILE" + fi +} diff --git a/scripts/registry/common.sh b/scripts/registry/common.sh index 7a99736c0e..1fc17cd107 100644 --- a/scripts/registry/common.sh +++ b/scripts/registry/common.sh @@ -46,6 +46,8 @@ BENCH_RUNS="${BENCH_RUNS:=5}" # Per-command timeout in seconds (default: 5 minutes) BENCH_TIMEOUT="${BENCH_TIMEOUT:=300}" BENCH_LOGLEVEL="${BENCH_LOGLEVEL:=http}" +BENCH_NETWORK_PROFILE="${BENCH_NETWORK_PROFILE:=registry-bandwidth}" +BENCH_TOXIPROXY_RATE_KBPS="${BENCH_TOXIPROXY_RATE_KBPS:=8192}" BENCH_OUTPUT_FOLDER="$BENCH_RESULTS/$BENCH_FIXTURE/$BENCH_VARIATION" # Add --force for large fixture to bypass peer dependency errors @@ -62,28 +64,19 @@ BENCH_NPM_INSTALL="npm install --prefer-online --no-audit --no-fund --no-update- # Registry definitions BENCH_REGISTRY_NPM_URL="https://registry.npmjs.org/" BENCH_REGISTRY_VLT_URL="https://registry.vlt.io/npm/" -BENCH_REGISTRY_VLT_URL_NPMRC_KEY="${BENCH_REGISTRY_VLT_URL#http*://}" BENCH_REGISTRY_AWS_URL="https://vlt-451504312483.d.codeartifact.us-east-1.amazonaws.com/npm/code-artifact-benchmark-test/" -BENCH_REGISTRY_AWS_NPMRC_KEY="${BENCH_REGISTRY_AWS_URL#http*://}" # GitHub registry URL is injected without the protocol prefix # (e.g. "//npm.pkg.github.com/..."), so we prepend "https:" for the registry config. # The auth .npmrc key uses the URL as-is (without protocol). BENCH_REGISTRY_GITHUB_URL="https:${GH_REGISTRY:-}" -BENCH_REGISTRY_GITHUB_NPMRC_KEY="${GH_REGISTRY#//}" - -# Registry setup commands run in hyperfine --prepare (untimed, before each run). -# Auth token is written as a literal placeholder so npm resolves it from env. -# For vlt registry, a random suffix is appended to the auth token on every -# iteration (separated by `:`) so that each run uses a unique token string. -# This ensures the registry does not serve cached responses across iterations. -BENCH_SETUP_REGISTRY_NPM="npm config set registry \"$BENCH_REGISTRY_NPM_URL\" --location=project" -# The vlt registry auth token is appended with a random suffix with the iteration number to -# ensure that a fresh cache is used for each iteration. A future update to the vlt registry -# will allow sharing a public cache regardless of the authorization header. -BENCH_SETUP_REGISTRY_VLT="npm config set registry \"$BENCH_REGISTRY_VLT_URL\" --location=project && npm config set \"//${BENCH_REGISTRY_VLT_URL_NPMRC_KEY}:_authToken=\\\${VLT_REGISTRY_AUTH_TOKEN}:$(head -c 16 /dev/urandom | xxd -p)_\\\${HYPERFINE_ITERATION}\" --location=project" -BENCH_SETUP_REGISTRY_AWS="npm config set registry \"$BENCH_REGISTRY_AWS_URL\" --location=project && npm config set \"//${BENCH_REGISTRY_AWS_NPMRC_KEY}:_authToken=\\\${CODEARTIFACT_AUTH_TOKEN}\" --location=project" -BENCH_SETUP_REGISTRY_GITHUB="npm config set registry \"$BENCH_REGISTRY_GITHUB_URL\" --location=project && npm config set \"//${BENCH_REGISTRY_GITHUB_NPMRC_KEY}:_authToken=\\\${GH_AUTH_TOKEN}\" --location=project" +BENCH_REGISTRY_NPM_EFFECTIVE_URL="$BENCH_REGISTRY_NPM_URL" +BENCH_REGISTRY_VLT_EFFECTIVE_URL="$BENCH_REGISTRY_VLT_URL" +BENCH_REGISTRY_AWS_EFFECTIVE_URL="$BENCH_REGISTRY_AWS_URL" +BENCH_REGISTRY_GITHUB_EFFECTIVE_URL="$BENCH_REGISTRY_GITHUB_URL" +BENCH_REGISTRY_VLT_EFFECTIVE_NPMRC_KEY="${BENCH_REGISTRY_VLT_EFFECTIVE_URL#http*://}" +BENCH_REGISTRY_AWS_EFFECTIVE_NPMRC_KEY="${BENCH_REGISTRY_AWS_EFFECTIVE_URL#http*://}" +BENCH_REGISTRY_GITHUB_EFFECTIVE_NPMRC_KEY="${BENCH_REGISTRY_GITHUB_EFFECTIVE_URL#http*://}" # Registry verification helper runs in hyperfine --conclude (untimed, after each run). BENCH_VERIFY_REGISTRY="npm config get registry && ((grep -m3 '\"resolved\"' package-lock.json 2>/dev/null | sed 's/^[[:space:]]*//') || echo 'no lockfile yet') && echo ''" @@ -153,7 +146,56 @@ if [ -n "$BENCH_INCLUDE_REG_GITHUB" ] && [ -z "${GH_REGISTRY:-}" ]; then exit 1 fi +if [ -n "$BENCH_NETWORK_PROFILE" ]; then + case "$BENCH_NETWORK_PROFILE" in + registry-bandwidth) + source "$BENCH_SCRIPTS/network-isolation.sh" + ensure_toxiproxy_installed + start_toxiproxy_server + reset_toxiproxy + trap cleanup_network_isolation EXIT + if [ -n "$BENCH_INCLUDE_REG_NPM" ]; then + BENCH_REGISTRY_NPM_EFFECTIVE_URL="$(setup_registry_bandwidth_proxy bench-npm "$BENCH_REGISTRY_NPM_URL" 7441)" + fi + if [ -n "$BENCH_INCLUDE_REG_VLT" ]; then + BENCH_REGISTRY_VLT_EFFECTIVE_URL="$(setup_registry_bandwidth_proxy bench-vlt "$BENCH_REGISTRY_VLT_URL" 7443)" + fi + if [ -n "$BENCH_INCLUDE_REG_AWS" ]; then + BENCH_REGISTRY_AWS_EFFECTIVE_URL="$(setup_registry_bandwidth_proxy bench-aws "$BENCH_REGISTRY_AWS_URL" 7444)" + fi + if [ -n "$BENCH_INCLUDE_REG_GITHUB" ]; then + BENCH_REGISTRY_GITHUB_EFFECTIVE_URL="$(setup_registry_bandwidth_proxy bench-github "$BENCH_REGISTRY_GITHUB_URL" 7445)" + fi + BENCH_REGISTRY_AWS_EFFECTIVE_NPMRC_KEY="${BENCH_REGISTRY_AWS_EFFECTIVE_URL#http*://}" + BENCH_REGISTRY_GITHUB_EFFECTIVE_NPMRC_KEY="${BENCH_REGISTRY_GITHUB_EFFECTIVE_URL#http*://}" + BENCH_REGISTRY_VLT_EFFECTIVE_NPMRC_KEY="${BENCH_REGISTRY_VLT_EFFECTIVE_URL#http*://}" + ;; + none) + ;; + *) + echo "Error: Unknown BENCH_NETWORK_PROFILE '$BENCH_NETWORK_PROFILE'" + exit 1 + ;; + esac +fi + echo "Registry benchmarks will run: $BENCH_INCLUDE_REGISTRY" +if [ -n "$BENCH_NETWORK_PROFILE" ] && [ "$BENCH_NETWORK_PROFILE" != "none" ]; then + echo "Network isolation profile: $BENCH_NETWORK_PROFILE (${BENCH_TOXIPROXY_RATE_KBPS} KB/s)" +fi + +# Registry setup commands run in hyperfine --prepare (untimed, before each run). +# Auth token is written as a literal placeholder so npm resolves it from env. +# For vlt registry, a random suffix is appended to the auth token on every +# iteration (separated by `:`) so that each run uses a unique token string. +# This ensures the registry does not serve cached responses across iterations. +BENCH_SETUP_REGISTRY_NPM="npm config set registry \"$BENCH_REGISTRY_NPM_EFFECTIVE_URL\" --location=project" +# The vlt registry auth token is appended with a random suffix with the iteration number to +# ensure that a fresh cache is used for each iteration. A future update to the vlt registry +# will allow sharing a public cache regardless of the authorization header. +BENCH_SETUP_REGISTRY_VLT="npm config set registry \"$BENCH_REGISTRY_VLT_EFFECTIVE_URL\" --location=project && npm config set \"//${BENCH_REGISTRY_VLT_EFFECTIVE_NPMRC_KEY}:_authToken=\\\${VLT_REGISTRY_AUTH_TOKEN}:$(head -c 16 /dev/urandom | xxd -p)_\\\${HYPERFINE_ITERATION}\" --location=project" +BENCH_SETUP_REGISTRY_AWS="npm config set registry \"$BENCH_REGISTRY_AWS_EFFECTIVE_URL\" --location=project && npm config set \"//${BENCH_REGISTRY_AWS_EFFECTIVE_NPMRC_KEY}:_authToken=\\\${CODEARTIFACT_AUTH_TOKEN}\" --location=project" +BENCH_SETUP_REGISTRY_GITHUB="npm config set registry \"$BENCH_REGISTRY_GITHUB_EFFECTIVE_URL\" --location=project && npm config set \"//${BENCH_REGISTRY_GITHUB_EFFECTIVE_NPMRC_KEY}:_authToken=\\\${GH_AUTH_TOKEN}\" --location=project" # Clean up & create the results directory rm -rf "$BENCH_OUTPUT_FOLDER" diff --git a/scripts/setup.sh b/scripts/setup.sh index 6cf0b0f85a..22d8e4652a 100644 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -37,11 +37,21 @@ wget -q "https://github.com/sharkdp/hyperfine/releases/download/${HYPERFINE_VERS sudo dpkg -i /tmp/hyperfine.deb rm -f /tmp/hyperfine.deb +# Install toxiproxy for opt-in network-isolated registry benchmarks +TOXIPROXY_VERSION_TAG="v2.12.0" +for binary in toxiproxy-server toxiproxy-cli; do + curl -fsSL "https://github.com/Shopify/toxiproxy/releases/download/${TOXIPROXY_VERSION_TAG}/${binary}-linux-${ARCH}" -o "/tmp/${binary}" + chmod +x "/tmp/${binary}" + sudo mv "/tmp/${binary}" "/usr/local/bin/${binary}" +done + echo "Required system dependencies installed successfully!" JQ_VERSION=$(jq --version) HYPERFINE_VERSION=$(hyperfine --version) +TOXIPROXY_SERVER_VERSION=$(toxiproxy-server -version 2>/dev/null || toxiproxy-server --version 2>/dev/null || echo "installed") echo "jq: $JQ_VERSION" echo "hyperfine: $HYPERFINE_VERSION" +echo "toxiproxy: $TOXIPROXY_SERVER_VERSION" # Install Node.js package managers and tools echo "Installing package managers and tools..."