Merge pull request #855 from containerd/add-stress-test-to-benchmark

Add stress-test to the benchmark CI

Author: Mossaka

.github/workflows/benchmarks.yml

@@ -115,3 +115,49 @@ jobs:
         shell: bash
         run: |
           sudo ctr task kill -s SIGKILL wasi-http
+
+  benchmark-stress:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          rustflags: '' #Disable.  By default this action sets environment variable is set to -D warnings.  We manage this in the Makefile
+      - uses: ./.github/actions/setup-env
+      - name: Setup build profile
+        shell: bash
+        run: echo "OPT_PROFILE=release" >> ${GITHUB_ENV}
+      - uses: ./.github/actions/build
+      - name: Run Stress Tests
+        shell: bash
+        run: |
+          set -euxo pipefail
+          for RUNTIME in wasmtime wasmedge wasmer wamr; do
+            # Run containerd stress test
+            make test/stress-c8d-$RUNTIME \
+              STRESS_TEST_COUNT=1000 \
+              STRESS_TEST_TIMEOUT=10s \
+              STRESS_TEST_JSON=stress-bench-c8d-$RUNTIME.json \
+              STRESS_TEST_IMAGE=ghcr.io/containerd/runwasi/wasi-demo-app:latest
+            
+            # Run non-containerd stress test
+            make OPT_PROFILE=release test/stress-$RUNTIME \
+              STRESS_TEST_COUNT=1000 \
+              STRESS_TEST_TIMEOUT=10s \
+              STRESS_TEST_JSON=stress-bench-$RUNTIME.json \
+              STRESS_TEST_IMAGE=ghcr.io/containerd/runwasi/wasi-demo-app:latest
+          done
+          cat stress-bench-*.json | jq -s 'flatten(1)' > stress-bench.json
+      - name: Store benchmark result
+        uses: benchmark-action/[email protected]
+        with:
+          name: Stress Test Benchmark
+          tool: 'customBiggerIsBetter'
+          output-file-path: stress-bench.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          alert-threshold: '130%'
+          fail-on-alert: ${{ github.event_name == 'schedule' }}
+          alert-comment-cc-users: '@runwasi-committers'
+          summary-always: true
+          auto-push: ${{ github.event_name == 'schedule' }}

Cargo.lock

@@ -23,7 +23,6 @@ homepage = "https://github.com/containerd/runwasi"
 
 [workspace.dependencies]
 anyhow = "1.0"
-cap-std = "1.0"
 chrono = { version = "0.4", default-features = false, features = ["clock"] }
 containerd-shim = "0.8"
 containerd-shim-wasm = { path = "crates/containerd-shim-wasm", version = "0.9.0" }

Cargo.toml

@@ -69,6 +69,13 @@ KIND_CLUSTER_NAME ?= containerd-wasm
 
 export
 
+STRESS_TEST_COUNT ?= 100
+STRESS_TEST_PARALLEL ?= $$(nproc || echo 10)
+STRESS_TEST_TIMEOUT ?= 5s
+STRESS_TEST_IMAGE ?= ghcr.io/containerd/runwasi/wasi-demo-app:latest
+STRESS_TEST_JSON ?= 
+STRESS_TEST_JSON_FLAG = $(if $(STRESS_TEST_JSON),--json-output $(STRESS_TEST_JSON),)
+
 .PHONY: build build-common build-wasm build-%
 build: build-wasm $(RUNTIMES:%=build-%);
 
@@ -142,12 +149,25 @@ test-doc:
 test/stress-%: dist-%
 	# Do not use trace logging as that negatively impacts performance.
 	# Do not use cross (always use cargo) to avoid the qemu environment.
-	cargo run -p stress-test $(TARGET_FLAG) -- $(PWD)/dist/bin/containerd-shim-$*-v1 --count=100 --parallel=$$(nproc || echo 10) --timeout=5s
+	cargo run -p stress-test $(TARGET_FLAG) -- \
+		$(PWD)/dist/bin/containerd-shim-$*-v1 \
+		--count=$(STRESS_TEST_COUNT) \
+		--parallel=$(STRESS_TEST_PARALLEL) \
+		--timeout=$(STRESS_TEST_TIMEOUT) \
+		--image=$(STRESS_TEST_IMAGE) \
+		$(STRESS_TEST_JSON_FLAG)
 
 test/stress-c8d-%: dist-%
 	# Do not use trace logging as that negatively impacts performance.
 	# Do not use cross (always use cargo) to avoid the qemu environment.
-	cargo run -p stress-test $(TARGET_FLAG) -- --containerd $(PWD)/dist/bin/containerd-shim-$*-v1 --count=100 --parallel=$$(nproc || echo 10) --timeout=5s
+	cargo run -p stress-test $(TARGET_FLAG) -- \
+		--containerd \
+		$(PWD)/dist/bin/containerd-shim-$*-v1 \
+		--count=$(STRESS_TEST_COUNT) \
+		--parallel=$(STRESS_TEST_PARALLEL) \
+		--timeout=$(STRESS_TEST_TIMEOUT) \
+		--image=$(STRESS_TEST_IMAGE) \
+		$(STRESS_TEST_JSON_FLAG)
 
 generate-doc:
 	RUST_LOG=trace $(CARGO) doc --workspace --all-features --no-deps --document-private-items --exclude wasi-demo-app

Makefile

@@ -17,17 +17,18 @@ tokio-async-drop = "0.1.0"
 humantime = "2.1.0"
 tempfile = { workspace = true }
 oci-spec = { workspace = true }
-clap = { version = "4", features = [ "derive"] }
+clap = { version = "4", features = ["derive"] }
 log = { workspace = true }
 env_logger = { workspace = true }
 nix = { workspace = true, features = ["process", "signal", "mount"] }
 trait-variant = "0.1"
 containerd-client = "0.6.0"
 tonic = "0.12"
-serde_json.workspace = true
+serde_json = { workspace = true }
+serde = { workspace = true }
 futures = "0.3"
 
 
 [package.metadata.cargo-machete]
 # used by the bindings generated by trapeze
-ignored = ["prost", "prost-types"]
+ignored = ["prost", "prost-types"]

crates/containerd-shim-wasm-test-modules/src/modules/fetch_rs.wasm

@@ -4,7 +4,8 @@ mod protos;
 mod traits;
 mod utils;
 
-use std::path::PathBuf;
+use std::fs::File;
+use std::path::{Path, PathBuf};
 use std::pin::pin;
 use std::sync::Arc;
 use std::time::Instant;
@@ -16,6 +17,7 @@ use futures::stream::FuturesUnordered;
 use futures::{FutureExt as _, StreamExt as _};
 use humantime::{format_duration, parse_duration};
 use nix::sys::prctl::set_child_subreaper;
+use serde::Serialize;
 use tokio::signal::ctrl_c;
 use tokio::sync::{Barrier, OnceCell, Semaphore};
 use tokio::time::Duration;
@@ -30,6 +32,15 @@ enum Step {
     Delete,
 }
 
+#[derive(Serialize)]
+struct BenchmarkResult {
+    name: String,
+    unit: String,
+    value: f64,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    extra: Option<String>,
+}
+
 #[derive(Parser)]
 #[command(version, about, long_about = None)]
 struct Cli {
@@ -65,6 +76,10 @@ struct Cli {
     /// Image to use for the test
     image: String,
 
+    #[arg(long)]
+    /// Output the benchmark results to a JSON file
+    json_output: Option<PathBuf>,
+
     /// Path to the shim binary
     shim: PathBuf,
 
@@ -95,21 +110,33 @@ async fn main_impl() -> Result<()> {
     }
 }
 
+fn get_runtime(path: &Path) -> Option<&str> {
+    let runtime = path
+        .file_name()?
+        .to_str()?
+        .strip_prefix("containerd-shim-")?
+        .split_once('-')?
+        .0;
+    Some(runtime)
+}
+
 async fn run_stress_test(cli: Cli, c8d: impl Containerd) -> Result<()> {
     let Cli {
-        shim,
+        containerd,
+        shim: shim_path,
         container_output,
         parallel,
         count,
         timeout,
         image,
+        json_output,
         args,
         ..
     } = cli;
 
     println!("\x1b[1mUsing image {image:?} with arguments {args:?}\x1b[0m");
 
-    let shim = c8d.start_shim(shim).await?;
+    let shim = c8d.start_shim(shim_path.clone()).await?;
     let shim = Arc::new(shim);
 
     // create a "pause" container to keep the shim running
@@ -222,11 +249,29 @@ async fn run_stress_test(cli: Cli, c8d: impl Containerd) -> Result<()> {
 
     let elapsed = start.get().unwrap().elapsed();
     let throuput = count as f64 / elapsed.as_secs_f64();
-    let elapsed = format_duration(elapsed);
+    let duration = format_duration(elapsed);
 
     println!("\x1b[32m{success} tasks succeeded\x1b[0m");
-    println!("\x1b[32m  elapsed time: {elapsed}\x1b[0m");
+    println!("\x1b[32m  elapsed time: {duration}\x1b[0m");
     println!("\x1b[32m  throuput: {throuput} tasks/s\x1b[0m");
 
+    let shim = get_runtime(&shim_path).unwrap_or("unknown");
+    let containerd_shim = if containerd { "containerd" } else { "mock" };
+
+    if let Some(json_output) = json_output {
+        let results = vec![BenchmarkResult {
+            name: format!(
+                "Stress Test Tasks Throughput with {} service - {}",
+                containerd_shim, shim
+            ),
+            unit: "tasks/s".to_string(),
+            value: throuput,
+            extra: Some(format!(
+                "Image: {}\nTasks: {}\nParallel: {}\nDuration: {}",
+                image, count, parallel, duration
+            )),
+        }];
+        serde_json::to_writer_pretty(&mut File::create(json_output)?, &results)?;
+    }
     Ok(())
 }