diff --git a/Makefile b/Makefile index 16383a62ec32..a1599aedde8d 100644 --- a/Makefile +++ b/Makefile @@ -17,10 +17,36 @@ else @echo "benchmark tool already installed..." endif -.PHONY: bench-put -bench-put: build install-benchmark - @echo "Running benchmark: put $(ARGS)" - ./scripts/benchmark_test.sh put $(ARGS) +.PHONY: bench-lease-keepalive bench-put bench-txn-mixed bench-watch-latency bench-watch bench-range-key +## ---------------------------------------------------------------------------- +## etcd Benchmark Operations +## Run the corresponding operation with the relevant arguments set through ARGS, after setting up an etcd server to bench against. +## Eg. make bench-put ARGS="--clients=100 --conns=10" +## Targets: +## bench-lease-keepalive: Run the benchmark lease-keepalive operation with optional ARGS. +bench-lease-keepalive: TEST := lease-keepalive +## bench-put: Run the benchmark put operation with optional ARGS. +bench-put: TEST := put +## bench-txn-mixed: Run the benchmark txn-mixed operation with optional ARGS. +bench-txn-mixed: TEST := txn-mixed +## bench-watch-latency: Run the benchmark watch-latency operation with optional ARGS. +bench-watch-latency: TEST := watch-latency +## bench-watch: Run the benchmark watch operation with optional ARGS. +bench-watch: TEST := watch +## bench-range-key: Run the benchmark range-key operation with optional ARGS. +bench-range-key: TEST := range +bench-range-key: ARGS := key + +bench-lease-keepalive bench-put bench-txn-mixed bench-watch-latency bench-watch bench-range-key: build install-benchmark + @echo "Running benchmark: $(TEST) $(ARGS)" + ./scripts/benchmark_test.sh "$(TEST):$(ARGS)" + +## bench: Run a custom benchmark with TEST_ARGS which can be a combination of tests with arguments. +## Eg. make bench TEST_ARGS='put:"--clients=1000" range:"key --conns=10"' +## ---------------------------------------------------------------------------- +.PHONY: bench +bench: build install-benchmark + ./scripts/benchmark_test.sh $(TEST_ARGS) PLATFORMS=linux-amd64 linux-386 linux-arm linux-arm64 linux-ppc64le linux-s390x darwin-amd64 darwin-arm64 windows-amd64 windows-arm64 @@ -253,3 +279,7 @@ markdown-diff-lint: .PHONY: update-go-workspace update-go-workspace: ./scripts/update_go_workspace.sh + +.PHONY: help +help: + @sed -ne '/@sed/!s/## //p' $(MAKEFILE_LIST) diff --git a/pkg/report/perfdash.go b/pkg/report/perfdash.go index bd0fd0ed3af8..794d0f0878e3 100644 --- a/pkg/report/perfdash.go +++ b/pkg/report/perfdash.go @@ -74,7 +74,7 @@ func (r *report) writePerfDashReport(benchmarkOp string) { artifactsDir = "./_artifacts" } - fileName := fmt.Sprintf("EtcdAPI_benchmark_%s.json", time.Now().UTC().Format(time.RFC3339)) + fileName := fmt.Sprintf("EtcdAPI_benchmark_%s_%s.json", benchmarkOp, time.Now().UTC().Format(time.RFC3339)) err := os.MkdirAll(artifactsDir, 0o755) if err != nil { fmt.Println("Error creating artifacts directory:", err) diff --git a/scripts/benchmark_test.sh b/scripts/benchmark_test.sh index bff071fcafd7..f4fcd0542880 100755 --- a/scripts/benchmark_test.sh +++ b/scripts/benchmark_test.sh @@ -13,53 +13,100 @@ # See the License for the specific language governing permissions and # limitations under the License. -# This script runs a benchmark on a locally started etcd server +# This utility script helps to set up the test environment for benchmark tests and runs the provided tests sequentially. set -euo pipefail source ./scripts/test_lib.sh -COMMON_BENCHMARK_FLAGS="--report-perfdash" +declare -A BENCHMARK_FLAGS +BENCHMARKS_TO_RUN=() +COMMON_BENCHMARK_FLAGS=(--report-perfdash) -if [[ $# -lt 1 ]]; then - echo "Usage: $0 [tester args...]" - exit 1 -fi +parse_args() { + for arg in "$@"; do + if [[ "$arg" == *:* ]]; then + key="${arg%%:*}" + value="${arg#*:}" + BENCHMARKS_TO_RUN+=("$key") + BENCHMARK_FLAGS["$key"]="$value" + else + BENCHMARKS_TO_RUN+=("$arg") + BENCHMARK_FLAGS["$arg"]="" + fi + done -BENCHMARK_NAME="$1" -ARGS="${*:2}" + if [ ${#BENCHMARKS_TO_RUN[@]} -eq 0 ]; then + log_error "Usage: ./benchmark_test.sh 'benchmark-operation:\"test_arguments\"'" + log_error "Example: ./benchmark_test.sh 'put:\"--clients=1000\" range:\"--conns=100\"'" + exit 1 + fi +} -echo "Starting the etcd server..." +start_etcd() { + log_callout "Setup: Starting the etcd server..." + # Create a directory for etcd data under /tmp/etcd + mkdir -p /tmp/etcd + DATA_DIR=$(mktemp -d /tmp/etcd/data-XXXXXX) + ARTIFACTS_DIR="${ARTIFACTS:-./_artifacts}" + mkdir -p "$ARTIFACTS_DIR" + ETCD_LOG_FILE="${ARTIFACTS_DIR}/etcd-${bench}-$(date +%Y%m%d-%H%M%S).log" + log_callout -e "Setup: etcd log file path set to $ETCD_LOG_FILE. DATA_DIR set to $DATA_DIR" + ./bin/etcd --data-dir="$DATA_DIR" > "${ETCD_LOG_FILE}" 2>&1 & + ETCD_PID=$! + RETRY=0 + MAX_RETRY_ATTEMPTS=10 -# Create a directory for etcd data under /tmp/etcd -mkdir -p /tmp/etcd -DATA_DIR=$(mktemp -d /tmp/etcd/data-XXXXXX) -./bin/etcd --data-dir="$DATA_DIR" > /tmp/etcd.log 2>&1 & -etcd_pid=$! + # Set up the trap to handle errors/interrupts which leads to cleaning up of the DATADIR and stopping the etcd server. + trap 'stop_and_cleanup_etcd' EXIT -trap 'log_warning -e "Stopping etcd server - PID $etcd_pid"; - kill $etcd_pid 2>/dev/null; - rm -rf $DATA_DIR; - log_success "Deleted the contents from $DATA_DIR related to benchmark test"' EXIT + # Poll until etcd is healthy + until curl -fs http://127.0.0.1:2379/health | grep -q '"health":"true"'; do + RETRY=$((RETRY + 1)) + if [[ $RETRY -gt $MAX_RETRY_ATTEMPTS ]]; then + log_error -e "Setup: Failed to confirm etcd health after $MAX_RETRY_ATTEMPTS attempts." + exit 1 + fi + log_warning -e "Setup: Waiting for etcd to be healthy... (retry: $RETRY/$MAX_RETRY_ATTEMPTS)" + sleep 1 + done + log_success -e "Setup: etcd is healthy and running on pid $ETCD_PID" +} -# Wait until etcd becomes healthy -for retry in {1..10}; do - if ./bin/etcdctl endpoint health --cluster> /dev/null 2>&1; then - log_success -e "\\netcd is healthy" - break - fi - log_warning -e "\\nWaiting for etcd to be healthy..." - sleep 1 - if [[ $retry -eq 10 ]]; then - log_error -e "\\nFailed to confirm etcd health after $retry attempts. Check /tmp/etcd.log for more information" - exit 1 +stop_and_cleanup_etcd() { + trap - EXIT + log_warning -e "Cleanup: Stopping etcd server - PID $ETCD_PID" + kill "$ETCD_PID" 2>/dev/null || true + rm -rf "$DATA_DIR" + log_success "Cleanup: Deleted the DATA_DIR contents from $DATA_DIR related to benchmark test" +} + +run_benchmark() { + local bench=$1 + local args="${BENCHMARK_FLAGS[$bench]:-}" + + if [[ -z "$args" ]]; then + log_callout -e "\\nPerforming benchmark $bench with default arguments" + benchmark "$bench" "${COMMON_BENCHMARK_FLAGS[@]}" + else + log_callout -e "\\nPerforming benchmark $bench with arguments: $args" + read -r -a TESTER_OPTIONS <<< "$args" + + printf "Running: benchmark %s %s %s\n" \ + "$bench" "${TESTER_OPTIONS[*]}" "${COMMON_BENCHMARK_FLAGS[*]}" + + benchmark "$bench" "${TESTER_OPTIONS[@]}" "${COMMON_BENCHMARK_FLAGS[@]}" fi -done +} + +main() { + parse_args "$@" -log_success -e "etcd process is running with PID $etcd_pid" + for bench in "${BENCHMARKS_TO_RUN[@]}"; do + start_etcd + run_benchmark "$bench" + stop_and_cleanup_etcd + done +} -log_callout -e "\\nPerforming benchmark $BENCHMARK_NAME with arguments: $ARGS" -read -r -a TESTER_OPTIONS <<< "$ARGS" -log_callout "Running: benchmark $BENCHMARK_NAME ${TESTER_OPTIONS[*]} $COMMON_BENCHMARK_FLAGS" -benchmark "$BENCHMARK_NAME" "${TESTER_OPTIONS[@]}" $COMMON_BENCHMARK_FLAGS -log_callout "Completed: benchmark $BENCHMARK_NAME ${TESTER_OPTIONS[*]} $COMMON_BENCHMARK_FLAGS" +main "$@"