Skip to content

Commit 025d09f

Browse files
ahmadelyousseffacebook-github-bot
ahmadelyoussef
authored and
facebook-github-bot
committed
Adding execution tests for DC Perf github CI (facebookresearch#207)
Summary: Adding execution testcases for github CI for feedsim, mediawiki, django, tao_bench, spark and syscall. Differential Revision: D80854990
1 parent 2b78258 commit 025d09f

21 files changed

+407
-19
lines changed

‎.github/scripts/dcperf_run_django.bash‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
9+
# shellcheck disable=SC1091,SC2128
10+
. "$( dirname -- "$BASH_SOURCE"; )/utils_base.bash"
11+
12+
13+
################################################################################
14+
# Django Run Functions
15+
################################################################################
16+
17+
run_django () {
18+
local env_name="$1"
19+
if [ "$env_name" == "" ]; then
20+
echo "Usage: ${FUNCNAME[0]} ENV_NAME"
21+
echo "Example(s):"
22+
echo " ${FUNCNAME[0]} build_env"
23+
return 1
24+
else
25+
echo "################################################################################"
26+
echo "# Run Django"
27+
echo "#"
28+
echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
29+
echo "################################################################################"
30+
echo ""
31+
fi
32+
33+
34+
# shellcheck disable=SC2155
35+
local env_prefix=$(env_name_or_prefix "${env_name}")
36+
37+
echo "[RUN] Running Django ..."
38+
# (print_exec conda run --no-capture-output ${env_prefix} \
39+
./benchmarks/django_workload/bin/run.sh -r standalone -d 5M -i 1 p 1000 -l /siege.log -s urls.txt -c 127.0.0.1 -I cpython || true
40+
cat benchmarks/django_workload/django-workload/django-workload/django-uwsgi.log
41+
}

‎.github/scripts/dcperf_run_feedsim.bash‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
9+
# shellcheck disable=SC1091,SC2128
10+
. "$( dirname -- "$BASH_SOURCE"; )/utils_base.bash"
11+
12+
13+
################################################################################
14+
# Run FeedSim
15+
################################################################################
16+
17+
run_feedsim () {
18+
local env_name="$1"
19+
if [ "$env_name" == "" ]; then
20+
echo "Usage: ${FUNCNAME[0]} ENV_NAME"
21+
echo "Example(s):"
22+
echo " ${FUNCNAME[0]} build_env"
23+
return 1
24+
else
25+
echo "################################################################################"
26+
echo "# Run FeedSim"
27+
echo "#"
28+
echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
29+
echo "################################################################################"
30+
echo ""
31+
fi
32+
33+
# shellcheck disable=SC2155
34+
local env_prefix=$(env_name_or_prefix "${env_name}")
35+
36+
echo "[RUN] Running FeedSim ..."
37+
(print_exec conda run --no-capture-output ${env_prefix} \
38+
python ./benchpress_cli.py run feedsim_autoscale -i "'{\"extra_args\": \"-q 5 -d 20\"}'") || true
39+
cat benchpress.log
40+
41+
# Print all feedsim results and logs from benchmark_metrics_* directories
42+
echo ""
43+
echo "[INFO] Searching for FeedSim results in benchmark_metrics_* directories..."
44+
for metrics_dir in benchmark_metrics_*/; do
45+
if [ -d "$metrics_dir" ]; then
46+
echo ""
47+
echo "=== Found directory: $metrics_dir ==="
48+
49+
# Print all feedsim_results*.txt files
50+
for results_file in "${metrics_dir}"feedsim_results*.txt; do
51+
if [ -f "$results_file" ]; then
52+
echo ""
53+
echo "--- Contents of $results_file ---"
54+
cat "$results_file"
55+
fi
56+
done
57+
58+
# Print all feedsim-multi-inst-*.log files
59+
for log_file in "${metrics_dir}"feedsim-multi-inst-*.log; do
60+
if [ -f "$log_file" ]; then
61+
echo ""
62+
echo "--- Contents of $log_file ---"
63+
cat "$log_file"
64+
fi
65+
done
66+
fi
67+
done
68+
69+
# Run benchpress and capture output
70+
# if output=$(print_exec conda run --no-capture-output ${env_prefix} python ./benchpress_cli.py run feedsim_autoscale -i "'{\"extra_args\": \"-q 5 -d 20\"}'" 2>&1); then
71+
# echo "$output"
72+
# else
73+
# echo "ERROR: Benchpress command failed to execute"
74+
# return 1
75+
# fi
76+
77+
# # Extract final_achieved_qps from the output and validate
78+
# final_qps=$(echo "$output" | grep -A3 '"overall"' | grep -o '"final_achieved_qps":[^,]*' | grep -o '[0-9.]*' | head -1)
79+
80+
# if [ -z "$final_qps" ]; then
81+
# echo "ERROR: Could not extract final_achieved_qps from output"
82+
# return 1
83+
# fi
84+
85+
# # Check if final_achieved_qps is greater than zero
86+
# if (( $(echo "$final_qps > 0" | bc -l) )); then
87+
# echo "SUCCESS: final_achieved_qps ($final_qps) is greater than zero"
88+
# return 0
89+
# else
90+
# echo "FAILURE: final_achieved_qps ($final_qps) is not greater than zero"
91+
# return 1
92+
# fi
93+
}

‎.github/scripts/dcperf_run_mediawiki.bash‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
9+
# shellcheck disable=SC1091,SC2128
10+
. "$( dirname -- "$BASH_SOURCE"; )/utils_base.bash"
11+
12+
13+
################################################################################
14+
# Run Mediawiki
15+
################################################################################
16+
17+
run_mediawiki () {
18+
local env_name="$1"
19+
if [ "$env_name" == "" ]; then
20+
echo "Usage: ${FUNCNAME[0]} ENV_NAME"
21+
echo "Example(s):"
22+
echo " ${FUNCNAME[0]} build_env"
23+
return 1
24+
else
25+
echo "################################################################################"
26+
echo "# Run Mediawiki"
27+
echo "#"
28+
echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
29+
echo "################################################################################"
30+
echo ""
31+
fi
32+
33+
# shellcheck disable=SC2155
34+
local env_prefix=$(env_name_or_prefix "${env_name}")
35+
export DEBIAN_FRONTEND=noninteractive
36+
echo "[RUN] Running Mediawiki ..."
37+
# (print_exec conda run --no-capture-output ${env_prefix} \
38+
# python ./benchpress_cli.py run oss_performance_mediawiki_mini) || return 1
39+
40+
./packages/mediawiki/run.sh \
41+
-r/usr/local/hphpi/legacy/bin/hhvm \
42+
-nnginx \
43+
-L wrk \
44+
-s benchmarks/oss_performance_mediawiki/wrk/wrk \
45+
-T default_no_temp_dir \
46+
-p -- \
47+
--mediawiki \
48+
--client-duration=4s \
49+
--client-timeout=60s \
50+
--run-as-root \
51+
--i-am-not-benchmarking \
52+
--shorten-health-check \
53+
--skip-single-request-warmup \
54+
--skip-sleep-between-warmups \
55+
--hhvm-extra-arguments="-vEval.JitRetranslateAllSeconds=5" \
56+
--num-multi-req-warmups=-1 \
57+
--no-load-if-pending-translate \
58+
--first-multi-warmup-duration=25 \
59+
--subseq-multi-warmup-duration=5 \
60+
--load-gen-seed=1000
61+
}

‎.github/scripts/dcperf_run_spark.bash‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
9+
# shellcheck disable=SC1091,SC2128
10+
. "$( dirname -- "$BASH_SOURCE"; )/utils_base.bash"
11+
12+
13+
################################################################################
14+
# Run Spark
15+
################################################################################
16+
17+
run_spark () {
18+
local env_name="$1"
19+
if [ "$env_name" == "" ]; then
20+
echo "Usage: ${FUNCNAME[0]} ENV_NAME"
21+
echo "Example(s):"
22+
echo " ${FUNCNAME[0]} build_env"
23+
return 1
24+
else
25+
echo "################################################################################"
26+
echo "# Run Spark"
27+
echo "#"
28+
echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
29+
echo "################################################################################"
30+
echo ""
31+
fi
32+
33+
# shellcheck disable=SC2155
34+
local env_prefix=$(env_name_or_prefix "${env_name}")
35+
36+
echo "[RUN] Running Spark ..."
37+
wget https://github.com/facebookresearch/DCPerf-datasets/releases/download/v1.0/bpc_t93586_s2_synthetic_1GB.tar.gz
38+
tar -xzf bpc_t93586_s2_synthetic_1GB.tar.gz
39+
mkdir /flash23
40+
cp -rf bpc_t93586_s2_synthetic_1GB /flash23/
41+
(print_exec conda run --no-capture-output ${env_prefix} \
42+
python ./benchpress_cli.py run spark_standalone_remote_mini -i "'{\"dataset_name\":\"bpc_t93586_s2_synthetic_1GB\"}'") || return 1
43+
}

‎.github/scripts/dcperf_run_syscall.bash‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
9+
# shellcheck disable=SC1091,SC2128
10+
. "$( dirname -- "$BASH_SOURCE"; )/utils_base.bash"
11+
12+
13+
################################################################################
14+
# Run syscall
15+
################################################################################
16+
17+
run_syscall () {
18+
local env_name="$1"
19+
if [ "$env_name" == "" ]; then
20+
echo "Usage: ${FUNCNAME[0]} ENV_NAME"
21+
echo "Example(s):"
22+
echo " ${FUNCNAME[0]} build_env"
23+
return 1
24+
else
25+
echo "################################################################################"
26+
echo "# Run Syscall"
27+
echo "#"
28+
echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
29+
echo "################################################################################"
30+
echo ""
31+
fi
32+
33+
# shellcheck disable=SC2155
34+
local env_prefix=$(env_name_or_prefix "${env_name}")
35+
36+
echo "[RUN] Running Syscall ..."
37+
(print_exec conda run --no-capture-output ${env_prefix} \
38+
python ./benchpress_cli.py -b system run syscall_single_core) || return 1
39+
}

‎.github/scripts/dcperf_run_tao_bench.bash‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
9+
# shellcheck disable=SC1091,SC2128
10+
. "$( dirname -- "$BASH_SOURCE"; )/utils_base.bash"
11+
12+
13+
################################################################################
14+
# tao_bench Run Functions
15+
################################################################################
16+
17+
run_tao_bench () {
18+
local env_name="$1"
19+
if [ "$env_name" == "" ]; then
20+
echo "Usage: ${FUNCNAME[0]} ENV_NAME"
21+
echo "Example(s):"
22+
echo " ${FUNCNAME[0]} build_env"
23+
return 1
24+
else
25+
echo "################################################################################"
26+
echo "# Run Tao_bench"
27+
echo "#"
28+
echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
29+
echo "################################################################################"
30+
echo ""
31+
fi
32+
33+
34+
# shellcheck disable=SC2155
35+
local env_prefix=$(env_name_or_prefix "${env_name}")
36+
37+
echo "[RUN] Running Tao_bench ..."
38+
39+
# Run benchpress and capture output
40+
if output=$(print_exec conda run --no-capture-output ${env_prefix} python ./benchpress_cli.py run tao_bench_standalone -i "'{\"memsize\": 0.5, \"stats_interval\":1000, \"warmup_time\": 60, \"test_time\": 60}'" 2>&1); then
41+
echo "$output"
42+
else
43+
echo "ERROR: Benchpress command failed to execute"
44+
return 1
45+
fi
46+
47+
# Extract total_qps from the output and validate
48+
total_qps=$(echo "$output" | grep -o '"total_qps":[^,]*' | grep -o '[0-9.]*' | head -1)
49+
50+
if [ -z "$total_qps" ]; then
51+
echo "ERROR: Could not extract total_qps from output"
52+
return 1
53+
fi
54+
55+
# Check if total_qps is greater than zero
56+
if (( $(echo "$total_qps > 0" | bc -l) )); then
57+
echo "SUCCESS: total_qps ($total_qps) is greater than zero"
58+
return 0
59+
else
60+
echo "FAILURE: total_qps ($total_qps) is not greater than zero"
61+
return 1
62+
fi
63+
}

‎.github/scripts/setup_env.bash‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,18 +16,30 @@
1616
# shellcheck disable=SC1091,SC2128
1717
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_django.bash"
1818
# shellcheck disable=SC1091,SC2128
19+
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_run_django.bash"
20+
# shellcheck disable=SC1091,SC2128
1921
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_feedsim.bash"
2022
# shellcheck disable=SC1091,SC2128
23+
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_run_feedsim.bash"
24+
# shellcheck disable=SC1091,SC2128
2125
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_tao_bench.bash"
2226
# shellcheck disable=SC1091,SC2128
27+
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_run_tao_bench.bash"
28+
# shellcheck disable=SC1091,SC2128
2329
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_spark.bash"
2430
# shellcheck disable=SC1091,SC2128
31+
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_run_spark.bash"
32+
# shellcheck disable=SC1091,SC2128
2533
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_video_transcode_bench.bash"
2634
# shellcheck disable=SC1091,SC2128
2735
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_mediawiki.bash"
2836
# shellcheck disable=SC1091,SC2128
37+
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_run_mediawiki.bash"
38+
# shellcheck disable=SC1091,SC2128
2939
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_wdl.bash"
3040
# shellcheck disable=SC1091,SC2128
3141
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_health_check.bash"
3242
# shellcheck disable=SC1091,SC2128
3343
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_syscall.bash"
44+
# shellcheck disable=SC1091,SC2128
45+
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_run_syscall.bash"

‎.github/workflows/dcperf_ci_django.yml‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,11 +55,11 @@ jobs:
5555
steps:
5656
- name: Setup Build Container (ubuntu-based)
5757
if: ${{ matrix.host-machine.type == 'ubuntu' }}
58-
run: apt update -y; apt install -y build-essential git pciutils socat sudo wget libpython2-dev python3-dev
58+
run: apt update -y; apt install -y build-essential git pciutils socat sudo wget libpython2-dev python3-dev numactl lshw dmidecode procps netcat bc
5959

6060
- name: Setup Build Container (centos-based)
6161
if: ${{ matrix.host-machine.type == 'centos' }}
62-
run: dnf update -y; dnf install -y git pciutils which python3-devel ; dnf group install -y "Development Tools" --exclude="texlive*"
62+
run: dnf update -y; dnf install -y git pciutils which python3-devel numactl lshw dmidecode procps-ng nmap-ncat bc; dnf group install -y "Development Tools" --exclude="texlive*"
6363

6464
- name: Checkout the Repository
6565
uses: actions/checkout@v4
@@ -86,3 +86,6 @@ jobs:
8686

8787
- name: Install Django
8888
run: . $PRELUDE; install_django $BUILD_ENV
89+
90+
- name: Run Django
91+
run: . $PRELUDE; run_django $BUILD_ENV

0 commit comments

Comments
 (0)