Skip to content

Commit 996f6c5

Browse files
ahmadelyousseffacebook-github-bot
ahmadelyoussef
authored and
facebook-github-bot
committed
Adding execution tests for DC Perf github CI (#207)
Summary: Adding execution testcases for github CI for feedsim, mediawiki, django, tao_bench, spark and syscall. Differential Revision: D80854990
1 parent 5313034 commit 996f6c5

21 files changed

+353
-19
lines changed

‎.github/scripts/dcperf_run_django.bash‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
9+
# shellcheck disable=SC1091,SC2128
10+
. "$( dirname -- "$BASH_SOURCE"; )/utils_base.bash"
11+
12+
13+
################################################################################
14+
# Django Run Functions
15+
################################################################################
16+
17+
run_django () {
18+
local env_name="$1"
19+
if [ "$env_name" == "" ]; then
20+
echo "Usage: ${FUNCNAME[0]} ENV_NAME"
21+
echo "Example(s):"
22+
echo " ${FUNCNAME[0]} build_env"
23+
return 1
24+
else
25+
echo "################################################################################"
26+
echo "# Run Django"
27+
echo "#"
28+
echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
29+
echo "################################################################################"
30+
echo ""
31+
fi
32+
33+
34+
# shellcheck disable=SC2155
35+
local env_prefix=$(env_name_or_prefix "${env_name}")
36+
37+
echo "[RUN] Running Django ..."
38+
(print_exec conda run --no-capture-output ${env_prefix} \
39+
python ./benchpress_cli.py run django_workload_default -r standalone -i "'{\"reps\": 1000, \"iterations\": 1}'") || return 1
40+
}

‎.github/scripts/dcperf_run_feedsim.bash‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
9+
# shellcheck disable=SC1091,SC2128
10+
. "$( dirname -- "$BASH_SOURCE"; )/utils_base.bash"
11+
12+
13+
################################################################################
14+
# Run FeedSim
15+
################################################################################
16+
17+
run_feedsim () {
18+
local env_name="$1"
19+
if [ "$env_name" == "" ]; then
20+
echo "Usage: ${FUNCNAME[0]} ENV_NAME"
21+
echo "Example(s):"
22+
echo " ${FUNCNAME[0]} build_env"
23+
return 1
24+
else
25+
echo "################################################################################"
26+
echo "# Run FeedSim"
27+
echo "#"
28+
echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
29+
echo "################################################################################"
30+
echo ""
31+
fi
32+
33+
# shellcheck disable=SC2155
34+
local env_prefix=$(env_name_or_prefix "${env_name}")
35+
36+
echo "[RUN] Running FeedSim ..."
37+
38+
# Run benchpress and capture output
39+
if output=$(print_exec conda run --no-capture-output ${env_prefix} python ./benchpress_cli.py run feedsim_autoscale -i "'{\"extra_args\": \"-q 5 -d 20\"}'" 2>&1); then
40+
echo "$output"
41+
else
42+
echo "ERROR: Benchpress command failed to execute"
43+
return 1
44+
fi
45+
46+
# Extract final_achieved_qps from the output and validate
47+
final_qps=$(echo "$output" | grep -A3 '"overall"' | grep -o '"final_achieved_qps":[^,]*' | grep -o '[0-9.]*' | head -1)
48+
49+
if [ -z "$final_qps" ]; then
50+
echo "ERROR: Could not extract final_achieved_qps from output"
51+
return 1
52+
fi
53+
54+
# Check if final_achieved_qps is greater than zero
55+
if (( $(echo "$final_qps > 0" | bc -l) )); then
56+
echo "SUCCESS: final_achieved_qps ($final_qps) is greater than zero"
57+
return 0
58+
else
59+
echo "FAILURE: final_achieved_qps ($final_qps) is not greater than zero"
60+
return 1
61+
fi
62+
}

‎.github/scripts/dcperf_run_mediawiki.bash‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
9+
# shellcheck disable=SC1091,SC2128
10+
. "$( dirname -- "$BASH_SOURCE"; )/utils_base.bash"
11+
12+
13+
################################################################################
14+
# Run Mediawiki
15+
################################################################################
16+
17+
run_mediawiki () {
18+
local env_name="$1"
19+
if [ "$env_name" == "" ]; then
20+
echo "Usage: ${FUNCNAME[0]} ENV_NAME"
21+
echo "Example(s):"
22+
echo " ${FUNCNAME[0]} build_env"
23+
return 1
24+
else
25+
echo "################################################################################"
26+
echo "# Run Mediawiki"
27+
echo "#"
28+
echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
29+
echo "################################################################################"
30+
echo ""
31+
fi
32+
33+
# shellcheck disable=SC2155
34+
local env_prefix=$(env_name_or_prefix "${env_name}")
35+
export DEBIAN_FRONTEND=noninteractive
36+
echo "[RUN] Running Mediawiki ..."
37+
(print_exec conda run --no-capture-output ${env_prefix} \
38+
python ./benchpress_cli.py run oss_performance_mediawiki_mini) || return 1
39+
}

‎.github/scripts/dcperf_run_spark.bash‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
9+
# shellcheck disable=SC1091,SC2128
10+
. "$( dirname -- "$BASH_SOURCE"; )/utils_base.bash"
11+
12+
13+
################################################################################
14+
# Run Spark
15+
################################################################################
16+
17+
run_spark () {
18+
local env_name="$1"
19+
if [ "$env_name" == "" ]; then
20+
echo "Usage: ${FUNCNAME[0]} ENV_NAME"
21+
echo "Example(s):"
22+
echo " ${FUNCNAME[0]} build_env"
23+
return 1
24+
else
25+
echo "################################################################################"
26+
echo "# Run Spark"
27+
echo "#"
28+
echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
29+
echo "################################################################################"
30+
echo ""
31+
fi
32+
33+
# shellcheck disable=SC2155
34+
local env_prefix=$(env_name_or_prefix "${env_name}")
35+
36+
echo "[RUN] Running Spark ..."
37+
wget https://github.com/facebookresearch/DCPerf-datasets/releases/download/v1.0/bpc_t93586_s2_synthetic_1GB.tar.gz
38+
tar -xzf bpc_t93586_s2_synthetic_1GB.tar.gz
39+
mkdir /flash23
40+
cp -rf bpc_t93586_s2_synthetic_1GB /flash23/
41+
(print_exec conda run --no-capture-output ${env_prefix} \
42+
python ./benchpress_cli.py run spark_standalone_remote_mini -i '{"dataset_name":"bpc_t93586_s2_synthetic_1GB"}') || return 1
43+
}

‎.github/scripts/dcperf_run_syscall.bash‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
9+
# shellcheck disable=SC1091,SC2128
10+
. "$( dirname -- "$BASH_SOURCE"; )/utils_base.bash"
11+
12+
13+
################################################################################
14+
# Run syscall
15+
################################################################################
16+
17+
run_syscall () {
18+
local env_name="$1"
19+
if [ "$env_name" == "" ]; then
20+
echo "Usage: ${FUNCNAME[0]} ENV_NAME"
21+
echo "Example(s):"
22+
echo " ${FUNCNAME[0]} build_env"
23+
return 1
24+
else
25+
echo "################################################################################"
26+
echo "# Run Syscall"
27+
echo "#"
28+
echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
29+
echo "################################################################################"
30+
echo ""
31+
fi
32+
33+
# shellcheck disable=SC2155
34+
local env_prefix=$(env_name_or_prefix "${env_name}")
35+
36+
echo "[RUN] Running Syscall ..."
37+
(print_exec conda run --no-capture-output ${env_prefix} \
38+
python ./benchpress_cli.py -b system run syscall_single_core) || return 1
39+
}

‎.github/scripts/dcperf_run_tao_bench.bash‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
9+
# shellcheck disable=SC1091,SC2128
10+
. "$( dirname -- "$BASH_SOURCE"; )/utils_base.bash"
11+
12+
13+
################################################################################
14+
# tao_bench Run Functions
15+
################################################################################
16+
17+
run_tao_bench () {
18+
local env_name="$1"
19+
if [ "$env_name" == "" ]; then
20+
echo "Usage: ${FUNCNAME[0]} ENV_NAME"
21+
echo "Example(s):"
22+
echo " ${FUNCNAME[0]} build_env"
23+
return 1
24+
else
25+
echo "################################################################################"
26+
echo "# Run Tao_bench"
27+
echo "#"
28+
echo "# [$(date --utc +%FT%T.%3NZ)] + ${FUNCNAME[0]} ${*}"
29+
echo "################################################################################"
30+
echo ""
31+
fi
32+
33+
34+
# shellcheck disable=SC2155
35+
local env_prefix=$(env_name_or_prefix "${env_name}")
36+
37+
echo "[RUN] Running Tao_bench ..."
38+
39+
# Run benchpress and capture output
40+
if output=$(print_exec conda run --no-capture-output ${env_prefix} python ./benchpress_cli.py run tao_bench_standalone -i "'{\"memsize\": 0.5, \"stats_interval\":1000, \"warmup_time\": 60, \"test_time\": 60}'" 2>&1); then
41+
echo "$output"
42+
else
43+
echo "ERROR: Benchpress command failed to execute"
44+
return 1
45+
fi
46+
47+
# Extract total_qps from the output and validate
48+
total_qps=$(echo "$output" | grep -o '"total_qps":[^,]*' | grep -o '[0-9.]*' | head -1)
49+
50+
if [ -z "$total_qps" ]; then
51+
echo "ERROR: Could not extract total_qps from output"
52+
return 1
53+
fi
54+
55+
# Check if total_qps is greater than zero
56+
if (( $(echo "$total_qps > 0" | bc -l) )); then
57+
echo "SUCCESS: total_qps ($total_qps) is greater than zero"
58+
return 0
59+
else
60+
echo "FAILURE: total_qps ($total_qps) is not greater than zero"
61+
return 1
62+
fi
63+
}

‎.github/scripts/setup_env.bash‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,18 +16,30 @@
1616
# shellcheck disable=SC1091,SC2128
1717
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_django.bash"
1818
# shellcheck disable=SC1091,SC2128
19+
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_run_django.bash"
20+
# shellcheck disable=SC1091,SC2128
1921
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_feedsim.bash"
2022
# shellcheck disable=SC1091,SC2128
23+
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_run_feedsim.bash"
24+
# shellcheck disable=SC1091,SC2128
2125
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_tao_bench.bash"
2226
# shellcheck disable=SC1091,SC2128
27+
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_run_tao_bench.bash"
28+
# shellcheck disable=SC1091,SC2128
2329
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_spark.bash"
2430
# shellcheck disable=SC1091,SC2128
31+
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_run_spark.bash"
32+
# shellcheck disable=SC1091,SC2128
2533
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_video_transcode_bench.bash"
2634
# shellcheck disable=SC1091,SC2128
2735
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_mediawiki.bash"
2836
# shellcheck disable=SC1091,SC2128
37+
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_run_mediawiki.bash"
38+
# shellcheck disable=SC1091,SC2128
2939
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_wdl.bash"
3040
# shellcheck disable=SC1091,SC2128
3141
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_health_check.bash"
3242
# shellcheck disable=SC1091,SC2128
3343
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_install_syscall.bash"
44+
# shellcheck disable=SC1091,SC2128
45+
. "$( dirname -- "$BASH_SOURCE"; )/dcperf_run_syscall.bash"

‎.github/workflows/dcperf_ci_django.yml‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,11 +55,11 @@ jobs:
5555
steps:
5656
- name: Setup Build Container (ubuntu-based)
5757
if: ${{ matrix.host-machine.type == 'ubuntu' }}
58-
run: apt update -y; apt install -y build-essential git pciutils socat sudo wget libpython2-dev python3-dev
58+
run: apt update -y; apt install -y build-essential git pciutils socat sudo wget libpython2-dev python3-dev numactl lshw dmidecode procps netcat bc
5959

6060
- name: Setup Build Container (centos-based)
6161
if: ${{ matrix.host-machine.type == 'centos' }}
62-
run: dnf update -y; dnf install -y git pciutils which python3-devel ; dnf group install -y "Development Tools" --exclude="texlive*"
62+
run: dnf update -y; dnf install -y git pciutils which python3-devel numactl lshw dmidecode procps-ng nmap-ncat bc; dnf group install -y "Development Tools" --exclude="texlive*"
6363

6464
- name: Checkout the Repository
6565
uses: actions/checkout@v4
@@ -86,3 +86,6 @@ jobs:
8686

8787
- name: Install Django
8888
run: . $PRELUDE; install_django $BUILD_ENV
89+
90+
- name: Run Django
91+
run: . $PRELUDE; run_django $BUILD_ENV

‎.github/workflows/dcperf_ci_feedsim.yml‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,11 +55,11 @@ jobs:
5555
steps:
5656
- name: Setup Build Container (ubuntu-based)
5757
if: ${{ matrix.host-machine.type == 'ubuntu' }}
58-
run: apt update -y; apt install -y build-essential git pciutils socat sudo wget dmidecode
58+
run: apt update -y; apt install -y build-essential git pciutils socat sudo wget dmidecode numactl lshw procps sysstat
5959

6060
- name: Setup Build Container (centos-based)
6161
if: ${{ matrix.host-machine.type == 'centos' }}
62-
run: dnf update -y; dnf install -y git pciutils which dmidecode
62+
run: dnf update -y; dnf install -y git pciutils which dmidecode numactl lshw procps-ng sysstat
6363

6464
- name: Checkout the Repository
6565
uses: actions/checkout@v4
@@ -86,3 +86,6 @@ jobs:
8686

8787
- name: Install FeedSim
8888
run: . $PRELUDE; install_feedsim $BUILD_ENV
89+
90+
- name: Run FeedSim
91+
run: . $PRELUDE; run_feedsim $BUILD_ENV

‎.github/workflows/dcperf_ci_mediawiki.yml‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,11 +53,11 @@ jobs:
5353
steps:
5454
- name: Setup Build Container (ubuntu-based)
5555
if: ${{ matrix.host-machine.type == 'ubuntu' }}
56-
run: apt update -y; apt install -y build-essential git pciutils socat sudo wget dmidecode curl
56+
run: apt update -y; apt install -y build-essential git pciutils socat sudo wget dmidecode curl numactl lshw procps bc
5757

5858
- name: Setup Build Container (centos-based)
5959
if: ${{ matrix.host-machine.type == 'centos' }}
60-
run: dnf update -y; dnf install -y git pciutils which dmidecode
60+
run: dnf update -y; dnf install -y git pciutils which numactl lshw procps-ng bc
6161

6262
- name: Checkout the Repository
6363
uses: actions/checkout@v4
@@ -87,3 +87,6 @@ jobs:
8787

8888
- name: Install Mediawiki
8989
run: . $PRELUDE; install_mediawiki $BUILD_ENV
90+
91+
- name: Run Mediawiki
92+
run: . $PRELUDE; run_mediawiki $BUILD_ENV

0 commit comments

Comments
 (0)