diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 86c0247d706e..572d44178a73 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -7,7 +7,7 @@ Documents the various GitHub Actions workflows, the role they fulfill and 3rd pa Builds and runs CCF performance tests, both end to end and micro-benchmarks. Results are posted to bencher.dev, and [plotted to make regressions obvious](https://bencher.dev/console/projects/ccf/plots). Triggered on every commit on `main`, but not on PR builds because the setup required to build from forks is complex and fragile in terms of security, and the increase in pool usage would be substantial. -Tests are run and published on two different testbeds for comparison: gha-vmss-d16av5-ci (d16av5 VMs) and gha-c-aci-ci (C-ACI with 16 cores and 32Gb RAM), and are labeled accordingly in the bencher UI. +Tests are run and published on two different testbeds for comparison: gha-vmss-d16av5-ci (d16ads_v5 VMs) and gha-c-aci-ci (C-ACI with 16 cores and 32Gb RAM), and are labeled accordingly in the bencher UI. File: `bencher.yml` 3rd party dependencies: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8d58a8e940bd..a5daeec8a116 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,8 +14,8 @@ concurrency: permissions: read-all jobs: - checks: - name: "Format and License Checks" + build_and_checks: + name: "Format, License, Static Analysis, Documentation, Unit and Partition Tests" runs-on: [ self-hosted, @@ -38,7 +38,7 @@ jobs: with: fetch-depth: 0 - - name: Run CI checks + - name: "Run CI checks" run: | set -ex git config --global --add safe.directory /__w/CCF/CCF @@ -47,33 +47,7 @@ jobs: ./scripts/ci-checks.sh shell: bash - ci-docs-tidy-unit-partitions: - name: "Docs/clang-tidy/Unit/Partitions" - needs: checks - runs-on: - [ - self-hosted, - 1ES.Pool=gha-vmss-d16av5-ci, - "JobId=ci_build_tidy-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}", - ] - container: - image: mcr.microsoft.com/azurelinux/base/core:3.0 - options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE - - steps: - - name: "Checkout dependencies" - shell: bash - run: | - set -ex - gpg --import /etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY - tdnf -y update - tdnf -y install ca-certificates git - - - uses: actions/checkout@v5 - with: - fetch-depth: 0 - - - name: "Install dependencies" + - name: "Install build dependencies" shell: bash run: | set -ex @@ -82,7 +56,6 @@ jobs: - name: "Build Debug with clang-tidy" run: | set -ex - git config --global --add safe.directory /__w/CCF/CCF mkdir build cd build cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DCLANG_TIDY=ON .. @@ -114,7 +87,7 @@ jobs: run: | set -ex cd build - ./tests.sh --output-on-failure -L unit -j$(nproc --all) + ctest --output-on-failure -L unit -j$(($(nproc --all) * 2)) # Note that those are only run on the virtual CI, as they require enough # privileges to configure iptables. ACI-based pools run unprivileged @@ -127,7 +100,6 @@ jobs: build_and_test_virtual: name: "Virtual CI" - needs: checks runs-on: [ self-hosted, @@ -209,7 +181,6 @@ jobs: 1ES.Pool=gha-c-aci-ci, "JobId=aci_snp_milan-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}", ] - needs: checks steps: - uses: actions/checkout@v5 @@ -291,7 +262,6 @@ jobs: 1ES.Pool=gha-aci-genoa, "JobId=aci_snp_genoa-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}", ] - needs: checks steps: - uses: actions/checkout@v5 @@ -354,7 +324,7 @@ jobs: - name: "Upload logs" uses: actions/upload-artifact@v5 with: - name: logs-caci-snp-genoa + name: logs-aci-snp-genoa path: | dmesg.log build/workspace/*/*.config.json diff --git a/CMakeLists.txt b/CMakeLists.txt index d4a0eefdad57..1e0947a08d46 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -848,7 +848,7 @@ if(BUILD_TESTS) add_e2e_test( NAME recovery_test_api_1 PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/recovery.py - ADDITIONAL_ARGS ${ADDITIONAL_RECOVERY_ARGS} --gov-api-version "2024-07-01" + ADDITIONAL_ARGS ${ADDITIONAL_RECOVERY_ARGS} ) add_e2e_test( @@ -900,12 +900,6 @@ if(BUILD_TESTS) ${CMAKE_SOURCE_DIR}/samples/templates ) - add_e2e_test( - NAME committable_suffix_test - PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/committable.py - ADDITIONAL_ARGS --sig-ms-interval 100 - ) - add_e2e_test( NAME commit_latency PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/commit_latency.py @@ -955,7 +949,7 @@ if(BUILD_TESTS) ) add_e2e_test( - NAME governance_test + NAME governance PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/governance.py CONSTITUTION ${CONSTITUTION_ARGS} ADDITIONAL_ARGS --initial-operator-count 1 --jinja-templates-path @@ -1013,6 +1007,7 @@ if(BUILD_TESTS) NAME programmability CONSTITUTION ${RBAC_CONSTITUTION_ARGS} PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/programmability.py + ADDITIONAL_ARGS --js-app-bundle ${CMAKE_SOURCE_DIR}/samples/apps/logging/js ) # This test uses large requests (so too slow for SAN) @@ -1022,12 +1017,6 @@ if(BUILD_TESTS) ) endif() - add_e2e_test( - NAME e2e_redirects - PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/redirects.py - ADDITIONAL_ARGS --js-app-bundle ${CMAKE_SOURCE_DIR}/samples/apps/logging/js - ) - add_e2e_test( NAME e2e_logging_http2 PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/e2e_logging.py @@ -1035,12 +1024,6 @@ if(BUILD_TESTS) --http2 ) - add_e2e_test( - NAME membership_api_1 - PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/membership.py - ADDITIONAL_ARGS --gov-api-version "2024-07-01" - ) - add_e2e_test( NAME partitions PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/partitions_test.py @@ -1073,15 +1056,6 @@ if(BUILD_TESTS) NAME connections PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/connections.py ) - add_e2e_test( - NAME consistency_trace_validation - PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/consistency_trace_validation.py - ) - - add_e2e_test( - NAME fuzz_test PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/fuzzing.py - ) - if(CLIENT_PROTOCOLS_TEST) add_e2e_test( NAME client_protocols @@ -1138,28 +1112,12 @@ if(BUILD_TESTS) ) endif() - if(LONG_TESTS) - set(ROTATION_TEST_ARGS --rotation-retirements 10) - endif() - - add_e2e_test( - NAME rotation_test - PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/rotation.py - LABEL rotation - ADDITIONAL_ARGS ${ROTATION_TEST_ARGS} - ) - set(RECONFIG_TEST_ARGS --ccf-version ${CCF_VERSION}) add_e2e_test( - NAME reconfiguration_test - PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/reconfiguration.py + NAME election_test + PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/election.py ADDITIONAL_ARGS ${RECONFIG_TEST_ARGS} ) - set_property(TEST reconfiguration_test PROPERTY LABELS reconfiguration) - - add_e2e_test( - NAME election_test PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/election.py - ) add_piccolo_test( NAME pi_ls @@ -1225,7 +1183,13 @@ if(BUILD_TESTS) add_e2e_test( NAME e2e_curl PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/e2e_curl.py ) + + add_e2e_test( + NAME consistency_trace_validation + PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/consistency_trace_validation.py + ) endif() + endif() # Generate and install CMake export file for consumers using CMake diff --git a/tests/amd.py b/tests/amd.py new file mode 100644 index 000000000000..c1d99d3bff51 --- /dev/null +++ b/tests/amd.py @@ -0,0 +1,20 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the Apache 2.0 License. +import infra.e2e_args +from infra.runner import ConcurrentRunner +import e2e_operations + + +if __name__ == "__main__": + cr = ConcurrentRunner() + + cr.add( + "platform", + e2e_operations.run_snp_tests, + package="samples/apps/logging/logging", + nodes=infra.e2e_args.min_nodes(cr.args, f=0), + initial_user_count=1, + ledger_chunk_bytes="1B", # Chunk ledger at every signature transaction + ) + + cr.run() diff --git a/tests/committable.py b/tests/committable.py index 575cc3f39432..3d898b893c45 100644 --- a/tests/committable.py +++ b/tests/committable.py @@ -106,9 +106,3 @@ def run(args): # Resume original primary, check that they rejoin correctly, including new transactions primary.resume() network.wait_for_node_commit_sync(timeout=16) - - -if __name__ == "__main__": - args = infra.e2e_args.cli_args() - args.package = "samples/apps/logging/logging" - run(args) diff --git a/tests/connections.py b/tests/connections.py index 666f34df5f25..3eaa8687bf90 100644 --- a/tests/connections.py +++ b/tests/connections.py @@ -20,6 +20,7 @@ from infra.runner import ConcurrentRunner from loguru import logger as LOG +import fuzzing class AllConnectionsCreatedException(Exception): @@ -448,14 +449,21 @@ def try_write(msg_bytes): "robustness", run_node_socket_robustness_tests, package="samples/apps/logging/logging", - nodes=infra.e2e_args.nodes(cr.args, 1), + nodes=infra.e2e_args.min_nodes(cr.args, f=0), ) cr.add( "idletimeout", run_idle_timeout_tests, package="samples/apps/logging/logging", - nodes=infra.e2e_args.nodes(cr.args, 1), + nodes=infra.e2e_args.min_nodes(cr.args, f=0), + ) + + cr.add( + "fuzzing", + fuzzing.run, + package="samples/apps/logging/logging", + nodes=infra.e2e_args.min_nodes(cr.args, f=0), ) # Need to modify args.max_open_sessions _before_ calling e2e_args.nodes for @@ -468,7 +476,7 @@ def try_write(msg_bytes): "caps", run_connection_caps_tests, package="samples/apps/logging/logging", - nodes=infra.e2e_args.nodes(cr.args, 1), + nodes=infra.e2e_args.min_nodes(cr.args, f=0), initial_user_count=1, ) diff --git a/tests/election.py b/tests/election.py index 19760b47ec33..3be12e743c8f 100644 --- a/tests/election.py +++ b/tests/election.py @@ -16,6 +16,9 @@ from infra.runner import ConcurrentRunner from loguru import logger as LOG +import committable +import reconfiguration + # This test starts from a given number of nodes (hosts), commits # a transaction, stops the current primary, waits for an election and repeats # this process until no progress can be made (i.e. no primary can be elected @@ -241,17 +244,54 @@ def run(args): LOG.success("Test ended successfully.") +def rotation(args): + with infra.network.network( + args.nodes, args.binary_dir, args.debug_nodes, pdb=args.pdb + ) as network: + network.start_and_open(args) + rotation_retirements = 2 + + # Replace primary repeatedly and check the network still operates + LOG.info(f"Retiring primary {rotation_retirements} times") + for i in range(rotation_retirements): + LOG.warning(f"Retirement {i}") + reconfiguration.test_add_node(network, args) + reconfiguration.test_retire_primary(network, args) + + if __name__ == "__main__": cr = ConcurrentRunner() args = copy.deepcopy(cr.args) cr.add( - "cft", + "election", run, package="samples/apps/logging/logging", nodes=infra.e2e_args.min_nodes(args, f=1), election_timeout_ms=1000, ) - cr.run(1) + cr.add( + "committable", + committable.run, + package="samples/apps/logging/logging", + nodes=[], + ) + + cr.add( + "rotation", + rotation, + package="samples/apps/logging/logging", + nodes=infra.e2e_args.max_nodes(args, f=0), + initial_member_count=1, + ) + + cr.add( + "reconfiguration", + reconfiguration.run_all, + package="samples/apps/logging/logging", + nodes=infra.e2e_args.min_nodes(cr.args, f=1), + ) + + cr.run() diff --git a/tests/fuzzing.py b/tests/fuzzing.py index 05249cded6ce..592f00eb02f9 100644 --- a/tests/fuzzing.py +++ b/tests/fuzzing.py @@ -214,11 +214,3 @@ def run(args): network.ignore_error_pattern_on_shutdown("Unknown frontend msg type") fuzz_node_to_node(network, args) - - -if __name__ == "__main__": - args = infra.e2e_args.cli_args() - args.package = "samples/apps/logging/logging" - - args.nodes = infra.e2e_args.min_nodes(args, f=0) - run(args) diff --git a/tests/governance.py b/tests/governance.py index 9c72aad9b709..c90537c3b28c 100644 --- a/tests/governance.py +++ b/tests/governance.py @@ -24,6 +24,8 @@ import governance_api from hashlib import md5 import random +import memberclient +import membership from loguru import logger as LOG @@ -808,4 +810,19 @@ def add(parser): nodes=infra.e2e_args.max_nodes(cr.args, f=0), ) - cr.run(2) + cr.add( + "membership", + membership.run, + package="samples/apps/logging/logging", + nodes=infra.e2e_args.max_nodes(cr.args, f=0), + initial_user_count=0, + ) + + cr.add( + "member_client", + memberclient.run, + package="samples/apps/logging/logging", + nodes=infra.e2e_args.max_nodes(cr.args, f=1), + ) + + cr.run() diff --git a/tests/membership.py b/tests/membership.py index 25e85bdc7782..8b6baee6b1d1 100644 --- a/tests/membership.py +++ b/tests/membership.py @@ -4,8 +4,6 @@ import infra.network import infra.consortium import random -from infra.runner import ConcurrentRunner -import memberclient import infra.proposal import infra.member @@ -637,24 +635,3 @@ def run(args): service_startups(args) recovery_shares_scenario(args) recovery_shares_with_owners_scenario(args) - - -if __name__ == "__main__": - cr = ConcurrentRunner() - - cr.add( - "membership", - run, - package="samples/apps/logging/logging", - nodes=infra.e2e_args.max_nodes(cr.args, f=0), - initial_user_count=0, - ) - - cr.add( - "member_client", - memberclient.run, - package="samples/apps/logging/logging", - nodes=infra.e2e_args.max_nodes(cr.args, f=1), - ) - - cr.run() diff --git a/tests/programmability.py b/tests/programmability.py index 415389944191..e57a6f680743 100644 --- a/tests/programmability.py +++ b/tests/programmability.py @@ -14,6 +14,7 @@ import infra.clients import npm_tests +import redirects from loguru import logger as LOG @@ -634,4 +635,34 @@ def run(args): initial_member_count=1, ) + cr.add( + "cpp_redirects_role", + redirects.run_redirect_tests_role, + package="samples/apps/logging/logging", + js_app_bundle=None, + nodes=infra.e2e_args.min_nodes(cr.args, f=1), + ) + + cr.add( + "cpp_redirects_static", + redirects.run_redirect_tests_static, + package="samples/apps/logging/logging", + js_app_bundle=None, + nodes=infra.e2e_args.min_nodes(cr.args, f=0), + ) + + cr.add( + "js_redirects_role", + redirects.run_redirect_tests_role, + package="js_generic", + nodes=infra.e2e_args.min_nodes(cr.args, f=1), + ) + + cr.add( + "js_redirects_static", + redirects.run_redirect_tests_static, + package="js_generic", + nodes=infra.e2e_args.min_nodes(cr.args, f=0), + ) + cr.run() diff --git a/tests/reconfiguration.py b/tests/reconfiguration.py index 85f9bab9e601..41a59ea6b546 100644 --- a/tests/reconfiguration.py +++ b/tests/reconfiguration.py @@ -20,7 +20,6 @@ from infra.checker import check_can_progress from governance_history import check_signatures from infra.snp import SNP_SUPPORT -from infra.runner import ConcurrentRunner import http import random @@ -922,15 +921,3 @@ def run_join_old_snapshot(args): fetch_recent_snapshot=True, timeout=3, ) - - -if __name__ == "__main__": - cr = ConcurrentRunner() - cr.add( - "reconfiguration", - run_all, - package="samples/apps/logging/logging", - nodes=infra.e2e_args.min_nodes(cr.args, f=1), - ) - - cr.run() diff --git a/tests/rotation.py b/tests/rotation.py deleted file mode 100644 index dda2e72e4f0c..000000000000 --- a/tests/rotation.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the Apache 2.0 License. -import infra.e2e_args -import infra.network -import infra.proc -import reconfiguration - -from loguru import logger as LOG - - -def run(args): - with infra.network.network( - args.nodes, args.binary_dir, args.debug_nodes, pdb=args.pdb - ) as network: - network.start_and_open(args) - - # Replace primary repeatedly and check the network still operates - LOG.info(f"Retiring primary {args.rotation_retirements} times") - for i in range(args.rotation_retirements): - LOG.warning(f"Retirement {i}") - reconfiguration.test_add_node(network, args) - reconfiguration.test_retire_primary(network, args) - - -if __name__ == "__main__": - - def add(parser): - parser.add_argument( - "--rotation-retirements", - help="Number of times to retire the primary", - type=int, - default=2, - ) - - args = infra.e2e_args.cli_args(add=add) - args.package = "samples/apps/logging/logging" - args.nodes = infra.e2e_args.max_nodes(args, f=0) - args.initial_member_count = 1 - run(args) diff --git a/tests/schema.py b/tests/schema.py index 8b969c09edf0..9d5809ef8721 100644 --- a/tests/schema.py +++ b/tests/schema.py @@ -4,6 +4,7 @@ import json import http import infra.network +import infra.platform_detection import infra.proc import infra.e2e_args import infra.checker