[Ignore] Workflow experiments

.github/workflows/README.md

@@ -7,7 +7,7 @@ Documents the various GitHub Actions workflows, the role they fulfill and 3rd pa
 Builds and runs CCF performance tests, both end to end and micro-benchmarks. Results are posted to bencher.dev, and [plotted to make regressions obvious](https://bencher.dev/console/projects/ccf/plots).
 Triggered on every commit on `main`, but not on PR builds because the setup required to build from forks is complex and fragile in terms of security, and the increase in pool usage would be substantial.
 
-Tests are run and published on two different testbeds for comparison: gha-vmss-d16av5-ci (d16av5 VMs) and gha-c-aci-ci (C-ACI with 16 cores and 32Gb RAM), and are labeled accordingly in the bencher UI.
+Tests are run and published on two different testbeds for comparison: gha-vmss-d16av5-ci (d16ads_v5 VMs) and gha-c-aci-ci (C-ACI with 16 cores and 32Gb RAM), and are labeled accordingly in the bencher UI.
 
 File: `bencher.yml`
 3rd party dependencies:

.github/workflows/ci.yml

@@ -14,8 +14,8 @@ concurrency:
 permissions: read-all
 
 jobs:
-  checks:
-    name: "Format and License Checks"
+  build_and_checks:
+    name: "Format, License, Static Analysis, Documentation, Unit and Partition Tests"
     runs-on:
       [
         self-hosted,
@@ -38,7 +38,7 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Run CI checks
+      - name: "Run CI checks"
         run: |
           set -ex
           git config --global --add safe.directory /__w/CCF/CCF
@@ -47,33 +47,7 @@ jobs:
           ./scripts/ci-checks.sh
         shell: bash
 
-  ci-docs-tidy-unit-partitions:
-    name: "Docs/clang-tidy/Unit/Partitions"
-    needs: checks
-    runs-on:
-      [
-        self-hosted,
-        1ES.Pool=gha-vmss-d16av5-ci,
-        "JobId=ci_build_tidy-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}",
-      ]
-    container:
-      image: mcr.microsoft.com/azurelinux/base/core:3.0
-      options: --user root --publish-all --cap-add NET_ADMIN --cap-add NET_RAW --cap-add SYS_PTRACE
-
-    steps:
-      - name: "Checkout dependencies"
-        shell: bash
-        run: |
-          set -ex
-          gpg --import /etc/pki/rpm-gpg/MICROSOFT-RPM-GPG-KEY
-          tdnf -y update
-          tdnf -y install ca-certificates git
-
-      - uses: actions/checkout@v5
-        with:
-          fetch-depth: 0
-
-      - name: "Install dependencies"
+      - name: "Install build dependencies"
         shell: bash
         run: |
           set -ex
@@ -82,7 +56,6 @@ jobs:
       - name: "Build Debug with clang-tidy"
         run: |
           set -ex
-          git config --global --add safe.directory /__w/CCF/CCF
           mkdir build
           cd build
           cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -DCLANG_TIDY=ON ..
@@ -114,7 +87,7 @@ jobs:
         run: |
           set -ex
           cd build
-          ./tests.sh --output-on-failure -L unit -j$(nproc --all)
+          ctest --output-on-failure -L unit -j$(($(nproc --all) * 2))
 
         # Note that those are only run on the virtual CI, as they require enough
         # privileges to configure iptables. ACI-based pools run unprivileged
@@ -127,7 +100,6 @@ jobs:
 
   build_and_test_virtual:
     name: "Virtual CI"
-    needs: checks
     runs-on:
       [
         self-hosted,
@@ -209,7 +181,6 @@ jobs:
         1ES.Pool=gha-c-aci-ci,
         "JobId=aci_snp_milan-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}",
       ]
-    needs: checks
 
     steps:
       - uses: actions/checkout@v5
@@ -291,7 +262,6 @@ jobs:
         1ES.Pool=gha-aci-genoa,
         "JobId=aci_snp_genoa-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}",
       ]
-    needs: checks
 
     steps:
       - uses: actions/checkout@v5
@@ -354,7 +324,7 @@ jobs:
       - name: "Upload logs"
         uses: actions/upload-artifact@v5
         with:
-          name: logs-caci-snp-genoa
+          name: logs-aci-snp-genoa
           path: |
             dmesg.log
             build/workspace/*/*.config.json

CMakeLists.txt

@@ -848,7 +848,7 @@ if(BUILD_TESTS)
   add_e2e_test(
     NAME recovery_test_api_1
     PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/recovery.py
-    ADDITIONAL_ARGS ${ADDITIONAL_RECOVERY_ARGS} --gov-api-version "2024-07-01"
+    ADDITIONAL_ARGS ${ADDITIONAL_RECOVERY_ARGS}
   )
 
   add_e2e_test(
@@ -900,12 +900,6 @@ if(BUILD_TESTS)
       ${CMAKE_SOURCE_DIR}/samples/templates
   )
 
-  add_e2e_test(
-    NAME committable_suffix_test
-    PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/committable.py
-    ADDITIONAL_ARGS --sig-ms-interval 100
-  )
-
   add_e2e_test(
     NAME commit_latency
     PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/commit_latency.py
@@ -955,7 +949,7 @@ if(BUILD_TESTS)
   )
 
   add_e2e_test(
-    NAME governance_test
+    NAME governance
     PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/governance.py
     CONSTITUTION ${CONSTITUTION_ARGS}
     ADDITIONAL_ARGS --initial-operator-count 1 --jinja-templates-path
@@ -1013,6 +1007,7 @@ if(BUILD_TESTS)
     NAME programmability
     CONSTITUTION ${RBAC_CONSTITUTION_ARGS}
     PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/programmability.py
+    ADDITIONAL_ARGS --js-app-bundle ${CMAKE_SOURCE_DIR}/samples/apps/logging/js
   )
 
   # This test uses large requests (so too slow for SAN)
@@ -1022,25 +1017,13 @@ if(BUILD_TESTS)
     )
   endif()
 
-  add_e2e_test(
-    NAME e2e_redirects
-    PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/redirects.py
-    ADDITIONAL_ARGS --js-app-bundle ${CMAKE_SOURCE_DIR}/samples/apps/logging/js
-  )
-
   add_e2e_test(
     NAME e2e_logging_http2
     PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/e2e_logging.py
     ADDITIONAL_ARGS --js-app-bundle ${CMAKE_SOURCE_DIR}/samples/apps/logging/js
                     --http2
   )
 
-  add_e2e_test(
-    NAME membership_api_1
-    PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/membership.py
-    ADDITIONAL_ARGS --gov-api-version "2024-07-01"
-  )
-
   add_e2e_test(
     NAME partitions
     PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/partitions_test.py
@@ -1073,15 +1056,6 @@ if(BUILD_TESTS)
     NAME connections PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/connections.py
   )
 
-  add_e2e_test(
-    NAME consistency_trace_validation
-    PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/consistency_trace_validation.py
-  )
-
-  add_e2e_test(
-    NAME fuzz_test PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/fuzzing.py
-  )
-
   if(CLIENT_PROTOCOLS_TEST)
     add_e2e_test(
       NAME client_protocols
@@ -1138,28 +1112,12 @@ if(BUILD_TESTS)
     )
   endif()
 
-  if(LONG_TESTS)
-    set(ROTATION_TEST_ARGS --rotation-retirements 10)
-  endif()
-
-  add_e2e_test(
-    NAME rotation_test
-    PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/rotation.py
-    LABEL rotation
-    ADDITIONAL_ARGS ${ROTATION_TEST_ARGS}
-  )
-
   set(RECONFIG_TEST_ARGS --ccf-version ${CCF_VERSION})
   add_e2e_test(
-    NAME reconfiguration_test
-    PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/reconfiguration.py
+    NAME election_test
+    PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/election.py
     ADDITIONAL_ARGS ${RECONFIG_TEST_ARGS}
   )
-  set_property(TEST reconfiguration_test PROPERTY LABELS reconfiguration)
-
-  add_e2e_test(
-    NAME election_test PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/election.py
-  )
 
   add_piccolo_test(
     NAME pi_ls
@@ -1225,7 +1183,13 @@ if(BUILD_TESTS)
     add_e2e_test(
       NAME e2e_curl PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/e2e_curl.py
     )
+
+    add_e2e_test(
+      NAME consistency_trace_validation
+      PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/consistency_trace_validation.py
+    )
   endif()
+
 endif()
 
 # Generate and install CMake export file for consumers using CMake

tests/amd.py

@@ -0,0 +1,20 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the Apache 2.0 License.
+import infra.e2e_args
+from infra.runner import ConcurrentRunner
+import e2e_operations
+
+
+if __name__ == "__main__":
+    cr = ConcurrentRunner()
+
+    cr.add(
+        "platform",
+        e2e_operations.run_snp_tests,
+        package="samples/apps/logging/logging",
+        nodes=infra.e2e_args.min_nodes(cr.args, f=0),
+        initial_user_count=1,
+        ledger_chunk_bytes="1B",  # Chunk ledger at every signature transaction
+    )
+
+    cr.run()

tests/committable.py

@@ -106,9 +106,3 @@ def run(args):
         # Resume original primary, check that they rejoin correctly, including new transactions
         primary.resume()
         network.wait_for_node_commit_sync(timeout=16)
-
-
-if __name__ == "__main__":
-    args = infra.e2e_args.cli_args()
-    args.package = "samples/apps/logging/logging"
-    run(args)

tests/connections.py

@@ -20,6 +20,7 @@
 from infra.runner import ConcurrentRunner
 
 from loguru import logger as LOG
+import fuzzing
 
 
 class AllConnectionsCreatedException(Exception):
@@ -448,14 +449,21 @@ def try_write(msg_bytes):
         "robustness",
         run_node_socket_robustness_tests,
         package="samples/apps/logging/logging",
-        nodes=infra.e2e_args.nodes(cr.args, 1),
+        nodes=infra.e2e_args.min_nodes(cr.args, f=0),
     )
 
     cr.add(
         "idletimeout",
         run_idle_timeout_tests,
         package="samples/apps/logging/logging",
-        nodes=infra.e2e_args.nodes(cr.args, 1),
+        nodes=infra.e2e_args.min_nodes(cr.args, f=0),
+    )
+
+    cr.add(
+        "fuzzing",
+        fuzzing.run,
+        package="samples/apps/logging/logging",
+        nodes=infra.e2e_args.min_nodes(cr.args, f=0),
     )
 
     # Need to modify args.max_open_sessions _before_ calling e2e_args.nodes for
@@ -468,7 +476,7 @@ def try_write(msg_bytes):
         "caps",
         run_connection_caps_tests,
         package="samples/apps/logging/logging",
-        nodes=infra.e2e_args.nodes(cr.args, 1),
+        nodes=infra.e2e_args.min_nodes(cr.args, f=0),
         initial_user_count=1,
     )
 

tests/election.py

@@ -16,6 +16,9 @@
 from infra.runner import ConcurrentRunner
 from loguru import logger as LOG
 
+import committable
+import reconfiguration
+
 # This test starts from a given number of nodes (hosts), commits
 # a transaction, stops the current primary, waits for an election and repeats
 # this process until no progress can be made (i.e. no primary can be elected
@@ -241,17 +244,54 @@ def run(args):
             LOG.success("Test ended successfully.")
 
 
+def rotation(args):
+    with infra.network.network(
+        args.nodes, args.binary_dir, args.debug_nodes, pdb=args.pdb
+    ) as network:
+        network.start_and_open(args)
+        rotation_retirements = 2
+
+        # Replace primary repeatedly and check the network still operates
+        LOG.info(f"Retiring primary {rotation_retirements} times")
+        for i in range(rotation_retirements):
+            LOG.warning(f"Retirement {i}")
+            reconfiguration.test_add_node(network, args)
+            reconfiguration.test_retire_primary(network, args)
+
+
 if __name__ == "__main__":
     cr = ConcurrentRunner()
 
     args = copy.deepcopy(cr.args)
 
     cr.add(
-        "cft",
+        "election",
         run,
         package="samples/apps/logging/logging",
         nodes=infra.e2e_args.min_nodes(args, f=1),
         election_timeout_ms=1000,
     )
 
-    cr.run(1)
+    cr.add(
+        "committable",
+        committable.run,
+        package="samples/apps/logging/logging",
+        nodes=[],
+    )
+
+    cr.add(
+        "rotation",
+        rotation,
+        package="samples/apps/logging/logging",
+        nodes=infra.e2e_args.max_nodes(args, f=0),
+        initial_member_count=1,
+    )
+
+    cr.add(
+        "reconfiguration",
+        reconfiguration.run_all,
+        package="samples/apps/logging/logging",
+        nodes=infra.e2e_args.min_nodes(cr.args, f=1),
+    )
+
+    cr.run()

tests/fuzzing.py

@@ -214,11 +214,3 @@ def run(args):
         network.ignore_error_pattern_on_shutdown("Unknown frontend msg type")
 
         fuzz_node_to_node(network, args)
-
-
-if __name__ == "__main__":
-    args = infra.e2e_args.cli_args()
-    args.package = "samples/apps/logging/logging"
-
-    args.nodes = infra.e2e_args.min_nodes(args, f=0)
-    run(args)