[release/6.x] Backport pre-vote capability (#7374, #7375, #7404, #7409, #7438, #7445, #7458) (#7436)

Author: cjen1-msft

.github/workflows/ci-verification.yml

@@ -39,7 +39,7 @@ jobs:
       - name: Install TLC dependencies
         run: |
           tdnf install -y jre wget
-          python3 tla/install_deps.py --skip-apt-packages
+          python3 tla/install_deps.py
 
       - run: cd tla && ./tlc.py mc consistency/MCSingleNode.tla
       - run: cd tla && ./tlc.py mc consistency/MCSingleNodeReads.tla
@@ -68,7 +68,7 @@ jobs:
       - name: Install TLC dependencies
         run: |
           sudo apt update
-          sudo apt install -y default-jre
+          sudo apt install -y default-jre wget
           python3 install_deps.py
 
       - run: ./tlc_debug.sh --config consistency/MCSingleNodeCommitReachability.cfg mc consistency/MCSingleNodeReads.tla
@@ -88,7 +88,7 @@ jobs:
       - name: Install TLC dependencies
         run: |
           sudo apt update
-          sudo apt install -y default-jre
+          sudo apt install -y default-jre wget
           python3 install_deps.py
 
       - run: ./tlc.py sim --num 500 --depth 50 consistency/MultiNodeReads.tla
@@ -121,7 +121,7 @@ jobs:
       - name: Install TLC dependencies
         run: |
           tdnf install -y jre wget
-          python3 tla/install_deps.py --skip-apt-packages
+          python3 tla/install_deps.py
 
       - run: cd tla && ./tlc.py mc consensus/MCabs.tla
       - run: cd tla && ./tlc.py --trace-name 1C2N mc --term-count 2 --request-count 2 --raft-configs 1C2N consensus/MCccfraft.tla
@@ -148,7 +148,7 @@ jobs:
       - name: Install TLC dependencies
         run: |
           sudo apt update
-          sudo apt install -y default-jre
+          sudo apt install -y default-jre wget
           python3 install_deps.py
 
       - run: ./tlc.py sim consensus/SIMccfraft.tla
@@ -181,22 +181,16 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Install TLC dependencies
-        run: |
-          tdnf install -y jre wget
-          python3 tla/install_deps.py --skip-apt-packages
-
       - name: "Install dependencies"
         shell: bash
         run: |
           set -ex
           ./scripts/setup-ci.sh
 
-          # Parallel
-          wget https://ftp.gnu.org/gnu/parallel/parallel-latest.tar.bz2
-          tar -xjf parallel-latest.tar.bz2
-          cd $(ls | grep 'parallel' | grep -v 'tar' | grep -v 'rpm')
-          ./configure && make && make install
+      - name: Install TLC dependencies
+        run: |
+          tdnf install -y jre wget
+          python3 tla/install_deps.py --tdnf-extended
 
       - name: "Build"
         run: |

.github/workflows/long-verification.yml

@@ -37,7 +37,7 @@ jobs:
       - name: Install TLC dependencies
         run: |
           tdnf install -y jre wget
-          python3 tla/install_deps.py --skip-apt-packages
+          python3 tla/install_deps.py
 
       - run: cd tla && ./tlc.py --trace-name 2C2N mc --term-count 2 --request-count 0 --raft-configs 2C2N --disable-check-quorum consensus/MCccfraft.tla
 
@@ -70,7 +70,7 @@ jobs:
       - name: Install TLC dependencies
         run: |
           tdnf install -y jre wget
-          python3 tla/install_deps.py --skip-apt-packages
+          python3 tla/install_deps.py
 
       - run: cd tla && ./tlc.py --trace-name 3C2N mc --term-count 2 --request-count 0 --raft-configs 3C2N --disable-check-quorum consensus/MCccfraft.tla
 
@@ -95,7 +95,7 @@ jobs:
       - uses: actions/checkout@v4
       - run: |
           sudo apt update
-          sudo apt install -y default-jre
+          sudo apt install -y default-jre wget
           python3 install_deps.py
 
       - run: ./tlc.py sim --max-seconds 3000 --depth 500 consensus/SIMccfraft.tla

CHANGELOG.md

@@ -5,6 +5,18 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [6.0.17]
+
+[6.0.17]: https://github.com/microsoft/CCF/releases/tag/6.0.17
+
+### Added
+
+- Support for PreVote optimisation. Nodes understand and are able to respond to PreVote messages, but will not become pre-vote candidates themselves. (#7419, #7445)
+
+### Fixed
+
+- CheckQuorum now requires a quorum in every configuration (#7375)
+
 ## [6.0.16]
 
 [6.0.16]: https://github.com/microsoft/CCF/releases/tag/6.0.16

doc/architecture/consensus/index.rst

@@ -25,6 +25,7 @@ Supported extensions include:
 
 - "CheckQuorum": the primary node automatically steps down, in the same view, if it does not hear back (via ``AppendEntriesResponse`` messages) from a majority of backups within a ``consensus.election_timeout`` period. This prevents an isolated primary node from still processing client write requests without being able to commit them.
 - "NoTimeoutRetirement": a primary node that completes its retirement sends a ProposeRequestVote message to the most up-to-date node in the new configuration, causing that node to run for election without waiting for time out.
+- "PreVote": followers must first request a pre-vote before starting a new election. This prevents followers from starting elections (and increasing the term) when they are isolated from the rest of the network.
 
 Replica State Machine
 ---------------------
@@ -206,3 +207,75 @@ Until the very last phase (``RetiredCommitted``) is reached, a retiring leader w
 
 Note that because the rollback triggered when a node becomes aware of a new term never preserves unsigned transactions,
 and because RCI is always the first signature after RI, RI and RCI are always both rolled back if RCI itself is rolled back.
+
+PreVote Extensions
+~~~~~~~~~~~~~~~~~~
+
+If a node's ``RequestVote`` requests are able to reach the cluster, but it is unable to hear the ``AppendEntries`` messages from the current leader (for example, due to network partitioning), it may start new elections, incrementing its term, which deposes the leader and disrupts the cluster.
+
+To mitigate this, the PreVote extension requires that a follower first become ``PreVoteCandidate`` and receive a quorum of speculative pre-votes, proving that they could be elected using the standard Raft election conditions, before becoming ``Candidate`` and potentially disrupting the cluster.
+
+More specifically, when a follower's election timeout elapses, it becomes a ``PreVoteCandidate`` for the current term  and sends out ``RequestVote`` messages with the ``electionType`` set to ``ElectionType::PreVote``.
+If the ``PreVoteCandidate`` hears from a current leader, or a new leader, it reverts back to being a ``Follower``.
+Nodes receive this pre-vote request, and respond positively if node would have voted for the ``PreVoteCandidate``'s ledger during an election, (ie. if the ``PreVoteCandidate``'s ledger is at least as up to date as the receiver's ledger).
+If the ``PreVoteCandidate`` receives a quorum of positive pre-vote responses, it then becomes a ``Candidate``, increments its term, sends a ``RequestVote`` message with ``election_type`` set to ``ElectionType::RegularVote`` and the election proceeds as normal from here.
+
+.. mermaid::
+
+    sequenceDiagram
+        participant Node 0
+        participant Node 1
+        participant Node 2
+
+        Note over Node 0: Leader for term 2
+
+        Note over Node 1: PreVoteCandidate in term 2
+        Node 1 ->> Node 2: RequestVote(ElectionType::PreVote, term=2)
+
+        Note right of Node 2: No changes to Node 2's state
+        Node 2 ->> Node 1: RequestVoteResponse(ElectionType::PreVote, term=2, granted=true)
+
+        Note over Node 1: Candidate in term 3
+        Node 1 ->> Node 2: RequestVote(ElectionType::RegularVote, term=3) 
+
+        Note right of Node 2: Updates term to 3 and votes for Node 1
+        Node 2 ->> Node 1: RequestVoteResponse(ElectionType::RegularVote, term=3, granted=true)
+
+        Note over Node 1: Leader for term 3
+
+The only state update in response to a pre-vote message is that if the node's term is older than the pre-vote messages's it will update it.
+This allows the pre-vote request to inform lagging nodes that a more recent term had a node succeed in its pre-vote, becoming a Candidate or a Leader.
+This can be viewed as piggybacking the term information from that previous Candidate or Leader, with the pre-vote request to the lagging node.
+
+.. mermaid::
+
+    sequenceDiagram
+        participant Node 0
+        participant Node 1
+        participant Node 2
+
+        Note over Node 0: Leader for term 2
+        Note over Node 1: Follower in term 2
+        Note over Node 2: Lagging Follower in term 1
+
+        Note over Node 1: PreVoteCandidate in term 2
+        Node 1 ->> Node 2: RequestVote(ElectionType::PreVote, term=2)
+
+        Note right of Node 2: Updates term to 2
+        Node 2 ->> Node 1: RequestVoteResponse(ElectionType::PreVote, term=2, granted=true)
+
+        Note over Node 1: Candidate in term 3
+        Node 1 ->> Node 2: RequestVote(ElectionType::RegularVote, term=3) 
+
+        Note right of Node 2: Updates to term 3 and votes for Node 1
+        Node 2 ->> Node 1: RequestVoteResponse(ElectionType::RegularVote, term=3, granted=true)
+
+        Note over Node 1: Leader for term 3
+
+Migration to PreVote
+~~~~~~~~~~~~~~~~~~~~
+
+Supposing we have a cluster of nodes which currently do not support PreVote, we must first migrate the cluster to support PreVote before we can enable it, as the nodes that do not support PreVote will respond incorrectly to PreVote requests.
+
+To enable PreVote safely, we must first migrate the cluster to support PreVote messages, and then enable PreVote.
+During the migration to enable PreVote, the pre-vote candidates will be less likely to be elected leader, as the other followers may preempt the pre-vote candidate and become candidates themselves.

doc/architecture/raft_tla.rst

@@ -44,4 +44,7 @@ It is possible to produce fresh traces quickly from the driver by running the ``
 
 Calling the trace validation on, for example, the ``append`` scenario can then be done with ``./tlc.py --driver-trace ../build/append.ndjson consensus/Traceccfraft.tla``.
 
+Generating a trace of a scenario and validating it in one go can be done with ``./tlc.py --workers 1 tv --scenario ../tests/raft_scenarios/append consensus/Traceccfraft.tla``.
+This runs the raft_driver on the scenario, cleans the trace and then validates it against the TLA+ specification.
+
 CCF also provides a command line trace visualizer to aid debugging, for example, the ``append`` scenario can be visualized with ``python ../tests/trace_viz.py ../build/append.ndjson``. 

doc/schemas/node_openapi.json

@@ -464,6 +464,7 @@
           "None",
           "Leader",
           "Follower",
+          "PreVoteCandidate",
           "Candidate"
         ],
         "type": "string"
@@ -904,7 +905,7 @@
   "info": {
     "description": "This API provides public, uncredentialed access to service and node state.",
     "title": "CCF Public Node API",
-    "version": "4.13.0"
+    "version": "4.14.0"
   },
   "openapi": "3.0.0",
   "paths": {

python/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ccf"
-version = "6.0.16"
+version = "6.0.17"
 authors = [
   { name="CCF Team", email="[email protected]" },
 ]

src/consensus/aft/impl/state.h

@@ -150,7 +150,10 @@ namespace aft
 
   struct State
   {
-    State(const ccf::NodeId& node_id_) : node_id(node_id_) {}
+    State(const ccf::NodeId& node_id_, bool pre_vote_enabled_ = false) :
+      node_id(node_id_),
+      pre_vote_enabled(pre_vote_enabled_)
+    {}
     State() = default;
 
     ccf::pal::Mutex lock;
@@ -188,6 +191,8 @@ namespace aft
     // Index at which this node observes its retired_committed, only set when
     // that index itself is committed
     std::optional<ccf::SeqNo> retired_committed_idx = std::nullopt;
+
+    bool pre_vote_enabled = false;
   };
   DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(State);
   DECLARE_JSON_REQUIRED_FIELDS(
@@ -197,7 +202,8 @@ namespace aft
     last_idx,
     commit_idx,
     leadership_state,
-    membership_state);
+    membership_state,
+    pre_vote_enabled);
   DECLARE_JSON_OPTIONAL_FIELDS(
     State,
     retirement_phase,