Add blaster test for endpoint performance evaluation

.github/workflows/ec2-leg.yml

@@ -5,6 +5,7 @@ name: Perf-eval EC2 leg (ephemeral)
 # Does not post to the PR -- the coordinator owns human-facing output.
 # Box bootstrap: perf-eval/bootstrap-common.sh + the leg script (inputs.leg_script);
 # runner-side polling: perf-eval/gather.
+# Legs can chain via keep_instance + adopt_run_label + the instance_ip output.
 
 on:
   # workflow_call is the only trigger: a leg runs through the coordinator.
@@ -62,6 +63,20 @@ on:
         required: false
         type: string
         default: ''
+      keep_instance:
+        description: 'Leave a passing leg''s box running for a chained successor; the box then owns its shutdown. A failing leg terminates as usual.'
+        required: false
+        type: boolean
+        default: false
+      adopt_run_label:
+        description: 'run_label of a prior keep_instance leg whose box this leg also terminates when it passes'
+        required: false
+        type: string
+        default: ''
+    outputs:
+      instance_ip:
+        description: 'Private IPv4 of the launched box, for chained legs'
+        value: ${{ jobs.ec2-leg.outputs.instance_ip }}
     secrets:
       AWS_GHA_ROLE_ARN:
         description: 'OIDC role ARN to assume for EC2 + S3 access'
@@ -75,6 +90,8 @@ permissions:
 jobs:
   ec2-leg:
     name: Launch + await ephemeral perf-eval box
+    outputs:
+      instance_ip: ${{ steps.launch.outputs.instance_ip }}
     # Non-cancelling group keyed on the test commit SHA + leg. An instance can run
     # until the box’s self-terminate ceiling, so new runs queue.
     concurrency:
@@ -149,7 +166,7 @@ jobs:
             echo '#!/usr/bin/env bash'
             echo "export TARGET_SHA=${{ steps.resolve.outputs.sha }} RUN_ID=${{ github.run_id }}-${{ github.run_attempt }}"
             echo "export BUCKET=$BUCKET RESULT_KEY=$RESULT_KEY"
-            echo "export SELF_TERMINATE_MINUTES=$SELF_TERMINATE_MINUTES"
+            echo "export SELF_TERMINATE_MINUTES=$SELF_TERMINATE_MINUTES BUDGET_MINUTES=${{ inputs.budget_minutes }}"
             if [ -n "$EXTRA_ENV" ]; then printf '%s\n' "$EXTRA_ENV"; fi
             cat "$PERF_EVAL_DIR/bootstrap-common.sh"
             cat "$LEG_SCRIPT"
@@ -190,6 +207,7 @@ jobs:
 
           INSTANCE_ID=$(printf '%s' "$RUN_INSTANCES_JSON" | jq -r '.Instances[0].InstanceId')
           echo "instance_id=$INSTANCE_ID" >> "$GITHUB_OUTPUT"
+          echo "instance_ip=$(printf '%s' "$RUN_INSTANCES_JSON" | jq -r '.Instances[0].PrivateIpAddress')" >> "$GITHUB_OUTPUT"
 
       - name: Wait for SSM agent to register
         env:
@@ -257,8 +275,29 @@ jobs:
             exit 1
           fi
 
+      # The adopted box is tag-discovered. Adoption fires only on a pass. Note that
+      # a failed leg leaves the box to its serve ceiling so a re-run can reuse it.
       - name: Terminate instance
-        if: always() && steps.launch.outputs.instance_id != ''
+        if: always()
+        env:
+          INSTANCE_ID: ${{ steps.launch.outputs.instance_id }}
+          KEEP_INSTANCE: ${{ inputs.keep_instance }}
+          LEG_PASSED: ${{ steps.results.outputs.passed }}
+          ADOPT_RUN_LABEL: ${{ inputs.adopt_run_label }}
         run: |
-          aws ec2 terminate-instances \
-            --instance-ids ${{ steps.launch.outputs.instance_id }} || true
+          if [ "$KEEP_INSTANCE" = "true" ] && [ "$LEG_PASSED" = "true" ]; then
+            echo "keep_instance: leaving $INSTANCE_ID running for the chained leg"
+          elif [ -n "$INSTANCE_ID" ]; then
+            aws ec2 terminate-instances --instance-ids "$INSTANCE_ID" || true
+          fi
+          if [ -n "$ADOPT_RUN_LABEL" ] && [ "$LEG_PASSED" = "true" ]; then
+            ADOPTED=$(aws ec2 describe-instances \
+              --filters "Name=tag:run-id,Values=${{ github.run_id }}" \
+                        "Name=tag:Name,Values=perf-eval-$ADOPT_RUN_LABEL" \
+                        "Name=instance-state-name,Values=pending,running" \
+              --query 'Reservations[].Instances[].InstanceId' --output text || true)
+            if [ -n "$ADOPTED" ]; then
+              echo "terminating adopted box(es): $ADOPTED"
+              aws ec2 terminate-instances --instance-ids $ADOPTED || true
+            fi
+          fi

.github/workflows/load-test-coordinator.yml

@@ -11,7 +11,7 @@ defaults:
 
 on:
   push:
-    branches: [release/**]
+    branches: [release/**, blaster-test]
   workflow_dispatch:
     inputs:
       target_ref:
@@ -46,6 +46,7 @@ jobs:
       target_sha: ${{ steps.ctx.outputs.target_sha }}
       pr_number: ${{ steps.ctx.outputs.pr_number }}
       legs: ${{ steps.ctx.outputs.legs }}
+      matrix_legs: ${{ steps.ctx.outputs.matrix_legs }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -62,24 +63,29 @@ jobs:
             PR="$(gh pr list --repo "${{ github.repository }}" --state open \
                     --head "$REF" --json number --jq '.[0].number // ""' 2>/dev/null || true)"
           fi
+          # One roster (comment order) drives both the report job and the
+          # matrix fan-out: entries carrying a script ride the matrix; the
+          # chained backfill + blaster pair runs as explicit jobs below.
           LEGS='[
             {"label":"Apply-load ingestion","run_label":"apply-load","script":"cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/ingest-load-test/run-load-test.sh","budget":225},
-            {"label":"Backfill ingestion","run_label":"backfill","script":"cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/backfill-test/run-backfill.sh","budget":345}
+            {"label":"Backfill ingestion","run_label":"backfill"},
+            {"label":"Endpoint load test","run_label":"blaster"}
           ]'
           {
             echo "target_sha=$SHA"
             echo "pr_number=$PR"
             echo "legs=$(jq -c . <<<"$LEGS")"
+            echo "matrix_legs=$(jq -c 'map(select(.script))' <<<"$LEGS")"
           } >> "$GITHUB_OUTPUT"
 
-  # The fan-out: one matrix entry per leg.
+  # The fan-out: one matrix entry per independent leg.
   load-test:
     name: ${{ matrix.leg.label }}
     needs: plan
     strategy:
       fail-fast: false # stops one leg's failure from cancelling other legs
       matrix:
-        leg: ${{ fromJSON(needs.plan.outputs.legs) }}
+        leg: ${{ fromJSON(needs.plan.outputs.matrix_legs) }}
     uses: ./.github/workflows/ec2-leg.yml
     secrets: inherit
     with:
@@ -88,9 +94,41 @@ jobs:
       leg_script: ${{ matrix.leg.script }}
       budget_minutes: ${{ matrix.leg.budget }}
 
+  # Chained pair, serving side (keep box's RPC serving for the blaster leg).
+  backfill:
+    name: Backfill ingestion
+    needs: plan
+    uses: ./.github/workflows/ec2-leg.yml
+    secrets: inherit
+    with:
+      target_ref: ${{ needs.plan.outputs.target_sha }}
+      run_label: backfill
+      leg_script: cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/backfill-test/run-backfill.sh
+      budget_minutes: 345
+      keep_instance: true
+      extra_env: |
+        export SERVE_AFTER_BACKFILL=true
+
+  # Chained pair, blasting side: starts once backfill's leg passes (its box then
+  # enters catchup), probes the handed-off address until healthy, then blasts.
+  blaster:
+    name: Endpoint load test
+    needs: [plan, backfill]
+    uses: ./.github/workflows/ec2-leg.yml
+    secrets: inherit
+    with:
+      target_ref: ${{ needs.plan.outputs.target_sha }}
+      run_label: blaster
+      leg_script: cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/endpoint-load-test/run-blaster.sh
+      budget_minutes: 255
+      root_volume_gb: 50
+      adopt_run_label: backfill
+      extra_env: |
+        export TARGET_RPC=http://${{ needs.backfill.outputs.instance_ip }}:8000
+
   report:
     name: Aggregate + report
-    needs: [plan, load-test]
+    needs: [plan, load-test, backfill, blaster]
     if: ${{ !cancelled() }}
     runs-on: ubuntu-latest
     timeout-minutes: 10
@@ -161,13 +199,14 @@ jobs:
             fi
           fi
 
-      # Authoritative gate: reflects the load-test legs, independent of whether
-      # the comment posted.
       - name: Set overall status
         if: always()
+        env:
+          NEEDS: ${{ toJSON(needs) }}
         run: |
-          if [ "${{ needs.load-test.result }}" != "success" ]; then
-            echo "::error::one or more legs did not pass (aggregate result: ${{ needs.load-test.result }})"
+          FAILED=$(jq -r '[to_entries[] | select(.value.result != "success") | .key] | join(", ")' <<<"$NEEDS")
+          if [ -n "$FAILED" ]; then
+            echo "::error::jobs did not pass: $FAILED"
             exit 1
           fi
           echo "all load-test legs passed"

cmd/stellar-rpc/internal/integrationtest/infrastructure/perf-eval/backfill-test/run-backfill.sh

@@ -1,8 +1,9 @@
 # Backfill ingestion leg. Concatenated after bootstrap-common.sh in the rendered
 # EC2 user-data, so it relies on that file's env, helpers, bootstrap_box, and
 # run_leg. It hands off to `runner instantiate`, which builds stellar-rpc and
-# times a backfill run against the pubnet datastore. The other half, `runner 
-# gather`, polls S3 for the result object.
+# times a backfill run against the pubnet datastore. The other half, `runner
+# gather`, polls S3 for the result object. With SERVE_AFTER_BACKFILL=true the
+# box then keeps serving for the chained blaster leg.
 LEG_TITLE="Backfill ingestion"
 
 bootstrap_box