From f23d5b7604bfb83f2380927deaf0dad7fe210edb Mon Sep 17 00:00:00 2001 From: Gregory Kimball Date: Mon, 24 Aug 2026 21:33:13 -0700 Subject: [PATCH 01/12] Add reproducible EC2 orchestration for Presto benchmarks Provide an SSM-managed cluster harness so distributed CPU baselines, cache studies, and controlled tuning runs can share one archived configuration and lifecycle. --- presto/README.md | 1 + presto/aws/ec2/README.md | 283 +++++ presto/aws/ec2/aws_cluster.py | 1069 ++++++++++++++++++ presto/aws/ec2/aws_config.env.example | 65 ++ presto/aws/ec2/remote/bootstrap_node.sh | 73 ++ presto/aws/ec2/remote/collect_node.sh | 67 ++ presto/aws/ec2/remote/configure_and_start.sh | 291 +++++ presto/aws/ec2/remote/run_benchmark.sh | 79 ++ presto/aws/ec2/remote/run_cache_series.sh | 61 + presto/aws/ec2/remote/sample_host.sh | 82 ++ presto/aws/ec2/remote/wait_for_cluster.sh | 42 + presto/aws/ec2/summarize_results.py | 117 ++ presto/aws/ec2/test_aws_cluster.py | 223 ++++ 13 files changed, 2453 insertions(+) create mode 100644 presto/aws/ec2/README.md create mode 100755 presto/aws/ec2/aws_cluster.py create mode 100644 presto/aws/ec2/aws_config.env.example create mode 100755 presto/aws/ec2/remote/bootstrap_node.sh create mode 100755 presto/aws/ec2/remote/collect_node.sh create mode 100755 presto/aws/ec2/remote/configure_and_start.sh create mode 100755 presto/aws/ec2/remote/run_benchmark.sh create mode 100755 presto/aws/ec2/remote/run_cache_series.sh create mode 100755 presto/aws/ec2/remote/sample_host.sh create mode 100755 presto/aws/ec2/remote/wait_for_cluster.sh create mode 100644 presto/aws/ec2/summarize_results.py create mode 100644 presto/aws/ec2/test_aws_cluster.py diff --git a/presto/README.md b/presto/README.md index c744e2f80..e68158e8b 100644 --- a/presto/README.md +++ b/presto/README.md @@ -106,6 +106,7 @@ pytest tpch_test.py ## Directory Structure - **`docker/`** - Docker Compose configurations and Dockerfiles for different Presto variants +- **`aws/ec2/`** - Direct EC2 + SSM orchestration for distributed CPU benchmarks - **`pbench/`** - Performance benchmarking utilities and TPC-H query definitions - **`scripts/`** - Shell scripts for building, deploying, and testing Presto - **`testing/`** - Python-based test framework using pytest diff --git a/presto/aws/ec2/README.md b/presto/aws/ec2/README.md new file mode 100644 index 000000000..53de71bcd --- /dev/null +++ b/presto/aws/ec2/README.md @@ -0,0 +1,283 @@ +# Presto CPU benchmarks on direct EC2 + +This directory manages an ephemeral Presto cluster with one coordinator-only +EC2 instance and one native CPU worker per worker EC2 instance. It uses AWS +Systems Manager (SSM) instead of SSH and keeps all Presto traffic on private +addresses. + +The initial implementation targets TPC-H SF1000 at 1, 2, 8, 16, or 32 workers. +Worker counts exclude the coordinator. + +## Status and source dependency + +The S3 benchmark path depends on +[velox-testing PR 376](https://github.com/rapidsai/velox-testing/pull/376). +The first qualified source revision is expected to use PR head +`7a2e66683fd7262d8ad2cb6614a6bac1a14e7afa`. Rebase this work onto `main` after +that PR merges, then repeat local checks and fresh-fleet 8-worker qualification. + +Do not use the broader `pioneer` branch as an implicit configuration bundle. +Apply experimental settings as explicit, archived overrides. + +## What the harness owns + +- EC2 launch, inventory, and RunId-scoped termination. +- SSM bootstrap and command status. +- Docker image pulls and immutable source checkout. +- Per-role Presto config generation and a recorded config diff. +- Coordinator-first startup and exact worker-count registration. +- Benchmark execution and S3 artifact upload. +- Host/container telemetry and node artifact collection. + +It does not create a VPC, subnet, security group, IAM role, S3 bucket, placement +group, or benchmark dataset. Those account-level resources are inputs. + +## Prerequisites + +The control machine needs Python 3.10+ and AWS CLI v2. The EC2 subnet needs +outbound access or VPC endpoints for SSM, S3, source checkout, and image pulls. + +The control identity needs permission to launch, describe, price, tag, and +terminate EC2 resources; pass the configured instance profile; resolve the AMI +SSM parameter; and send/read SSM commands. Resolve permissions and prices +before EC2 launch so access failures do not leave a partial fleet. + +The instance profile needs: + +- SSM managed-instance permissions; +- read access to the benchmark data and metastore; +- write access to the configured results prefix; +- ECR permissions only when private ECR images are selected. + +The security group needs no SSH ingress. Add a self-referencing TCP rule for +port 8080 so workers can register and exchange data with the coordinator. +Restrict all rules to the benchmark security group and required egress paths. + +Use a same-AZ subnet. A placement group is optional and must be recorded as a +separate experiment dimension. + +## Configuration + +Copy the example outside the repository and fill in real values: + +```bash +mkdir -p ~/.config/velox-testing +cp aws_config.env.example ~/.config/velox-testing/aws-cpu.env +$EDITOR ~/.config/velox-testing/aws-cpu.env +``` + +Do not commit the populated file. Pin coordinator and worker images by digest +before qualification even if initial smoke tests use tags. + +Set `VELOX_TESTING_REPOSITORY` and `VELOX_TESTING_FETCH_REF` to the fork and +branch containing this harness. Set `VELOX_TESTING_REF` to that branch's exact +40-character commit SHA. Bootstrap verifies that the fetched ref resolves to +the expected SHA before checkout. + +For a pre-commit smoke test only, `HARNESS_BUNDLE_S3_URI` and +`HARNESS_BUNDLE_SHA256` may point to an immutable tar overlay containing +`presto/aws/ec2/`. Bootstrap verifies and extracts it after checking out the +pinned base commit. The run manifest records both values. Qualification runs +must use a committed source revision instead. + +`m7a.4xlarge` has less memory than `r6i.4xlarge`; use a named worker-memory +overlay for each family. Keep the coordinator type and coordinator settings +fixed when comparing worker families. + +Leave `AMI_ID` empty to resolve the configured Canonical Ubuntu SSM parameter +at launch. The resolved AMI and live on-demand Linux prices are written to the +run manifest. + +Mixed-architecture fleets can instead set `COORDINATOR_AMI_SSM_PARAMETER` and +`WORKER_AMI_SSM_PARAMETER` (or their `_ID` equivalents). Bootstrap selects the +AWS CLI binary for each host architecture and pulls only the image used by that +role. + +## Local validation and dry run + +Validation and dry-run launch do not contact AWS: + +```bash +python3 aws_cluster.py \ + --config ~/.config/velox-testing/aws-cpu.env \ + --run-id aws102-dry-run \ + validate + +python3 aws_cluster.py \ + --config ~/.config/velox-testing/aws-cpu.env \ + --run-id aws102-dry-run \ + --workers 8 \ + --dry-run \ + launch + +python3 -m unittest -v test_aws_cluster.py +``` + +Review both generated `run-instances` commands, especially tags, count, +metadata options, instance types, subnet, security group, role, and placement. + +## Fleet workflow + +Choose one unique RunId and use it for the full lifecycle: + +```bash +CONFIG=~/.config/velox-testing/aws-cpu.env +RUN_ID=aws102-m7a8-$(date -u +%Y%m%dT%H%M%SZ) + +python3 aws_cluster.py --config "$CONFIG" --run-id "$RUN_ID" --workers 8 launch +python3 aws_cluster.py --config "$CONFIG" --run-id "$RUN_ID" --workers 8 status +python3 aws_cluster.py --config "$CONFIG" --run-id "$RUN_ID" --workers 8 bootstrap +python3 aws_cluster.py --config "$CONFIG" --run-id "$RUN_ID" --workers 8 start +python3 aws_cluster.py --config "$CONFIG" --run-id "$RUN_ID" --workers 8 run +python3 aws_cluster.py --config "$CONFIG" --run-id "$RUN_ID" --workers 8 collect +python3 aws_cluster.py --config "$CONFIG" --run-id "$RUN_ID" --workers 8 stop +python3 aws_cluster.py --config "$CONFIG" --run-id "$RUN_ID" --workers 8 terminate +``` + +`launch` waits for EC2 health and SSM registration. `bootstrap` installs Docker +and AWS CLI v2, checks out the pinned source revision, pulls images, restores +the metastore, and probes IMDSv2 credentials. `start` generates configs on each +role, starts the coordinator first, then starts workers and waits for exactly N +workers. + +All mutating commands select or tag resources with the exact RunId. `launch` +uses separate idempotency tokens for coordinator and worker requests. +Destructive inventory also includes the Owner tag to prevent a RunId collision +from crossing users. + +`SSM_COMMAND_TIMEOUT_SECONDS` applies to both the remote document and the local +poller. Its default is six hours, which avoids SSM's one-hour default while +still bounding a hung benchmark. + +Always run `collect` before `terminate`, including after failed benchmarks. +Collection accepts a partial fleet and gathers every still-running node. +Instances carry an expiry tag and an OS shutdown behavior, but neither replaces +explicit teardown and billing verification. + +List expired AWS 102 instances without mutating them: + +```bash +python3 aws_cluster.py \ + --config "$CONFIG" --run-id ignored-for-listing list-stale +``` + +## Correctness smoke + +The first cloud sequence is one coordinator plus one `m7a.4xlarge` worker: + +```bash +python3 aws_cluster.py \ + --config "$CONFIG" --run-id "$RUN_ID" --workers 1 \ + run --queries 6,18 --iterations 1 --tag c1_q6_q18 +``` + +Then use two workers for one full Q1-Q22 pass before allocating eight workers. + +The current benchmark runner collects query and task metrics. Provide expected +result files through the normal `velox-testing` benchmark configuration when +semantic validation artifacts are required for qualification. + +## Baseline timing + +For cache-off AWS 101-compatible timing, run five query-major iterations. Treat +iteration 1 as warm-up. For every query, average iterations 2-5, then sum the 22 +means. Provisioning, startup, registration, and warm-up are excluded from that +primary runtime and must be reported separately. + +The harness archives raw output; top-line reduction should happen in the +experiment repository so rejected runs and the reduction rule remain visible. +Use `summarize_results.py --baseline ` to apply the +one-warm-up/four-measured rule without relying on the runner's aggregate field. + +For repeated data-cold suites with cache off, drop host page caches before each +single-iteration Q1-Q22 pass: + +```bash +python3 aws_cluster.py \ + --config "$CONFIG" --run-id "$RUN_ID" --workers 8 \ + run-cold-series --repetitions 3 --tag-prefix sf1k +``` + +This preserves the Presto processes and worker membership while issuing +`sync` and dropping Linux page cache on every coordinator and worker before +each pass. Each repetition has a separate result tag. + +## Controlled CPU tuning + +The environment file exposes the generated CPU override as explicit controls: + +- `TASK_MAX_DRIVERS_PER_TASK`; +- `HIVE_MAX_SPLIT_SIZE`; +- `HIVE_SPLIT_LOADER_CONCURRENCY`; +- `DYNAMIC_FILTERING_ENABLED`; +- `CPU_EXCHANGE_TUNING_ENABLED`. + +The example values reproduce the normal generated CPU configuration. Change +one value at a time, rerun `start` to generate and archive the candidate +configuration, then use `run-cold-series` with a diagnostic query list. Setting +`CPU_EXCHANGE_TUNING_ENABLED=false` removes the generated CPU exchange +properties so the native worker uses its defaults. + +## Async data cache + +Do not compare cache-on and cache-off results as the same workload. + +For cache-on qualification, use a fresh fleet and an explicit cache capacity. +Run one complete Q1-Q22 suite with one iteration per query as cold fill. Without +clearing, restarting, or changing the worker session, run four more complete +one-iteration suites. Average each query across those four warm suites and sum +the query means. + +Do not use `--iterations 5` for this cache-on sequence because that produces +query-major order rather than suite-major order. Give each pass a unique tag, +and preserve one uninterrupted cluster session. + +The harness encodes that sequence: + +```bash +python3 aws_cluster.py \ + --config "$CONFIG" --run-id "$RUN_ID" --workers 8 \ + run-cache-series --tag-prefix sf1k_cache +``` + +It requires `ASYNC_DATA_CACHE_ENABLED=true`, runs one cold-fill suite followed +by four warm suites, and rejects a run if worker membership changes. +Pass the five resulting `benchmark_result.json` files to +`summarize_results.py --cache-series` in cold, warm-1, warm-2, warm-3, warm-4 +order. + +## Generated state and artifacts + +Local mutable state defaults to: + +```text +~/.cache/velox-testing/aws-ec2// +``` + +It contains the launch manifest and SSM command IDs. It is intentionally +outside the repository. + +Remote state is under `/opt/presto-aws`. Collection uploads: + +- generated base and final configs plus their diff and hashes; +- Presto and container logs; +- worker-registration response; +- host and container telemetry; +- CPU, memory, filesystem, network, Docker, kernel, and journal snapshots; +- benchmark raw output and detailed Presto/Velox metrics. + +SSM command output also goes to the run-specific S3 prefix when +`RESULTS_S3_ROOT` is configured. + +## Failure handling + +- A benchmark never starts unless exactly N workers register. +- SSM failures identify the command and instance. +- Config patching requires exactly one occurrence of each expected property. +- A failed benchmark still attempts to sync partial results. +- `terminate` describes resources by the exact Project, Experiment, and RunId + tags; it does not read a broad local instance list. + +If coordinator sizing must change during 16/32-worker work, restart the scaling +series at eight workers. Only required-worker count and aggregate coordinator +query limits should vary mechanically with N. diff --git a/presto/aws/ec2/aws_cluster.py b/presto/aws/ec2/aws_cluster.py new file mode 100755 index 000000000..a93899e40 --- /dev/null +++ b/presto/aws/ec2/aws_cluster.py @@ -0,0 +1,1069 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +"""Manage an ephemeral Presto CPU benchmark fleet on direct EC2 instances.""" + +from __future__ import annotations + +import argparse +import datetime as dt +import json +import shlex +import subprocess +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +SCRIPT_DIR = Path(__file__).resolve().parent +PROJECT_TAG = "cudf-performance" +EXPERIMENT_TAG = "20260824_aws_102" +DEFAULT_STATE_ROOT = Path.home() / ".cache" / "velox-testing" / "aws-ec2" + + +class ClusterError(RuntimeError): + """A user-actionable cluster orchestration failure.""" + + +def load_env(path: Path) -> dict[str, str]: + values: dict[str, str] = {} + for line_number, raw_line in enumerate(path.read_text().splitlines(), 1): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + if "=" not in line: + raise ClusterError(f"{path}:{line_number}: expected KEY=VALUE") + key, value = line.split("=", 1) + key = key.strip() + value = value.strip() + if not key.replace("_", "").isalnum() or not key[0].isalpha(): + raise ClusterError(f"{path}:{line_number}: invalid key {key!r}") + if len(value) >= 2 and value[0] == value[-1] and value[0] in "'\"": + value = value[1:-1] + values[key] = value + return values + + +def require(config: dict[str, str], *keys: str) -> None: + missing = [key for key in keys if not config.get(key)] + if missing: + raise ClusterError(f"missing required configuration: {', '.join(missing)}") + + +def positive_int(config: dict[str, str], key: str) -> int: + try: + value = int(config[key]) + except (KeyError, ValueError) as error: + raise ClusterError(f"{key} must be an integer") from error + if value <= 0: + raise ClusterError(f"{key} must be greater than zero") + return value + + +def nonnegative_int(config: dict[str, str], key: str) -> int: + try: + value = int(config[key]) + except (KeyError, ValueError) as error: + raise ClusterError(f"{key} must be an integer") from error + if value < 0: + raise ClusterError(f"{key} must be greater than or equal to zero") + return value + + +def utc_now() -> dt.datetime: + return dt.datetime.now(dt.timezone.utc) + + +def shell_join(parts: list[str]) -> str: + return shlex.join(parts) + + +@dataclass +class Instance: + instance_id: str + role: str + state: str + private_ip: str + private_dns: str + instance_type: str + launch_time: str + + +class Aws: + def __init__(self, config: dict[str, str], dry_run: bool = False): + self.config = config + self.dry_run = dry_run + + def base(self, region: str | None = None) -> list[str]: + command = ["aws"] + if self.config.get("AWS_PROFILE"): + command += ["--profile", self.config["AWS_PROFILE"]] + command += ["--region", region or self.config["AWS_REGION"]] + return command + + def run( + self, + service_args: list[str], + *, + json_output: bool = False, + allow_dry_run: bool = True, + region: str | None = None, + ) -> Any: + command = self.base(region=region) + service_args + if self.dry_run and allow_dry_run: + print(f"DRY-RUN {shell_join(command)}") + return {} if json_output else "" + if json_output: + command += ["--output", "json"] + completed = subprocess.run( + command, + check=False, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + if completed.returncode: + detail = completed.stderr.strip() or completed.stdout.strip() + raise ClusterError(f"command failed: {shell_join(command)}\n{detail}") + if json_output: + return json.loads(completed.stdout or "{}") + return completed.stdout + + +class Cluster: + def __init__( + self, + config: dict[str, str], + run_id: str, + workers: int, + state_root: Path, + dry_run: bool, + ): + self.config = config + self.run_id = run_id + self.workers = workers + self.state_dir = state_root / run_id + self.aws = Aws(config, dry_run=dry_run) + self.dry_run = dry_run + + def validate(self, cloud: bool = True) -> None: + require( + self.config, + "AWS_REGION", + "COORDINATOR_INSTANCE_TYPE", + "WORKER_INSTANCE_TYPE", + "VELOX_TESTING_REPOSITORY", + "VELOX_TESTING_FETCH_REF", + "VELOX_TESTING_REF", + "COORDINATOR_IMAGE", + "WORKER_IMAGE", + "HIVE_METASTORE_S3_URI", + "SCHEMA_NAME", + "OWNER", + "HIVE_MAX_SPLIT_SIZE", + "DYNAMIC_FILTERING_ENABLED", + "CPU_EXCHANGE_TUNING_ENABLED", + "ASYNC_CACHE_SSD_GIB", + "ASYNC_CACHE_NUM_SHARDS", + ) + if cloud: + require( + self.config, + "SUBNET_ID", + "SECURITY_GROUP_ID", + "IAM_INSTANCE_PROFILE", + ) + generic_ami = self.config.get("AMI_ID") or self.config.get("AMI_SSM_PARAMETER") + coordinator_ami = self.config.get("COORDINATOR_AMI_ID") or self.config.get( + "COORDINATOR_AMI_SSM_PARAMETER" + ) + worker_ami = self.config.get("WORKER_AMI_ID") or self.config.get( + "WORKER_AMI_SSM_PARAMETER" + ) + if not generic_ami and not (coordinator_ami and worker_ami): + raise ClusterError( + "set AMI_ID/AMI_SSM_PARAMETER or role-specific coordinator and worker AMIs" + ) + for key in ( + "ROOT_VOLUME_GIB", + "EXPIRY_HOURS", + "SSM_READY_TIMEOUT_SECONDS", + "SSM_COMMAND_TIMEOUT_SECONDS", + "WORKER_READY_TIMEOUT_SECONDS", + "VCPU_PER_WORKER", + "TASK_MAX_DRIVERS_PER_TASK", + "HIVE_SPLIT_LOADER_CONCURRENCY", + "COORDINATOR_HEAP_GIB", + "COORDINATOR_HEADROOM_GIB", + "COORDINATOR_QUERY_TOTAL_MEMORY_PER_NODE_GIB", + "COORDINATOR_QUERY_MEMORY_PER_NODE_GIB", + "WORKER_SYSTEM_MEMORY_GIB", + "WORKER_QUERY_MEMORY_GIB", + "WORKER_MEMORY_LIMIT_GIB", + "WORKER_MEMORY_SHRINK_GIB", + ): + positive_int(self.config, key) + if self.workers not in (1, 2, 8, 16, 32): + raise ClusterError("worker count must be one of: 1, 2, 8, 16, 32") + if not self.run_id.replace("-", "").replace("_", "").isalnum(): + raise ClusterError("RunId may contain only letters, digits, '-' and '_'") + source_ref = self.config["VELOX_TESTING_REF"] + if len(source_ref) != 40 or any(character not in "0123456789abcdefABCDEF" for character in source_ref): + raise ClusterError("VELOX_TESTING_REF must be a full 40-character commit SHA") + bundle_uri = self.config.get("HARNESS_BUNDLE_S3_URI", "") + bundle_sha = self.config.get("HARNESS_BUNDLE_SHA256", "") + if bool(bundle_uri) != bool(bundle_sha): + raise ClusterError("set both HARNESS_BUNDLE_S3_URI and HARNESS_BUNDLE_SHA256, or neither") + if bundle_sha and ( + len(bundle_sha) != 64 or any(character not in "0123456789abcdefABCDEF" for character in bundle_sha) + ): + raise ClusterError("HARNESS_BUNDLE_SHA256 must contain 64 hexadecimal characters") + if self.config["ASYNC_DATA_CACHE_ENABLED"] not in ("true", "false"): + raise ClusterError("ASYNC_DATA_CACHE_ENABLED must be true or false") + for key in ("DYNAMIC_FILTERING_ENABLED", "CPU_EXCHANGE_TUNING_ENABLED"): + if self.config[key] not in ("true", "false"): + raise ClusterError(f"{key} must be true or false") + split_size = self.config["HIVE_MAX_SPLIT_SIZE"] + if not split_size.endswith("MB") or not split_size[:-2].isdigit(): + raise ClusterError("HIVE_MAX_SPLIT_SIZE must be an integer number of MB") + if int(split_size[:-2]) <= 0: + raise ClusterError("HIVE_MAX_SPLIT_SIZE must be greater than zero") + ssd_gib = nonnegative_int(self.config, "ASYNC_CACHE_SSD_GIB") + positive_int(self.config, "ASYNC_CACHE_NUM_SHARDS") + if ssd_gib and not self.config.get("ASYNC_CACHE_SSD_PATH"): + raise ClusterError("ASYNC_CACHE_SSD_PATH is required when ASYNC_CACHE_SSD_GIB is nonzero") + if ssd_gib and not self.config["ASYNC_CACHE_SSD_PATH"].startswith("/mnt/nvme/"): + raise ClusterError("ASYNC_CACHE_SSD_PATH must be under /mnt/nvme/") + if positive_int(self.config, "WORKER_QUERY_MEMORY_GIB") >= positive_int( + self.config, "WORKER_SYSTEM_MEMORY_GIB" + ): + raise ClusterError("WORKER_QUERY_MEMORY_GIB must be less than WORKER_SYSTEM_MEMORY_GIB") + if positive_int(self.config, "COORDINATOR_QUERY_MEMORY_PER_NODE_GIB") >= positive_int( + self.config, "COORDINATOR_QUERY_TOTAL_MEMORY_PER_NODE_GIB" + ): + raise ClusterError( + "COORDINATOR_QUERY_MEMORY_PER_NODE_GIB must be less than COORDINATOR_QUERY_TOTAL_MEMORY_PER_NODE_GIB" + ) + coordinator_required_heap = positive_int( + self.config, "COORDINATOR_QUERY_TOTAL_MEMORY_PER_NODE_GIB" + ) + positive_int(self.config, "COORDINATOR_HEADROOM_GIB") + if coordinator_required_heap > positive_int(self.config, "COORDINATOR_HEAP_GIB"): + raise ClusterError( + "COORDINATOR_HEAP_GIB must be at least " + "COORDINATOR_QUERY_TOTAL_MEMORY_PER_NODE_GIB + COORDINATOR_HEADROOM_GIB" + ) + + def tags(self, role: str, expiry: str) -> list[dict[str, str]]: + return [ + {"Key": "Name", "Value": f"presto-cpu-{role}-{self.run_id}"}, + {"Key": "Project", "Value": PROJECT_TAG}, + {"Key": "Experiment", "Value": EXPERIMENT_TAG}, + {"Key": "RunId", "Value": self.run_id}, + {"Key": "Role", "Value": role}, + {"Key": "Owner", "Value": self.config["OWNER"]}, + {"Key": "ExpiresAt", "Value": expiry}, + ] + + def resolve_ami(self, role: str) -> str: + role_prefix = role.upper() + ami_id = self.config.get(f"{role_prefix}_AMI_ID") or self.config.get("AMI_ID") + if ami_id: + return ami_id + parameter = self.config.get(f"{role_prefix}_AMI_SSM_PARAMETER") or self.config.get( + "AMI_SSM_PARAMETER" + ) + if not parameter: + raise ClusterError(f"no AMI configured for {role}") + response = self.aws.run( + [ + "ssm", + "get-parameter", + "--name", + parameter, + ], + json_output=True, + ) + return "ami-dry-run" if self.dry_run else response["Parameter"]["Value"] + + def resolve_hourly_price(self, instance_type: str) -> float | None: + filters = [ + {"Type": "TERM_MATCH", "Field": "instanceType", "Value": instance_type}, + { + "Type": "TERM_MATCH", + "Field": "regionCode", + "Value": self.config["AWS_REGION"], + }, + {"Type": "TERM_MATCH", "Field": "operatingSystem", "Value": "Linux"}, + {"Type": "TERM_MATCH", "Field": "tenancy", "Value": "Shared"}, + {"Type": "TERM_MATCH", "Field": "preInstalledSw", "Value": "NA"}, + {"Type": "TERM_MATCH", "Field": "capacitystatus", "Value": "Used"}, + ] + response = self.aws.run( + [ + "pricing", + "get-products", + "--service-code", + "AmazonEC2", + "--filters", + json.dumps(filters, separators=(",", ":")), + "--max-results", + "100", + ], + json_output=True, + region="us-east-1", + ) + if self.dry_run: + return None + prices: set[float] = set() + for encoded_product in response.get("PriceList", []): + product = json.loads(encoded_product) + for term in product.get("terms", {}).get("OnDemand", {}).values(): + for dimension in term.get("priceDimensions", {}).values(): + usd = dimension.get("pricePerUnit", {}).get("USD") + if usd is not None: + prices.add(float(usd)) + nonzero = sorted(price for price in prices if price > 0) + if len(nonzero) != 1: + raise ClusterError( + f"expected one on-demand Linux price for {instance_type} in " + f"{self.config['AWS_REGION']}; found {nonzero}" + ) + return nonzero[0] + + def launch_role(self, role: str, count: int, expiry: str, ami_id: str) -> list[str]: + instance_type = self.config["COORDINATOR_INSTANCE_TYPE" if role == "coordinator" else "WORKER_INSTANCE_TYPE"] + mappings = [ + { + "DeviceName": "/dev/sda1", + "Ebs": { + "VolumeSize": positive_int(self.config, "ROOT_VOLUME_GIB"), + "VolumeType": "gp3", + "DeleteOnTermination": True, + }, + } + ] + tag_specifications = [ + {"ResourceType": "instance", "Tags": self.tags(role, expiry)}, + {"ResourceType": "volume", "Tags": self.tags(role, expiry)}, + ] + args = [ + "ec2", + "run-instances", + "--image-id", + ami_id, + "--instance-type", + instance_type, + "--count", + str(count), + "--subnet-id", + self.config["SUBNET_ID"], + "--security-group-ids", + self.config["SECURITY_GROUP_ID"], + "--iam-instance-profile", + f"Name={self.config['IAM_INSTANCE_PROFILE']}", + "--metadata-options", + "HttpTokens=required,HttpEndpoint=enabled,HttpPutResponseHopLimit=2", + "--block-device-mappings", + json.dumps(mappings, separators=(",", ":")), + "--instance-initiated-shutdown-behavior", + "terminate", + "--user-data", + (f"#!/usr/bin/env bash\nshutdown -h +{positive_int(self.config, 'EXPIRY_HOURS') * 60}\n"), + "--client-token", + f"{self.run_id}-{role}", + "--tag-specifications", + json.dumps(tag_specifications, separators=(",", ":")), + ] + if self.config.get("PLACEMENT_GROUP"): + args += ["--placement", f"GroupName={self.config['PLACEMENT_GROUP']}"] + response = self.aws.run(args, json_output=True) + if self.dry_run: + return [] + return [item["InstanceId"] for item in response["Instances"]] + + def launch(self) -> None: + self.validate(cloud=True) + expiry = (utc_now() + dt.timedelta(hours=positive_int(self.config, "EXPIRY_HOURS"))).isoformat() + coordinator_ami_id = self.resolve_ami("coordinator") + worker_ami_id = self.resolve_ami("worker") + coordinator_price = self.resolve_hourly_price(self.config["COORDINATOR_INSTANCE_TYPE"]) + worker_price = self.resolve_hourly_price(self.config["WORKER_INSTANCE_TYPE"]) + coordinator_ids: list[str] = [] + worker_ids: list[str] = [] + try: + coordinator_ids = self.launch_role("coordinator", 1, expiry, coordinator_ami_id) + worker_ids = self.launch_role("worker", self.workers, expiry, worker_ami_id) + if self.dry_run: + return + instance_ids = coordinator_ids + worker_ids + self.aws.run(["ec2", "wait", "instance-running", "--instance-ids", *instance_ids]) + self.aws.run(["ec2", "wait", "instance-status-ok", "--instance-ids", *instance_ids]) + self.write_manifest( + { + "run_id": self.run_id, + "created_at": utc_now().isoformat(), + "expires_at": expiry, + "worker_count": self.workers, + "coordinator_ami_id": coordinator_ami_id, + "worker_ami_id": worker_ami_id, + "coordinator_instance_ids": coordinator_ids, + "worker_instance_ids": worker_ids, + "coordinator_hourly_price_usd": coordinator_price, + "worker_hourly_price_usd": worker_price, + "source_ref": self.config["VELOX_TESTING_REF"], + "coordinator_image": self.config["COORDINATOR_IMAGE"], + "worker_image": self.config["WORKER_IMAGE"], + "harness_bundle_s3_uri": self.config.get("HARNESS_BUNDLE_S3_URI", ""), + "harness_bundle_sha256": self.config.get("HARNESS_BUNDLE_SHA256", ""), + "async_data_cache": self.config["ASYNC_DATA_CACHE_ENABLED"] == "true", + } + ) + self.wait_for_ssm(instance_ids) + print(json.dumps(self.inventory_json(), indent=2)) + except (ClusterError, OSError): + if not self.dry_run: + try: + self.terminate() + except (ClusterError, OSError) as cleanup_error: + print( + f"warning: launch rollback failed: {cleanup_error}", + file=sys.stderr, + ) + raise + + def inventory_json(self) -> list[dict[str, str]]: + return [item.__dict__ for item in self.inventory()] + + def inventory(self) -> list[Instance]: + response = self.aws.run( + [ + "ec2", + "describe-instances", + "--filters", + f"Name=tag:Project,Values={PROJECT_TAG}", + f"Name=tag:Experiment,Values={EXPERIMENT_TAG}", + f"Name=tag:RunId,Values={self.run_id}", + f"Name=tag:Owner,Values={self.config['OWNER']}", + "Name=instance-state-name,Values=pending,running,stopping,stopped", + ], + json_output=True, + allow_dry_run=False, + ) + instances: list[Instance] = [] + for reservation in response.get("Reservations", []): + for raw in reservation.get("Instances", []): + tags = {tag["Key"]: tag["Value"] for tag in raw.get("Tags", [])} + instances.append( + Instance( + instance_id=raw["InstanceId"], + role=tags.get("Role", "unknown"), + state=raw["State"]["Name"], + private_ip=raw.get("PrivateIpAddress", ""), + private_dns=raw.get("PrivateDnsName", ""), + instance_type=raw["InstanceType"], + launch_time=str(raw["LaunchTime"]), + ) + ) + return sorted(instances, key=lambda item: (item.role, item.instance_id)) + + def expected_inventory(self) -> tuple[Instance, list[Instance]]: + items = self.inventory() + coordinators = [item for item in items if item.role == "coordinator"] + workers = [item for item in items if item.role == "worker"] + if len(coordinators) != 1 or len(workers) != self.workers: + raise ClusterError( + f"RunId {self.run_id}: expected 1 coordinator and {self.workers} " + f"workers; found {len(coordinators)} and {len(workers)}" + ) + non_running = [item.instance_id for item in items if item.state != "running"] + if non_running: + raise ClusterError(f"instances are not running: {', '.join(non_running)}") + return coordinators[0], workers + + def wait_for_ssm(self, instance_ids: list[str]) -> None: + deadline = time.monotonic() + positive_int(self.config, "SSM_READY_TIMEOUT_SECONDS") + while time.monotonic() < deadline: + response = self.aws.run( + [ + "ssm", + "describe-instance-information", + "--filters", + "Key=InstanceIds,Values=" + ",".join(instance_ids), + ], + json_output=True, + ) + online = { + item["InstanceId"] + for item in response.get("InstanceInformationList", []) + if item.get("PingStatus") == "Online" + } + if online == set(instance_ids): + return + time.sleep(10) + missing = sorted(set(instance_ids) - online) + raise ClusterError(f"instances did not become SSM Online: {', '.join(missing)}") + + def send_command( + self, + instance_ids: list[str], + commands: list[str], + comment: str, + *, + wait: bool = True, + ) -> str: + execution_timeout = positive_int(self.config, "SSM_COMMAND_TIMEOUT_SECONDS") + parameters = json.dumps( + { + "commands": commands, + "executionTimeout": [str(execution_timeout)], + }, + separators=(",", ":"), + ) + args = [ + "ssm", + "send-command", + "--document-name", + "AWS-RunShellScript", + "--instance-ids", + *instance_ids, + "--comment", + comment[:100], + "--parameters", + parameters, + ] + if self.config.get("RESULTS_S3_ROOT"): + parsed = self.config["RESULTS_S3_ROOT"].removeprefix("s3://") + bucket, _, prefix = parsed.partition("/") + args += ["--output-s3-bucket-name", bucket] + if prefix: + args += [ + "--output-s3-key-prefix", + f"{prefix.rstrip('/')}/{self.run_id}/ssm", + ] + response = self.aws.run(args, json_output=True) + if self.dry_run: + return "dry-run-command" + command_id = response["Command"]["CommandId"] + if wait: + for instance_id in instance_ids: + self.wait_for_command( + command_id, + instance_id, + timeout_seconds=execution_timeout + 300, + ) + return command_id + + def wait_for_command(self, command_id: str, instance_id: str, timeout_seconds: int) -> None: + deadline = time.monotonic() + timeout_seconds + terminal = {"Success", "Cancelled", "TimedOut", "Failed", "Cancelling"} + while time.monotonic() < deadline: + response = self.aws.run( + [ + "ssm", + "list-command-invocations", + "--command-id", + command_id, + "--instance-id", + instance_id, + "--details", + ], + json_output=True, + ) + invocations = response.get("CommandInvocations", []) + if not invocations: + time.sleep(5) + continue + status = invocations[0].get("Status", "") + if status not in terminal: + time.sleep(5) + continue + if status == "Success": + return + invocation = self.aws.run( + [ + "ssm", + "get-command-invocation", + "--command-id", + command_id, + "--instance-id", + instance_id, + ], + json_output=True, + ) + raise ClusterError( + f"SSM command {command_id} ended as {status} on {instance_id}: " + f"{invocation.get('StandardErrorContent', '').strip()}" + ) + raise ClusterError( + f"timed out waiting {timeout_seconds}s for SSM command {command_id} " + f"on {instance_id}; the remote command may still be running" + ) + + def bootstrap(self) -> None: + self.validate(cloud=True) + coordinator, workers = self.expected_inventory() + repository = shlex.quote(self.config["VELOX_TESTING_REPOSITORY"]) + fetch_ref = shlex.quote(self.config["VELOX_TESTING_FETCH_REF"]) + ref = shlex.quote(self.config["VELOX_TESTING_REF"]) + commands = [ + "set -eu", + "export DEBIAN_FRONTEND=noninteractive", + "apt-get update -y", + ("apt-get install -y ca-certificates curl docker.io git jq numactl python3 python3-venv unzip"), + "systemctl enable --now docker", + ( + "if ! command -v aws >/dev/null; then " + "case \"$(uname -m)\" in " + "x86_64) aws_arch=x86_64 ;; " + "aarch64|arm64) aws_arch=aarch64 ;; " + "*) echo 'unsupported architecture for AWS CLI' >&2; exit 1 ;; " + "esac; " + "curl -fsSLo /tmp/awscliv2.zip " + "\"https://awscli.amazonaws.com/awscli-exe-linux-${aws_arch}.zip\"; " + "rm -rf /tmp/aws; unzip -q /tmp/awscliv2.zip -d /tmp; " + "/tmp/aws/install; fi" + ), + "rm -rf /opt/velox-testing", + f"git clone --filter=blob:none {repository} /opt/velox-testing", + f"git -C /opt/velox-testing fetch origin {fetch_ref}", + (f'test "$(git -C /opt/velox-testing rev-parse FETCH_HEAD)" = {ref}'), + f"git -C /opt/velox-testing checkout --detach {ref}", + ] + if self.config.get("HARNESS_BUNDLE_S3_URI"): + bundle_uri = shlex.quote(self.config["HARNESS_BUNDLE_S3_URI"]) + bundle_sha = shlex.quote(self.config["HARNESS_BUNDLE_SHA256"]) + commands += [ + f"aws s3 cp {bundle_uri} /tmp/presto-aws-harness.tar.gz --only-show-errors", + (f"echo '{bundle_sha} /tmp/presto-aws-harness.tar.gz' | sha256sum --check --strict"), + "tar -xzf /tmp/presto-aws-harness.tar.gz -C /opt/velox-testing", + ] + bootstrap_prefix = ( + "bash /opt/velox-testing/presto/aws/ec2/remote/bootstrap_node.sh " + ) + bootstrap_suffix = ( + shlex.quote(self.config["COORDINATOR_IMAGE"]) + + " " + + shlex.quote(self.config["WORKER_IMAGE"]) + + " " + + shlex.quote(self.config["HIVE_METASTORE_S3_URI"]) + ) + coordinator_command_id = self.send_command( + [coordinator.instance_id], + [*commands, f"{bootstrap_prefix}coordinator {bootstrap_suffix}"], + f"bootstrap coordinator {self.run_id}", + ) + self.record_command("bootstrap_coordinator", coordinator_command_id) + worker_command_id = self.send_command( + [item.instance_id for item in workers], + [*commands, f"{bootstrap_prefix}worker {bootstrap_suffix}"], + f"bootstrap workers {self.run_id}", + ) + self.record_command("bootstrap_workers", worker_command_id) + + def remote_env(self, coordinator_address: str, role: str, worker_index: int | None = None) -> str: + values = { + "RUN_ID": self.run_id, + "ROLE": role, + "WORKER_COUNT": str(self.workers), + "WORKER_INDEX": "" if worker_index is None else str(worker_index), + "COORDINATOR_ADDRESS": coordinator_address, + "AWS_REGION": self.config["AWS_REGION"], + "VCPU_PER_WORKER": self.config["VCPU_PER_WORKER"], + "TASK_MAX_DRIVERS_PER_TASK": self.config["TASK_MAX_DRIVERS_PER_TASK"], + "HIVE_MAX_SPLIT_SIZE": self.config["HIVE_MAX_SPLIT_SIZE"], + "HIVE_SPLIT_LOADER_CONCURRENCY": self.config[ + "HIVE_SPLIT_LOADER_CONCURRENCY" + ], + "DYNAMIC_FILTERING_ENABLED": self.config["DYNAMIC_FILTERING_ENABLED"], + "CPU_EXCHANGE_TUNING_ENABLED": self.config[ + "CPU_EXCHANGE_TUNING_ENABLED" + ], + "COORDINATOR_HEAP_GIB": self.config["COORDINATOR_HEAP_GIB"], + "COORDINATOR_HEADROOM_GIB": self.config["COORDINATOR_HEADROOM_GIB"], + "COORDINATOR_QUERY_TOTAL_MEMORY_PER_NODE_GIB": self.config["COORDINATOR_QUERY_TOTAL_MEMORY_PER_NODE_GIB"], + "COORDINATOR_QUERY_MEMORY_PER_NODE_GIB": self.config["COORDINATOR_QUERY_MEMORY_PER_NODE_GIB"], + "WORKER_SYSTEM_MEMORY_GIB": self.config["WORKER_SYSTEM_MEMORY_GIB"], + "WORKER_QUERY_MEMORY_GIB": self.config["WORKER_QUERY_MEMORY_GIB"], + "WORKER_MEMORY_LIMIT_GIB": self.config["WORKER_MEMORY_LIMIT_GIB"], + "WORKER_MEMORY_SHRINK_GIB": self.config["WORKER_MEMORY_SHRINK_GIB"], + "ASYNC_DATA_CACHE_ENABLED": self.config["ASYNC_DATA_CACHE_ENABLED"], + "ASYNC_CACHE_SSD_GIB": self.config["ASYNC_CACHE_SSD_GIB"], + "ASYNC_CACHE_SSD_PATH": self.config.get("ASYNC_CACHE_SSD_PATH", ""), + "ASYNC_CACHE_NUM_SHARDS": self.config["ASYNC_CACHE_NUM_SHARDS"], + "COORDINATOR_IMAGE": self.config["COORDINATOR_IMAGE"], + "WORKER_IMAGE": self.config["WORKER_IMAGE"], + } + return " ".join(f"{key}={shlex.quote(value)}" for key, value in values.items()) + + def start(self) -> None: + self.validate(cloud=True) + coordinator, workers = self.expected_inventory() + address = coordinator.private_ip or coordinator.private_dns + if nonnegative_int(self.config, "ASYNC_CACHE_SSD_GIB"): + cache_path = shlex.quote(self.config["ASYNC_CACHE_SSD_PATH"]) + prepare_command_id = self.send_command( + [worker.instance_id for worker in workers], + [ + "set -eu", + ( + "device=$(lsblk -dpno NAME,MODEL | " + "awk '$0 ~ /Amazon EC2 NVMe Instance Storage/ {print $1; exit}'); " + "test -n \"$device\"; test -b \"$device\"; " + "if ! blkid \"$device\" >/dev/null 2>&1; then " + "mkfs.ext4 -F \"$device\"; fi" + ), + "mkdir -p /mnt/nvme", + "mountpoint -q /mnt/nvme || mount \"$device\" /mnt/nvme", + f"mkdir -p {cache_path}", + f"chmod 0777 /mnt/nvme {cache_path}", + ], + f"prepare NVMe cache {self.run_id}", + wait=True, + ) + self.record_command("prepare_nvme_cache", prepare_command_id) + command_id = self.send_command( + [coordinator.instance_id], + [ + "set -eu", + ( + f"{self.remote_env(address, 'coordinator')} " + "bash /opt/velox-testing/presto/aws/ec2/remote/configure_and_start.sh" + ), + ], + f"start coordinator {self.run_id}", + ) + self.record_command("start_coordinator", command_id) + for index, worker in enumerate(workers): + command_id = self.send_command( + [worker.instance_id], + [ + "set -eu", + ( + f"{self.remote_env(address, 'worker', index)} " + "bash /opt/velox-testing/presto/aws/ec2/remote/configure_and_start.sh" + ), + ], + f"start worker {index} {self.run_id}", + ) + self.record_command(f"start_worker_{index}", command_id) + timeout = self.config["WORKER_READY_TIMEOUT_SECONDS"] + command_id = self.send_command( + [coordinator.instance_id], + [ + "set -eu", + ( + "bash /opt/velox-testing/presto/aws/ec2/remote/" + f"wait_for_cluster.sh {shlex.quote(address)} {self.workers} {timeout}" + ), + ], + f"wait for workers {self.run_id}", + ) + self.record_command("wait_for_cluster", command_id) + + def run_benchmark( + self, + queries: str, + iterations: int, + tag: str, + record_name: str = "benchmark", + ) -> None: + self.validate(cloud=True) + coordinator, _ = self.expected_inventory() + address = coordinator.private_ip or coordinator.private_dns + result_uri = self.result_uri() + commands = [ + "set -eu", + ( + f"AWS_DEFAULT_REGION={shlex.quote(self.config['AWS_REGION'])} " + f"AWS_REGION={shlex.quote(self.config['AWS_REGION'])} " + "HOME=/root " + "bash /opt/velox-testing/presto/aws/ec2/remote/run_benchmark.sh " + f"{shlex.quote(address)} {shlex.quote(self.config['SCHEMA_NAME'])} " + f"{shlex.quote(queries)} {iterations} {shlex.quote(tag)} " + f"{shlex.quote(result_uri)}" + ), + ] + command_id = self.send_command([coordinator.instance_id], commands, f"benchmark {self.run_id}", wait=True) + self.record_command(record_name, command_id) + + def run_cold_series(self, queries: str, repetitions: int, tag_prefix: str) -> None: + self.validate(cloud=True) + if self.config["ASYNC_DATA_CACHE_ENABLED"] != "false": + raise ClusterError("run-cold-series requires ASYNC_DATA_CACHE_ENABLED=false") + coordinator, workers = self.expected_inventory() + instance_ids = [coordinator.instance_id, *[item.instance_id for item in workers]] + for repetition in range(1, repetitions + 1): + clear_command_id = self.send_command( + instance_ids, + [ + "set -eu", + "sync", + "echo 3 > /proc/sys/vm/drop_caches", + ], + f"drop host caches repetition {repetition} {self.run_id}", + wait=True, + ) + self.record_command(f"drop_host_caches_{repetition}", clear_command_id) + self.run_benchmark( + queries, + 1, + f"{tag_prefix}_cold_{repetition}", + record_name=f"cold_benchmark_{repetition}", + ) + + def run_cache_series(self, queries: str, tag_prefix: str) -> None: + self.validate(cloud=True) + coordinator, workers = self.expected_inventory() + address = coordinator.private_ip or coordinator.private_dns + instance_ids = [coordinator.instance_id, *[worker.instance_id for worker in workers]] + drop_command_id = self.send_command( + instance_ids, + [ + "set -eu", + "sync", + "echo 3 > /proc/sys/vm/drop_caches", + ], + f"drop host caches {self.run_id}", + wait=True, + ) + self.record_command("drop_host_caches", drop_command_id) + if self.config["ASYNC_DATA_CACHE_ENABLED"] == "true": + cache_types = ["memory"] + if nonnegative_int(self.config, "ASYNC_CACHE_SSD_GIB"): + cache_types.append("ssd") + for cache_type in cache_types: + expected = f"Cleared {cache_type} cache" + clear_command_id = self.send_command( + [worker.instance_id for worker in workers], + [ + "set -eu", + ( + "response=$(curl -fsS " + f"'http://localhost:8080/v1/operation/server/clearCache?type={cache_type}'" + "); printf '%s\\n' \"$response\"; " + f"test \"$response\" = {shlex.quote(expected)}" + ), + ], + f"clear worker {cache_type} caches {self.run_id}", + wait=True, + ) + self.record_command(f"clear_worker_{cache_type}_caches", clear_command_id) + commands = [ + "set -eu", + ( + f"AWS_DEFAULT_REGION={shlex.quote(self.config['AWS_REGION'])} " + f"AWS_REGION={shlex.quote(self.config['AWS_REGION'])} " + "HOME=/root " + "bash /opt/velox-testing/presto/aws/ec2/remote/run_cache_series.sh " + f"{shlex.quote(address)} {shlex.quote(self.config['SCHEMA_NAME'])} " + f"{shlex.quote(queries)} {shlex.quote(tag_prefix)} " + f"{shlex.quote(self.result_uri())}" + ), + ] + command_id = self.send_command( + [coordinator.instance_id], + commands, + f"cache series {self.run_id}", + wait=True, + ) + self.record_command("cache_series", command_id) + + def collect(self) -> None: + self.validate(cloud=True) + inventory = self.inventory() + running = [item for item in inventory if item.state == "running"] + if not running: + raise ClusterError(f"RunId {self.run_id}: no running instances to collect") + expected = self.workers + 1 + if len(running) != expected: + print( + f"warning: collecting {len(running)} of {expected} expected running instances", + file=sys.stderr, + ) + result_uri = self.result_uri() + for item in running: + commands = [ + "set -eu", + ( + "bash /opt/velox-testing/presto/aws/ec2/remote/collect_node.sh " + f"{shlex.quote(self.run_id)} {shlex.quote(item.role)} " + f"{shlex.quote(item.instance_id)} {shlex.quote(result_uri)}" + ), + ] + command_id = self.send_command([item.instance_id], commands, f"collect {item.instance_id}") + self.record_command(f"collect_{item.instance_id}", command_id) + + def stop(self) -> None: + coordinator, workers = self.expected_inventory() + command_id = self.send_command( + [coordinator.instance_id, *[item.instance_id for item in workers]], + [ + "set -eu", + "docker rm -f presto-coordinator presto-native-worker-cpu 2>/dev/null || true", + ], + f"stop {self.run_id}", + ) + self.record_command("stop", command_id) + + def terminate(self) -> None: + instances = self.inventory() + if not instances: + print(f"RunId {self.run_id}: no live instances found") + return + ids = [item.instance_id for item in instances] + self.aws.run(["ec2", "terminate-instances", "--instance-ids", *ids]) + if not self.dry_run: + self.aws.run(["ec2", "wait", "instance-terminated", "--instance-ids", *ids]) + print(f"RunId {self.run_id}: terminated {len(ids)} instances") + + def list_stale(self) -> None: + response = self.aws.run( + [ + "ec2", + "describe-instances", + "--filters", + f"Name=tag:Project,Values={PROJECT_TAG}", + f"Name=tag:Experiment,Values={EXPERIMENT_TAG}", + "Name=instance-state-name,Values=pending,running,stopping,stopped", + ], + json_output=True, + allow_dry_run=False, + ) + now = utc_now() + stale: list[dict[str, str]] = [] + for reservation in response.get("Reservations", []): + for raw in reservation.get("Instances", []): + tags = {tag["Key"]: tag["Value"] for tag in raw.get("Tags", [])} + expiry_text = tags.get("ExpiresAt", "") + if not expiry_text: + continue + try: + expiry = dt.datetime.fromisoformat(expiry_text) + except ValueError: + continue + if expiry <= now: + stale.append( + { + "instance_id": raw["InstanceId"], + "run_id": tags.get("RunId", ""), + "role": tags.get("Role", ""), + "owner": tags.get("Owner", ""), + "expires_at": expiry_text, + "state": raw["State"]["Name"], + } + ) + print(json.dumps(sorted(stale, key=lambda item: item["expires_at"]), indent=2)) + + def result_uri(self) -> str: + require(self.config, "RESULTS_S3_ROOT") + return f"{self.config['RESULTS_S3_ROOT'].rstrip('/')}/{self.run_id}" + + def write_manifest(self, payload: dict[str, Any]) -> None: + self.state_dir.mkdir(parents=True, exist_ok=True) + (self.state_dir / "run_manifest.json").write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + + def record_command(self, name: str, command_id: str) -> None: + if self.dry_run: + return + self.state_dir.mkdir(parents=True, exist_ok=True) + path = self.state_dir / "ssm_commands.json" + commands = json.loads(path.read_text()) if path.exists() else {} + commands[name] = {"command_id": command_id, "recorded_at": utc_now().isoformat()} + path.write_text(json.dumps(commands, indent=2, sort_keys=True) + "\n") + + +def default_run_id() -> str: + return "aws102-" + utc_now().strftime("%Y%m%dT%H%M%SZ").lower() + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path, required=True) + parser.add_argument("--run-id", default=default_run_id()) + parser.add_argument("--workers", type=int) + parser.add_argument("--state-root", type=Path, default=DEFAULT_STATE_ROOT) + parser.add_argument("--dry-run", action="store_true") + subparsers = parser.add_subparsers(dest="command", required=True) + for name in ( + "validate", + "launch", + "status", + "bootstrap", + "start", + "collect", + "stop", + "terminate", + "list-stale", + ): + subparsers.add_parser(name) + run_parser = subparsers.add_parser("run") + run_parser.add_argument("--queries", default="1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22") + run_parser.add_argument("--iterations", type=int, default=5) + run_parser.add_argument("--tag", default="sf1k_q1q22") + cache_parser = subparsers.add_parser("run-cache-series") + cache_parser.add_argument( + "--queries", + default="1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22", + ) + cache_parser.add_argument("--tag-prefix", default="sf1k_cache") + cold_parser = subparsers.add_parser("run-cold-series") + cold_parser.add_argument( + "--queries", + default="1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22", + ) + cold_parser.add_argument("--repetitions", type=int, default=3) + cold_parser.add_argument("--tag-prefix", default="sf1k") + return parser + + +def main() -> int: + args = build_parser().parse_args() + try: + config = load_env(args.config) + workers = args.workers or positive_int(config, "WORKER_COUNT") + cluster = Cluster( + config, + run_id=args.run_id, + workers=workers, + state_root=args.state_root, + dry_run=args.dry_run, + ) + if args.command == "validate": + cluster.validate(cloud=True) + print("configuration is valid") + elif args.command == "launch": + cluster.launch() + elif args.command == "status": + print(json.dumps(cluster.inventory_json(), indent=2)) + elif args.command == "bootstrap": + cluster.bootstrap() + elif args.command == "start": + cluster.start() + elif args.command == "run": + if args.iterations <= 0: + raise ClusterError("--iterations must be greater than zero") + cluster.run_benchmark(args.queries, args.iterations, args.tag) + elif args.command == "run-cache-series": + cluster.run_cache_series(args.queries, args.tag_prefix) + elif args.command == "run-cold-series": + if args.repetitions <= 0: + raise ClusterError("--repetitions must be greater than zero") + cluster.run_cold_series(args.queries, args.repetitions, args.tag_prefix) + elif args.command == "collect": + cluster.collect() + elif args.command == "stop": + cluster.stop() + elif args.command == "terminate": + cluster.terminate() + elif args.command == "list-stale": + cluster.list_stale() + return 0 + except (ClusterError, OSError) as error: + print(f"error: {error}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/presto/aws/ec2/aws_config.env.example b/presto/aws/ec2/aws_config.env.example new file mode 100644 index 000000000..9b78ee924 --- /dev/null +++ b/presto/aws/ec2/aws_config.env.example @@ -0,0 +1,65 @@ +# AWS account and placement. Keep real values outside the repository. +AWS_PROFILE= +AWS_REGION=us-east-2 +AMI_ID= +AMI_SSM_PARAMETER=/aws/service/canonical/ubuntu/server/24.04/stable/current/amd64/hvm/ebs-gp3/ami-id +# Optional role-specific overrides for mixed x86 coordinator / ARM worker fleets. +COORDINATOR_AMI_ID= +COORDINATOR_AMI_SSM_PARAMETER= +WORKER_AMI_ID= +WORKER_AMI_SSM_PARAMETER= +SUBNET_ID= +SECURITY_GROUP_ID= +IAM_INSTANCE_PROFILE= + +# Fleet shape. WORKER_COUNT is overridden by --workers when needed. +COORDINATOR_INSTANCE_TYPE=m7a.4xlarge +WORKER_INSTANCE_TYPE=m7a.4xlarge +WORKER_COUNT=8 +ROOT_VOLUME_GIB=100 + +# Reproducible software inputs. +VELOX_TESTING_REPOSITORY=https://github.com//velox-testing.git +VELOX_TESTING_FETCH_REF=refs/heads/aws-102-ec2-cpu-cluster +VELOX_TESTING_REF= +# Optional for an uncommitted smoke-test overlay. Set both or neither. +HARNESS_BUNDLE_S3_URI= +HARNESS_BUNDLE_SHA256= +COORDINATOR_IMAGE=ghcr.io/y-scope/presto:0.299 +WORKER_IMAGE=ghcr.io/y-scope/presto-native:0.299 + +# Benchmark data and output. Pin image digests before qualification. +HIVE_METASTORE_S3_URI=s3://rapids-tpch/presto-gpu/sf1k_v2_float/hive_metastore/ +SCHEMA_NAME=tpch_sf1k_v2_float_s3 +RESULTS_S3_ROOT= + +# Runtime sizing. One native worker runs on every worker EC2 instance. +VCPU_PER_WORKER=16 +# Explicit CPU tuning overlay. Defaults reproduce the generated CPU override. +TASK_MAX_DRIVERS_PER_TASK=16 +HIVE_MAX_SPLIT_SIZE=256MB +HIVE_SPLIT_LOADER_CONCURRENCY=32 +DYNAMIC_FILTERING_ENABLED=false +CPU_EXCHANGE_TUNING_ENABLED=true +COORDINATOR_HEAP_GIB=54 +COORDINATOR_HEADROOM_GIB=4 +COORDINATOR_QUERY_TOTAL_MEMORY_PER_NODE_GIB=48 +COORDINATOR_QUERY_MEMORY_PER_NODE_GIB=44 +WORKER_SYSTEM_MEMORY_GIB=50 +WORKER_QUERY_MEMORY_GIB=44 +WORKER_MEMORY_LIMIT_GIB=54 +WORKER_MEMORY_SHRINK_GIB=16 +ASYNC_DATA_CACHE_ENABLED=false +ASYNC_CACHE_SSD_GIB=0 +ASYNC_CACHE_SSD_PATH= +ASYNC_CACHE_NUM_SHARDS=16 + +# Fleet safety and ownership. +OWNER= +EXPIRY_HOURS=8 +SSM_READY_TIMEOUT_SECONDS=900 +SSM_COMMAND_TIMEOUT_SECONDS=21600 +WORKER_READY_TIMEOUT_SECONDS=600 + +# Optional. Leave empty for ordinary same-AZ placement. +PLACEMENT_GROUP= diff --git a/presto/aws/ec2/remote/bootstrap_node.sh b/presto/aws/ec2/remote/bootstrap_node.sh new file mode 100755 index 000000000..2b1f1c9ac --- /dev/null +++ b/presto/aws/ec2/remote/bootstrap_node.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +if [[ $# -ne 4 ]]; then + echo "usage: bootstrap_node.sh " >&2 + exit 2 +fi + +role=$1 +coordinator_image=$2 +worker_image=$3 +metastore_uri=$4 +runtime_root=/opt/presto-aws + +if [[ ${role} != coordinator && ${role} != worker ]]; then + echo "role must be coordinator or worker" >&2 + exit 2 +fi + +mkdir -p \ + "${runtime_root}/base_config" \ + "${runtime_root}/final_config" \ + "${runtime_root}/logs" \ + "${runtime_root}/metastore" \ + "${runtime_root}/results" \ + "${runtime_root}/telemetry" + +python3 -m venv "${runtime_root}/venv" +"${runtime_root}/venv/bin/pip" install --disable-pip-version-check \ + -r /opt/velox-testing/presto/testing/requirements.txt + +if [[ ${role} == coordinator ]]; then + docker pull "${coordinator_image}" + local_image=${coordinator_image} +else + docker pull "${worker_image}" + local_image=${worker_image} +fi + +aws s3 sync "${metastore_uri}" "${runtime_root}/metastore/" --only-show-errors + +token=$(curl -fsS -X PUT \ + -H 'X-aws-ec2-metadata-token-ttl-seconds: 21600' \ + http://169.254.169.254/latest/api/token) +role_name=$(curl -fsS \ + -H "X-aws-ec2-metadata-token: ${token}" \ + http://169.254.169.254/latest/meta-data/iam/security-credentials/) +credential_document=$(curl -fsS \ + -H "X-aws-ec2-metadata-token: ${token}" \ + "http://169.254.169.254/latest/meta-data/iam/security-credentials/${role_name}") +credential_code=$(jq -r .Code <<<"${credential_document}") +if [[ "${credential_code}" != "Success" ]]; then + echo "instance-profile credential probe failed: ${credential_code}" >&2 + exit 1 +fi + +aws sts get-caller-identity >/dev/null + +cat >"${runtime_root}/bootstrap_info.json" < " >&2 + exit 2 +fi + +run_id=$1 +role=$2 +instance_id=$3 +result_uri=$4 +runtime_root=/opt/presto-aws +artifact_root="${runtime_root}/artifacts/${role}-${instance_id}" + +rm -rf "${artifact_root}" +mkdir -p "${artifact_root}" + +cp -a "${runtime_root}/logs" "${artifact_root}/" 2>/dev/null || true +cp -a "${runtime_root}/telemetry" "${artifact_root}/" 2>/dev/null || true +cp -a "${runtime_root}/base_config" "${artifact_root}/" 2>/dev/null || true +cp -a "${runtime_root}/final_config" "${artifact_root}/" 2>/dev/null || true +cp -a "${runtime_root}/config_history" "${artifact_root}/" 2>/dev/null || true +cp -a "${runtime_root}/results" "${artifact_root}/" 2>/dev/null || true +cp "${runtime_root}"/*.json "${artifact_root}/" 2>/dev/null || true +cp "${runtime_root}"/*.sha256 "${artifact_root}/" 2>/dev/null || true +cp "${runtime_root}"/*.diff "${artifact_root}/" 2>/dev/null || true + +docker ps -a --no-trunc >"${artifact_root}/docker_ps.txt" 2>&1 || true +docker inspect presto-coordinator presto-native-worker-cpu \ + >"${artifact_root}/docker_inspect.json" 2>/dev/null || true +docker logs presto-coordinator >"${artifact_root}/coordinator_container.log" 2>&1 || true +docker logs presto-native-worker-cpu >"${artifact_root}/worker_container.log" 2>&1 || true + +uname -a >"${artifact_root}/uname.txt" +lscpu --json >"${artifact_root}/lscpu.json" +lsmem --json >"${artifact_root}/lsmem.json" +ip -json address >"${artifact_root}/ip_address.json" +ip -s -json link >"${artifact_root}/ip_link_stats.json" +lsblk --json --bytes --output NAME,PATH,MODEL,SERIAL,SIZE,FSTYPE,MOUNTPOINTS \ + >"${artifact_root}/lsblk.json" +cp /proc/diskstats "${artifact_root}/diskstats.txt" +df -B1 --output=source,fstype,size,used,avail,target \ + >"${artifact_root}/filesystems.txt" +if [[ ${role} == worker ]]; then + curl -fsS http://localhost:8080/v1/info/metrics \ + >"${artifact_root}/worker_metrics.prom" 2>&1 || true + curl -fsS http://localhost:8080/v1/status \ + >"${artifact_root}/worker_status.json" 2>&1 || true +fi +journalctl -u docker --no-pager >"${artifact_root}/docker_journal.log" 2>&1 || true +dmesg --ctime >"${artifact_root}/dmesg.log" 2>&1 || true + +cat >"${artifact_root}/collection.json" <&2 + exit 2 + fi +done + +if [[ ${ROLE} != coordinator && ${ROLE} != worker ]]; then + echo "ROLE must be coordinator or worker" >&2 + exit 2 +fi +if [[ ${ROLE} == worker && -z ${WORKER_INDEX:-} ]]; then + echo "WORKER_INDEX is required for worker role" >&2 + exit 2 +fi + +repo=/opt/velox-testing +runtime_root=/opt/presto-aws +generated="${repo}/presto/docker/config/generated/cpu" +base="${runtime_root}/base_config/${ROLE}" +final="${runtime_root}/final_config/${ROLE}" +assembled="${runtime_root}/runtime_etc" + +set_property() { + local path=$1 key=$2 value=$3 + python3 - "${path}" "${key}" "${value}" <<'PY' +import sys +from pathlib import Path + +path = Path(sys.argv[1]) +key = sys.argv[2] +value = sys.argv[3] +prefix = f"{key}=" +lines = path.read_text().splitlines() +matches = [index for index, line in enumerate(lines) if line.startswith(prefix)] +if len(matches) != 1: + raise SystemExit(f"expected exactly one {key} in {path}; found {len(matches)}") +lines[matches[0]] = f"{key}={value}" +path.write_text("\n".join(lines) + "\n") +PY +} + +upsert_property() { + local path=$1 key=$2 value=$3 + python3 - "${path}" "${key}" "${value}" <<'PY' +import sys +from pathlib import Path + +path = Path(sys.argv[1]) +key = sys.argv[2] +value = sys.argv[3] +prefix = f"{key}=" +lines = path.read_text().splitlines() +matches = [index for index, line in enumerate(lines) if line.startswith(prefix)] +if len(matches) > 1: + raise SystemExit(f"expected at most one {key} in {path}; found {len(matches)}") +if matches: + lines[matches[0]] = f"{key}={value}" +else: + lines.append(f"{key}={value}") +path.write_text("\n".join(lines) + "\n") +PY +} + +delete_property() { + local path=$1 key=$2 + python3 - "${path}" "${key}" <<'PY' +import sys +from pathlib import Path + +path = Path(sys.argv[1]) +prefix = f"{sys.argv[2]}=" +lines = [line for line in path.read_text().splitlines() if not line.startswith(prefix)] +path.write_text("\n".join(lines) + "\n") +PY +} + +cd "${repo}/presto/scripts" +OVERWRITE_CONFIG=true \ +NUM_WORKERS="${WORKER_COUNT}" \ +VARIANT_TYPE=cpu \ +VCPU_PER_WORKER="${VCPU_PER_WORKER}" \ + ./generate_presto_config.sh + +rm -rf "${base}" "${final}" "${assembled}" +mkdir -p "${base}" "${final}" "${assembled}" "${runtime_root}/data" + +if [[ ${ROLE} == coordinator ]]; then + cp -a "${generated}/etc_common/." "${base}/" + cp "${generated}/etc_coordinator/config_native.properties" "${base}/config.properties" + cp "${generated}/etc_coordinator/node.properties" "${base}/node.properties" + mkdir -p "${base}/catalog" + cp "${generated}/etc_coordinator/catalog/hive.properties" "${base}/catalog/hive.properties" +else + worker_source="${generated}/etc_worker_0" + cp -a "${generated}/etc_common/." "${base}/" + cp "${worker_source}/config_native.properties" "${base}/config.properties" + cp "${worker_source}/node.properties" "${base}/node.properties" + mkdir -p "${base}/catalog" + cp "${worker_source}/catalog/hive.properties" "${base}/catalog/hive.properties" +fi +cp -a "${base}/." "${final}/" +delete_property "${final}/catalog/hive.properties" hive.max-split-size +printf 'hive.max-split-size=%s\n' "${HIVE_MAX_SPLIT_SIZE}" \ + >>"${final}/catalog/hive.properties" +delete_property "${final}/catalog/hive.properties" hive.split-loader-concurrency +printf 'hive.split-loader-concurrency=%s\n' "${HIVE_SPLIT_LOADER_CONCURRENCY}" \ + >>"${final}/catalog/hive.properties" + +if [[ ${ROLE} == coordinator ]]; then + set_property "${final}/config.properties" discovery.uri \ + "http://${COORDINATOR_ADDRESS}:8080" + set_property "${final}/config.properties" query-manager.required-workers \ + "${WORKER_COUNT}" + set_property "${final}/config.properties" \ + query-manager.required-workers-max-wait 10m + set_property "${final}/config.properties" query.max-total-memory-per-node \ + "${COORDINATOR_QUERY_TOTAL_MEMORY_PER_NODE_GIB}GB" + set_property "${final}/config.properties" query.max-total-memory \ + "$((COORDINATOR_QUERY_TOTAL_MEMORY_PER_NODE_GIB * WORKER_COUNT))GB" + set_property "${final}/config.properties" query.max-memory-per-node \ + "${COORDINATOR_QUERY_MEMORY_PER_NODE_GIB}GB" + set_property "${final}/config.properties" query.max-memory \ + "$((COORDINATOR_QUERY_MEMORY_PER_NODE_GIB * WORKER_COUNT))GB" + set_property "${final}/config.properties" memory.heap-headroom-per-node \ + "${COORDINATOR_HEADROOM_GIB}GB" + set_property "${final}/config.properties" single-node-execution-enabled false + set_property "${final}/config.properties" experimental.enable-dynamic-filtering \ + "${DYNAMIC_FILTERING_ENABLED}" + python3 - "${final}/jvm.config" "${COORDINATOR_HEAP_GIB}" <<'PY' +import sys +from pathlib import Path + +path = Path(sys.argv[1]) +heap = sys.argv[2] +lines = path.read_text().splitlines() +lines = [ + f"-Xmx{heap}G" if line.startswith("-Xmx") else + f"-Xms{heap}G" if line.startswith("-Xms") else line + for line in lines +] +path.write_text("\n".join(lines) + "\n") +PY +else + set_property "${final}/config.properties" http-server.http.port 8080 + set_property "${final}/config.properties" discovery.uri \ + "http://${COORDINATOR_ADDRESS}:8080" + set_property "${final}/config.properties" single-node-execution-enabled false + set_property "${final}/config.properties" task.max-drivers-per-task \ + "${TASK_MAX_DRIVERS_PER_TASK}" + set_property "${final}/config.properties" system-memory-gb \ + "${WORKER_SYSTEM_MEMORY_GIB}" + set_property "${final}/config.properties" query-memory-gb \ + "${WORKER_QUERY_MEMORY_GIB}" + set_property "${final}/config.properties" query.max-memory-per-node \ + "${WORKER_QUERY_MEMORY_GIB}GB" + set_property "${final}/config.properties" system-mem-limit-gb \ + "${WORKER_MEMORY_LIMIT_GIB}" + set_property "${final}/config.properties" system-mem-shrink-gb \ + "${WORKER_MEMORY_SHRINK_GIB}" + set_property "${final}/config.properties" async-data-cache-enabled \ + "${ASYNC_DATA_CACHE_ENABLED}" + upsert_property "${final}/config.properties" async-cache-ssd-gb \ + "${ASYNC_CACHE_SSD_GIB}" + upsert_property "${final}/config.properties" async-cache-num-shards \ + "${ASYNC_CACHE_NUM_SHARDS}" + upsert_property "${final}/config.properties" runtime-metrics-collection-enabled \ + true + if [[ ${CPU_EXCHANGE_TUNING_ENABLED} == false ]]; then + for key in \ + exchange.http-client.enable-connection-pool \ + exchange.max-buffer-size \ + sink.max-buffer-size \ + exchange.max-response-size \ + exchange.client-threads \ + exchange.http-client.max-connections \ + exchange.http-client.max-connections-per-server \ + exchange.http-client.max-requests-queued-per-destination \ + exchange.http-client.request-timeout; do + delete_property "${final}/config.properties" "${key}" + done + fi + if ((ASYNC_CACHE_SSD_GIB > 0)); then + if [[ -z ${ASYNC_CACHE_SSD_PATH:-} ]]; then + echo "ASYNC_CACHE_SSD_PATH is required for SSD cache" >&2 + exit 2 + fi + upsert_property "${final}/config.properties" async-cache-ssd-path \ + "${ASYNC_CACHE_SSD_PATH}" + fi + set_property "${final}/node.properties" node.id \ + "aws-${RUN_ID}-worker-${WORKER_INDEX}" +fi + +cp -a "${final}/." "${assembled}/" +find "${base}" -type f -print0 | sort -z | xargs -0 sha256sum \ + >"${runtime_root}/base_config_${ROLE}.sha256" +find "${final}" -type f -print0 | sort -z | xargs -0 sha256sum \ + >"${runtime_root}/final_config_${ROLE}.sha256" +diff -ru "${base}" "${final}" >"${runtime_root}/config_${ROLE}.diff" || true +tuning_id=$( + printf '%s\n' \ + "drivers=${TASK_MAX_DRIVERS_PER_TASK}" \ + "split=${HIVE_MAX_SPLIT_SIZE}" \ + "loader=${HIVE_SPLIT_LOADER_CONCURRENCY}" \ + "dynamic_filtering=${DYNAMIC_FILTERING_ENABLED}" \ + "exchange_tuning=${CPU_EXCHANGE_TUNING_ENABLED}" | + sha256sum | cut -c1-12 +) +history="${runtime_root}/config_history/${tuning_id}/${ROLE}" +rm -rf "${history}" +mkdir -p "${history}" +cp -a "${final}/." "${history}/" +cat >"${history}/tuning.json" </dev/null || true +timestamp=$(date -u +%Y%m%dT%H%M%SZ) + +if [[ ${ROLE} == coordinator ]]; then + docker run -d \ + --name presto-coordinator \ + --network host \ + --restart no \ + -e "AWS_DEFAULT_REGION=${AWS_REGION}" \ + -e "AWS_REGION=${AWS_REGION}" \ + -e "SERVER_START_TIMESTAMP=${timestamp}" \ + -v "${assembled}:/opt/presto-server/etc:ro" \ + -v "${runtime_root}/metastore:/var/lib/presto/data/hive/metastore" \ + -v "${runtime_root}/data:/var/lib/presto/data/local" \ + -v "${runtime_root}/logs:/opt/presto-server/logs" \ + -v "${repo}/presto/docker/launch_coordinator.sh:/opt/launch_coordinator.sh:ro" \ + --entrypoint bash \ + "${COORDINATOR_IMAGE}" \ + /opt/launch_coordinator.sh +else + cache_mount_args=() + if ((ASYNC_CACHE_SSD_GIB > 0)); then + cache_mount_args=(-v /mnt/nvme:/mnt/nvme) + fi + docker run -d \ + --name presto-native-worker-cpu \ + --network host \ + --restart no \ + --cap-add SYS_NICE \ + -e "AWS_DEFAULT_REGION=${AWS_REGION}" \ + -e "AWS_REGION=${AWS_REGION}" \ + -e "SERVER_START_TIMESTAMP=${timestamp}" \ + -v "${assembled}:/opt/presto-server/etc:ro" \ + -v "${runtime_root}/metastore:/var/lib/presto/data/hive/metastore" \ + -v "${runtime_root}/data:/var/lib/presto/data/local" \ + -v "${runtime_root}/logs:/opt/presto-server/logs" \ + "${cache_mount_args[@]}" \ + -v "${repo}/presto/docker/launch_presto_servers.sh:/opt/launch_presto_servers.sh:ro" \ + --entrypoint bash \ + "${WORKER_IMAGE}" \ + /opt/launch_presto_servers.sh +fi + +if [[ -f ${runtime_root}/telemetry/${ROLE}.sampler.pid ]]; then + kill "$(cat "${runtime_root}/telemetry/${ROLE}.sampler.pid")" 2>/dev/null || true +fi +nohup bash "${repo}/presto/aws/ec2/remote/sample_host.sh" \ + "${ROLE}" "${runtime_root}/telemetry/${ROLE}.jsonl" \ + >"${runtime_root}/telemetry/${ROLE}.sampler.log" 2>&1 & +echo "$!" >"${runtime_root}/telemetry/${ROLE}.sampler.pid" diff --git a/presto/aws/ec2/remote/run_benchmark.sh b/presto/aws/ec2/remote/run_benchmark.sh new file mode 100755 index 000000000..fbea56052 --- /dev/null +++ b/presto/aws/ec2/remote/run_benchmark.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +if [[ $# -ne 6 ]]; then + echo "usage: run_benchmark.sh " >&2 + exit 2 +fi + +coordinator=$1 +schema=$2 +queries=$3 +iterations=$4 +tag=$5 +result_uri=$6 +repo=/opt/velox-testing +runtime_root=/opt/presto-aws +output="${runtime_root}/results/${tag}" + +mkdir -p "${output}" +cat >"${output}/driver_info.json" <"${output}/benchmark.log" 2>&1 || status=$? + +python3 - "${output}/driver_info.json" "${status}" <<'PY' +import json +import sys +from datetime import datetime, timezone +from pathlib import Path + +path = Path(sys.argv[1]) +payload = json.loads(path.read_text()) +payload["exit_code"] = int(sys.argv[2]) +payload["finished_at"] = datetime.now(timezone.utc).isoformat() +path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") +PY + +upload_status=0 +sync_results || upload_status=$? +trap - EXIT +if ((upload_status != 0)); then + echo "benchmark artifact upload failed with exit code ${upload_status}" >&2 + exit "${upload_status}" +fi +exit "${status}" diff --git a/presto/aws/ec2/remote/run_cache_series.sh b/presto/aws/ec2/remote/run_cache_series.sh new file mode 100755 index 000000000..ae8176023 --- /dev/null +++ b/presto/aws/ec2/remote/run_cache_series.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +if [[ $# -ne 5 ]]; then + echo "usage: run_cache_series.sh " >&2 + exit 2 +fi + +coordinator=$1 +schema=$2 +queries=$3 +tag_prefix=$4 +result_uri=$5 +runner=/opt/velox-testing/presto/aws/ec2/remote/run_benchmark.sh +runtime_root=/opt/presto-aws + +before_nodes=$(curl -fsS "http://${coordinator}:8080/v1/node" \ + | jq -c '[.[].uri] | sort | unique') +printf '%s\n' "${before_nodes}" >"${runtime_root}/results/${tag_prefix}_nodes_before.json" + +snapshot_worker_metrics() { + local label=$1 output_dir uri base_url worker_name + output_dir="${runtime_root}/results/${tag_prefix}_metrics/${label}" + mkdir -p "${output_dir}" + while IFS= read -r uri; do + base_url=${uri%/v1/status} + worker_name=$(sed 's#^http://##; s#[/:]#_#g' <<<"${base_url}") + curl -fsS "${base_url}/v1/info/metrics" \ + >"${output_dir}/${worker_name}.prom" || true + curl -fsS "${base_url}/v1/status" \ + >"${output_dir}/${worker_name}_status.json" || true + done < <(jq -r '.[]' <<<"${before_nodes}") +} + +snapshot_worker_metrics before_cold +bash "${runner}" "${coordinator}" "${schema}" "${queries}" 1 \ + "${tag_prefix}_cold_fill" "${result_uri}" +snapshot_worker_metrics after_cold + +for pass in 1 2 3 4; do + bash "${runner}" "${coordinator}" "${schema}" "${queries}" 1 \ + "${tag_prefix}_warm_${pass}" "${result_uri}" + snapshot_worker_metrics "after_warm_${pass}" +done + +after_nodes=$(curl -fsS "http://${coordinator}:8080/v1/node" \ + | jq -c '[.[].uri] | sort | unique') +printf '%s\n' "${after_nodes}" >"${runtime_root}/results/${tag_prefix}_nodes_after.json" +if [[ ${before_nodes} != "${after_nodes}" ]]; then + echo "worker membership changed during cache series" >&2 + diff -u \ + "${runtime_root}/results/${tag_prefix}_nodes_before.json" \ + "${runtime_root}/results/${tag_prefix}_nodes_after.json" >&2 || true + exit 1 +fi + +aws s3 sync "${runtime_root}/results/" "${result_uri}/benchmark/" \ + --only-show-errors diff --git a/presto/aws/ec2/remote/sample_host.sh b/presto/aws/ec2/remote/sample_host.sh new file mode 100755 index 000000000..3e7fcaf13 --- /dev/null +++ b/presto/aws/ec2/remote/sample_host.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +if [[ $# -ne 2 ]]; then + echo "usage: sample_host.sh " >&2 + exit 2 +fi + +role=$1 +output=$2 +mkdir -p "$(dirname "${output}")" + +while true; do + python3 - "${role}" >>"${output}" <<'PY' +import json +import os +import resource +import time +from pathlib import Path + +role = os.sys.argv[1] +meminfo = {} +for line in Path("/proc/meminfo").read_text().splitlines(): + key, value = line.split(":", 1) + meminfo[key] = int(value.strip().split()[0]) * 1024 + +load1, load5, load15 = os.getloadavg() +network = {} +for line in Path("/proc/net/dev").read_text().splitlines()[2:]: + interface, counters = line.split(":", 1) + fields = counters.split() + interface = interface.strip() + if interface != "lo": + network[interface] = { + "rx_bytes": int(fields[0]), + "rx_packets": int(fields[1]), + "tx_bytes": int(fields[8]), + "tx_packets": int(fields[9]), + } + +disks = {} +for line in Path("/proc/diskstats").read_text().splitlines(): + fields = line.split() + name = fields[2] + if name.startswith("nvme"): + disks[name] = { + "reads_completed": int(fields[3]), + "sectors_read": int(fields[5]), + "writes_completed": int(fields[7]), + "sectors_written": int(fields[9]), + "io_time_ms": int(fields[12]), + } + +print(json.dumps({ + "timestamp_unix": time.time(), + "role": role, + "load": [load1, load5, load15], + "mem_total_bytes": meminfo["MemTotal"], + "mem_available_bytes": meminfo["MemAvailable"], + "swap_free_bytes": meminfo.get("SwapFree", 0), + "network": network, + "disks": disks, + "sampler_max_rss_kib": resource.getrusage(resource.RUSAGE_SELF).ru_maxrss, +}, separators=(",", ":"))) +PY + docker stats --no-stream --format '{{json .}}' 2>/dev/null \ + | python3 -c ' +import json +import sys +import time +for line in sys.stdin: + print(json.dumps({ + "timestamp_unix": time.time(), + "role": sys.argv[1], + "docker": json.loads(line), + }, separators=(",", ":"))) +' "${role}" >>"${output}" || true + sleep 5 +done diff --git a/presto/aws/ec2/remote/wait_for_cluster.sh b/presto/aws/ec2/remote/wait_for_cluster.sh new file mode 100755 index 000000000..cbd731a3d --- /dev/null +++ b/presto/aws/ec2/remote/wait_for_cluster.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +if [[ $# -ne 3 ]]; then + echo "usage: wait_for_cluster.sh " >&2 + exit 2 +fi + +coordinator=$1 +workers=$2 +timeout=$3 +deadline=$((SECONDS + timeout)) + +until [[ $(curl -fsS "http://${coordinator}:8080/v1/info/state" || true) == '"ACTIVE"' ]]; do + if ((SECONDS >= deadline)); then + echo "coordinator did not become ACTIVE within ${timeout}s" >&2 + docker logs presto-coordinator >&2 || true + exit 1 + fi + sleep 5 +done + +while true; do + node_json=$(curl -fsS "http://${coordinator}:8080/v1/node" || echo '[]') + # Discovery can retain stale descriptors briefly after workers restart. + # Count unique worker endpoints so an in-place service restart does not + # make a healthy cluster appear oversized. + count=$(jq '[.[].uri] | unique | length' <<<"${node_json}") + if [[ ${count} -eq ${workers} ]]; then + printf '%s\n' "${node_json}" >/opt/presto-aws/registered_workers.json + exit 0 + fi + if ((SECONDS >= deadline)); then + echo "expected ${workers} workers; observed ${count}" >&2 + printf '%s\n' "${node_json}" >&2 + exit 1 + fi + sleep 5 +done diff --git a/presto/aws/ec2/summarize_results.py b/presto/aws/ec2/summarize_results.py new file mode 100644 index 000000000..f085b0179 --- /dev/null +++ b/presto/aws/ec2/summarize_results.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +"""Apply the AWS 102 timing rules to benchmark_result.json files.""" + +from __future__ import annotations + +import argparse +import json +import statistics +from pathlib import Path +from typing import Any + + +def load_result(path: Path) -> dict[str, Any]: + payload = json.loads(path.read_text()) + tpch = payload.get("tpch", {}) + failures = tpch.get("failed_queries", {}) + if failures: + raise ValueError(f"{path}: failed queries are present: {sorted(failures)}") + raw = tpch.get("raw_times_ms") + if not isinstance(raw, dict) or not raw: + raise ValueError(f"{path}: tpch.raw_times_ms is missing or empty") + return raw + + +def query_sort_key(name: str) -> tuple[int, str]: + try: + return int(name.removeprefix("Q")), name + except ValueError: + return 10**9, name + + +def baseline(path: Path) -> dict[str, Any]: + raw = load_result(path) + per_query_ms: dict[str, float] = {} + for query, values in raw.items(): + if not isinstance(values, list) or len(values) != 5: + raise ValueError(f"{path}: {query} must contain exactly five iterations") + per_query_ms[query] = statistics.fmean(values[1:]) + ordered = dict(sorted(per_query_ms.items(), key=lambda item: query_sort_key(item[0]))) + return { + "mode": "cache_off_s3", + "source": str(path), + "warmup_iterations_excluded": 1, + "measured_iterations": 4, + "per_query_mean_ms": ordered, + "suite_runtime_ms": sum(ordered.values()), + "suite_runtime_seconds": sum(ordered.values()) / 1000, + } + + +def cache_series(paths: list[Path]) -> dict[str, Any]: + if len(paths) != 5: + raise ValueError("cache series requires one cold-fill and four warm results") + loaded = [load_result(path) for path in paths] + query_sets = [set(result) for result in loaded] + if any(query_set != query_sets[0] for query_set in query_sets[1:]): + raise ValueError("cache-series result files contain different query sets") + for path, result in zip(paths, loaded, strict=True): + for query, values in result.items(): + if not isinstance(values, list) or len(values) != 1: + raise ValueError(f"{path}: {query} must contain one suite-major iteration") + + cold_per_query = {query: loaded[0][query][0] for query in sorted(loaded[0], key=query_sort_key)} + warm_per_query = { + query: statistics.fmean(result[query][0] for result in loaded[1:]) + for query in sorted(loaded[0], key=query_sort_key) + } + return { + "mode": "cache_on_cold_fill_and_warm_reuse", + "sources": [str(path) for path in paths], + "cold_fill": { + "per_query_ms": cold_per_query, + "suite_runtime_ms": sum(cold_per_query.values()), + "suite_runtime_seconds": sum(cold_per_query.values()) / 1000, + }, + "warm_reuse": { + "measured_suite_passes": 4, + "per_query_mean_ms": warm_per_query, + "suite_runtime_ms": sum(warm_per_query.values()), + "suite_runtime_seconds": sum(warm_per_query.values()) / 1000, + }, + } + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + modes = parser.add_mutually_exclusive_group(required=True) + modes.add_argument("--baseline", type=Path, metavar="RESULT") + modes.add_argument( + "--cache-series", + type=Path, + nargs=5, + metavar=("COLD", "WARM1", "WARM2", "WARM3", "WARM4"), + ) + parser.add_argument("--output", type=Path) + return parser + + +def main() -> int: + args = build_parser().parse_args() + try: + summary = baseline(args.baseline) if args.baseline else cache_series(args.cache_series) + except (OSError, ValueError, json.JSONDecodeError) as error: + raise SystemExit(f"error: {error}") from error + rendered = json.dumps(summary, indent=2, sort_keys=True) + "\n" + if args.output: + args.output.write_text(rendered) + else: + print(rendered, end="") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/presto/aws/ec2/test_aws_cluster.py b/presto/aws/ec2/test_aws_cluster.py new file mode 100644 index 000000000..82ee6d724 --- /dev/null +++ b/presto/aws/ec2/test_aws_cluster.py @@ -0,0 +1,223 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import contextlib +import io +import json +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import aws_cluster +import summarize_results + + +def valid_config() -> dict[str, str]: + return { + "AWS_PROFILE": "", + "AWS_REGION": "us-east-2", + "AMI_ID": "ami-example", + "SUBNET_ID": "subnet-example", + "SECURITY_GROUP_ID": "sg-example", + "IAM_INSTANCE_PROFILE": "benchmark-role", + "COORDINATOR_INSTANCE_TYPE": "m7a.4xlarge", + "WORKER_INSTANCE_TYPE": "m7a.4xlarge", + "WORKER_COUNT": "8", + "ROOT_VOLUME_GIB": "100", + "VELOX_TESTING_REPOSITORY": "https://example.com/velox-testing.git", + "VELOX_TESTING_FETCH_REF": "refs/heads/example", + "VELOX_TESTING_REF": "d" * 40, + "COORDINATOR_IMAGE": "example/coordinator@sha256:abc", + "WORKER_IMAGE": "example/worker@sha256:def", + "HIVE_METASTORE_S3_URI": "s3://example/metastore/", + "SCHEMA_NAME": "tpch_sf1k", + "RESULTS_S3_ROOT": "s3://example/results", + "VCPU_PER_WORKER": "16", + "TASK_MAX_DRIVERS_PER_TASK": "16", + "HIVE_MAX_SPLIT_SIZE": "256MB", + "HIVE_SPLIT_LOADER_CONCURRENCY": "32", + "DYNAMIC_FILTERING_ENABLED": "false", + "CPU_EXCHANGE_TUNING_ENABLED": "true", + "COORDINATOR_HEAP_GIB": "16", + "COORDINATOR_HEADROOM_GIB": "4", + "COORDINATOR_QUERY_TOTAL_MEMORY_PER_NODE_GIB": "12", + "COORDINATOR_QUERY_MEMORY_PER_NODE_GIB": "10", + "WORKER_SYSTEM_MEMORY_GIB": "50", + "WORKER_QUERY_MEMORY_GIB": "44", + "WORKER_MEMORY_LIMIT_GIB": "54", + "WORKER_MEMORY_SHRINK_GIB": "16", + "ASYNC_DATA_CACHE_ENABLED": "false", + "ASYNC_CACHE_SSD_GIB": "0", + "ASYNC_CACHE_SSD_PATH": "", + "ASYNC_CACHE_NUM_SHARDS": "16", + "OWNER": "tester", + "EXPIRY_HOURS": "8", + "SSM_READY_TIMEOUT_SECONDS": "900", + "SSM_COMMAND_TIMEOUT_SECONDS": "21600", + "WORKER_READY_TIMEOUT_SECONDS": "600", + "PLACEMENT_GROUP": "", + } + + +class EnvTest(unittest.TestCase): + def test_load_env_handles_comments_and_quotes(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "config.env" + path.write_text("# comment\nA=one\nB='two words'\n\n") + self.assertEqual(aws_cluster.load_env(path), {"A": "one", "B": "two words"}) + + def test_load_env_rejects_non_assignments(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "config.env" + path.write_text("not an assignment\n") + with self.assertRaises(aws_cluster.ClusterError): + aws_cluster.load_env(path) + + +class ClusterTest(unittest.TestCase): + def make_cluster(self, config: dict[str, str] | None = None, workers: int = 8) -> aws_cluster.Cluster: + return aws_cluster.Cluster( + config or valid_config(), + run_id="test-run", + workers=workers, + state_root=Path("/tmp/unused-state"), + dry_run=True, + ) + + def test_validation_accepts_supported_shapes(self) -> None: + for workers in (1, 2, 8, 16, 32): + self.make_cluster(workers=workers).validate() + + def test_validation_accepts_role_specific_amis(self) -> None: + config = valid_config() + config["AMI_ID"] = "" + config["COORDINATOR_AMI_ID"] = "ami-x86" + config["WORKER_AMI_ID"] = "ami-arm" + cluster = self.make_cluster(config) + cluster.validate() + self.assertEqual(cluster.resolve_ami("coordinator"), "ami-x86") + self.assertEqual(cluster.resolve_ami("worker"), "ami-arm") + + def test_validation_rejects_unplanned_shape(self) -> None: + with self.assertRaisesRegex(aws_cluster.ClusterError, "worker count"): + self.make_cluster(workers=3).validate() + + def test_validation_rejects_unsafe_worker_memory(self) -> None: + config = valid_config() + config["WORKER_QUERY_MEMORY_GIB"] = "50" + with self.assertRaisesRegex(aws_cluster.ClusterError, "must be less"): + self.make_cluster(config).validate() + + def test_validation_rejects_coordinator_memory_larger_than_heap(self) -> None: + config = valid_config() + config["COORDINATOR_QUERY_TOTAL_MEMORY_PER_NODE_GIB"] = "13" + with self.assertRaisesRegex(aws_cluster.ClusterError, "must be at least"): + self.make_cluster(config).validate() + + def test_validation_requires_complete_harness_bundle_identity(self) -> None: + config = valid_config() + config["HARNESS_BUNDLE_S3_URI"] = "s3://example/harness.tar.gz" + with self.assertRaisesRegex(aws_cluster.ClusterError, "set both"): + self.make_cluster(config).validate() + + def test_launch_dry_run_is_two_idempotent_fleet_calls(self) -> None: + output = io.StringIO() + with contextlib.redirect_stdout(output): + self.make_cluster().launch() + rendered = output.getvalue() + self.assertEqual(rendered.count("ec2 run-instances"), 2) + self.assertIn("--client-token test-run-coordinator", rendered) + self.assertIn("--client-token test-run-worker", rendered) + self.assertIn("--count 8", rendered) + self.assertIn("HttpTokens=required", rendered) + self.assertNotIn("terminate-instances", rendered) + + def test_tags_scope_resources_to_run(self) -> None: + tags = {item["Key"]: item["Value"] for item in self.make_cluster().tags("worker", "2026-08-25T00:00:00+00:00")} + self.assertEqual(tags["Project"], "cudf-performance") + self.assertEqual(tags["Experiment"], "20260824_aws_102") + self.assertEqual(tags["RunId"], "test-run") + self.assertEqual(tags["Role"], "worker") + + def test_inventory_filters_by_owner_before_destructive_use(self) -> None: + cluster = self.make_cluster() + calls: list[list[str]] = [] + + def fake_run(arguments: list[str], **_: object) -> dict[str, list[object]]: + calls.append(arguments) + return {"Reservations": []} + + cluster.aws.run = fake_run # type: ignore[method-assign] + self.assertEqual(cluster.inventory(), []) + rendered = " ".join(calls[0]) + self.assertIn("Name=tag:RunId,Values=test-run", rendered) + self.assertIn("Name=tag:Owner,Values=tester", rendered) + + def test_remote_env_contains_fixed_scale_rules(self) -> None: + rendered = self.make_cluster(workers=16).remote_env("10.0.0.1", "worker", 3) + self.assertIn("WORKER_COUNT=16", rendered) + self.assertIn("WORKER_INDEX=3", rendered) + self.assertIn("COORDINATOR_ADDRESS=10.0.0.1", rendered) + self.assertIn("ASYNC_DATA_CACHE_ENABLED=false", rendered) + self.assertIn("TASK_MAX_DRIVERS_PER_TASK=16", rendered) + self.assertIn("HIVE_MAX_SPLIT_SIZE=256MB", rendered) + + def test_validation_rejects_invalid_cpu_tuning_values(self) -> None: + config = valid_config() + config["HIVE_MAX_SPLIT_SIZE"] = "256" + with self.assertRaisesRegex(aws_cluster.ClusterError, "integer number of MB"): + self.make_cluster(config).validate() + + config = valid_config() + config["CPU_EXCHANGE_TUNING_ENABLED"] = "yes" + with self.assertRaisesRegex(aws_cluster.ClusterError, "true or false"): + self.make_cluster(config).validate() + + def test_ssm_dry_run_sets_long_execution_timeout(self) -> None: + output = io.StringIO() + with contextlib.redirect_stdout(output): + self.make_cluster().send_command(["i-example"], ["sleep 7200"], "long benchmark") + rendered = output.getvalue() + self.assertIn("executionTimeout", rendered) + self.assertIn("21600", rendered) + + +class SummaryTest(unittest.TestCase): + def write_result(self, directory: str, name: str, raw: dict[str, list[int]]) -> Path: + path = Path(directory) / name + path.write_text(json.dumps({"tpch": {"raw_times_ms": raw, "failed_queries": {}}})) + return path + + def test_baseline_discards_first_iteration(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = self.write_result( + directory, + "baseline.json", + {"Q1": [100, 20, 30, 40, 50], "Q2": [200, 10, 20, 30, 40]}, + ) + summary = summarize_results.baseline(path) + self.assertEqual(summary["per_query_mean_ms"], {"Q1": 35, "Q2": 25}) + self.assertEqual(summary["suite_runtime_ms"], 60) + + def test_cache_series_uses_four_suite_major_passes(self) -> None: + with tempfile.TemporaryDirectory() as directory: + paths = [ + self.write_result( + directory, + f"pass-{index}.json", + {"Q1": [100 - index * 10], "Q2": [200 - index * 20]}, + ) + for index in range(5) + ] + summary = summarize_results.cache_series(paths) + self.assertEqual(summary["cold_fill"]["suite_runtime_ms"], 300) + self.assertEqual(summary["warm_reuse"]["per_query_mean_ms"]["Q1"], 75) + self.assertEqual(summary["warm_reuse"]["suite_runtime_ms"], 225) + + +if __name__ == "__main__": + unittest.main() From 1849100ef654a17556d327fea43e14e58d173fac Mon Sep 17 00:00:00 2001 From: Gregory Kimball Date: Mon, 24 Aug 2026 21:39:43 -0700 Subject: [PATCH 02/12] Support distributed GPU workers in the EC2 harness Reuse the existing GPU config and launch path so G7e fleets get NVIDIA preflight checks, pinned Kvikio settings, GPU telemetry, and the same safe lifecycle as CPU fleets. --- presto/README.md | 2 +- presto/aws/ec2/README.md | 37 +++++----- presto/aws/ec2/aws_cluster.py | 49 +++++++++++-- presto/aws/ec2/aws_config.env.example | 12 ++++ presto/aws/ec2/remote/bootstrap_node.sh | 26 ++++++- presto/aws/ec2/remote/collect_node.sh | 6 +- presto/aws/ec2/remote/configure_and_start.sh | 75 +++++++++++++++++--- presto/aws/ec2/remote/sample_host.sh | 26 +++++++ presto/aws/ec2/test_aws_cluster.py | 26 ++++++- 9 files changed, 221 insertions(+), 38 deletions(-) diff --git a/presto/README.md b/presto/README.md index e68158e8b..2aa13e2fe 100644 --- a/presto/README.md +++ b/presto/README.md @@ -106,7 +106,7 @@ pytest tpch_test.py ## Directory Structure - **`docker/`** - Docker Compose configurations and Dockerfiles for different Presto variants -- **`aws/ec2/`** - Direct EC2 + SSM orchestration for distributed CPU benchmarks +- **`aws/ec2/`** - Direct EC2 + SSM orchestration for distributed CPU and GPU benchmarks - **`pbench/`** - Performance benchmarking utilities and TPC-H query definitions - **`scripts/`** - Shell scripts for building, deploying, and testing Presto - **`testing/`** - Python-based test framework using pytest diff --git a/presto/aws/ec2/README.md b/presto/aws/ec2/README.md index 53de71bcd..0b147964d 100644 --- a/presto/aws/ec2/README.md +++ b/presto/aws/ec2/README.md @@ -1,23 +1,20 @@ -# Presto CPU benchmarks on direct EC2 +# Distributed Presto benchmarks on direct EC2 This directory manages an ephemeral Presto cluster with one coordinator-only -EC2 instance and one native CPU worker per worker EC2 instance. It uses AWS -Systems Manager (SSM) instead of SSH and keeps all Presto traffic on private -addresses. +EC2 instance and one native CPU or GPU worker per worker EC2 instance. It uses +AWS Systems Manager (SSM) instead of SSH and keeps all Presto traffic on +private addresses. The initial implementation targets TPC-H SF1000 at 1, 2, 8, 16, or 32 workers. Worker counts exclude the coordinator. -## Status and source dependency +## Source independence -The S3 benchmark path depends on -[velox-testing PR 376](https://github.com/rapidsai/velox-testing/pull/376). -The first qualified source revision is expected to use PR head -`7a2e66683fd7262d8ad2cb6614a6bac1a14e7afa`. Rebase this work onto `main` after -that PR merges, then repeat local checks and fresh-fleet 8-worker qualification. - -Do not use the broader `pioneer` branch as an implicit configuration bundle. -Apply experimental settings as explicit, archived overrides. +The harness has no dependency on a particular benchmark or engine-development +branch. Every run supplies and archives an exact source repository, fetch ref, +commit, coordinator image, and worker image. Experimental S3 or GPU runs may +pin another branch, but that dependency belongs to the run manifest rather +than this harness. ## What the harness owns @@ -49,9 +46,10 @@ The instance profile needs: - write access to the configured results prefix; - ECR permissions only when private ECR images are selected. -The security group needs no SSH ingress. Add a self-referencing TCP rule for -port 8080 so workers can register and exchange data with the coordinator. -Restrict all rules to the benchmark security group and required egress paths. +The security group needs no SSH ingress. Add self-referencing TCP rules for +port 8080 and any native exchange port selected by the generated worker config +(GPU currently uses 10003). Restrict all rules to the benchmark security group +and required egress paths. Use a same-AZ subnet. A placement group is optional and must be recorded as a separate experiment dimension. @@ -93,6 +91,13 @@ Mixed-architecture fleets can instead set `COORDINATOR_AMI_SSM_PARAMETER` and AWS CLI binary for each host architecture and pulls only the image used by that role. +Set `ENGINE_VARIANT=cpu` for CPU workers or `ENGINE_VARIANT=gpu` for one GPU +worker per EC2 instance. GPU workers require an NVIDIA-driver AMI with NVIDIA +Container Toolkit and a GPU-enabled worker image. Bootstrap verifies both host +and container GPU visibility before startup. The example's Kvikio values +reproduce the initial G7e S3 configuration and should remain explicit in each +run manifest. + ## Local validation and dry run Validation and dry-run launch do not contact AWS: diff --git a/presto/aws/ec2/aws_cluster.py b/presto/aws/ec2/aws_cluster.py index a93899e40..d4cb1ba5c 100755 --- a/presto/aws/ec2/aws_cluster.py +++ b/presto/aws/ec2/aws_cluster.py @@ -19,7 +19,6 @@ SCRIPT_DIR = Path(__file__).resolve().parent PROJECT_TAG = "cudf-performance" -EXPERIMENT_TAG = "20260824_aws_102" DEFAULT_STATE_ROOT = Path.home() / ".cache" / "velox-testing" / "aws-ec2" @@ -154,6 +153,7 @@ def validate(self, cloud: bool = True) -> None: "AWS_REGION", "COORDINATOR_INSTANCE_TYPE", "WORKER_INSTANCE_TYPE", + "ENGINE_VARIANT", "VELOX_TESTING_REPOSITORY", "VELOX_TESTING_FETCH_REF", "VELOX_TESTING_REF", @@ -162,9 +162,13 @@ def validate(self, cloud: bool = True) -> None: "HIVE_METASTORE_S3_URI", "SCHEMA_NAME", "OWNER", + "EXPERIMENT_TAG", "HIVE_MAX_SPLIT_SIZE", "DYNAMIC_FILTERING_ENABLED", "CPU_EXCHANGE_TUNING_ENABLED", + "GPU_DEVICE_ID", + "KVIKIO_REMOTE_IO_BACKEND", + "CUDA_MODULE_LOADING", "ASYNC_CACHE_SSD_GIB", "ASYNC_CACHE_NUM_SHARDS", ) @@ -195,6 +199,12 @@ def validate(self, cloud: bool = True) -> None: "VCPU_PER_WORKER", "TASK_MAX_DRIVERS_PER_TASK", "HIVE_SPLIT_LOADER_CONCURRENCY", + "KVIKIO_NTHREADS", + "KVIKIO_TASK_SIZE", + "KVIKIO_BOUNCE_BUFFER_SIZE", + "KVIKIO_REMOTE_IO_NUM_REACTORS", + "KVIKIO_REMOTE_IO_MAX_CONCURRENT_REQUESTS", + "LIBCUDF_NUM_HOST_WORKERS", "COORDINATOR_HEAP_GIB", "COORDINATOR_HEADROOM_GIB", "COORDINATOR_QUERY_TOTAL_MEMORY_PER_NODE_GIB", @@ -222,6 +232,10 @@ def validate(self, cloud: bool = True) -> None: raise ClusterError("HARNESS_BUNDLE_SHA256 must contain 64 hexadecimal characters") if self.config["ASYNC_DATA_CACHE_ENABLED"] not in ("true", "false"): raise ClusterError("ASYNC_DATA_CACHE_ENABLED must be true or false") + if self.config["ENGINE_VARIANT"] not in ("cpu", "gpu"): + raise ClusterError("ENGINE_VARIANT must be cpu or gpu") + if self.config["ENGINE_VARIANT"] == "gpu": + nonnegative_int(self.config, "GPU_DEVICE_ID") for key in ("DYNAMIC_FILTERING_ENABLED", "CPU_EXCHANGE_TUNING_ENABLED"): if self.config[key] not in ("true", "false"): raise ClusterError(f"{key} must be true or false") @@ -256,10 +270,11 @@ def validate(self, cloud: bool = True) -> None: ) def tags(self, role: str, expiry: str) -> list[dict[str, str]]: + engine = self.config["ENGINE_VARIANT"] return [ - {"Key": "Name", "Value": f"presto-cpu-{role}-{self.run_id}"}, + {"Key": "Name", "Value": f"presto-{engine}-{role}-{self.run_id}"}, {"Key": "Project", "Value": PROJECT_TAG}, - {"Key": "Experiment", "Value": EXPERIMENT_TAG}, + {"Key": "Experiment", "Value": self.config["EXPERIMENT_TAG"]}, {"Key": "RunId", "Value": self.run_id}, {"Key": "Role", "Value": role}, {"Key": "Owner", "Value": self.config["OWNER"]}, @@ -418,6 +433,7 @@ def launch(self) -> None: "harness_bundle_s3_uri": self.config.get("HARNESS_BUNDLE_S3_URI", ""), "harness_bundle_sha256": self.config.get("HARNESS_BUNDLE_SHA256", ""), "async_data_cache": self.config["ASYNC_DATA_CACHE_ENABLED"] == "true", + "engine_variant": self.config["ENGINE_VARIANT"], } ) self.wait_for_ssm(instance_ids) @@ -443,7 +459,7 @@ def inventory(self) -> list[Instance]: "describe-instances", "--filters", f"Name=tag:Project,Values={PROJECT_TAG}", - f"Name=tag:Experiment,Values={EXPERIMENT_TAG}", + f"Name=tag:Experiment,Values={self.config['EXPERIMENT_TAG']}", f"Name=tag:RunId,Values={self.run_id}", f"Name=tag:Owner,Values={self.config['OWNER']}", "Name=instance-state-name,Values=pending,running,stopping,stopped", @@ -648,6 +664,8 @@ def bootstrap(self) -> None: + shlex.quote(self.config["WORKER_IMAGE"]) + " " + shlex.quote(self.config["HIVE_METASTORE_S3_URI"]) + + " " + + shlex.quote(self.config["ENGINE_VARIANT"]) ) coordinator_command_id = self.send_command( [coordinator.instance_id], @@ -670,6 +688,7 @@ def remote_env(self, coordinator_address: str, role: str, worker_index: int | No "WORKER_INDEX": "" if worker_index is None else str(worker_index), "COORDINATOR_ADDRESS": coordinator_address, "AWS_REGION": self.config["AWS_REGION"], + "ENGINE_VARIANT": self.config["ENGINE_VARIANT"], "VCPU_PER_WORKER": self.config["VCPU_PER_WORKER"], "TASK_MAX_DRIVERS_PER_TASK": self.config["TASK_MAX_DRIVERS_PER_TASK"], "HIVE_MAX_SPLIT_SIZE": self.config["HIVE_MAX_SPLIT_SIZE"], @@ -680,6 +699,21 @@ def remote_env(self, coordinator_address: str, role: str, worker_index: int | No "CPU_EXCHANGE_TUNING_ENABLED": self.config[ "CPU_EXCHANGE_TUNING_ENABLED" ], + "GPU_DEVICE_ID": self.config["GPU_DEVICE_ID"], + "KVIKIO_REMOTE_IO_BACKEND": self.config["KVIKIO_REMOTE_IO_BACKEND"], + "KVIKIO_NTHREADS": self.config["KVIKIO_NTHREADS"], + "KVIKIO_TASK_SIZE": self.config["KVIKIO_TASK_SIZE"], + "KVIKIO_BOUNCE_BUFFER_SIZE": self.config[ + "KVIKIO_BOUNCE_BUFFER_SIZE" + ], + "KVIKIO_REMOTE_IO_NUM_REACTORS": self.config[ + "KVIKIO_REMOTE_IO_NUM_REACTORS" + ], + "KVIKIO_REMOTE_IO_MAX_CONCURRENT_REQUESTS": self.config[ + "KVIKIO_REMOTE_IO_MAX_CONCURRENT_REQUESTS" + ], + "LIBCUDF_NUM_HOST_WORKERS": self.config["LIBCUDF_NUM_HOST_WORKERS"], + "CUDA_MODULE_LOADING": self.config["CUDA_MODULE_LOADING"], "COORDINATOR_HEAP_GIB": self.config["COORDINATOR_HEAP_GIB"], "COORDINATOR_HEADROOM_GIB": self.config["COORDINATOR_HEADROOM_GIB"], "COORDINATOR_QUERY_TOTAL_MEMORY_PER_NODE_GIB": self.config["COORDINATOR_QUERY_TOTAL_MEMORY_PER_NODE_GIB"], @@ -901,7 +935,10 @@ def stop(self) -> None: [coordinator.instance_id, *[item.instance_id for item in workers]], [ "set -eu", - "docker rm -f presto-coordinator presto-native-worker-cpu 2>/dev/null || true", + ( + "docker rm -f presto-coordinator presto-native-worker-cpu " + "presto-native-worker-gpu 2>/dev/null || true" + ), ], f"stop {self.run_id}", ) @@ -925,7 +962,7 @@ def list_stale(self) -> None: "describe-instances", "--filters", f"Name=tag:Project,Values={PROJECT_TAG}", - f"Name=tag:Experiment,Values={EXPERIMENT_TAG}", + f"Name=tag:Experiment,Values={self.config['EXPERIMENT_TAG']}", "Name=instance-state-name,Values=pending,running,stopping,stopped", ], json_output=True, diff --git a/presto/aws/ec2/aws_config.env.example b/presto/aws/ec2/aws_config.env.example index 9b78ee924..07d8f32c9 100644 --- a/presto/aws/ec2/aws_config.env.example +++ b/presto/aws/ec2/aws_config.env.example @@ -17,6 +17,7 @@ COORDINATOR_INSTANCE_TYPE=m7a.4xlarge WORKER_INSTANCE_TYPE=m7a.4xlarge WORKER_COUNT=8 ROOT_VOLUME_GIB=100 +ENGINE_VARIANT=cpu # Reproducible software inputs. VELOX_TESTING_REPOSITORY=https://github.com//velox-testing.git @@ -41,6 +42,16 @@ HIVE_MAX_SPLIT_SIZE=256MB HIVE_SPLIT_LOADER_CONCURRENCY=32 DYNAMIC_FILTERING_ENABLED=false CPU_EXCHANGE_TUNING_ENABLED=true +# GPU-only worker settings. These are ignored when ENGINE_VARIANT=cpu. +GPU_DEVICE_ID=0 +KVIKIO_REMOTE_IO_BACKEND=EASY_THREADPOOL +KVIKIO_NTHREADS=128 +KVIKIO_TASK_SIZE=16777216 +KVIKIO_BOUNCE_BUFFER_SIZE=16777216 +KVIKIO_REMOTE_IO_NUM_REACTORS=16 +KVIKIO_REMOTE_IO_MAX_CONCURRENT_REQUESTS=256 +LIBCUDF_NUM_HOST_WORKERS=32 +CUDA_MODULE_LOADING=LAZY COORDINATOR_HEAP_GIB=54 COORDINATOR_HEADROOM_GIB=4 COORDINATOR_QUERY_TOTAL_MEMORY_PER_NODE_GIB=48 @@ -56,6 +67,7 @@ ASYNC_CACHE_NUM_SHARDS=16 # Fleet safety and ownership. OWNER= +EXPERIMENT_TAG=20260824_aws_102 EXPIRY_HOURS=8 SSM_READY_TIMEOUT_SECONDS=900 SSM_COMMAND_TIMEOUT_SECONDS=21600 diff --git a/presto/aws/ec2/remote/bootstrap_node.sh b/presto/aws/ec2/remote/bootstrap_node.sh index 2b1f1c9ac..55deb5ac8 100755 --- a/presto/aws/ec2/remote/bootstrap_node.sh +++ b/presto/aws/ec2/remote/bootstrap_node.sh @@ -4,8 +4,8 @@ set -euo pipefail -if [[ $# -ne 4 ]]; then - echo "usage: bootstrap_node.sh " >&2 +if [[ $# -ne 5 ]]; then + echo "usage: bootstrap_node.sh " >&2 exit 2 fi @@ -13,12 +13,17 @@ role=$1 coordinator_image=$2 worker_image=$3 metastore_uri=$4 +engine_variant=$5 runtime_root=/opt/presto-aws if [[ ${role} != coordinator && ${role} != worker ]]; then echo "role must be coordinator or worker" >&2 exit 2 fi +if [[ ${engine_variant} != cpu && ${engine_variant} != gpu ]]; then + echo "engine variant must be cpu or gpu" >&2 + exit 2 +fi mkdir -p \ "${runtime_root}/base_config" \ @@ -32,12 +37,28 @@ python3 -m venv "${runtime_root}/venv" "${runtime_root}/venv/bin/pip" install --disable-pip-version-check \ -r /opt/velox-testing/presto/testing/requirements.txt +login_if_ecr() { + local image=$1 registry region + registry=${image%%/*} + if [[ ${registry} == *.dkr.ecr.*.amazonaws.com ]]; then + region=$(cut -d. -f4 <<<"${registry}") + aws ecr get-login-password --region "${region}" | + docker login --username AWS --password-stdin "${registry}" + fi +} + if [[ ${role} == coordinator ]]; then + login_if_ecr "${coordinator_image}" docker pull "${coordinator_image}" local_image=${coordinator_image} else + login_if_ecr "${worker_image}" docker pull "${worker_image}" local_image=${worker_image} + if [[ ${engine_variant} == gpu ]]; then + nvidia-smi -L + docker run --rm --gpus all --entrypoint nvidia-smi "${worker_image}" -L + fi fi aws s3 sync "${metastore_uri}" "${runtime_root}/metastore/" --only-show-errors @@ -63,6 +84,7 @@ cat >"${runtime_root}/bootstrap_info.json" </dev/null || true cp "${runtime_root}"/*.diff "${artifact_root}/" 2>/dev/null || true docker ps -a --no-trunc >"${artifact_root}/docker_ps.txt" 2>&1 || true -docker inspect presto-coordinator presto-native-worker-cpu \ +docker inspect presto-coordinator presto-native-worker-cpu presto-native-worker-gpu \ >"${artifact_root}/docker_inspect.json" 2>/dev/null || true docker logs presto-coordinator >"${artifact_root}/coordinator_container.log" 2>&1 || true docker logs presto-native-worker-cpu >"${artifact_root}/worker_container.log" 2>&1 || true +docker logs presto-native-worker-gpu >"${artifact_root}/worker_gpu_container.log" 2>&1 || true uname -a >"${artifact_root}/uname.txt" lscpu --json >"${artifact_root}/lscpu.json" @@ -50,6 +51,9 @@ if [[ ${role} == worker ]]; then >"${artifact_root}/worker_metrics.prom" 2>&1 || true curl -fsS http://localhost:8080/v1/status \ >"${artifact_root}/worker_status.json" 2>&1 || true + nvidia-smi -q >"${artifact_root}/nvidia_smi_q.txt" 2>&1 || true + nvidia-smi dmon -s pucvmt -c 1 \ + >"${artifact_root}/nvidia_smi_dmon.txt" 2>&1 || true fi journalctl -u docker --no-pager >"${artifact_root}/docker_journal.log" 2>&1 || true dmesg --ctime >"${artifact_root}/dmesg.log" 2>&1 || true diff --git a/presto/aws/ec2/remote/configure_and_start.sh b/presto/aws/ec2/remote/configure_and_start.sh index 1a24fd6bd..aacaf26f3 100755 --- a/presto/aws/ec2/remote/configure_and_start.sh +++ b/presto/aws/ec2/remote/configure_and_start.sh @@ -6,6 +6,10 @@ set -euo pipefail required=( RUN_ID ROLE WORKER_COUNT COORDINATOR_ADDRESS AWS_REGION VCPU_PER_WORKER + ENGINE_VARIANT GPU_DEVICE_ID KVIKIO_REMOTE_IO_BACKEND KVIKIO_NTHREADS + KVIKIO_TASK_SIZE KVIKIO_BOUNCE_BUFFER_SIZE + KVIKIO_REMOTE_IO_NUM_REACTORS KVIKIO_REMOTE_IO_MAX_CONCURRENT_REQUESTS + LIBCUDF_NUM_HOST_WORKERS CUDA_MODULE_LOADING TASK_MAX_DRIVERS_PER_TASK HIVE_MAX_SPLIT_SIZE HIVE_SPLIT_LOADER_CONCURRENCY DYNAMIC_FILTERING_ENABLED CPU_EXCHANGE_TUNING_ENABLED @@ -27,6 +31,10 @@ if [[ ${ROLE} != coordinator && ${ROLE} != worker ]]; then echo "ROLE must be coordinator or worker" >&2 exit 2 fi +if [[ ${ENGINE_VARIANT} != cpu && ${ENGINE_VARIANT} != gpu ]]; then + echo "ENGINE_VARIANT must be cpu or gpu" >&2 + exit 2 +fi if [[ ${ROLE} == worker && -z ${WORKER_INDEX:-} ]]; then echo "WORKER_INDEX is required for worker role" >&2 exit 2 @@ -34,7 +42,7 @@ fi repo=/opt/velox-testing runtime_root=/opt/presto-aws -generated="${repo}/presto/docker/config/generated/cpu" +generated="${repo}/presto/docker/config/generated/${ENGINE_VARIANT}" base="${runtime_root}/base_config/${ROLE}" final="${runtime_root}/final_config/${ROLE}" assembled="${runtime_root}/runtime_etc" @@ -96,7 +104,7 @@ PY cd "${repo}/presto/scripts" OVERWRITE_CONFIG=true \ NUM_WORKERS="${WORKER_COUNT}" \ -VARIANT_TYPE=cpu \ +VARIANT_TYPE="${ENGINE_VARIANT}" \ VCPU_PER_WORKER="${VCPU_PER_WORKER}" \ ./generate_presto_config.sh @@ -118,12 +126,20 @@ else cp "${worker_source}/catalog/hive.properties" "${base}/catalog/hive.properties" fi cp -a "${base}/." "${final}/" -delete_property "${final}/catalog/hive.properties" hive.max-split-size -printf 'hive.max-split-size=%s\n' "${HIVE_MAX_SPLIT_SIZE}" \ - >>"${final}/catalog/hive.properties" -delete_property "${final}/catalog/hive.properties" hive.split-loader-concurrency -printf 'hive.split-loader-concurrency=%s\n' "${HIVE_SPLIT_LOADER_CONCURRENCY}" \ - >>"${final}/catalog/hive.properties" +if [[ ${ENGINE_VARIANT} == cpu ]]; then + delete_property "${final}/catalog/hive.properties" hive.max-split-size + printf 'hive.max-split-size=%s\n' "${HIVE_MAX_SPLIT_SIZE}" \ + >>"${final}/catalog/hive.properties" + delete_property "${final}/catalog/hive.properties" hive.split-loader-concurrency + printf 'hive.split-loader-concurrency=%s\n' "${HIVE_SPLIT_LOADER_CONCURRENCY}" \ + >>"${final}/catalog/hive.properties" +else + delete_property "${final}/catalog/hive.properties" hive.file-splittable + printf 'hive.file-splittable=false\n' >>"${final}/catalog/hive.properties" + delete_property "${final}/catalog/hive.properties" cudf.hive.use-buffered-input + printf 'cudf.hive.use-buffered-input=false\n' \ + >>"${final}/catalog/hive.properties" +fi if [[ ${ROLE} == coordinator ]]; then set_property "${final}/config.properties" discovery.uri \ @@ -184,7 +200,13 @@ else "${ASYNC_CACHE_NUM_SHARDS}" upsert_property "${final}/config.properties" runtime-metrics-collection-enabled \ true - if [[ ${CPU_EXCHANGE_TUNING_ENABLED} == false ]]; then + if [[ ${ENGINE_VARIANT} == gpu ]]; then + delete_property "${final}/config.properties" cudf.batch_size_min_threshold + printf 'cudf.batch_size_min_threshold=40000000\n' \ + >>"${final}/config.properties" + delete_property "${final}/config.properties" cudf.s3.use_kvikio + printf 'cudf.s3.use_kvikio=true\n' >>"${final}/config.properties" + elif [[ ${CPU_EXCHANGE_TUNING_ENABLED} == false ]]; then for key in \ exchange.http-client.enable-connection-pool \ exchange.max-buffer-size \ @@ -218,6 +240,7 @@ find "${final}" -type f -print0 | sort -z | xargs -0 sha256sum \ diff -ru "${base}" "${final}" >"${runtime_root}/config_${ROLE}.diff" || true tuning_id=$( printf '%s\n' \ + "engine=${ENGINE_VARIANT}" \ "drivers=${TASK_MAX_DRIVERS_PER_TASK}" \ "split=${HIVE_MAX_SPLIT_SIZE}" \ "loader=${HIVE_SPLIT_LOADER_CONCURRENCY}" \ @@ -231,6 +254,7 @@ mkdir -p "${history}" cp -a "${final}/." "${history}/" cat >"${history}/tuning.json" <"${history}/tuning.json" </dev/null || true +docker rm -f \ + presto-coordinator presto-native-worker-cpu presto-native-worker-gpu \ + 2>/dev/null || true timestamp=$(date -u +%Y%m%dT%H%M%SZ) if [[ ${ROLE} == coordinator ]]; then @@ -260,17 +286,44 @@ if [[ ${ROLE} == coordinator ]]; then /opt/launch_coordinator.sh else cache_mount_args=() + gpu_args=() + gpu_env=() if ((ASYNC_CACHE_SSD_GIB > 0)); then cache_mount_args=(-v /mnt/nvme:/mnt/nvme) fi + if [[ ${ENGINE_VARIANT} == gpu ]]; then + gpu_args=( + --gpus "device=${GPU_DEVICE_ID}" + --cap-add IPC_LOCK + --ulimit memlock=-1:-1 + --shm-size 1g + -v /sys/devices/system/node:/sys/devices/system/node:ro + ) + gpu_env=( + -e "CUDA_VISIBLE_DEVICES=${GPU_DEVICE_ID}" + -e "NVIDIA_VISIBLE_DEVICES=${GPU_DEVICE_ID}" + -e "KVIKIO_REMOTE_IO_BACKEND=${KVIKIO_REMOTE_IO_BACKEND}" + -e "KVIKIO_NTHREADS=${KVIKIO_NTHREADS}" + -e "KVIKIO_TASK_SIZE=${KVIKIO_TASK_SIZE}" + -e "KVIKIO_BOUNCE_BUFFER_SIZE=${KVIKIO_BOUNCE_BUFFER_SIZE}" + -e "KVIKIO_REMOTE_IO_NUM_REACTORS=${KVIKIO_REMOTE_IO_NUM_REACTORS}" + -e "KVIKIO_REMOTE_IO_MAX_CONCURRENT_REQUESTS=${KVIKIO_REMOTE_IO_MAX_CONCURRENT_REQUESTS}" + -e "LIBCUDF_NUM_HOST_WORKERS=${LIBCUDF_NUM_HOST_WORKERS}" + -e "CUDA_MODULE_LOADING=${CUDA_MODULE_LOADING}" + -e UCX_TLS=tcp,cuda_copy,cuda_ipc + -e UCX_TCP_CM_REUSEADDR=y + ) + fi docker run -d \ - --name presto-native-worker-cpu \ + --name "presto-native-worker-${ENGINE_VARIANT}" \ --network host \ --restart no \ --cap-add SYS_NICE \ + "${gpu_args[@]}" \ -e "AWS_DEFAULT_REGION=${AWS_REGION}" \ -e "AWS_REGION=${AWS_REGION}" \ -e "SERVER_START_TIMESTAMP=${timestamp}" \ + "${gpu_env[@]}" \ -v "${assembled}:/opt/presto-server/etc:ro" \ -v "${runtime_root}/metastore:/var/lib/presto/data/hive/metastore" \ -v "${runtime_root}/data:/var/lib/presto/data/local" \ diff --git a/presto/aws/ec2/remote/sample_host.sh b/presto/aws/ec2/remote/sample_host.sh index 3e7fcaf13..c2fe5fbc7 100755 --- a/presto/aws/ec2/remote/sample_host.sh +++ b/presto/aws/ec2/remote/sample_host.sh @@ -78,5 +78,31 @@ for line in sys.stdin: "docker": json.loads(line), }, separators=(",", ":"))) ' "${role}" >>"${output}" || true + if [[ ${role} == worker ]] && command -v nvidia-smi >/dev/null; then + nvidia-smi \ + --query-gpu=index,utilization.gpu,memory.used,memory.total,power.draw,temperature.gpu \ + --format=csv,noheader,nounits 2>/dev/null | + python3 -c ' +import json +import sys +import time +for line in sys.stdin: + index, util, used, total, power, temperature = [ + value.strip() for value in line.split(",") + ] + print(json.dumps({ + "timestamp_unix": time.time(), + "role": "worker", + "gpu": { + "index": int(index), + "utilization_percent": float(util), + "memory_used_mib": float(used), + "memory_total_mib": float(total), + "power_watts": float(power), + "temperature_c": float(temperature), + }, + }, separators=(",", ":"))) +' >>"${output}" || true + fi sleep 5 done diff --git a/presto/aws/ec2/test_aws_cluster.py b/presto/aws/ec2/test_aws_cluster.py index 82ee6d724..a57b01204 100644 --- a/presto/aws/ec2/test_aws_cluster.py +++ b/presto/aws/ec2/test_aws_cluster.py @@ -28,6 +28,7 @@ def valid_config() -> dict[str, str]: "WORKER_INSTANCE_TYPE": "m7a.4xlarge", "WORKER_COUNT": "8", "ROOT_VOLUME_GIB": "100", + "ENGINE_VARIANT": "cpu", "VELOX_TESTING_REPOSITORY": "https://example.com/velox-testing.git", "VELOX_TESTING_FETCH_REF": "refs/heads/example", "VELOX_TESTING_REF": "d" * 40, @@ -42,6 +43,16 @@ def valid_config() -> dict[str, str]: "HIVE_SPLIT_LOADER_CONCURRENCY": "32", "DYNAMIC_FILTERING_ENABLED": "false", "CPU_EXCHANGE_TUNING_ENABLED": "true", + "GPU_DEVICE_ID": "0", + "KVIKIO_REMOTE_IO_BACKEND": "EASY_THREADPOOL", + "KVIKIO_NTHREADS": "128", + "KVIKIO_TASK_SIZE": "16777216", + "KVIKIO_BOUNCE_BUFFER_SIZE": "16777216", + "KVIKIO_REMOTE_IO_NUM_REACTORS": "16", + "KVIKIO_REMOTE_IO_MAX_CONCURRENT_REQUESTS": "256", + "LIBCUDF_NUM_HOST_WORKERS": "32", + "CUDA_MODULE_LOADING": "LAZY", + "EXPERIMENT_TAG": "test-experiment", "COORDINATOR_HEAP_GIB": "16", "COORDINATOR_HEADROOM_GIB": "4", "COORDINATOR_QUERY_TOTAL_MEMORY_PER_NODE_GIB": "12", @@ -139,7 +150,7 @@ def test_launch_dry_run_is_two_idempotent_fleet_calls(self) -> None: def test_tags_scope_resources_to_run(self) -> None: tags = {item["Key"]: item["Value"] for item in self.make_cluster().tags("worker", "2026-08-25T00:00:00+00:00")} self.assertEqual(tags["Project"], "cudf-performance") - self.assertEqual(tags["Experiment"], "20260824_aws_102") + self.assertEqual(tags["Experiment"], "test-experiment") self.assertEqual(tags["RunId"], "test-run") self.assertEqual(tags["Role"], "worker") @@ -165,6 +176,7 @@ def test_remote_env_contains_fixed_scale_rules(self) -> None: self.assertIn("ASYNC_DATA_CACHE_ENABLED=false", rendered) self.assertIn("TASK_MAX_DRIVERS_PER_TASK=16", rendered) self.assertIn("HIVE_MAX_SPLIT_SIZE=256MB", rendered) + self.assertIn("ENGINE_VARIANT=cpu", rendered) def test_validation_rejects_invalid_cpu_tuning_values(self) -> None: config = valid_config() @@ -177,6 +189,18 @@ def test_validation_rejects_invalid_cpu_tuning_values(self) -> None: with self.assertRaisesRegex(aws_cluster.ClusterError, "true or false"): self.make_cluster(config).validate() + def test_validation_accepts_gpu_variant(self) -> None: + config = valid_config() + config["ENGINE_VARIANT"] = "gpu" + config["GPU_DEVICE_ID"] = "0" + self.make_cluster(config).validate() + + def test_validation_rejects_unknown_engine_variant(self) -> None: + config = valid_config() + config["ENGINE_VARIANT"] = "fpga" + with self.assertRaisesRegex(aws_cluster.ClusterError, "cpu or gpu"): + self.make_cluster(config).validate() + def test_ssm_dry_run_sets_long_execution_timeout(self) -> None: output = io.StringIO() with contextlib.redirect_stdout(output): From 310b7879b6b6cf40edf6dfa34f5f9f32917efa86 Mon Sep 17 00:00:00 2001 From: Gregory Kimball Date: Mon, 24 Aug 2026 21:41:31 -0700 Subject: [PATCH 03/12] Remove experiment-specific naming from the EC2 harness Keep the reusable branch independent of local benchmark series by using generic examples, tags, run IDs, and timing terminology. --- presto/aws/ec2/README.md | 10 +++++----- presto/aws/ec2/aws_cluster.py | 2 +- presto/aws/ec2/aws_config.env.example | 4 ++-- presto/aws/ec2/summarize_results.py | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/presto/aws/ec2/README.md b/presto/aws/ec2/README.md index 0b147964d..17e68b87a 100644 --- a/presto/aws/ec2/README.md +++ b/presto/aws/ec2/README.md @@ -105,12 +105,12 @@ Validation and dry-run launch do not contact AWS: ```bash python3 aws_cluster.py \ --config ~/.config/velox-testing/aws-cpu.env \ - --run-id aws102-dry-run \ + --run-id presto-ec2-dry-run \ validate python3 aws_cluster.py \ --config ~/.config/velox-testing/aws-cpu.env \ - --run-id aws102-dry-run \ + --run-id presto-ec2-dry-run \ --workers 8 \ --dry-run \ launch @@ -127,7 +127,7 @@ Choose one unique RunId and use it for the full lifecycle: ```bash CONFIG=~/.config/velox-testing/aws-cpu.env -RUN_ID=aws102-m7a8-$(date -u +%Y%m%dT%H%M%SZ) +RUN_ID=presto-m7a8-$(date -u +%Y%m%dT%H%M%SZ) python3 aws_cluster.py --config "$CONFIG" --run-id "$RUN_ID" --workers 8 launch python3 aws_cluster.py --config "$CONFIG" --run-id "$RUN_ID" --workers 8 status @@ -159,7 +159,7 @@ Collection accepts a partial fleet and gathers every still-running node. Instances carry an expiry tag and an OS shutdown behavior, but neither replaces explicit teardown and billing verification. -List expired AWS 102 instances without mutating them: +List expired benchmark instances without mutating them: ```bash python3 aws_cluster.py \ @@ -184,7 +184,7 @@ semantic validation artifacts are required for qualification. ## Baseline timing -For cache-off AWS 101-compatible timing, run five query-major iterations. Treat +For cache-off query-major timing, run five iterations. Treat iteration 1 as warm-up. For every query, average iterations 2-5, then sum the 22 means. Provisioning, startup, registration, and warm-up are excluded from that primary runtime and must be reported separately. diff --git a/presto/aws/ec2/aws_cluster.py b/presto/aws/ec2/aws_cluster.py index d4cb1ba5c..12db27dc4 100755 --- a/presto/aws/ec2/aws_cluster.py +++ b/presto/aws/ec2/aws_cluster.py @@ -1012,7 +1012,7 @@ def record_command(self, name: str, command_id: str) -> None: def default_run_id() -> str: - return "aws102-" + utc_now().strftime("%Y%m%dT%H%M%SZ").lower() + return "presto-ec2-" + utc_now().strftime("%Y%m%dT%H%M%SZ").lower() def build_parser() -> argparse.ArgumentParser: diff --git a/presto/aws/ec2/aws_config.env.example b/presto/aws/ec2/aws_config.env.example index 07d8f32c9..2513f45bb 100644 --- a/presto/aws/ec2/aws_config.env.example +++ b/presto/aws/ec2/aws_config.env.example @@ -21,7 +21,7 @@ ENGINE_VARIANT=cpu # Reproducible software inputs. VELOX_TESTING_REPOSITORY=https://github.com//velox-testing.git -VELOX_TESTING_FETCH_REF=refs/heads/aws-102-ec2-cpu-cluster +VELOX_TESTING_FETCH_REF=refs/heads/aws-ec2-presto-cluster VELOX_TESTING_REF= # Optional for an uncommitted smoke-test overlay. Set both or neither. HARNESS_BUNDLE_S3_URI= @@ -67,7 +67,7 @@ ASYNC_CACHE_NUM_SHARDS=16 # Fleet safety and ownership. OWNER= -EXPERIMENT_TAG=20260824_aws_102 +EXPERIMENT_TAG=presto_ec2_benchmark EXPIRY_HOURS=8 SSM_READY_TIMEOUT_SECONDS=900 SSM_COMMAND_TIMEOUT_SECONDS=21600 diff --git a/presto/aws/ec2/summarize_results.py b/presto/aws/ec2/summarize_results.py index f085b0179..a2d975377 100644 --- a/presto/aws/ec2/summarize_results.py +++ b/presto/aws/ec2/summarize_results.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -"""Apply the AWS 102 timing rules to benchmark_result.json files.""" +"""Apply benchmark timing rules to benchmark_result.json files.""" from __future__ import annotations From 69c9ad22eb5eecb93899554209b819b6c458a66a Mon Sep 17 00:00:00 2001 From: Gregory Kimball Date: Mon, 24 Aug 2026 21:42:56 -0700 Subject: [PATCH 04/12] Preserve preinstalled Docker on GPU AMIs Avoid replacing the DLAMI Docker installation so its NVIDIA runtime integration remains intact during worker bootstrap. --- presto/aws/ec2/aws_cluster.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/presto/aws/ec2/aws_cluster.py b/presto/aws/ec2/aws_cluster.py index 12db27dc4..cb6e2161c 100755 --- a/presto/aws/ec2/aws_cluster.py +++ b/presto/aws/ec2/aws_cluster.py @@ -627,7 +627,14 @@ def bootstrap(self) -> None: "set -eu", "export DEBIAN_FRONTEND=noninteractive", "apt-get update -y", - ("apt-get install -y ca-certificates curl docker.io git jq numactl python3 python3-venv unzip"), + ( + "apt-get install -y ca-certificates curl git jq numactl " + "python3 python3-venv unzip" + ), + ( + "if ! command -v docker >/dev/null; then " + "apt-get install -y docker.io; fi" + ), "systemctl enable --now docker", ( "if ! command -v aws >/dev/null; then " From ca61352aece3ca8d80fca3cc507b05bfdfa0b2c5 Mon Sep 17 00:00:00 2001 From: Gregory Kimball Date: Mon, 24 Aug 2026 21:49:54 -0700 Subject: [PATCH 05/12] Verify active UCX exchange on GPU fleets Capture protocol logs, socket state, and exchange counters so distributed GPU qualification proves that cuDF exchange transferred data instead of merely accepting its configuration. --- presto/aws/ec2/README.md | 23 +++- presto/aws/ec2/aws_cluster.py | 26 ++++- presto/aws/ec2/remote/collect_node.sh | 1 + presto/aws/ec2/remote/configure_and_start.sh | 5 + presto/aws/ec2/remote/verify_gpu_exchange.sh | 107 +++++++++++++++++++ presto/aws/ec2/test_aws_cluster.py | 9 ++ 6 files changed, 168 insertions(+), 3 deletions(-) create mode 100644 presto/aws/ec2/remote/verify_gpu_exchange.sh diff --git a/presto/aws/ec2/README.md b/presto/aws/ec2/README.md index 17e68b87a..aae4182c0 100644 --- a/presto/aws/ec2/README.md +++ b/presto/aws/ec2/README.md @@ -95,8 +95,7 @@ Set `ENGINE_VARIANT=cpu` for CPU workers or `ENGINE_VARIANT=gpu` for one GPU worker per EC2 instance. GPU workers require an NVIDIA-driver AMI with NVIDIA Container Toolkit and a GPU-enabled worker image. Bootstrap verifies both host and container GPU visibility before startup. The example's Kvikio values -reproduce the initial G7e S3 configuration and should remain explicit in each -run manifest. +are explicit so each GPU run records its remote-I/O behavior. ## Local validation and dry run @@ -207,6 +206,26 @@ This preserves the Presto processes and worker membership while issuing `sync` and dropping Linux page cache on every coordinator and worker before each pass. Each repetition has a separate result tag. +## GPU exchange verification + +For two or more GPU workers, run a shuffle-heavy query and then require +observable evidence that cuDF's UCX exchange path handled traffic: + +```bash +python3 aws_cluster.py \ + --config "$CONFIG" --run-id "$RUN_ID" --workers 2 \ + run --queries 9 --iterations 1 --tag gpu_exchange_probe +python3 aws_cluster.py \ + --config "$CONFIG" --run-id "$RUN_ID" --workers 2 \ + verify-gpu-exchange +``` + +Verification fails unless every worker has `cudf.exchange=true`, the configured +UCX server port is listening, and worker logs or positive runtime counters show +active UCX/exchange protocol use. Collection preserves the full worker log, +filtered metrics, socket snapshots, and `gpu_exchange_verification.json`. +Configuration alone is not accepted as proof of active exchange. + ## Controlled CPU tuning The environment file exposes the generated CPU override as explicit controls: diff --git a/presto/aws/ec2/aws_cluster.py b/presto/aws/ec2/aws_cluster.py index cb6e2161c..8994ff668 100755 --- a/presto/aws/ec2/aws_cluster.py +++ b/presto/aws/ec2/aws_cluster.py @@ -628,7 +628,7 @@ def bootstrap(self) -> None: "export DEBIAN_FRONTEND=noninteractive", "apt-get update -y", ( - "apt-get install -y ca-certificates curl git jq numactl " + "apt-get install -y ca-certificates curl git iproute2 jq numactl " "python3 python3-venv unzip" ), ( @@ -911,6 +911,27 @@ def run_cache_series(self, queries: str, tag_prefix: str) -> None: ) self.record_command("cache_series", command_id) + def verify_gpu_exchange(self) -> None: + self.validate(cloud=True) + if self.config["ENGINE_VARIANT"] != "gpu": + raise ClusterError("verify-gpu-exchange requires ENGINE_VARIANT=gpu") + if self.workers < 2: + raise ClusterError("verify-gpu-exchange requires at least two workers") + _, workers = self.expected_inventory() + command_id = self.send_command( + [worker.instance_id for worker in workers], + [ + "set -eu", + ( + "bash /opt/velox-testing/presto/aws/ec2/remote/" + "verify_gpu_exchange.sh" + ), + ], + f"verify GPU exchange {self.run_id}", + wait=True, + ) + self.record_command("verify_gpu_exchange", command_id) + def collect(self) -> None: self.validate(cloud=True) inventory = self.inventory() @@ -1036,6 +1057,7 @@ def build_parser() -> argparse.ArgumentParser: "status", "bootstrap", "start", + "verify-gpu-exchange", "collect", "stop", "terminate", @@ -1095,6 +1117,8 @@ def main() -> int: if args.repetitions <= 0: raise ClusterError("--repetitions must be greater than zero") cluster.run_cold_series(args.queries, args.repetitions, args.tag_prefix) + elif args.command == "verify-gpu-exchange": + cluster.verify_gpu_exchange() elif args.command == "collect": cluster.collect() elif args.command == "stop": diff --git a/presto/aws/ec2/remote/collect_node.sh b/presto/aws/ec2/remote/collect_node.sh index 7cb72e143..e48ccc775 100755 --- a/presto/aws/ec2/remote/collect_node.sh +++ b/presto/aws/ec2/remote/collect_node.sh @@ -28,6 +28,7 @@ cp -a "${runtime_root}/results" "${artifact_root}/" 2>/dev/null || true cp "${runtime_root}"/*.json "${artifact_root}/" 2>/dev/null || true cp "${runtime_root}"/*.sha256 "${artifact_root}/" 2>/dev/null || true cp "${runtime_root}"/*.diff "${artifact_root}/" 2>/dev/null || true +cp "${runtime_root}"/gpu_exchange_* "${artifact_root}/" 2>/dev/null || true docker ps -a --no-trunc >"${artifact_root}/docker_ps.txt" 2>&1 || true docker inspect presto-coordinator presto-native-worker-cpu presto-native-worker-gpu \ diff --git a/presto/aws/ec2/remote/configure_and_start.sh b/presto/aws/ec2/remote/configure_and_start.sh index aacaf26f3..435502e1f 100755 --- a/presto/aws/ec2/remote/configure_and_start.sh +++ b/presto/aws/ec2/remote/configure_and_start.sh @@ -312,6 +312,11 @@ else -e "CUDA_MODULE_LOADING=${CUDA_MODULE_LOADING}" -e UCX_TLS=tcp,cuda_copy,cuda_ipc -e UCX_TCP_CM_REUSEADDR=y + -e UCX_LOG_LEVEL=info + -e UCX_PROTO_INFO=y + -e UCX_RNDV_PIPELINE_ERROR_HANDLING=y + -e UCX_TCP_KEEPINTVL=1ms + -e UCX_KEEPALIVE_INTERVAL=1ms ) fi docker run -d \ diff --git a/presto/aws/ec2/remote/verify_gpu_exchange.sh b/presto/aws/ec2/remote/verify_gpu_exchange.sh new file mode 100644 index 000000000..9b02da17a --- /dev/null +++ b/presto/aws/ec2/remote/verify_gpu_exchange.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +runtime_root=/opt/presto-aws +config="${runtime_root}/final_config/worker/config.properties" +output="${runtime_root}/gpu_exchange_verification.json" +container=presto-native-worker-gpu + +test -f "${config}" +enabled=$(awk -F= '$1 == "cudf.exchange" {value=$2} END {print value}' "${config}") +port=$(awk -F= '$1 == "cudf.exchange.server.port" {value=$2} END {print value}' "${config}") +test -n "${port}" + +docker logs "${container}" >"${runtime_root}/gpu_exchange_container.log" 2>&1 +curl -fsS http://localhost:8080/v1/info/metrics \ + >"${runtime_root}/gpu_exchange_metrics.prom" +ss -ltnp >"${runtime_root}/gpu_exchange_listeners.txt" +ss -tinp >"${runtime_root}/gpu_exchange_connections.txt" + +python3 - \ + "${enabled}" \ + "${port}" \ + "${runtime_root}/gpu_exchange_container.log" \ + "${runtime_root}/gpu_exchange_metrics.prom" \ + "${runtime_root}/gpu_exchange_listeners.txt" \ + "${runtime_root}/gpu_exchange_connections.txt" \ + "${output}" <<'PY' +import json +import re +import sys +from datetime import datetime, timezone +from pathlib import Path + +enabled, port, log_path, metrics_path, listeners_path, connections_path, output_path = ( + sys.argv[1:] +) +log_text = Path(log_path).read_text(errors="replace") +metrics_text = Path(metrics_path).read_text(errors="replace") +listeners_text = Path(listeners_path).read_text(errors="replace") +connections_text = Path(connections_path).read_text(errors="replace") + +ucx_lines = [ + line + for line in log_text.splitlines() + if re.search(r"\bucx\b|cuda_(?:copy|ipc)|UCX_", line, re.IGNORECASE) +] +protocol_lines = [ + line + for line in ucx_lines + if re.search( + r"proto|rndv|eager|endpoint|listener|transport|cuda_copy|cuda_ipc", + line, + re.IGNORECASE, + ) +] +exchange_metrics = [] +positive_exchange_metrics = [] +for line in metrics_text.splitlines(): + if not line or line.startswith("#"): + continue + name = line.split("{", 1)[0].split(None, 1)[0] + if not re.search(r"ucx|cudf.*exchange|exchange.*cudf", name, re.IGNORECASE): + continue + exchange_metrics.append(name) + try: + value = float(line.rsplit(None, 1)[1]) + except (ValueError, IndexError): + continue + if value > 0: + positive_exchange_metrics.append({"name": name, "value": value}) + +listener_active = bool( + re.search(rf"[:.]({re.escape(port)})\s", listeners_text) +) +established_connections = sum( + 1 + for line in connections_text.splitlines() + if re.search(rf"[:.]({re.escape(port)})\s", line) +) +active_evidence = bool(protocol_lines or positive_exchange_metrics) + +payload = { + "checked_at": datetime.now(timezone.utc).isoformat(), + "configured": enabled == "true", + "server_port": int(port), + "listener_active": listener_active, + "established_connection_lines": established_connections, + "ucx_log_line_count": len(ucx_lines), + "ucx_protocol_line_count": len(protocol_lines), + "ucx_protocol_examples": protocol_lines[:20], + "exchange_metric_names": sorted(set(exchange_metrics)), + "positive_exchange_metrics": positive_exchange_metrics[:50], + "active_transfer_evidence": active_evidence, +} +Path(output_path).write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") +print(json.dumps(payload, indent=2, sort_keys=True)) + +if not payload["configured"]: + raise SystemExit(3) +if not listener_active: + raise SystemExit(4) +if not active_evidence: + raise SystemExit(5) +PY diff --git a/presto/aws/ec2/test_aws_cluster.py b/presto/aws/ec2/test_aws_cluster.py index a57b01204..5c71c1e2d 100644 --- a/presto/aws/ec2/test_aws_cluster.py +++ b/presto/aws/ec2/test_aws_cluster.py @@ -201,6 +201,15 @@ def test_validation_rejects_unknown_engine_variant(self) -> None: with self.assertRaisesRegex(aws_cluster.ClusterError, "cpu or gpu"): self.make_cluster(config).validate() + def test_gpu_exchange_verification_rejects_cpu_and_single_worker(self) -> None: + with self.assertRaisesRegex(aws_cluster.ClusterError, "ENGINE_VARIANT=gpu"): + self.make_cluster(workers=2).verify_gpu_exchange() + + config = valid_config() + config["ENGINE_VARIANT"] = "gpu" + with self.assertRaisesRegex(aws_cluster.ClusterError, "at least two"): + self.make_cluster(config, workers=1).verify_gpu_exchange() + def test_ssm_dry_run_sets_long_execution_timeout(self) -> None: output = io.StringIO() with contextlib.redirect_stdout(output): From 7c19526000f0e1afbe60eb9b15386857d4d83309 Mon Sep 17 00:00:00 2001 From: Gregory Kimball Date: Mon, 24 Aug 2026 21:56:12 -0700 Subject: [PATCH 06/12] Scope cuDF Hive settings to GPU workers Keep GPU-native connector properties out of the Java coordinator catalog so mixed-role fleets start with strict configuration validation enabled. --- presto/aws/ec2/remote/configure_and_start.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/presto/aws/ec2/remote/configure_and_start.sh b/presto/aws/ec2/remote/configure_and_start.sh index 435502e1f..0320fda48 100755 --- a/presto/aws/ec2/remote/configure_and_start.sh +++ b/presto/aws/ec2/remote/configure_and_start.sh @@ -136,9 +136,11 @@ if [[ ${ENGINE_VARIANT} == cpu ]]; then else delete_property "${final}/catalog/hive.properties" hive.file-splittable printf 'hive.file-splittable=false\n' >>"${final}/catalog/hive.properties" - delete_property "${final}/catalog/hive.properties" cudf.hive.use-buffered-input - printf 'cudf.hive.use-buffered-input=false\n' \ - >>"${final}/catalog/hive.properties" + if [[ ${ROLE} == worker ]]; then + delete_property "${final}/catalog/hive.properties" cudf.hive.use-buffered-input + printf 'cudf.hive.use-buffered-input=false\n' \ + >>"${final}/catalog/hive.properties" + fi fi if [[ ${ROLE} == coordinator ]]; then From 04578f808347b853aad61009240a9da7a14771c4 Mon Sep 17 00:00:00 2001 From: Gregory Kimball Date: Tue, 25 Aug 2026 16:00:09 -0700 Subject: [PATCH 07/12] Stabilize GPU cache and S3 benchmark paths Make GPU input and batching behavior reproducible while supporting DLAMI NVMe layouts, instance-role S3 access, and valid single-worker execution. --- presto/aws/ec2/README.md | 4 ++ presto/aws/ec2/aws_cluster.py | 40 +++++++++++++++++--- presto/aws/ec2/aws_config.env.example | 3 ++ presto/aws/ec2/remote/configure_and_start.sh | 32 +++++++++++++--- presto/aws/ec2/test_aws_cluster.py | 12 ++++++ 5 files changed, 79 insertions(+), 12 deletions(-) diff --git a/presto/aws/ec2/README.md b/presto/aws/ec2/README.md index aae4182c0..5d6c88eb7 100644 --- a/presto/aws/ec2/README.md +++ b/presto/aws/ec2/README.md @@ -96,6 +96,10 @@ worker per EC2 instance. GPU workers require an NVIDIA-driver AMI with NVIDIA Container Toolkit and a GPU-enabled worker image. Bootstrap verifies both host and container GPU visibility before startup. The example's Kvikio values are explicit so each GPU run records its remote-I/O behavior. +`GPU_USE_BUFFERED_INPUT` and `GPU_USE_KVIKIO` select between Velox buffered +input (required for AsyncDataCache) and direct KvikIO S3 reads. +`GPU_BATCH_SIZE_MIN_THRESHOLD` controls GPU rebatching and can be reduced when +buffered-input workloads exceed device memory. ## Local validation and dry run diff --git a/presto/aws/ec2/aws_cluster.py b/presto/aws/ec2/aws_cluster.py index 8994ff668..c8fbf16b8 100755 --- a/presto/aws/ec2/aws_cluster.py +++ b/presto/aws/ec2/aws_cluster.py @@ -167,6 +167,9 @@ def validate(self, cloud: bool = True) -> None: "DYNAMIC_FILTERING_ENABLED", "CPU_EXCHANGE_TUNING_ENABLED", "GPU_DEVICE_ID", + "GPU_BATCH_SIZE_MIN_THRESHOLD", + "GPU_USE_BUFFERED_INPUT", + "GPU_USE_KVIKIO", "KVIKIO_REMOTE_IO_BACKEND", "CUDA_MODULE_LOADING", "ASYNC_CACHE_SSD_GIB", @@ -236,7 +239,13 @@ def validate(self, cloud: bool = True) -> None: raise ClusterError("ENGINE_VARIANT must be cpu or gpu") if self.config["ENGINE_VARIANT"] == "gpu": nonnegative_int(self.config, "GPU_DEVICE_ID") - for key in ("DYNAMIC_FILTERING_ENABLED", "CPU_EXCHANGE_TUNING_ENABLED"): + positive_int(self.config, "GPU_BATCH_SIZE_MIN_THRESHOLD") + for key in ( + "DYNAMIC_FILTERING_ENABLED", + "CPU_EXCHANGE_TUNING_ENABLED", + "GPU_USE_BUFFERED_INPUT", + "GPU_USE_KVIKIO", + ): if self.config[key] not in ("true", "false"): raise ClusterError(f"{key} must be true or false") split_size = self.config["HIVE_MAX_SPLIT_SIZE"] @@ -707,6 +716,11 @@ def remote_env(self, coordinator_address: str, role: str, worker_index: int | No "CPU_EXCHANGE_TUNING_ENABLED" ], "GPU_DEVICE_ID": self.config["GPU_DEVICE_ID"], + "GPU_BATCH_SIZE_MIN_THRESHOLD": self.config[ + "GPU_BATCH_SIZE_MIN_THRESHOLD" + ], + "GPU_USE_BUFFERED_INPUT": self.config["GPU_USE_BUFFERED_INPUT"], + "GPU_USE_KVIKIO": self.config["GPU_USE_KVIKIO"], "KVIKIO_REMOTE_IO_BACKEND": self.config["KVIKIO_REMOTE_IO_BACKEND"], "KVIKIO_NTHREADS": self.config["KVIKIO_NTHREADS"], "KVIKIO_TASK_SIZE": self.config["KVIKIO_TASK_SIZE"], @@ -748,15 +762,29 @@ def start(self) -> None: [worker.instance_id for worker in workers], [ "set -eu", + "mkdir -p /mnt/nvme", ( + "if mountpoint -q /opt/dlami/nvme; then " + "mountpoint -q /mnt/nvme || " + "mount --bind /opt/dlami/nvme /mnt/nvme; " + "else " "device=$(lsblk -dpno NAME,MODEL | " - "awk '$0 ~ /Amazon EC2 NVMe Instance Storage/ {print $1; exit}'); " + "awk '$0 ~ /Amazon EC2 NVMe Instance Storage/ " + "{print $1; exit}'); " "test -n \"$device\"; test -b \"$device\"; " - "if ! blkid \"$device\" >/dev/null 2>&1; then " - "mkfs.ext4 -F \"$device\"; fi" + "existing_mount=$(lsblk -nrpo MOUNTPOINT \"$device\" | " + "awk 'NF {print; exit}'); " + "if test -n \"$existing_mount\"; then " + "mountpoint -q /mnt/nvme || " + "mount --bind \"$existing_mount\" /mnt/nvme; " + "else " + "fstype=$(blkid -s TYPE -o value \"$device\" || true); " + "test \"$fstype\" != LVM2_member; " + "if test -z \"$fstype\"; then mkfs.ext4 -F \"$device\"; fi; " + "mountpoint -q /mnt/nvme || mount \"$device\" /mnt/nvme; " + "fi; " + "fi" ), - "mkdir -p /mnt/nvme", - "mountpoint -q /mnt/nvme || mount \"$device\" /mnt/nvme", f"mkdir -p {cache_path}", f"chmod 0777 /mnt/nvme {cache_path}", ], diff --git a/presto/aws/ec2/aws_config.env.example b/presto/aws/ec2/aws_config.env.example index 2513f45bb..42bb065b2 100644 --- a/presto/aws/ec2/aws_config.env.example +++ b/presto/aws/ec2/aws_config.env.example @@ -44,6 +44,9 @@ DYNAMIC_FILTERING_ENABLED=false CPU_EXCHANGE_TUNING_ENABLED=true # GPU-only worker settings. These are ignored when ENGINE_VARIANT=cpu. GPU_DEVICE_ID=0 +GPU_BATCH_SIZE_MIN_THRESHOLD=40000000 +GPU_USE_BUFFERED_INPUT=false +GPU_USE_KVIKIO=true KVIKIO_REMOTE_IO_BACKEND=EASY_THREADPOOL KVIKIO_NTHREADS=128 KVIKIO_TASK_SIZE=16777216 diff --git a/presto/aws/ec2/remote/configure_and_start.sh b/presto/aws/ec2/remote/configure_and_start.sh index 0320fda48..fd07f7481 100755 --- a/presto/aws/ec2/remote/configure_and_start.sh +++ b/presto/aws/ec2/remote/configure_and_start.sh @@ -6,7 +6,9 @@ set -euo pipefail required=( RUN_ID ROLE WORKER_COUNT COORDINATOR_ADDRESS AWS_REGION VCPU_PER_WORKER - ENGINE_VARIANT GPU_DEVICE_ID KVIKIO_REMOTE_IO_BACKEND KVIKIO_NTHREADS + ENGINE_VARIANT GPU_DEVICE_ID GPU_BATCH_SIZE_MIN_THRESHOLD + GPU_USE_BUFFERED_INPUT GPU_USE_KVIKIO + KVIKIO_REMOTE_IO_BACKEND KVIKIO_NTHREADS KVIKIO_TASK_SIZE KVIKIO_BOUNCE_BUFFER_SIZE KVIKIO_REMOTE_IO_NUM_REACTORS KVIKIO_REMOTE_IO_MAX_CONCURRENT_REQUESTS LIBCUDF_NUM_HOST_WORKERS CUDA_MODULE_LOADING @@ -138,7 +140,7 @@ else printf 'hive.file-splittable=false\n' >>"${final}/catalog/hive.properties" if [[ ${ROLE} == worker ]]; then delete_property "${final}/catalog/hive.properties" cudf.hive.use-buffered-input - printf 'cudf.hive.use-buffered-input=false\n' \ + printf 'cudf.hive.use-buffered-input=%s\n' "${GPU_USE_BUFFERED_INPUT}" \ >>"${final}/catalog/hive.properties" fi fi @@ -160,7 +162,11 @@ if [[ ${ROLE} == coordinator ]]; then "$((COORDINATOR_QUERY_MEMORY_PER_NODE_GIB * WORKER_COUNT))GB" set_property "${final}/config.properties" memory.heap-headroom-per-node \ "${COORDINATOR_HEADROOM_GIB}GB" - set_property "${final}/config.properties" single-node-execution-enabled false + if ((WORKER_COUNT == 1)); then + set_property "${final}/config.properties" single-node-execution-enabled true + else + set_property "${final}/config.properties" single-node-execution-enabled false + fi set_property "${final}/config.properties" experimental.enable-dynamic-filtering \ "${DYNAMIC_FILTERING_ENABLED}" python3 - "${final}/jvm.config" "${COORDINATOR_HEAP_GIB}" <<'PY' @@ -181,7 +187,11 @@ else set_property "${final}/config.properties" http-server.http.port 8080 set_property "${final}/config.properties" discovery.uri \ "http://${COORDINATOR_ADDRESS}:8080" - set_property "${final}/config.properties" single-node-execution-enabled false + if ((WORKER_COUNT == 1)); then + set_property "${final}/config.properties" single-node-execution-enabled true + else + set_property "${final}/config.properties" single-node-execution-enabled false + fi set_property "${final}/config.properties" task.max-drivers-per-task \ "${TASK_MAX_DRIVERS_PER_TASK}" set_property "${final}/config.properties" system-memory-gb \ @@ -204,10 +214,12 @@ else true if [[ ${ENGINE_VARIANT} == gpu ]]; then delete_property "${final}/config.properties" cudf.batch_size_min_threshold - printf 'cudf.batch_size_min_threshold=40000000\n' \ + printf 'cudf.batch_size_min_threshold=%s\n' \ + "${GPU_BATCH_SIZE_MIN_THRESHOLD}" \ >>"${final}/config.properties" delete_property "${final}/config.properties" cudf.s3.use_kvikio - printf 'cudf.s3.use_kvikio=true\n' >>"${final}/config.properties" + printf 'cudf.s3.use_kvikio=%s\n' "${GPU_USE_KVIKIO}" \ + >>"${final}/config.properties" elif [[ ${CPU_EXCHANGE_TUNING_ENABLED} == false ]]; then for key in \ exchange.http-client.enable-connection-pool \ @@ -294,6 +306,10 @@ else cache_mount_args=(-v /mnt/nvme:/mnt/nvme) fi if [[ ${ENGINE_VARIANT} == gpu ]]; then + # KvikIO performs S3 reads inside the worker container. Export the + # instance-profile credentials because containerized processes cannot + # reliably reach IMDS when the instance hop limit is one. + eval "$(aws configure export-credentials --format env)" gpu_args=( --gpus "device=${GPU_DEVICE_ID}" --cap-add IPC_LOCK @@ -310,8 +326,12 @@ else -e "KVIKIO_BOUNCE_BUFFER_SIZE=${KVIKIO_BOUNCE_BUFFER_SIZE}" -e "KVIKIO_REMOTE_IO_NUM_REACTORS=${KVIKIO_REMOTE_IO_NUM_REACTORS}" -e "KVIKIO_REMOTE_IO_MAX_CONCURRENT_REQUESTS=${KVIKIO_REMOTE_IO_MAX_CONCURRENT_REQUESTS}" + -e KVIKIO_REMOTE_IO_REACTOR_DISPATCH=PER_CHUNK -e "LIBCUDF_NUM_HOST_WORKERS=${LIBCUDF_NUM_HOST_WORKERS}" -e "CUDA_MODULE_LOADING=${CUDA_MODULE_LOADING}" + -e AWS_ACCESS_KEY_ID + -e AWS_SECRET_ACCESS_KEY + -e AWS_SESSION_TOKEN -e UCX_TLS=tcp,cuda_copy,cuda_ipc -e UCX_TCP_CM_REUSEADDR=y -e UCX_LOG_LEVEL=info diff --git a/presto/aws/ec2/test_aws_cluster.py b/presto/aws/ec2/test_aws_cluster.py index 5c71c1e2d..9deacac7c 100644 --- a/presto/aws/ec2/test_aws_cluster.py +++ b/presto/aws/ec2/test_aws_cluster.py @@ -44,6 +44,9 @@ def valid_config() -> dict[str, str]: "DYNAMIC_FILTERING_ENABLED": "false", "CPU_EXCHANGE_TUNING_ENABLED": "true", "GPU_DEVICE_ID": "0", + "GPU_BATCH_SIZE_MIN_THRESHOLD": "40000000", + "GPU_USE_BUFFERED_INPUT": "false", + "GPU_USE_KVIKIO": "true", "KVIKIO_REMOTE_IO_BACKEND": "EASY_THREADPOOL", "KVIKIO_NTHREADS": "128", "KVIKIO_TASK_SIZE": "16777216", @@ -195,6 +198,15 @@ def test_validation_accepts_gpu_variant(self) -> None: config["GPU_DEVICE_ID"] = "0" self.make_cluster(config).validate() + def test_validation_rejects_invalid_gpu_io_toggle(self) -> None: + config = valid_config() + config["GPU_USE_BUFFERED_INPUT"] = "yes" + with self.assertRaisesRegex( + aws_cluster.ClusterError, + "GPU_USE_BUFFERED_INPUT must be true or false", + ): + self.make_cluster(config).validate() + def test_validation_rejects_unknown_engine_variant(self) -> None: config = valid_config() config["ENGINE_VARIANT"] = "fpga" From 412e5a8fe2cb646d96781d34e864bf30b2497477 Mon Sep 17 00:00:00 2001 From: Gregory Kimball Date: Tue, 25 Aug 2026 16:25:38 -0700 Subject: [PATCH 08/12] Remove conflicting suite repetition modes Keep benchmark repetition semantics aligned with velox-testing by using query-major --iterations for both cache configurations. --- presto/aws/ec2/README.md | 38 ++------ presto/aws/ec2/aws_cluster.py | 101 ---------------------- presto/aws/ec2/remote/run_cache_series.sh | 61 ------------- presto/aws/ec2/summarize_results.py | 49 +---------- presto/aws/ec2/test_aws_cluster.py | 19 +--- 5 files changed, 13 insertions(+), 255 deletions(-) delete mode 100755 presto/aws/ec2/remote/run_cache_series.sh diff --git a/presto/aws/ec2/README.md b/presto/aws/ec2/README.md index 5d6c88eb7..138abd80a 100644 --- a/presto/aws/ec2/README.md +++ b/presto/aws/ec2/README.md @@ -194,22 +194,9 @@ primary runtime and must be reported separately. The harness archives raw output; top-line reduction should happen in the experiment repository so rejected runs and the reduction rule remain visible. -Use `summarize_results.py --baseline ` to apply the +Use `summarize_results.py ` to apply the one-warm-up/four-measured rule without relying on the runner's aggregate field. -For repeated data-cold suites with cache off, drop host page caches before each -single-iteration Q1-Q22 pass: - -```bash -python3 aws_cluster.py \ - --config "$CONFIG" --run-id "$RUN_ID" --workers 8 \ - run-cold-series --repetitions 3 --tag-prefix sf1k -``` - -This preserves the Presto processes and worker membership while issuing -`sync` and dropping Linux page cache on every coordinator and worker before -each pass. Each repetition has a separate result tag. - ## GPU exchange verification For two or more GPU workers, run a shuffle-heavy query and then require @@ -242,7 +229,7 @@ The environment file exposes the generated CPU override as explicit controls: The example values reproduce the normal generated CPU configuration. Change one value at a time, rerun `start` to generate and archive the candidate -configuration, then use `run-cold-series` with a diagnostic query list. Setting +configuration, then use `run --iterations 5` with a diagnostic query list. Setting `CPU_EXCHANGE_TUNING_ENABLED=false` removes the generated CPU exchange properties so the native worker uses its defaults. @@ -251,28 +238,17 @@ properties so the native worker uses its defaults. Do not compare cache-on and cache-off results as the same workload. For cache-on qualification, use a fresh fleet and an explicit cache capacity. -Run one complete Q1-Q22 suite with one iteration per query as cold fill. Without -clearing, restarting, or changing the worker session, run four more complete -one-iteration suites. Average each query across those four warm suites and sum -the query means. - -Do not use `--iterations 5` for this cache-on sequence because that produces -query-major order rather than suite-major order. Give each pass a unique tag, -and preserve one uninterrupted cluster session. - -The harness encodes that sequence: +Use the same query-major iteration protocol as cache-off timing: ```bash python3 aws_cluster.py \ --config "$CONFIG" --run-id "$RUN_ID" --workers 8 \ - run-cache-series --tag-prefix sf1k_cache + run --iterations 5 --tag sf1k_cache ``` -It requires `ASYNC_DATA_CACHE_ENABLED=true`, runs one cold-fill suite followed -by four warm suites, and rejects a run if worker membership changes. -Pass the five resulting `benchmark_result.json` files to -`summarize_results.py --cache-series` in cold, warm-1, warm-2, warm-3, warm-4 -order. +For each query, plot iteration 1 separately and average iterations 2-5 for the +reported result. The hot/cold distinction comes only from the archived cache +and input-path configuration, not from a second repetition model. ## Generated state and artifacts diff --git a/presto/aws/ec2/aws_cluster.py b/presto/aws/ec2/aws_cluster.py index c8fbf16b8..7701361c7 100755 --- a/presto/aws/ec2/aws_cluster.py +++ b/presto/aws/ec2/aws_cluster.py @@ -857,88 +857,6 @@ def run_benchmark( command_id = self.send_command([coordinator.instance_id], commands, f"benchmark {self.run_id}", wait=True) self.record_command(record_name, command_id) - def run_cold_series(self, queries: str, repetitions: int, tag_prefix: str) -> None: - self.validate(cloud=True) - if self.config["ASYNC_DATA_CACHE_ENABLED"] != "false": - raise ClusterError("run-cold-series requires ASYNC_DATA_CACHE_ENABLED=false") - coordinator, workers = self.expected_inventory() - instance_ids = [coordinator.instance_id, *[item.instance_id for item in workers]] - for repetition in range(1, repetitions + 1): - clear_command_id = self.send_command( - instance_ids, - [ - "set -eu", - "sync", - "echo 3 > /proc/sys/vm/drop_caches", - ], - f"drop host caches repetition {repetition} {self.run_id}", - wait=True, - ) - self.record_command(f"drop_host_caches_{repetition}", clear_command_id) - self.run_benchmark( - queries, - 1, - f"{tag_prefix}_cold_{repetition}", - record_name=f"cold_benchmark_{repetition}", - ) - - def run_cache_series(self, queries: str, tag_prefix: str) -> None: - self.validate(cloud=True) - coordinator, workers = self.expected_inventory() - address = coordinator.private_ip or coordinator.private_dns - instance_ids = [coordinator.instance_id, *[worker.instance_id for worker in workers]] - drop_command_id = self.send_command( - instance_ids, - [ - "set -eu", - "sync", - "echo 3 > /proc/sys/vm/drop_caches", - ], - f"drop host caches {self.run_id}", - wait=True, - ) - self.record_command("drop_host_caches", drop_command_id) - if self.config["ASYNC_DATA_CACHE_ENABLED"] == "true": - cache_types = ["memory"] - if nonnegative_int(self.config, "ASYNC_CACHE_SSD_GIB"): - cache_types.append("ssd") - for cache_type in cache_types: - expected = f"Cleared {cache_type} cache" - clear_command_id = self.send_command( - [worker.instance_id for worker in workers], - [ - "set -eu", - ( - "response=$(curl -fsS " - f"'http://localhost:8080/v1/operation/server/clearCache?type={cache_type}'" - "); printf '%s\\n' \"$response\"; " - f"test \"$response\" = {shlex.quote(expected)}" - ), - ], - f"clear worker {cache_type} caches {self.run_id}", - wait=True, - ) - self.record_command(f"clear_worker_{cache_type}_caches", clear_command_id) - commands = [ - "set -eu", - ( - f"AWS_DEFAULT_REGION={shlex.quote(self.config['AWS_REGION'])} " - f"AWS_REGION={shlex.quote(self.config['AWS_REGION'])} " - "HOME=/root " - "bash /opt/velox-testing/presto/aws/ec2/remote/run_cache_series.sh " - f"{shlex.quote(address)} {shlex.quote(self.config['SCHEMA_NAME'])} " - f"{shlex.quote(queries)} {shlex.quote(tag_prefix)} " - f"{shlex.quote(self.result_uri())}" - ), - ] - command_id = self.send_command( - [coordinator.instance_id], - commands, - f"cache series {self.run_id}", - wait=True, - ) - self.record_command("cache_series", command_id) - def verify_gpu_exchange(self) -> None: self.validate(cloud=True) if self.config["ENGINE_VARIANT"] != "gpu": @@ -1096,19 +1014,6 @@ def build_parser() -> argparse.ArgumentParser: run_parser.add_argument("--queries", default="1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22") run_parser.add_argument("--iterations", type=int, default=5) run_parser.add_argument("--tag", default="sf1k_q1q22") - cache_parser = subparsers.add_parser("run-cache-series") - cache_parser.add_argument( - "--queries", - default="1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22", - ) - cache_parser.add_argument("--tag-prefix", default="sf1k_cache") - cold_parser = subparsers.add_parser("run-cold-series") - cold_parser.add_argument( - "--queries", - default="1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22", - ) - cold_parser.add_argument("--repetitions", type=int, default=3) - cold_parser.add_argument("--tag-prefix", default="sf1k") return parser @@ -1139,12 +1044,6 @@ def main() -> int: if args.iterations <= 0: raise ClusterError("--iterations must be greater than zero") cluster.run_benchmark(args.queries, args.iterations, args.tag) - elif args.command == "run-cache-series": - cluster.run_cache_series(args.queries, args.tag_prefix) - elif args.command == "run-cold-series": - if args.repetitions <= 0: - raise ClusterError("--repetitions must be greater than zero") - cluster.run_cold_series(args.queries, args.repetitions, args.tag_prefix) elif args.command == "verify-gpu-exchange": cluster.verify_gpu_exchange() elif args.command == "collect": diff --git a/presto/aws/ec2/remote/run_cache_series.sh b/presto/aws/ec2/remote/run_cache_series.sh deleted file mode 100755 index ae8176023..000000000 --- a/presto/aws/ec2/remote/run_cache_series.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -if [[ $# -ne 5 ]]; then - echo "usage: run_cache_series.sh " >&2 - exit 2 -fi - -coordinator=$1 -schema=$2 -queries=$3 -tag_prefix=$4 -result_uri=$5 -runner=/opt/velox-testing/presto/aws/ec2/remote/run_benchmark.sh -runtime_root=/opt/presto-aws - -before_nodes=$(curl -fsS "http://${coordinator}:8080/v1/node" \ - | jq -c '[.[].uri] | sort | unique') -printf '%s\n' "${before_nodes}" >"${runtime_root}/results/${tag_prefix}_nodes_before.json" - -snapshot_worker_metrics() { - local label=$1 output_dir uri base_url worker_name - output_dir="${runtime_root}/results/${tag_prefix}_metrics/${label}" - mkdir -p "${output_dir}" - while IFS= read -r uri; do - base_url=${uri%/v1/status} - worker_name=$(sed 's#^http://##; s#[/:]#_#g' <<<"${base_url}") - curl -fsS "${base_url}/v1/info/metrics" \ - >"${output_dir}/${worker_name}.prom" || true - curl -fsS "${base_url}/v1/status" \ - >"${output_dir}/${worker_name}_status.json" || true - done < <(jq -r '.[]' <<<"${before_nodes}") -} - -snapshot_worker_metrics before_cold -bash "${runner}" "${coordinator}" "${schema}" "${queries}" 1 \ - "${tag_prefix}_cold_fill" "${result_uri}" -snapshot_worker_metrics after_cold - -for pass in 1 2 3 4; do - bash "${runner}" "${coordinator}" "${schema}" "${queries}" 1 \ - "${tag_prefix}_warm_${pass}" "${result_uri}" - snapshot_worker_metrics "after_warm_${pass}" -done - -after_nodes=$(curl -fsS "http://${coordinator}:8080/v1/node" \ - | jq -c '[.[].uri] | sort | unique') -printf '%s\n' "${after_nodes}" >"${runtime_root}/results/${tag_prefix}_nodes_after.json" -if [[ ${before_nodes} != "${after_nodes}" ]]; then - echo "worker membership changed during cache series" >&2 - diff -u \ - "${runtime_root}/results/${tag_prefix}_nodes_before.json" \ - "${runtime_root}/results/${tag_prefix}_nodes_after.json" >&2 || true - exit 1 -fi - -aws s3 sync "${runtime_root}/results/" "${result_uri}/benchmark/" \ - --only-show-errors diff --git a/presto/aws/ec2/summarize_results.py b/presto/aws/ec2/summarize_results.py index a2d975377..d1699869f 100644 --- a/presto/aws/ec2/summarize_results.py +++ b/presto/aws/ec2/summarize_results.py @@ -32,7 +32,7 @@ def query_sort_key(name: str) -> tuple[int, str]: return 10**9, name -def baseline(path: Path) -> dict[str, Any]: +def summarize(path: Path) -> dict[str, Any]: raw = load_result(path) per_query_ms: dict[str, float] = {} for query, values in raw.items(): @@ -41,7 +41,7 @@ def baseline(path: Path) -> dict[str, Any]: per_query_ms[query] = statistics.fmean(values[1:]) ordered = dict(sorted(per_query_ms.items(), key=lambda item: query_sort_key(item[0]))) return { - "mode": "cache_off_s3", + "mode": "query_major_iterations", "source": str(path), "warmup_iterations_excluded": 1, "measured_iterations": 4, @@ -51,50 +51,9 @@ def baseline(path: Path) -> dict[str, Any]: } -def cache_series(paths: list[Path]) -> dict[str, Any]: - if len(paths) != 5: - raise ValueError("cache series requires one cold-fill and four warm results") - loaded = [load_result(path) for path in paths] - query_sets = [set(result) for result in loaded] - if any(query_set != query_sets[0] for query_set in query_sets[1:]): - raise ValueError("cache-series result files contain different query sets") - for path, result in zip(paths, loaded, strict=True): - for query, values in result.items(): - if not isinstance(values, list) or len(values) != 1: - raise ValueError(f"{path}: {query} must contain one suite-major iteration") - - cold_per_query = {query: loaded[0][query][0] for query in sorted(loaded[0], key=query_sort_key)} - warm_per_query = { - query: statistics.fmean(result[query][0] for result in loaded[1:]) - for query in sorted(loaded[0], key=query_sort_key) - } - return { - "mode": "cache_on_cold_fill_and_warm_reuse", - "sources": [str(path) for path in paths], - "cold_fill": { - "per_query_ms": cold_per_query, - "suite_runtime_ms": sum(cold_per_query.values()), - "suite_runtime_seconds": sum(cold_per_query.values()) / 1000, - }, - "warm_reuse": { - "measured_suite_passes": 4, - "per_query_mean_ms": warm_per_query, - "suite_runtime_ms": sum(warm_per_query.values()), - "suite_runtime_seconds": sum(warm_per_query.values()) / 1000, - }, - } - - def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) - modes = parser.add_mutually_exclusive_group(required=True) - modes.add_argument("--baseline", type=Path, metavar="RESULT") - modes.add_argument( - "--cache-series", - type=Path, - nargs=5, - metavar=("COLD", "WARM1", "WARM2", "WARM3", "WARM4"), - ) + parser.add_argument("result", type=Path, metavar="RESULT") parser.add_argument("--output", type=Path) return parser @@ -102,7 +61,7 @@ def build_parser() -> argparse.ArgumentParser: def main() -> int: args = build_parser().parse_args() try: - summary = baseline(args.baseline) if args.baseline else cache_series(args.cache_series) + summary = summarize(args.result) except (OSError, ValueError, json.JSONDecodeError) as error: raise SystemExit(f"error: {error}") from error rendered = json.dumps(summary, indent=2, sort_keys=True) + "\n" diff --git a/presto/aws/ec2/test_aws_cluster.py b/presto/aws/ec2/test_aws_cluster.py index 9deacac7c..3221255b3 100644 --- a/presto/aws/ec2/test_aws_cluster.py +++ b/presto/aws/ec2/test_aws_cluster.py @@ -237,32 +237,17 @@ def write_result(self, directory: str, name: str, raw: dict[str, list[int]]) -> path.write_text(json.dumps({"tpch": {"raw_times_ms": raw, "failed_queries": {}}})) return path - def test_baseline_discards_first_iteration(self) -> None: + def test_summary_discards_first_iteration(self) -> None: with tempfile.TemporaryDirectory() as directory: path = self.write_result( directory, "baseline.json", {"Q1": [100, 20, 30, 40, 50], "Q2": [200, 10, 20, 30, 40]}, ) - summary = summarize_results.baseline(path) + summary = summarize_results.summarize(path) self.assertEqual(summary["per_query_mean_ms"], {"Q1": 35, "Q2": 25}) self.assertEqual(summary["suite_runtime_ms"], 60) - def test_cache_series_uses_four_suite_major_passes(self) -> None: - with tempfile.TemporaryDirectory() as directory: - paths = [ - self.write_result( - directory, - f"pass-{index}.json", - {"Q1": [100 - index * 10], "Q2": [200 - index * 20]}, - ) - for index in range(5) - ] - summary = summarize_results.cache_series(paths) - self.assertEqual(summary["cold_fill"]["suite_runtime_ms"], 300) - self.assertEqual(summary["warm_reuse"]["per_query_mean_ms"]["Q1"], 75) - self.assertEqual(summary["warm_reuse"]["suite_runtime_ms"], 225) - if __name__ == "__main__": unittest.main() From 8b829c83b0605fc5b38c872bac01219f2704f4cd Mon Sep 17 00:00:00 2001 From: Gregory Kimball Date: Wed, 26 Aug 2026 11:08:59 -0700 Subject: [PATCH 09/12] Stabilize GPU exchange tuning and verification Expose reproducible UCX controls, prove active peer traffic, and tolerate package-manager startup races found during multi-node qualification. --- presto/aws/ec2/README.md | 32 +++- presto/aws/ec2/aws_cluster.py | 40 ++++- presto/aws/ec2/aws_config.env.example | 10 ++ presto/aws/ec2/remote/configure_and_start.sh | 26 +++ presto/aws/ec2/remote/verify_gpu_exchange.sh | 158 +++++++++++++++---- 5 files changed, 228 insertions(+), 38 deletions(-) diff --git a/presto/aws/ec2/README.md b/presto/aws/ec2/README.md index 138abd80a..519e51415 100644 --- a/presto/aws/ec2/README.md +++ b/presto/aws/ec2/README.md @@ -100,6 +100,22 @@ are explicit so each GPU run records its remote-I/O behavior. input (required for AsyncDataCache) and direct KvikIO S3 reads. `GPU_BATCH_SIZE_MIN_THRESHOLD` controls GPU rebatching and can be reduced when buffered-input workloads exceed device memory. +`GPU_PARTITIONED_OUTPUT_BATCH_ROWS` controls how many rows the GPU partitioned +output operator accumulates before flushing to exchange. +`GPU_UCXX_BLOCKING_POLLING` selects event-driven or continuously progressing +UCXX worker polling. +`GPU_UCX_NET_DEVICES` pins UCX to a network interface. The TCP TX/RX staging +chunk sizes are controlled by `GPU_UCX_TCP_TX_SEG_SIZE` and +`GPU_UCX_TCP_RX_SEG_SIZE`; larger values can improve large GPU transfers over +TCP by reducing host-staging fragmentation. `GPU_UCX_RNDV_FRAG_SIZE` controls +the UCX rendezvous fragment size by memory type and can be matched to the TCP +staging size for large CUDA transfers. +`GPU_UCX_CUDA_COPY_MAX_REG_RATIO` limits how much host memory the CUDA-copy +transport may register for staging. +`GPU_UCX_TCP_MAX_BW` supplies UCX's protocol selector with the expected TCP +transport bandwidth. +`GPU_UCX_TCP_MAX_POLL` and `GPU_UCX_CUDA_COPY_MAX_POLL` bound the number of +transport completions processed per UCX progress call. ## Local validation and dry run @@ -199,23 +215,25 @@ one-warm-up/four-measured rule without relying on the runner's aggregate field. ## GPU exchange verification -For two or more GPU workers, run a shuffle-heavy query and then require -observable evidence that cuDF's UCX exchange path handled traffic: +For two or more GPU workers, run a shuffle-heavy query and invoke verification +while that query is actively exchanging data: ```bash python3 aws_cluster.py \ --config "$CONFIG" --run-id "$RUN_ID" --workers 2 \ - run --queries 9 --iterations 1 --tag gpu_exchange_probe + run --queries 18 --iterations 1 --tag gpu_exchange_probe & python3 aws_cluster.py \ --config "$CONFIG" --run-id "$RUN_ID" --workers 2 \ verify-gpu-exchange +wait ``` Verification fails unless every worker has `cudf.exchange=true`, the configured -UCX server port is listening, and worker logs or positive runtime counters show -active UCX/exchange protocol use. Collection preserves the full worker log, -filtered metrics, socket snapshots, and `gpu_exchange_verification.json`. -Configuration alone is not accepted as proof of active exchange. +UCX server port is listening, remote `presto_server` peer connections exist, +and their kernel TCP byte counters increase during the sample window. +Collection preserves metrics, socket snapshots, and +`gpu_exchange_verification.json`. Configuration or protocol-looking logs alone +are not accepted as proof of active exchange. ## Controlled CPU tuning diff --git a/presto/aws/ec2/aws_cluster.py b/presto/aws/ec2/aws_cluster.py index 7701361c7..a29574d74 100755 --- a/presto/aws/ec2/aws_cluster.py +++ b/presto/aws/ec2/aws_cluster.py @@ -635,14 +635,15 @@ def bootstrap(self) -> None: commands = [ "set -eu", "export DEBIAN_FRONTEND=noninteractive", - "apt-get update -y", + "apt-get -o DPkg::Lock::Timeout=300 update -y", ( - "apt-get install -y ca-certificates curl git iproute2 jq numactl " + "apt-get -o DPkg::Lock::Timeout=300 install -y " + "ca-certificates curl git iproute2 jq numactl " "python3 python3-venv unzip" ), ( "if ! command -v docker >/dev/null; then " - "apt-get install -y docker.io; fi" + "apt-get -o DPkg::Lock::Timeout=300 install -y docker.io; fi" ), "systemctl enable --now docker", ( @@ -719,6 +720,34 @@ def remote_env(self, coordinator_address: str, role: str, worker_index: int | No "GPU_BATCH_SIZE_MIN_THRESHOLD": self.config[ "GPU_BATCH_SIZE_MIN_THRESHOLD" ], + "GPU_PARTITIONED_OUTPUT_BATCH_ROWS": self.config.get( + "GPU_PARTITIONED_OUTPUT_BATCH_ROWS", "100000000" + ), + "GPU_UCXX_BLOCKING_POLLING": self.config.get( + "GPU_UCXX_BLOCKING_POLLING", "true" + ), + "GPU_UCX_NET_DEVICES": self.config.get("GPU_UCX_NET_DEVICES", "all"), + "GPU_UCX_TCP_TX_SEG_SIZE": self.config.get( + "GPU_UCX_TCP_TX_SEG_SIZE", "8K" + ), + "GPU_UCX_TCP_RX_SEG_SIZE": self.config.get( + "GPU_UCX_TCP_RX_SEG_SIZE", "64K" + ), + "GPU_UCX_RNDV_FRAG_SIZE": self.config.get( + "GPU_UCX_RNDV_FRAG_SIZE", "host:512K,cuda:4M" + ), + "GPU_UCX_CUDA_COPY_MAX_REG_RATIO": self.config.get( + "GPU_UCX_CUDA_COPY_MAX_REG_RATIO", "0.100" + ), + "GPU_UCX_TCP_MAX_BW": self.config.get( + "GPU_UCX_TCP_MAX_BW", "2200MBps" + ), + "GPU_UCX_TCP_MAX_POLL": self.config.get( + "GPU_UCX_TCP_MAX_POLL", "16" + ), + "GPU_UCX_CUDA_COPY_MAX_POLL": self.config.get( + "GPU_UCX_CUDA_COPY_MAX_POLL", "16" + ), "GPU_USE_BUFFERED_INPUT": self.config["GPU_USE_BUFFERED_INPUT"], "GPU_USE_KVIKIO": self.config["GPU_USE_KVIKIO"], "KVIKIO_REMOTE_IO_BACKEND": self.config["KVIKIO_REMOTE_IO_BACKEND"], @@ -864,13 +893,16 @@ def verify_gpu_exchange(self) -> None: if self.workers < 2: raise ClusterError("verify-gpu-exchange requires at least two workers") _, workers = self.expected_inventory() + peer_addresses = ",".join( + item.private_ip or item.private_dns for item in workers + ) command_id = self.send_command( [worker.instance_id for worker in workers], [ "set -eu", ( "bash /opt/velox-testing/presto/aws/ec2/remote/" - "verify_gpu_exchange.sh" + f"verify_gpu_exchange.sh {shlex.quote(peer_addresses)}" ), ], f"verify GPU exchange {self.run_id}", diff --git a/presto/aws/ec2/aws_config.env.example b/presto/aws/ec2/aws_config.env.example index 42bb065b2..1689dfee2 100644 --- a/presto/aws/ec2/aws_config.env.example +++ b/presto/aws/ec2/aws_config.env.example @@ -45,6 +45,16 @@ CPU_EXCHANGE_TUNING_ENABLED=true # GPU-only worker settings. These are ignored when ENGINE_VARIANT=cpu. GPU_DEVICE_ID=0 GPU_BATCH_SIZE_MIN_THRESHOLD=40000000 +GPU_PARTITIONED_OUTPUT_BATCH_ROWS=100000000 +GPU_UCXX_BLOCKING_POLLING=true +GPU_UCX_NET_DEVICES=all +GPU_UCX_TCP_TX_SEG_SIZE=8K +GPU_UCX_TCP_RX_SEG_SIZE=64K +GPU_UCX_RNDV_FRAG_SIZE=host:512K,cuda:4M +GPU_UCX_CUDA_COPY_MAX_REG_RATIO=0.100 +GPU_UCX_TCP_MAX_BW=2200MBps +GPU_UCX_TCP_MAX_POLL=16 +GPU_UCX_CUDA_COPY_MAX_POLL=16 GPU_USE_BUFFERED_INPUT=false GPU_USE_KVIKIO=true KVIKIO_REMOTE_IO_BACKEND=EASY_THREADPOOL diff --git a/presto/aws/ec2/remote/configure_and_start.sh b/presto/aws/ec2/remote/configure_and_start.sh index fd07f7481..bad8913aa 100755 --- a/presto/aws/ec2/remote/configure_and_start.sh +++ b/presto/aws/ec2/remote/configure_and_start.sh @@ -7,6 +7,13 @@ set -euo pipefail required=( RUN_ID ROLE WORKER_COUNT COORDINATOR_ADDRESS AWS_REGION VCPU_PER_WORKER ENGINE_VARIANT GPU_DEVICE_ID GPU_BATCH_SIZE_MIN_THRESHOLD + GPU_PARTITIONED_OUTPUT_BATCH_ROWS + GPU_UCXX_BLOCKING_POLLING + GPU_UCX_NET_DEVICES GPU_UCX_TCP_TX_SEG_SIZE GPU_UCX_TCP_RX_SEG_SIZE + GPU_UCX_RNDV_FRAG_SIZE + GPU_UCX_CUDA_COPY_MAX_REG_RATIO + GPU_UCX_TCP_MAX_BW + GPU_UCX_TCP_MAX_POLL GPU_UCX_CUDA_COPY_MAX_POLL GPU_USE_BUFFERED_INPUT GPU_USE_KVIKIO KVIKIO_REMOTE_IO_BACKEND KVIKIO_NTHREADS KVIKIO_TASK_SIZE KVIKIO_BOUNCE_BUFFER_SIZE @@ -187,6 +194,11 @@ else set_property "${final}/config.properties" http-server.http.port 8080 set_property "${final}/config.properties" discovery.uri \ "http://${COORDINATOR_ADDRESS}:8080" + if [[ ${ENGINE_VARIANT} == gpu ]]; then + # cuDF exchange advertises HTTP port + 3. EC2 runs one worker per host, + # so every worker can use the same host-local exchange port. + set_property "${final}/config.properties" cudf.exchange.server.port 8083 + fi if ((WORKER_COUNT == 1)); then set_property "${final}/config.properties" single-node-execution-enabled true else @@ -213,6 +225,12 @@ else upsert_property "${final}/config.properties" runtime-metrics-collection-enabled \ true if [[ ${ENGINE_VARIANT} == gpu ]]; then + set_property "${final}/config.properties" \ + cudf.partitioned_output_batch_rows \ + "${GPU_PARTITIONED_OUTPUT_BATCH_ROWS}" + upsert_property "${final}/config.properties" \ + ucxx.blocking_polling \ + "${GPU_UCXX_BLOCKING_POLLING}" delete_property "${final}/config.properties" cudf.batch_size_min_threshold printf 'cudf.batch_size_min_threshold=%s\n' \ "${GPU_BATCH_SIZE_MIN_THRESHOLD}" \ @@ -333,6 +351,14 @@ else -e AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN -e UCX_TLS=tcp,cuda_copy,cuda_ipc + -e "UCX_NET_DEVICES=${GPU_UCX_NET_DEVICES}" + -e "UCX_TCP_TX_SEG_SIZE=${GPU_UCX_TCP_TX_SEG_SIZE}" + -e "UCX_TCP_RX_SEG_SIZE=${GPU_UCX_TCP_RX_SEG_SIZE}" + -e "UCX_RNDV_FRAG_SIZE=${GPU_UCX_RNDV_FRAG_SIZE}" + -e "UCX_CUDA_COPY_MAX_REG_RATIO=${GPU_UCX_CUDA_COPY_MAX_REG_RATIO}" + -e "UCX_TCP_MAX_BW=${GPU_UCX_TCP_MAX_BW}" + -e "UCX_TCP_MAX_POLL=${GPU_UCX_TCP_MAX_POLL}" + -e "UCX_CUDA_COPY_MAX_POLL=${GPU_UCX_CUDA_COPY_MAX_POLL}" -e UCX_TCP_CM_REUSEADDR=y -e UCX_LOG_LEVEL=info -e UCX_PROTO_INFO=y diff --git a/presto/aws/ec2/remote/verify_gpu_exchange.sh b/presto/aws/ec2/remote/verify_gpu_exchange.sh index 9b02da17a..984337a8a 100644 --- a/presto/aws/ec2/remote/verify_gpu_exchange.sh +++ b/presto/aws/ec2/remote/verify_gpu_exchange.sh @@ -4,6 +4,14 @@ set -euo pipefail +if [[ $# -lt 1 || $# -gt 2 ]]; then + echo "usage: verify_gpu_exchange.sh [sample-seconds]" >&2 + exit 2 +fi + +expected_peers=$1 +sample_seconds=${2:-10} +local_address=$(hostname -I | awk '{print $1}') runtime_root=/opt/presto-aws config="${runtime_root}/final_config/worker/config.properties" output="${runtime_root}/gpu_exchange_verification.json" @@ -16,17 +24,26 @@ test -n "${port}" docker logs "${container}" >"${runtime_root}/gpu_exchange_container.log" 2>&1 curl -fsS http://localhost:8080/v1/info/metrics \ - >"${runtime_root}/gpu_exchange_metrics.prom" + >"${runtime_root}/gpu_exchange_metrics_before.prom" ss -ltnp >"${runtime_root}/gpu_exchange_listeners.txt" -ss -tinp >"${runtime_root}/gpu_exchange_connections.txt" +ss -tinp >"${runtime_root}/gpu_exchange_connections_before.txt" +sleep "${sample_seconds}" +curl -fsS http://localhost:8080/v1/info/metrics \ + >"${runtime_root}/gpu_exchange_metrics_after.prom" +ss -tinp >"${runtime_root}/gpu_exchange_connections_after.txt" python3 - \ "${enabled}" \ "${port}" \ + "${expected_peers}" \ + "${local_address}" \ + "${sample_seconds}" \ "${runtime_root}/gpu_exchange_container.log" \ - "${runtime_root}/gpu_exchange_metrics.prom" \ + "${runtime_root}/gpu_exchange_metrics_before.prom" \ + "${runtime_root}/gpu_exchange_metrics_after.prom" \ "${runtime_root}/gpu_exchange_listeners.txt" \ - "${runtime_root}/gpu_exchange_connections.txt" \ + "${runtime_root}/gpu_exchange_connections_before.txt" \ + "${runtime_root}/gpu_exchange_connections_after.txt" \ "${output}" <<'PY' import json import re @@ -34,13 +51,32 @@ import sys from datetime import datetime, timezone from pathlib import Path -enabled, port, log_path, metrics_path, listeners_path, connections_path, output_path = ( - sys.argv[1:] -) +( + enabled, + port, + expected_peers_csv, + local_address, + sample_seconds, + log_path, + metrics_before_path, + metrics_after_path, + listeners_path, + connections_before_path, + connections_after_path, + output_path, +) = sys.argv[1:] log_text = Path(log_path).read_text(errors="replace") -metrics_text = Path(metrics_path).read_text(errors="replace") +metrics_before_text = Path(metrics_before_path).read_text(errors="replace") +metrics_after_text = Path(metrics_after_path).read_text(errors="replace") listeners_text = Path(listeners_path).read_text(errors="replace") -connections_text = Path(connections_path).read_text(errors="replace") +connections_before_text = Path(connections_before_path).read_text(errors="replace") +connections_after_text = Path(connections_after_path).read_text(errors="replace") +connections_text = connections_before_text + "\n" + connections_after_text +expected_peers = sorted( + peer + for peer in set(filter(None, expected_peers_csv.split(","))) + if peer != local_address +) ucx_lines = [ line @@ -56,21 +92,41 @@ protocol_lines = [ re.IGNORECASE, ) ] -exchange_metrics = [] -positive_exchange_metrics = [] -for line in metrics_text.splitlines(): - if not line or line.startswith("#"): - continue - name = line.split("{", 1)[0].split(None, 1)[0] - if not re.search(r"ucx|cudf.*exchange|exchange.*cudf", name, re.IGNORECASE): - continue - exchange_metrics.append(name) - try: - value = float(line.rsplit(None, 1)[1]) - except (ValueError, IndexError): + + +def parse_exchange_metrics(text): + metrics = {} + for line in text.splitlines(): + if not line or line.startswith("#"): + continue + try: + key, value_text = line.rsplit(None, 1) + value = float(value_text) + except (ValueError, IndexError): + continue + name = key.split("{", 1)[0] + if re.search(r"ucx|exchange", name, re.IGNORECASE): + metrics[key] = value + return metrics + + +metrics_before = parse_exchange_metrics(metrics_before_text) +metrics_after = parse_exchange_metrics(metrics_after_text) +byte_counter_deltas = [] +for key in sorted(metrics_before.keys() & metrics_after.keys()): + name = key.split("{", 1)[0] + if not re.search(r"bytes?", name, re.IGNORECASE): continue - if value > 0: - positive_exchange_metrics.append({"name": name, "value": value}) + delta = metrics_after[key] - metrics_before[key] + if delta > 0: + byte_counter_deltas.append( + { + "metric": key, + "before": metrics_before[key], + "after": metrics_after[key], + "delta": delta, + } + ) listener_active = bool( re.search(rf"[:.]({re.escape(port)})\s", listeners_text) @@ -80,7 +136,42 @@ established_connections = sum( for line in connections_text.splitlines() if re.search(rf"[:.]({re.escape(port)})\s", line) ) -active_evidence = bool(protocol_lines or positive_exchange_metrics) +connected_peers = [ + peer + for peer in expected_peers + if any( + peer in line and "presto_server" in line + for line in connections_text.splitlines() + ) +] +remote_peer_connections = bool(connected_peers) + + +def socket_exchange_bytes(text): + total = 0 + matching_connection = False + for line in text.splitlines(): + if line and not line[0].isspace(): + matching_connection = ( + "presto_server" in line + and any(peer in line for peer in expected_peers) + ) + elif matching_connection: + total += sum( + int(value) + for value in re.findall( + r"bytes_(?:sent|received):(\d+)", + line, + ) + ) + return total + + +socket_bytes_before = socket_exchange_bytes(connections_before_text) +socket_bytes_after = socket_exchange_bytes(connections_after_text) +socket_byte_delta = socket_bytes_after - socket_bytes_before +increasing_exchange_bytes = bool(byte_counter_deltas) or socket_byte_delta > 0 +active_evidence = remote_peer_connections and increasing_exchange_bytes payload = { "checked_at": datetime.now(timezone.utc).isoformat(), @@ -88,11 +179,22 @@ payload = { "server_port": int(port), "listener_active": listener_active, "established_connection_lines": established_connections, + "expected_peer_addresses": expected_peers, + "local_address": local_address, + "connected_peer_addresses": connected_peers, + "remote_peer_connections": remote_peer_connections, + "sample_seconds": int(sample_seconds), "ucx_log_line_count": len(ucx_lines), "ucx_protocol_line_count": len(protocol_lines), "ucx_protocol_examples": protocol_lines[:20], - "exchange_metric_names": sorted(set(exchange_metrics)), - "positive_exchange_metrics": positive_exchange_metrics[:50], + "exchange_metric_names": sorted( + {key.split("{", 1)[0] for key in metrics_before | metrics_after} + ), + "positive_exchange_byte_deltas": byte_counter_deltas[:100], + "exchange_socket_bytes_before": socket_bytes_before, + "exchange_socket_bytes_after": socket_bytes_after, + "exchange_socket_byte_delta": socket_byte_delta, + "increasing_exchange_bytes": increasing_exchange_bytes, "active_transfer_evidence": active_evidence, } Path(output_path).write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") @@ -102,6 +204,8 @@ if not payload["configured"]: raise SystemExit(3) if not listener_active: raise SystemExit(4) -if not active_evidence: +if not remote_peer_connections: raise SystemExit(5) +if not increasing_exchange_bytes: + raise SystemExit(6) PY From 2795cd145fdf67695a98799a59ecbc29eb57ec18 Mon Sep 17 00:00:00 2001 From: Gregory Kimball Date: Wed, 26 Aug 2026 21:54:07 -0700 Subject: [PATCH 10/12] Support multi-GPU EC2 workers and EFA devices Run one pinned worker per local GPU while preserving distinct ports, cache paths, and artifacts, and allow GPU fleets to attach EFA devices for inter-node exchange testing. --- presto/aws/ec2/README.md | 9 ++ presto/aws/ec2/aws_cluster.py | 108 ++++++++++--- presto/aws/ec2/aws_config.env.example | 4 + presto/aws/ec2/remote/collect_node.sh | 16 +- presto/aws/ec2/remote/configure_and_start.sh | 154 ++++++++++++++----- presto/aws/ec2/test_aws_cluster.py | 37 +++++ 6 files changed, 259 insertions(+), 69 deletions(-) diff --git a/presto/aws/ec2/README.md b/presto/aws/ec2/README.md index 519e51415..1ea331ff0 100644 --- a/presto/aws/ec2/README.md +++ b/presto/aws/ec2/README.md @@ -104,6 +104,15 @@ buffered-input workloads exceed device memory. output operator accumulates before flushing to exchange. `GPU_UCXX_BLOCKING_POLLING` selects event-driven or continuously progressing UCXX worker polling. +`GPU_WORKERS_PER_INSTANCE` launches multiple native workers on each GPU host. +Local worker N is pinned to GPU N, receives a unique node ID, and uses an HTTP +port spaced ten ports apart with its UCX listener at HTTP port + 3. The +coordinator's required worker count is the number of EC2 worker instances +multiplied by this value. Set `VCPU_PER_WORKER` and worker memory limits for one +local process, not the whole instance. +`GPU_UCX_TLS` controls the enabled UCX transports. Use +`cuda_ipc,cuda_copy` for an intra-node-only test, or include `tcp` for +inter-node runs. `GPU_UCX_NET_DEVICES` pins UCX to a network interface. The TCP TX/RX staging chunk sizes are controlled by `GPU_UCX_TCP_TX_SEG_SIZE` and `GPU_UCX_TCP_RX_SEG_SIZE`; larger values can improve large GPU transfers over diff --git a/presto/aws/ec2/aws_cluster.py b/presto/aws/ec2/aws_cluster.py index a29574d74..ffaed861f 100755 --- a/presto/aws/ec2/aws_cluster.py +++ b/presto/aws/ec2/aws_cluster.py @@ -147,6 +147,16 @@ def __init__( self.aws = Aws(config, dry_run=dry_run) self.dry_run = dry_run + @property + def workers_per_instance(self) -> int: + if self.config.get("ENGINE_VARIANT") != "gpu": + return 1 + return int(self.config.get("GPU_WORKERS_PER_INSTANCE", "1")) + + @property + def logical_workers(self) -> int: + return self.workers * self.workers_per_instance + def validate(self, cloud: bool = True) -> None: require( self.config, @@ -240,14 +250,25 @@ def validate(self, cloud: bool = True) -> None: if self.config["ENGINE_VARIANT"] == "gpu": nonnegative_int(self.config, "GPU_DEVICE_ID") positive_int(self.config, "GPU_BATCH_SIZE_MIN_THRESHOLD") + workers_per_instance = positive_int( + self.config, "GPU_WORKERS_PER_INSTANCE" + ) if "GPU_WORKERS_PER_INSTANCE" in self.config else 1 + if workers_per_instance > 8: + raise ClusterError("GPU_WORKERS_PER_INSTANCE must be at most 8") + elif int(self.config.get("GPU_WORKERS_PER_INSTANCE", "1")) != 1: + raise ClusterError("GPU_WORKERS_PER_INSTANCE must be 1 for CPU") for key in ( "DYNAMIC_FILTERING_ENABLED", "CPU_EXCHANGE_TUNING_ENABLED", "GPU_USE_BUFFERED_INPUT", "GPU_USE_KVIKIO", + "EFA_ENABLED", ): - if self.config[key] not in ("true", "false"): + value = self.config.get(key, "false") + if value not in ("true", "false"): raise ClusterError(f"{key} must be true or false") + if self.config.get("EFA_ENABLED", "false") == "true" and self.config["ENGINE_VARIANT"] != "gpu": + raise ClusterError("EFA_ENABLED requires ENGINE_VARIANT=gpu") split_size = self.config["HIVE_MAX_SPLIT_SIZE"] if not split_size.endswith("MB") or not split_size[:-2].isdigit(): raise ClusterError("HIVE_MAX_SPLIT_SIZE must be an integer number of MB") @@ -381,10 +402,6 @@ def launch_role(self, role: str, count: int, expiry: str, ami_id: str) -> list[s instance_type, "--count", str(count), - "--subnet-id", - self.config["SUBNET_ID"], - "--security-group-ids", - self.config["SECURITY_GROUP_ID"], "--iam-instance-profile", f"Name={self.config['IAM_INSTANCE_PROFILE']}", "--metadata-options", @@ -400,6 +417,27 @@ def launch_role(self, role: str, count: int, expiry: str, ami_id: str) -> list[s "--tag-specifications", json.dumps(tag_specifications, separators=(",", ":")), ] + if role == "worker" and self.config.get("EFA_ENABLED", "false") == "true": + network_interface = [ + { + "DeviceIndex": 0, + "SubnetId": self.config["SUBNET_ID"], + "Groups": [self.config["SECURITY_GROUP_ID"]], + "InterfaceType": "efa", + "DeleteOnTermination": True, + } + ] + args += [ + "--network-interfaces", + json.dumps(network_interface, separators=(",", ":")), + ] + else: + args += [ + "--subnet-id", + self.config["SUBNET_ID"], + "--security-group-ids", + self.config["SECURITY_GROUP_ID"], + ] if self.config.get("PLACEMENT_GROUP"): args += ["--placement", f"GroupName={self.config['PLACEMENT_GROUP']}"] response = self.aws.run(args, json_output=True) @@ -430,6 +468,8 @@ def launch(self) -> None: "created_at": utc_now().isoformat(), "expires_at": expiry, "worker_count": self.workers, + "logical_worker_count": self.logical_workers, + "gpu_workers_per_instance": self.workers_per_instance, "coordinator_ami_id": coordinator_ami_id, "worker_ami_id": worker_ami_id, "coordinator_instance_ids": coordinator_ids, @@ -442,6 +482,7 @@ def launch(self) -> None: "harness_bundle_s3_uri": self.config.get("HARNESS_BUNDLE_S3_URI", ""), "harness_bundle_sha256": self.config.get("HARNESS_BUNDLE_SHA256", ""), "async_data_cache": self.config["ASYNC_DATA_CACHE_ENABLED"] == "true", + "efa_enabled": self.config.get("EFA_ENABLED", "false") == "true", "engine_variant": self.config["ENGINE_VARIANT"], } ) @@ -697,12 +738,22 @@ def bootstrap(self) -> None: ) self.record_command("bootstrap_workers", worker_command_id) - def remote_env(self, coordinator_address: str, role: str, worker_index: int | None = None) -> str: + def remote_env( + self, + coordinator_address: str, + role: str, + worker_index: int | None = None, + local_worker_index: int | None = None, + ) -> str: values = { "RUN_ID": self.run_id, "ROLE": role, - "WORKER_COUNT": str(self.workers), + "WORKER_COUNT": str(self.logical_workers), + "WORKERS_PER_INSTANCE": str(self.workers_per_instance), "WORKER_INDEX": "" if worker_index is None else str(worker_index), + "LOCAL_WORKER_INDEX": ( + "" if local_worker_index is None else str(local_worker_index) + ), "COORDINATOR_ADDRESS": coordinator_address, "AWS_REGION": self.config["AWS_REGION"], "ENGINE_VARIANT": self.config["ENGINE_VARIANT"], @@ -716,7 +767,11 @@ def remote_env(self, coordinator_address: str, role: str, worker_index: int | No "CPU_EXCHANGE_TUNING_ENABLED": self.config[ "CPU_EXCHANGE_TUNING_ENABLED" ], - "GPU_DEVICE_ID": self.config["GPU_DEVICE_ID"], + "GPU_DEVICE_ID": ( + self.config["GPU_DEVICE_ID"] + if local_worker_index is None + else str(local_worker_index) + ), "GPU_BATCH_SIZE_MIN_THRESHOLD": self.config[ "GPU_BATCH_SIZE_MIN_THRESHOLD" ], @@ -727,6 +782,9 @@ def remote_env(self, coordinator_address: str, role: str, worker_index: int | No "GPU_UCXX_BLOCKING_POLLING", "true" ), "GPU_UCX_NET_DEVICES": self.config.get("GPU_UCX_NET_DEVICES", "all"), + "GPU_UCX_TLS": self.config.get( + "GPU_UCX_TLS", "tcp,cuda_copy,cuda_ipc" + ), "GPU_UCX_TCP_TX_SEG_SIZE": self.config.get( "GPU_UCX_TCP_TX_SEG_SIZE", "8K" ), @@ -833,19 +891,21 @@ def start(self) -> None: f"start coordinator {self.run_id}", ) self.record_command("start_coordinator", command_id) - for index, worker in enumerate(workers): - command_id = self.send_command( - [worker.instance_id], - [ - "set -eu", - ( - f"{self.remote_env(address, 'worker', index)} " - "bash /opt/velox-testing/presto/aws/ec2/remote/configure_and_start.sh" - ), - ], - f"start worker {index} {self.run_id}", - ) - self.record_command(f"start_worker_{index}", command_id) + for instance_index, worker in enumerate(workers): + for local_index in range(self.workers_per_instance): + worker_index = instance_index * self.workers_per_instance + local_index + command_id = self.send_command( + [worker.instance_id], + [ + "set -eu", + ( + f"{self.remote_env(address, 'worker', worker_index, local_index)} " + "bash /opt/velox-testing/presto/aws/ec2/remote/configure_and_start.sh" + ), + ], + f"start worker {worker_index} {self.run_id}", + ) + self.record_command(f"start_worker_{worker_index}", command_id) timeout = self.config["WORKER_READY_TIMEOUT_SECONDS"] command_id = self.send_command( [coordinator.instance_id], @@ -853,7 +913,7 @@ def start(self) -> None: "set -eu", ( "bash /opt/velox-testing/presto/aws/ec2/remote/" - f"wait_for_cluster.sh {shlex.quote(address)} {self.workers} {timeout}" + f"wait_for_cluster.sh {shlex.quote(address)} {self.logical_workers} {timeout}" ), ], f"wait for workers {self.run_id}", @@ -942,8 +1002,8 @@ def stop(self) -> None: [ "set -eu", ( - "docker rm -f presto-coordinator presto-native-worker-cpu " - "presto-native-worker-gpu 2>/dev/null || true" + "containers=$(docker ps -aq --filter 'name=^presto-'); " + "if test -n \"$containers\"; then docker rm -f $containers; fi" ), ], f"stop {self.run_id}", diff --git a/presto/aws/ec2/aws_config.env.example b/presto/aws/ec2/aws_config.env.example index 1689dfee2..142c923f6 100644 --- a/presto/aws/ec2/aws_config.env.example +++ b/presto/aws/ec2/aws_config.env.example @@ -44,9 +44,13 @@ DYNAMIC_FILTERING_ENABLED=false CPU_EXCHANGE_TUNING_ENABLED=true # GPU-only worker settings. These are ignored when ENGINE_VARIANT=cpu. GPU_DEVICE_ID=0 +# Launch this many native worker processes on each GPU worker instance. +# Values above one pin local worker N to GPU N and use distinct HTTP/UCX ports. +GPU_WORKERS_PER_INSTANCE=1 GPU_BATCH_SIZE_MIN_THRESHOLD=40000000 GPU_PARTITIONED_OUTPUT_BATCH_ROWS=100000000 GPU_UCXX_BLOCKING_POLLING=true +GPU_UCX_TLS=tcp,cuda_copy,cuda_ipc GPU_UCX_NET_DEVICES=all GPU_UCX_TCP_TX_SEG_SIZE=8K GPU_UCX_TCP_RX_SEG_SIZE=64K diff --git a/presto/aws/ec2/remote/collect_node.sh b/presto/aws/ec2/remote/collect_node.sh index e48ccc775..b29435568 100755 --- a/presto/aws/ec2/remote/collect_node.sh +++ b/presto/aws/ec2/remote/collect_node.sh @@ -31,11 +31,17 @@ cp "${runtime_root}"/*.diff "${artifact_root}/" 2>/dev/null || true cp "${runtime_root}"/gpu_exchange_* "${artifact_root}/" 2>/dev/null || true docker ps -a --no-trunc >"${artifact_root}/docker_ps.txt" 2>&1 || true -docker inspect presto-coordinator presto-native-worker-cpu presto-native-worker-gpu \ - >"${artifact_root}/docker_inspect.json" 2>/dev/null || true -docker logs presto-coordinator >"${artifact_root}/coordinator_container.log" 2>&1 || true -docker logs presto-native-worker-cpu >"${artifact_root}/worker_container.log" 2>&1 || true -docker logs presto-native-worker-gpu >"${artifact_root}/worker_gpu_container.log" 2>&1 || true +mapfile -t presto_containers < <( + docker ps -a --format '{{.Names}}' | awk '/^presto-/' +) +if ((${#presto_containers[@]})); then + docker inspect "${presto_containers[@]}" \ + >"${artifact_root}/docker_inspect.json" 2>/dev/null || true + for container in "${presto_containers[@]}"; do + docker logs "${container}" \ + >"${artifact_root}/${container}.log" 2>&1 || true + done +fi uname -a >"${artifact_root}/uname.txt" lscpu --json >"${artifact_root}/lscpu.json" diff --git a/presto/aws/ec2/remote/configure_and_start.sh b/presto/aws/ec2/remote/configure_and_start.sh index bad8913aa..c6c28a13d 100755 --- a/presto/aws/ec2/remote/configure_and_start.sh +++ b/presto/aws/ec2/remote/configure_and_start.sh @@ -5,10 +5,10 @@ set -euo pipefail required=( - RUN_ID ROLE WORKER_COUNT COORDINATOR_ADDRESS AWS_REGION VCPU_PER_WORKER + RUN_ID ROLE WORKER_COUNT WORKERS_PER_INSTANCE COORDINATOR_ADDRESS AWS_REGION VCPU_PER_WORKER ENGINE_VARIANT GPU_DEVICE_ID GPU_BATCH_SIZE_MIN_THRESHOLD GPU_PARTITIONED_OUTPUT_BATCH_ROWS - GPU_UCXX_BLOCKING_POLLING + GPU_UCXX_BLOCKING_POLLING GPU_UCX_TLS GPU_UCX_NET_DEVICES GPU_UCX_TCP_TX_SEG_SIZE GPU_UCX_TCP_RX_SEG_SIZE GPU_UCX_RNDV_FRAG_SIZE GPU_UCX_CUDA_COPY_MAX_REG_RATIO @@ -44,17 +44,35 @@ if [[ ${ENGINE_VARIANT} != cpu && ${ENGINE_VARIANT} != gpu ]]; then echo "ENGINE_VARIANT must be cpu or gpu" >&2 exit 2 fi -if [[ ${ROLE} == worker && -z ${WORKER_INDEX:-} ]]; then - echo "WORKER_INDEX is required for worker role" >&2 - exit 2 +if [[ ${ROLE} == worker ]]; then + if [[ -z ${WORKER_INDEX:-} || -z ${LOCAL_WORKER_INDEX:-} ]]; then + echo "WORKER_INDEX and LOCAL_WORKER_INDEX are required for worker role" >&2 + exit 2 + fi fi repo=/opt/velox-testing runtime_root=/opt/presto-aws generated="${repo}/presto/docker/config/generated/${ENGINE_VARIANT}" -base="${runtime_root}/base_config/${ROLE}" -final="${runtime_root}/final_config/${ROLE}" -assembled="${runtime_root}/runtime_etc" +role_key=${ROLE} +container_name="presto-${ROLE}" +http_port=8080 +exchange_port=8083 +if [[ ${ROLE} == worker ]]; then + if ((WORKERS_PER_INSTANCE > 1)); then + role_key="worker_${WORKER_INDEX}" + container_name="presto-native-worker-${ENGINE_VARIANT}" + http_port=$((8080 + LOCAL_WORKER_INDEX * 10)) + exchange_port=$((http_port + 3)) + else + container_name="presto-native-worker-${ENGINE_VARIANT}" + fi +fi +base="${runtime_root}/base_config/${role_key}" +final="${runtime_root}/final_config/${role_key}" +assembled="${runtime_root}/runtime_etc/${role_key}" +data_dir="${runtime_root}/data/${role_key}" +logs_dir="${runtime_root}/logs/${role_key}" set_property() { local path=$1 key=$2 value=$3 @@ -118,7 +136,7 @@ VCPU_PER_WORKER="${VCPU_PER_WORKER}" \ ./generate_presto_config.sh rm -rf "${base}" "${final}" "${assembled}" -mkdir -p "${base}" "${final}" "${assembled}" "${runtime_root}/data" +mkdir -p "${base}" "${final}" "${assembled}" "${data_dir}" "${logs_dir}" if [[ ${ROLE} == coordinator ]]; then cp -a "${generated}/etc_common/." "${base}/" @@ -127,7 +145,7 @@ if [[ ${ROLE} == coordinator ]]; then mkdir -p "${base}/catalog" cp "${generated}/etc_coordinator/catalog/hive.properties" "${base}/catalog/hive.properties" else - worker_source="${generated}/etc_worker_0" + worker_source="${generated}/etc_worker_${WORKER_INDEX}" cp -a "${generated}/etc_common/." "${base}/" cp "${worker_source}/config_native.properties" "${base}/config.properties" cp "${worker_source}/node.properties" "${base}/node.properties" @@ -191,13 +209,14 @@ lines = [ path.write_text("\n".join(lines) + "\n") PY else - set_property "${final}/config.properties" http-server.http.port 8080 + set_property "${final}/config.properties" http-server.http.port "${http_port}" set_property "${final}/config.properties" discovery.uri \ "http://${COORDINATOR_ADDRESS}:8080" if [[ ${ENGINE_VARIANT} == gpu ]]; then - # cuDF exchange advertises HTTP port + 3. EC2 runs one worker per host, - # so every worker can use the same host-local exchange port. - set_property "${final}/config.properties" cudf.exchange.server.port 8083 + # Keep the cuDF exchange listener three ports above each worker's HTTP + # endpoint while spacing local workers far enough apart to avoid collisions. + set_property "${final}/config.properties" cudf.exchange.server.port \ + "${exchange_port}" fi if ((WORKER_COUNT == 1)); then set_property "${final}/config.properties" single-node-execution-enabled true @@ -257,8 +276,12 @@ else echo "ASYNC_CACHE_SSD_PATH is required for SSD cache" >&2 exit 2 fi - upsert_property "${final}/config.properties" async-cache-ssd-path \ - "${ASYNC_CACHE_SSD_PATH}" + cache_path=${ASYNC_CACHE_SSD_PATH} + if ((WORKERS_PER_INSTANCE > 1)); then + cache_path="${ASYNC_CACHE_SSD_PATH}/worker-${WORKER_INDEX}" + mkdir -p "${cache_path}" + fi + upsert_property "${final}/config.properties" async-cache-ssd-path "${cache_path}" fi set_property "${final}/node.properties" node.id \ "aws-${RUN_ID}-worker-${WORKER_INDEX}" @@ -266,10 +289,10 @@ fi cp -a "${final}/." "${assembled}/" find "${base}" -type f -print0 | sort -z | xargs -0 sha256sum \ - >"${runtime_root}/base_config_${ROLE}.sha256" + >"${runtime_root}/base_config_${role_key}.sha256" find "${final}" -type f -print0 | sort -z | xargs -0 sha256sum \ - >"${runtime_root}/final_config_${ROLE}.sha256" -diff -ru "${base}" "${final}" >"${runtime_root}/config_${ROLE}.diff" || true + >"${runtime_root}/final_config_${role_key}.sha256" +diff -ru "${base}" "${final}" >"${runtime_root}/config_${role_key}.diff" || true tuning_id=$( printf '%s\n' \ "engine=${ENGINE_VARIANT}" \ @@ -280,7 +303,7 @@ tuning_id=$( "exchange_tuning=${CPU_EXCHANGE_TUNING_ENABLED}" | sha256sum | cut -c1-12 ) -history="${runtime_root}/config_history/${tuning_id}/${ROLE}" +history="${runtime_root}/config_history/${tuning_id}/${role_key}" rm -rf "${history}" mkdir -p "${history}" cp -a "${final}/." "${history}/" @@ -295,9 +318,23 @@ cat >"${history}/tuning.json" </dev/null || true +if [[ ${ROLE} == coordinator ]]; then + docker rm -f presto-coordinator 2>/dev/null || true +elif ((WORKERS_PER_INSTANCE > 1 && LOCAL_WORKER_INDEX == 0)); then + stale_workers=$(docker ps -aq --filter "name=^presto-native-worker-${ENGINE_VARIANT}-") + if [[ -n ${stale_workers} ]]; then + docker rm -f ${stale_workers} 2>/dev/null || true + fi +else + docker rm -f "${container_name}" 2>/dev/null || true +fi +if [[ ${ROLE} == worker ]] && + ((WORKERS_PER_INSTANCE > 1 && LOCAL_WORKER_INDEX + 1 < WORKERS_PER_INSTANCE)); then + # aws_cluster invokes this script once per local GPU. Earlier invocations + # only materialize configs; the final invocation starts one container with + # all GPUs so launch_presto_servers.sh can apply GPU-local NUMA affinity. + exit 0 +fi timestamp=$(date -u +%Y%m%dT%H%M%SZ) if [[ ${ROLE} == coordinator ]]; then @@ -310,8 +347,8 @@ if [[ ${ROLE} == coordinator ]]; then -e "SERVER_START_TIMESTAMP=${timestamp}" \ -v "${assembled}:/opt/presto-server/etc:ro" \ -v "${runtime_root}/metastore:/var/lib/presto/data/hive/metastore" \ - -v "${runtime_root}/data:/var/lib/presto/data/local" \ - -v "${runtime_root}/logs:/opt/presto-server/logs" \ + -v "${data_dir}:/var/lib/presto/data/local" \ + -v "${logs_dir}:/opt/presto-server/logs" \ -v "${repo}/presto/docker/launch_coordinator.sh:/opt/launch_coordinator.sh:ro" \ --entrypoint bash \ "${COORDINATOR_IMAGE}" \ @@ -320,6 +357,10 @@ else cache_mount_args=() gpu_args=() gpu_env=() + config_mount_args=(-v "${assembled}:/opt/presto-server/etc:ro") + worker_launch_args=() + worker_logs_dir=${logs_dir} + worker_data_dir=${data_dir} if ((ASYNC_CACHE_SSD_GIB > 0)); then cache_mount_args=(-v /mnt/nvme:/mnt/nvme) fi @@ -329,15 +370,36 @@ else # reliably reach IMDS when the instance hop limit is one. eval "$(aws configure export-credentials --format env)" gpu_args=( - --gpus "device=${GPU_DEVICE_ID}" --cap-add IPC_LOCK + --cap-add NET_RAW --ulimit memlock=-1:-1 --shm-size 1g -v /sys/devices/system/node:/sys/devices/system/node:ro ) + for efa_device in /dev/infiniband/*; do + if [[ -c ${efa_device} ]]; then + gpu_args+=(--device "${efa_device}") + fi + done + if ((WORKERS_PER_INSTANCE > 1)); then + gpu_args=(--gpus all "${gpu_args[@]}") + config_mount_args=() + worker_launch_args=() + worker_index_base=$((WORKER_INDEX - LOCAL_WORKER_INDEX)) + for ((local_index = 0; local_index < WORKERS_PER_INSTANCE; local_index++)); do + global_index=$((worker_index_base + local_index)) + config_mount_args+=( + -v "${runtime_root}/runtime_etc/worker_${global_index}:/opt/presto-server/etc${local_index}:ro" + ) + worker_launch_args+=("${local_index}") + done + worker_logs_dir="${runtime_root}/logs/worker_host_${worker_index_base}" + worker_data_dir="${runtime_root}/data/worker_host_${worker_index_base}" + mkdir -p "${worker_logs_dir}" "${worker_data_dir}" + else + gpu_args=(--gpus "device=${GPU_DEVICE_ID}" "${gpu_args[@]}") + fi gpu_env=( - -e "CUDA_VISIBLE_DEVICES=${GPU_DEVICE_ID}" - -e "NVIDIA_VISIBLE_DEVICES=${GPU_DEVICE_ID}" -e "KVIKIO_REMOTE_IO_BACKEND=${KVIKIO_REMOTE_IO_BACKEND}" -e "KVIKIO_NTHREADS=${KVIKIO_NTHREADS}" -e "KVIKIO_TASK_SIZE=${KVIKIO_TASK_SIZE}" @@ -350,7 +412,7 @@ else -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_SESSION_TOKEN - -e UCX_TLS=tcp,cuda_copy,cuda_ipc + -e "UCX_TLS=${GPU_UCX_TLS}" -e "UCX_NET_DEVICES=${GPU_UCX_NET_DEVICES}" -e "UCX_TCP_TX_SEG_SIZE=${GPU_UCX_TCP_TX_SEG_SIZE}" -e "UCX_TCP_RX_SEG_SIZE=${GPU_UCX_TCP_RX_SEG_SIZE}" @@ -366,10 +428,19 @@ else -e UCX_TCP_KEEPINTVL=1ms -e UCX_KEEPALIVE_INTERVAL=1ms ) + if ((WORKERS_PER_INSTANCE > 1)); then + gpu_env+=(-e NVIDIA_VISIBLE_DEVICES=all) + else + gpu_env+=( + -e "CUDA_VISIBLE_DEVICES=${GPU_DEVICE_ID}" + -e "NVIDIA_VISIBLE_DEVICES=${GPU_DEVICE_ID}" + ) + fi fi docker run -d \ - --name "presto-native-worker-${ENGINE_VARIANT}" \ + --name "${container_name}" \ --network host \ + --ipc host \ --restart no \ --cap-add SYS_NICE \ "${gpu_args[@]}" \ @@ -377,21 +448,24 @@ else -e "AWS_REGION=${AWS_REGION}" \ -e "SERVER_START_TIMESTAMP=${timestamp}" \ "${gpu_env[@]}" \ - -v "${assembled}:/opt/presto-server/etc:ro" \ + "${config_mount_args[@]}" \ -v "${runtime_root}/metastore:/var/lib/presto/data/hive/metastore" \ - -v "${runtime_root}/data:/var/lib/presto/data/local" \ - -v "${runtime_root}/logs:/opt/presto-server/logs" \ + -v "${worker_data_dir}:/var/lib/presto/data/local" \ + -v "${worker_logs_dir}:/opt/presto-server/logs" \ "${cache_mount_args[@]}" \ -v "${repo}/presto/docker/launch_presto_servers.sh:/opt/launch_presto_servers.sh:ro" \ --entrypoint bash \ "${WORKER_IMAGE}" \ - /opt/launch_presto_servers.sh + /opt/launch_presto_servers.sh "${worker_launch_args[@]}" fi -if [[ -f ${runtime_root}/telemetry/${ROLE}.sampler.pid ]]; then - kill "$(cat "${runtime_root}/telemetry/${ROLE}.sampler.pid")" 2>/dev/null || true +if [[ ${ROLE} == coordinator || ${LOCAL_WORKER_INDEX:-0} == 0 ]]; then + mkdir -p "${runtime_root}/telemetry" + if [[ -f ${runtime_root}/telemetry/${ROLE}.sampler.pid ]]; then + kill "$(cat "${runtime_root}/telemetry/${ROLE}.sampler.pid")" 2>/dev/null || true + fi + nohup bash "${repo}/presto/aws/ec2/remote/sample_host.sh" \ + "${ROLE}" "${runtime_root}/telemetry/${ROLE}.jsonl" \ + >"${runtime_root}/telemetry/${ROLE}.sampler.log" 2>&1 & + echo "$!" >"${runtime_root}/telemetry/${ROLE}.sampler.pid" fi -nohup bash "${repo}/presto/aws/ec2/remote/sample_host.sh" \ - "${ROLE}" "${runtime_root}/telemetry/${ROLE}.jsonl" \ - >"${runtime_root}/telemetry/${ROLE}.sampler.log" 2>&1 & -echo "$!" >"${runtime_root}/telemetry/${ROLE}.sampler.pid" diff --git a/presto/aws/ec2/test_aws_cluster.py b/presto/aws/ec2/test_aws_cluster.py index 3221255b3..eb6ff494c 100644 --- a/presto/aws/ec2/test_aws_cluster.py +++ b/presto/aws/ec2/test_aws_cluster.py @@ -44,6 +44,7 @@ def valid_config() -> dict[str, str]: "DYNAMIC_FILTERING_ENABLED": "false", "CPU_EXCHANGE_TUNING_ENABLED": "true", "GPU_DEVICE_ID": "0", + "GPU_WORKERS_PER_INSTANCE": "1", "GPU_BATCH_SIZE_MIN_THRESHOLD": "40000000", "GPU_USE_BUFFERED_INPUT": "false", "GPU_USE_KVIKIO": "true", @@ -74,6 +75,7 @@ def valid_config() -> dict[str, str]: "SSM_COMMAND_TIMEOUT_SECONDS": "21600", "WORKER_READY_TIMEOUT_SECONDS": "600", "PLACEMENT_GROUP": "", + "EFA_ENABLED": "false", } @@ -150,6 +152,17 @@ def test_launch_dry_run_is_two_idempotent_fleet_calls(self) -> None: self.assertIn("HttpTokens=required", rendered) self.assertNotIn("terminate-instances", rendered) + def test_efa_launch_uses_worker_network_interface(self) -> None: + config = valid_config() + config["ENGINE_VARIANT"] = "gpu" + config["EFA_ENABLED"] = "true" + output = io.StringIO() + with contextlib.redirect_stdout(output): + self.make_cluster(config, workers=1).launch() + rendered = output.getvalue() + self.assertEqual(rendered.count('"InterfaceType":"efa"'), 1) + self.assertIn("--network-interfaces", rendered) + def test_tags_scope_resources_to_run(self) -> None: tags = {item["Key"]: item["Value"] for item in self.make_cluster().tags("worker", "2026-08-25T00:00:00+00:00")} self.assertEqual(tags["Project"], "cudf-performance") @@ -198,6 +211,30 @@ def test_validation_accepts_gpu_variant(self) -> None: config["GPU_DEVICE_ID"] = "0" self.make_cluster(config).validate() + def test_gpu_workers_per_instance_sets_logical_worker_count(self) -> None: + config = valid_config() + config["ENGINE_VARIANT"] = "gpu" + config["GPU_WORKERS_PER_INSTANCE"] = "4" + cluster = self.make_cluster(config, workers=1) + cluster.validate() + self.assertEqual(cluster.logical_workers, 4) + rendered = cluster.remote_env( + "10.0.0.1", "worker", worker_index=3, local_worker_index=3 + ) + self.assertIn("WORKER_COUNT=4", rendered) + self.assertIn("WORKERS_PER_INSTANCE=4", rendered) + self.assertIn("WORKER_INDEX=3", rendered) + self.assertIn("LOCAL_WORKER_INDEX=3", rendered) + self.assertIn("GPU_DEVICE_ID=3", rendered) + + def test_cpu_rejects_multiple_workers_per_instance(self) -> None: + config = valid_config() + config["GPU_WORKERS_PER_INSTANCE"] = "2" + with self.assertRaisesRegex( + aws_cluster.ClusterError, "must be 1 for CPU" + ): + self.make_cluster(config).validate() + def test_validation_rejects_invalid_gpu_io_toggle(self) -> None: config = valid_config() config["GPU_USE_BUFFERED_INPUT"] = "yes" From 57b5a58265c6f168796c8461de09fe2a19e058ef Mon Sep 17 00:00:00 2001 From: Gregory Kimball Date: Mon, 31 Aug 2026 11:53:20 -0700 Subject: [PATCH 11/12] Enable native worker spilling on EC2 Configure Prestissimo's native spill gate and worker address so memory-heavy queries can use instance-store NVMe independently of async cache. --- presto/aws/ec2/README.md | 11 +++++ presto/aws/ec2/aws_cluster.py | 39 +++++++++++++++--- presto/aws/ec2/aws_config.env.example | 5 +++ presto/aws/ec2/remote/configure_and_start.sh | 42 ++++++++++++++++++-- presto/aws/ec2/test_aws_cluster.py | 42 ++++++++++++++++++++ 5 files changed, 130 insertions(+), 9 deletions(-) diff --git a/presto/aws/ec2/README.md b/presto/aws/ec2/README.md index 1ea331ff0..2a3d0d099 100644 --- a/presto/aws/ec2/README.md +++ b/presto/aws/ec2/README.md @@ -277,6 +277,17 @@ For each query, plot iteration 1 separately and average iterations 2-5 for the reported result. The hot/cold distinction comes only from the archived cache and input-path configuration, not from a second repetition model. +## Local NVMe spill + +`SPILL_ENABLED=true` mounts the worker's instance-store NVMe device at +`/mnt/nvme` and configures native workers to spill under `SPILL_PATH`. +Spilling is independent of async data cache, so cache-off runs can use +`ASYNC_DATA_CACHE_ENABLED=false`, `ASYNC_CACHE_SSD_GIB=0`, and still spill. + +When SSD cache and spill share one device, use separate directories and size +`ASYNC_CACHE_SSD_GIB`, `MAX_SPILL_GIB`, and `QUERY_MAX_SPILL_GIB` to leave +filesystem headroom. + ## Generated state and artifacts Local mutable state defaults to: diff --git a/presto/aws/ec2/aws_cluster.py b/presto/aws/ec2/aws_cluster.py index ffaed861f..f8f8700a8 100755 --- a/presto/aws/ec2/aws_cluster.py +++ b/presto/aws/ec2/aws_cluster.py @@ -184,6 +184,10 @@ def validate(self, cloud: bool = True) -> None: "CUDA_MODULE_LOADING", "ASYNC_CACHE_SSD_GIB", "ASYNC_CACHE_NUM_SHARDS", + "SPILL_ENABLED", + "SPILL_PATH", + "MAX_SPILL_GIB", + "QUERY_MAX_SPILL_GIB", ) if cloud: require( @@ -226,6 +230,8 @@ def validate(self, cloud: bool = True) -> None: "WORKER_QUERY_MEMORY_GIB", "WORKER_MEMORY_LIMIT_GIB", "WORKER_MEMORY_SHRINK_GIB", + "MAX_SPILL_GIB", + "QUERY_MAX_SPILL_GIB", ): positive_int(self.config, key) if self.workers not in (1, 2, 8, 16, 32): @@ -263,6 +269,7 @@ def validate(self, cloud: bool = True) -> None: "GPU_USE_BUFFERED_INPUT", "GPU_USE_KVIKIO", "EFA_ENABLED", + "SPILL_ENABLED", ): value = self.config.get(key, "false") if value not in ("true", "false"): @@ -280,6 +287,15 @@ def validate(self, cloud: bool = True) -> None: raise ClusterError("ASYNC_CACHE_SSD_PATH is required when ASYNC_CACHE_SSD_GIB is nonzero") if ssd_gib and not self.config["ASYNC_CACHE_SSD_PATH"].startswith("/mnt/nvme/"): raise ClusterError("ASYNC_CACHE_SSD_PATH must be under /mnt/nvme/") + if self.config["SPILL_ENABLED"] == "true": + if not self.config["SPILL_PATH"].startswith("/mnt/nvme/"): + raise ClusterError("SPILL_PATH must be under /mnt/nvme/") + if ( + ssd_gib + and self.config["SPILL_PATH"].rstrip("/") + == self.config["ASYNC_CACHE_SSD_PATH"].rstrip("/") + ): + raise ClusterError("SPILL_PATH and ASYNC_CACHE_SSD_PATH must differ") if positive_int(self.config, "WORKER_QUERY_MEMORY_GIB") >= positive_int( self.config, "WORKER_SYSTEM_MEMORY_GIB" ): @@ -834,6 +850,10 @@ def remote_env( "ASYNC_CACHE_SSD_GIB": self.config["ASYNC_CACHE_SSD_GIB"], "ASYNC_CACHE_SSD_PATH": self.config.get("ASYNC_CACHE_SSD_PATH", ""), "ASYNC_CACHE_NUM_SHARDS": self.config["ASYNC_CACHE_NUM_SHARDS"], + "SPILL_ENABLED": self.config["SPILL_ENABLED"], + "SPILL_PATH": self.config["SPILL_PATH"], + "MAX_SPILL_GIB": self.config["MAX_SPILL_GIB"], + "QUERY_MAX_SPILL_GIB": self.config["QUERY_MAX_SPILL_GIB"], "COORDINATOR_IMAGE": self.config["COORDINATOR_IMAGE"], "WORKER_IMAGE": self.config["WORKER_IMAGE"], } @@ -843,8 +863,15 @@ def start(self) -> None: self.validate(cloud=True) coordinator, workers = self.expected_inventory() address = coordinator.private_ip or coordinator.private_dns - if nonnegative_int(self.config, "ASYNC_CACHE_SSD_GIB"): - cache_path = shlex.quote(self.config["ASYNC_CACHE_SSD_PATH"]) + ssd_gib = nonnegative_int(self.config, "ASYNC_CACHE_SSD_GIB") + spill_enabled = self.config["SPILL_ENABLED"] == "true" + if ssd_gib or spill_enabled: + storage_paths = [] + if ssd_gib: + storage_paths.append(self.config["ASYNC_CACHE_SSD_PATH"]) + if spill_enabled: + storage_paths.append(self.config["SPILL_PATH"]) + quoted_paths = " ".join(shlex.quote(path) for path in storage_paths) prepare_command_id = self.send_command( [worker.instance_id for worker in workers], [ @@ -872,13 +899,13 @@ def start(self) -> None: "fi; " "fi" ), - f"mkdir -p {cache_path}", - f"chmod 0777 /mnt/nvme {cache_path}", + f"mkdir -p {quoted_paths}", + f"chmod 0777 /mnt/nvme {quoted_paths}", ], - f"prepare NVMe cache {self.run_id}", + f"prepare NVMe storage {self.run_id}", wait=True, ) - self.record_command("prepare_nvme_cache", prepare_command_id) + self.record_command("prepare_nvme_storage", prepare_command_id) command_id = self.send_command( [coordinator.instance_id], [ diff --git a/presto/aws/ec2/aws_config.env.example b/presto/aws/ec2/aws_config.env.example index 142c923f6..609dc16f6 100644 --- a/presto/aws/ec2/aws_config.env.example +++ b/presto/aws/ec2/aws_config.env.example @@ -81,6 +81,11 @@ ASYNC_DATA_CACHE_ENABLED=false ASYNC_CACHE_SSD_GIB=0 ASYNC_CACHE_SSD_PATH= ASYNC_CACHE_NUM_SHARDS=16 +# Independent local-NVMe spilling. This can remain enabled when async cache is off. +SPILL_ENABLED=false +SPILL_PATH=/mnt/nvme/spill +MAX_SPILL_GIB=150 +QUERY_MAX_SPILL_GIB=150 # Fleet safety and ownership. OWNER= diff --git a/presto/aws/ec2/remote/configure_and_start.sh b/presto/aws/ec2/remote/configure_and_start.sh index c6c28a13d..102ea60b6 100755 --- a/presto/aws/ec2/remote/configure_and_start.sh +++ b/presto/aws/ec2/remote/configure_and_start.sh @@ -27,6 +27,7 @@ required=( COORDINATOR_QUERY_MEMORY_PER_NODE_GIB WORKER_SYSTEM_MEMORY_GIB WORKER_QUERY_MEMORY_GIB WORKER_MEMORY_LIMIT_GIB WORKER_MEMORY_SHRINK_GIB ASYNC_DATA_CACHE_ENABLED ASYNC_CACHE_SSD_GIB ASYNC_CACHE_NUM_SHARDS + SPILL_ENABLED SPILL_PATH MAX_SPILL_GIB QUERY_MAX_SPILL_GIB COORDINATOR_IMAGE WORKER_IMAGE ) for name in "${required[@]}"; do @@ -194,6 +195,14 @@ if [[ ${ROLE} == coordinator ]]; then fi set_property "${final}/config.properties" experimental.enable-dynamic-filtering \ "${DYNAMIC_FILTERING_ENABLED}" + upsert_property "${final}/config.properties" experimental.spill-enabled \ + "${SPILL_ENABLED}" + if [[ ${SPILL_ENABLED} == true ]]; then + upsert_property "${final}/config.properties" experimental.max-spill-per-node \ + "${MAX_SPILL_GIB}GB" + upsert_property "${final}/config.properties" experimental.query-max-spill-per-node \ + "${QUERY_MAX_SPILL_GIB}GB" + fi python3 - "${final}/jvm.config" "${COORDINATOR_HEAP_GIB}" <<'PY' import sys from pathlib import Path @@ -241,6 +250,23 @@ else "${ASYNC_CACHE_SSD_GIB}" upsert_property "${final}/config.properties" async-cache-num-shards \ "${ASYNC_CACHE_NUM_SHARDS}" + # Prestissimo uses its native spill gate on workers. The + # experimental.spill-enabled property is the Java coordinator setting and + # does not enable Velox operator spilling. + upsert_property "${final}/config.properties" spill-enabled \ + "${SPILL_ENABLED}" + if [[ ${SPILL_ENABLED} == true ]]; then + upsert_property "${final}/config.properties" max-spill-bytes \ + "$((MAX_SPILL_GIB * 1024 * 1024 * 1024))" + upsert_property "${final}/config.properties" experimental.spiller-spill-path \ + "${SPILL_PATH}" + upsert_property "${final}/config.properties" experimental.max-spill-per-node \ + "${MAX_SPILL_GIB}GB" + upsert_property "${final}/config.properties" experimental.query-max-spill-per-node \ + "${QUERY_MAX_SPILL_GIB}GB" + upsert_property "${final}/config.properties" \ + experimental.spiller-max-used-space-threshold 0.9 + fi upsert_property "${final}/config.properties" runtime-metrics-collection-enabled \ true if [[ ${ENGINE_VARIANT} == gpu ]]; then @@ -285,6 +311,8 @@ else fi set_property "${final}/node.properties" node.id \ "aws-${RUN_ID}-worker-${WORKER_INDEX}" + upsert_property "${final}/node.properties" node.internal-address \ + "$(hostname -I | awk '{print $1}')" fi cp -a "${final}/." "${assembled}/" @@ -300,7 +328,11 @@ tuning_id=$( "split=${HIVE_MAX_SPLIT_SIZE}" \ "loader=${HIVE_SPLIT_LOADER_CONCURRENCY}" \ "dynamic_filtering=${DYNAMIC_FILTERING_ENABLED}" \ - "exchange_tuning=${CPU_EXCHANGE_TUNING_ENABLED}" | + "exchange_tuning=${CPU_EXCHANGE_TUNING_ENABLED}" \ + "spill_enabled=${SPILL_ENABLED}" \ + "spill_path=${SPILL_PATH}" \ + "max_spill_gib=${MAX_SPILL_GIB}" \ + "query_max_spill_gib=${QUERY_MAX_SPILL_GIB}" | sha256sum | cut -c1-12 ) history="${runtime_root}/config_history/${tuning_id}/${role_key}" @@ -314,7 +346,11 @@ cat >"${history}/tuning.json" < 0)); then + if ((ASYNC_CACHE_SSD_GIB > 0)) || [[ ${SPILL_ENABLED} == true ]]; then cache_mount_args=(-v /mnt/nvme:/mnt/nvme) fi if [[ ${ENGINE_VARIANT} == gpu ]]; then diff --git a/presto/aws/ec2/test_aws_cluster.py b/presto/aws/ec2/test_aws_cluster.py index eb6ff494c..4733430bc 100644 --- a/presto/aws/ec2/test_aws_cluster.py +++ b/presto/aws/ec2/test_aws_cluster.py @@ -69,6 +69,10 @@ def valid_config() -> dict[str, str]: "ASYNC_CACHE_SSD_GIB": "0", "ASYNC_CACHE_SSD_PATH": "", "ASYNC_CACHE_NUM_SHARDS": "16", + "SPILL_ENABLED": "false", + "SPILL_PATH": "/mnt/nvme/spill", + "MAX_SPILL_GIB": "150", + "QUERY_MAX_SPILL_GIB": "150", "OWNER": "tester", "EXPIRY_HOURS": "8", "SSM_READY_TIMEOUT_SECONDS": "900", @@ -190,6 +194,7 @@ def test_remote_env_contains_fixed_scale_rules(self) -> None: self.assertIn("WORKER_INDEX=3", rendered) self.assertIn("COORDINATOR_ADDRESS=10.0.0.1", rendered) self.assertIn("ASYNC_DATA_CACHE_ENABLED=false", rendered) + self.assertIn("SPILL_ENABLED=false", rendered) self.assertIn("TASK_MAX_DRIVERS_PER_TASK=16", rendered) self.assertIn("HIVE_MAX_SPLIT_SIZE=256MB", rendered) self.assertIn("ENGINE_VARIANT=cpu", rendered) @@ -205,6 +210,43 @@ def test_validation_rejects_invalid_cpu_tuning_values(self) -> None: with self.assertRaisesRegex(aws_cluster.ClusterError, "true or false"): self.make_cluster(config).validate() + def test_validation_accepts_spill_without_async_cache(self) -> None: + config = valid_config() + config["SPILL_ENABLED"] = "true" + cluster = self.make_cluster(config) + cluster.validate() + rendered = cluster.remote_env("10.0.0.1", "worker", 0) + self.assertIn("SPILL_ENABLED=true", rendered) + self.assertIn("SPILL_PATH=/mnt/nvme/spill", rendered) + + def test_validation_rejects_non_nvme_spill_path(self) -> None: + config = valid_config() + config["SPILL_ENABLED"] = "true" + config["SPILL_PATH"] = "/tmp/spill" + with self.assertRaisesRegex(aws_cluster.ClusterError, "under /mnt/nvme"): + self.make_cluster(config).validate() + + def test_remote_worker_config_uses_native_spill_gate(self) -> None: + configure_script = ( + Path(__file__).parent / "remote" / "configure_and_start.sh" + ).read_text() + worker_config = configure_script.split( + "else\n set_property \"${final}/config.properties\" http-server.http.port", + 1, + )[1] + self.assertIn( + 'upsert_property "${final}/config.properties" spill-enabled', + worker_config, + ) + self.assertIn( + 'upsert_property "${final}/config.properties" max-spill-bytes', + worker_config, + ) + self.assertIn( + 'upsert_property "${final}/node.properties" node.internal-address', + worker_config, + ) + def test_validation_accepts_gpu_variant(self) -> None: config = valid_config() config["ENGINE_VARIANT"] = "gpu" From 56a564cc8321256aca04e0a52b1eac6cd016136f Mon Sep 17 00:00:00 2001 From: Gregory Kimball Date: Mon, 31 Aug 2026 11:56:28 -0700 Subject: [PATCH 12/12] Deduplicate restarted workers in benchmark metadata Count unique worker endpoints so stale node records left by service restarts do not inflate reported cluster size. --- .../performance_benchmarks/presto_api.py | 10 +++++ .../performance_benchmarks/run_context.py | 8 ++-- .../performance_benchmarks/test_presto_api.py | 41 +++++++++++++++++++ 3 files changed, 55 insertions(+), 4 deletions(-) create mode 100644 presto/testing/performance_benchmarks/test_presto_api.py diff --git a/presto/testing/performance_benchmarks/presto_api.py b/presto/testing/performance_benchmarks/presto_api.py index da64d9ef5..ee93c88b1 100644 --- a/presto/testing/performance_benchmarks/presto_api.py +++ b/presto/testing/performance_benchmarks/presto_api.py @@ -58,3 +58,13 @@ def get_nodes(hostname: str, port: int) -> list | None: if not isinstance(raw, list): return None return raw + + +def count_unique_node_uris(nodes: list) -> int: + """Count worker endpoints, ignoring stale duplicate node records.""" + uris = { + node["uri"] + for node in nodes + if isinstance(node, dict) and isinstance(node.get("uri"), str) and node["uri"] + } + return len(uris) diff --git a/presto/testing/performance_benchmarks/run_context.py b/presto/testing/performance_benchmarks/run_context.py index 17acec259..76c46a5e6 100644 --- a/presto/testing/performance_benchmarks/run_context.py +++ b/presto/testing/performance_benchmarks/run_context.py @@ -16,7 +16,7 @@ import prestodb from ..common import test_utils -from .presto_api import get_cluster_tag, get_nodes +from .presto_api import count_unique_node_uris, get_cluster_tag, get_nodes # Enabled by run_benchmark.sh --verbose (sets PRESTO_BENCHMARK_DEBUG=1) _DEBUG = os.environ.get("PRESTO_BENCHMARK_DEBUG", "") == "1" or os.environ.get("DEBUG", "") == "1" @@ -28,12 +28,12 @@ def _debug(msg: str) -> None: def _get_node_count(hostname: str, port: int) -> int | None: - """Return number of nodes in the Presto /v1/node list (workers only; coordinator not listed).""" + """Return unique worker endpoints from Presto's /v1/node response.""" nodes = get_nodes(hostname, port) if nodes is None: return None - n = len(nodes) - _debug(f"get_node_count: {n} node(s)") + n = count_unique_node_uris(nodes) + _debug(f"get_node_count: {n} unique endpoint(s) from {len(nodes)} node record(s)") return n diff --git a/presto/testing/performance_benchmarks/test_presto_api.py b/presto/testing/performance_benchmarks/test_presto_api.py new file mode 100644 index 000000000..a66601798 --- /dev/null +++ b/presto/testing/performance_benchmarks/test_presto_api.py @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +import sys +import types +import unittest + +sys.modules.setdefault("requests", types.SimpleNamespace()) + +import presto_api # noqa: E402 + + +class CountUniqueNodeUrisTest(unittest.TestCase): + def test_ignores_stale_duplicate_records(self) -> None: + nodes = [ + {"uri": "http://10.0.0.1:8080/v1/status"}, + { + "uri": "http://10.0.0.1:8080/v1/status", + "lastFailureInfo": {"message": "connection refused"}, + }, + {"uri": "http://10.0.0.2:8080/v1/status"}, + ] + + self.assertEqual(presto_api.count_unique_node_uris(nodes), 2) + + def test_distinguishes_workers_on_different_ports(self) -> None: + nodes = [ + {"uri": "http://10.0.0.1:8080/v1/status"}, + {"uri": "http://10.0.0.1:8090/v1/status"}, + ] + + self.assertEqual(presto_api.count_unique_node_uris(nodes), 2) + + def test_ignores_malformed_records(self) -> None: + nodes = [{}, {"uri": None}, {"uri": ""}, "not-a-node"] + + self.assertEqual(presto_api.count_unique_node_uris(nodes), 0) + + +if __name__ == "__main__": + unittest.main()