From e3d1265ebdda8a099f7bd5ec7e9ee4489c8630e2 Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Tue, 11 Aug 2026 11:51:01 -0400 Subject: [PATCH] Add agent commands Co-authored-by: Cursor --- README.md | 17 + .../container_scripts/run_functional_tests.sh | 12 +- client/oci_env/agent.py | 554 ++++++++++++++++++ client/oci_env/main.py | 127 ++++ docs/dev/guides/agent-envs.md | 145 +++++ docs/dev/guides/create-multiple-envs.md | 4 + docs/dev/reference/profiles/lean.md | 14 + profiles/lean/README.md | 56 ++ profiles/lean/compose.yaml | 4 + profiles/lean/profile_default_config.env | 3 + profiles/lean/pulp_config.env | 1 + 11 files changed, 936 insertions(+), 1 deletion(-) create mode 100644 client/oci_env/agent.py create mode 100644 docs/dev/guides/agent-envs.md create mode 100644 docs/dev/reference/profiles/lean.md create mode 100644 profiles/lean/README.md create mode 100644 profiles/lean/compose.yaml create mode 100644 profiles/lean/profile_default_config.env create mode 100644 profiles/lean/pulp_config.env diff --git a/README.md b/README.md index 1076f11..bb1f2df 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,23 @@ A developer environment for pulp based off of the [Pulp OCI Images](https://gith A detailed guide on setting up the development environment is available [here](docs/dev/tutorials/quickstart.md). +## Lean multi-agent environments + +For parallel agents that each need an isolated live Pulp API (own DB, volumes, and source +worktrees) with lower resource use, see the [agent environments guide](docs/dev/guides/agent-envs.md) +and the `lean` profile (`oci-env profile docs lean`). + +```bash +oci-env agent create pr-123 --plugins pulpcore +oci-env agent up pr-123 +oci-env agent test pr-123 -p pulpcore -- -k test_crud_repos +oci-env agent destroy pr-123 + +# Or reuse a Cursor / other AI tool worktree for plugins under test: +oci-env agent create pr-123 \ + --plugins pulpcore=~/.cursor/worktrees/pulpcore/my-wt +``` + ## Multiple environments oci-env supports running multiple environments simultaneously. To do this, simply create a new .env file such as: diff --git a/base/container_scripts/run_functional_tests.sh b/base/container_scripts/run_functional_tests.sh index 78210a3..c13b20a 100755 --- a/base/container_scripts/run_functional_tests.sh +++ b/base/container_scripts/run_functional_tests.sh @@ -42,4 +42,14 @@ EOF check_pytest check_client -sudo -u pulp -E pytest -r sx --rootdir=/var/lib/pulp --color=yes --pyargs "${PROJECT}.tests.functional" "${@:2}" +API_URL="${API_PROTOCOL:-http}://${API_HOST:-localhost}:${API_PORT:-5001}" +echo "Running functional tests for ${PROJECT} against live API ${API_URL}" +echo "Auth: ${DJANGO_SUPERUSER_USERNAME:-admin} (password from DJANGO_SUPERUSER_PASSWORD)" + +sudo -u pulp -E \ + API_PROTOCOL="${API_PROTOCOL:-http}" \ + API_HOST="${API_HOST:-localhost}" \ + API_PORT="${API_PORT:-5001}" \ + DJANGO_SUPERUSER_USERNAME="${DJANGO_SUPERUSER_USERNAME:-admin}" \ + DJANGO_SUPERUSER_PASSWORD="${DJANGO_SUPERUSER_PASSWORD:-password}" \ + pytest -r sx --rootdir=/var/lib/pulp --color=yes --pyargs "${PROJECT}.tests.functional" "${@:2}" diff --git a/client/oci_env/agent.py b/client/oci_env/agent.py new file mode 100644 index 0000000..2cf372c --- /dev/null +++ b/client/oci_env/agent.py @@ -0,0 +1,554 @@ +"""Lifecycle helpers for lean, isolated multi-agent Pulp environments.""" + +import os +import re +import shutil +import socket +import subprocess + +from oci_env.logger import logger +from oci_env.utils import ( + Compose, + exit_with_error, + get_config, + get_oci_env_path, + read_env_file, +) + + +AGENT_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$") +DEFAULT_PORT_START = 5100 +DEFAULT_PORT_END = 5199 +AGENT_ENV_FILENAME = "compose.env" +AGENT_OVERLAY_FILENAME = "plugin_volumes_compose.yaml" + +# Agent environments always use podman (rootless-friendly for parallel stacks). +AGENT_COMPOSE_BINARY = "podman" + +# Keys owned by agent create; --env cannot override these. +_AGENT_LOCKED_KEYS = { + "OCI_AGENT_ID", + "OCI_AGENT_HOST_SRC_DIR", + "OCI_AGENT_PLUGIN_PATHS", + "OCI_AGENT_VOLUME_OVERLAY", + "COMPOSE_BINARY", + "COMPOSE_PROJECT_NAME", + "DEV_SOURCE_PATH", + "SRC_DIR", + "OCI_ENV_DIR", + "OCI_ENV_CONFIG_FILE", +} + + +def agent_project_name(agent_id): + return f"agent_{agent_id}" + + +def agent_compiled_dir(oci_env_path, agent_id): + """Per-agent directory under .compiled/ (also used as SRC_DIR and compose project).""" + return os.path.join(oci_env_path, ".compiled", agent_project_name(agent_id)) + + +def agent_env_path(oci_env_path, agent_id): + return os.path.join(agent_compiled_dir(oci_env_path, agent_id), AGENT_ENV_FILENAME) + + +def agent_volume_overlay_path(oci_env_path, agent_id): + return os.path.join(agent_compiled_dir(oci_env_path, agent_id), AGENT_OVERLAY_FILENAME) + + +def validate_agent_id(agent_id): + if not AGENT_ID_RE.match(agent_id): + exit_with_error( + f"Invalid agent id {agent_id!r}. Use letters, numbers, '_', '-', or '.' " + "(must start with alphanumeric)." + ) + + +def list_agent_ids(oci_env_path): + compiled = os.path.join(oci_env_path, ".compiled") + if not os.path.isdir(compiled): + return [] + ids = [] + for name in sorted(os.listdir(compiled)): + env_path = os.path.join(compiled, name, AGENT_ENV_FILENAME) + if not os.path.isfile(env_path): + continue + data = read_env_file(env_path) + agent_id = data.get("OCI_AGENT_ID") + if agent_id: + ids.append(agent_id) + return ids + + +def load_agent_env(oci_env_path, agent_id): + path = agent_env_path(oci_env_path, agent_id) + if not os.path.isfile(path): + exit_with_error(f"Agent {agent_id!r} does not exist (missing {path})") + return path, read_env_file(path) + + +def write_env_file(path, values): + lines = [ + "# AUTOGENERATED by oci-env agent", + f"# Agent id: {values.get('OCI_AGENT_ID', '')}", + "", + ] + # Stable order for core keys, then any extras (e.g. from --env). + preferred = ( + "OCI_AGENT_ID", + "COMPOSE_PROFILE", + "DEV_SOURCE_PATH", + "COMPOSE_PROJECT_NAME", + "API_PORT", + "SRC_DIR", + "COMPOSE_BINARY", + "API_HOST", + "API_PROTOCOL", + "OCI_AGENT_VOLUME_OVERLAY", + "OCI_AGENT_PLUGIN_PATHS", + "OCI_AGENT_HOST_SRC_DIR", + ) + written = set() + for key in preferred: + if key in values and values[key] not in (None, ""): + lines.append(f"{key}={values[key]}") + written.add(key) + for key in sorted(values.keys()): + if key in written or values[key] in (None, ""): + continue + lines.append(f"{key}={values[key]}") + + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w") as f: + f.write("\n".join(lines) + "\n") + + +def _port_in_use(port, host="127.0.0.1"): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + try: + sock.bind((host, port)) + except OSError: + return True + return False + + +def allocated_ports(oci_env_path): + ports = set() + for agent_id in list_agent_ids(oci_env_path): + _, data = load_agent_env(oci_env_path, agent_id) + port = data.get("API_PORT") + if port and str(port).isdigit(): + ports.add(int(port)) + return ports + + +def allocate_port(oci_env_path, preferred=None, start=DEFAULT_PORT_START, end=DEFAULT_PORT_END): + used = allocated_ports(oci_env_path) + if preferred is not None: + preferred = int(preferred) + if preferred in used: + exit_with_error(f"Port {preferred} is already allocated to another agent") + if _port_in_use(preferred): + exit_with_error(f"Port {preferred} is already in use on the host") + return preferred + + for port in range(start, end + 1): + if port in used: + continue + if not _port_in_use(port): + return port + exit_with_error(f"No free agent API ports left in range {start}-{end}") + + +def host_compose_config(oci_env_path): + """ + Load the user's normal compose.env/.compose.env if present. + + Falls back to empty dict when no host env file exists (agent create must + still work without one). Environment variable overrides are applied the + same way as get_config() when a file is found. + """ + for name in ("compose.env", ".compose.env"): + path = os.path.join(oci_env_path, name) + if os.path.isfile(path): + return get_config(path) + return {} + + +def default_host_src_dir(oci_env_path): + """Best-effort SRC_DIR from an existing compose.env, else parent of oci_env.""" + fallback = os.path.abspath(os.path.join(oci_env_path, "..")) + cfg = host_compose_config(oci_env_path) + return cfg.get("SRC_DIR") or fallback + + +def parse_env_arg(value): + """Parse KEY=VALUE from --env.""" + if "=" not in value: + exit_with_error(f"Invalid --env {value!r}. Expected KEY=VALUE.") + key, val = value.split("=", 1) + key = key.strip() + if not key: + exit_with_error(f"Invalid --env {value!r}: key is empty") + if key in _AGENT_LOCKED_KEYS: + exit_with_error( + f"--env cannot set {key}; it is managed by oci-env agent create" + ) + return key, val + + +def collect_env_overrides(args): + overrides = {} + for value in getattr(args, "env_vars", None) or []: + key, val = parse_env_arg(value) + overrides[key] = val + return overrides + + +def resolve_repo_path(host_src_dir, repo): + path = os.path.join(host_src_dir, repo) + if not os.path.isdir(path): + exit_with_error(f"Repository checkout not found at {path}") + if not os.path.isdir(os.path.join(path, ".git")) and not os.path.isfile( + os.path.join(path, ".git") + ): + # worktree checkouts have a .git file; plain dirs without git are invalid + exit_with_error(f"{path} is not a git repository") + return path + + +def parse_plugins_arg(spec, host_src_dir): + """ + Parse --plugins as colon-separated PLUGIN[=PATH] entries. + + Examples: + pulpcore + pulpcore:pulp_rpm + pulpcore=~/.cursor/worktrees/pulpcore/my-wt + pulpcore=/path/to/wt:pulp_rpm + """ + if not spec or not spec.strip(): + exit_with_error("At least one plugin is required via --plugins") + + plugins = [] + plugin_paths = {} + for part in spec.split(":"): + part = part.strip() + if not part: + continue + if "=" in part: + plugin, path = part.split("=", 1) + plugin = plugin.strip() + path = os.path.abspath(os.path.expanduser(path.strip())) + if not plugin: + exit_with_error(f"Invalid --plugins entry {part!r}: plugin name is empty") + if not os.path.isdir(path): + exit_with_error(f"Plugin path for {plugin!r} is not a directory: {path}") + else: + plugin = part + path = resolve_repo_path(host_src_dir, plugin) + if plugin in plugin_paths: + exit_with_error(f"Plugin {plugin!r} listed more than once in --plugins") + plugins.append(plugin) + plugin_paths[plugin] = path + + if not plugins: + exit_with_error("At least one plugin is required via --plugins") + return plugins, plugin_paths + + +def encode_plugin_paths(plugin_paths): + """Serialize plugin->path mapping for the agent env file.""" + return ",".join(f"{plugin}={path}" for plugin, path in sorted(plugin_paths.items())) + + +def write_plugin_volume_overlay(overlay_path, plugin_paths): + """ + Write a compose overlay that bind-mounts plugin checkouts into /src/. + + Nested under the base `{SRC_DIR}:/src` mount; podman creates the destination + path when missing, so no host-side mount-point directories are required. + """ + lines = [ + "# AUTOGENERATED by oci-env agent — bind mounts for plugin checkouts", + "services:", + " pulp:", + " volumes:", + ] + for plugin, path in sorted(plugin_paths.items()): + real = os.path.realpath(path) + lines.append(f' - "{real}:/src/{plugin}:z"') + + os.makedirs(os.path.dirname(overlay_path), exist_ok=True) + with open(overlay_path, "w") as f: + f.write("\n".join(lines) + "\n") + logger.info(f"Wrote plugin volume overlay: {overlay_path}") + return overlay_path + + +def compose_for_agent(is_verbose, env_file): + client = Compose(is_verbose, env_file) + overlay = client.config.get("OCI_AGENT_VOLUME_OVERLAY") + if overlay and os.path.isfile(overlay): + client.compose_files.append(overlay) + if is_verbose: + logger.info(f"Including agent volume overlay: {overlay}") + return client + + +def agent_status(oci_env_path, agent_id, env_data, is_verbose=False): + """Return a short status string for an agent environment.""" + project = env_data.get("COMPOSE_PROJECT_NAME", agent_project_name(agent_id)) + cmd = [AGENT_COMPOSE_BINARY, "ps", "--filter", f"name={project}", "--format", "{{.Names}}"] + try: + proc = subprocess.run(cmd, capture_output=True, text=True) + except FileNotFoundError: + return "unknown" + + names = [line.strip() for line in proc.stdout.splitlines() if line.strip()] + if not names: + return "down" + return "up" + + +def agent_create(args): + oci_env_path = get_oci_env_path(args.is_verbose) + validate_agent_id(args.agent_id) + + compiled_dir = agent_compiled_dir(oci_env_path, args.agent_id) + env_path = agent_env_path(oci_env_path, args.agent_id) + if os.path.exists(env_path) or ( + os.path.isdir(compiled_dir) and os.listdir(compiled_dir) + ): + exit_with_error( + f"Agent {args.agent_id!r} already exists at {compiled_dir}" + ) + + host_src_dir = os.path.abspath(args.host_src_dir or default_host_src_dir(oci_env_path)) + plugins, plugin_paths = parse_plugins_arg(args.plugins, host_src_dir) + env_overrides = collect_env_overrides(args) + + if "API_PORT" in env_overrides and args.port is not None: + exit_with_error("Specify API port with either --port or --env API_PORT, not both") + api_port = allocate_port(oci_env_path, preferred=env_overrides.get("API_PORT", args.port)) + + project_name = agent_project_name(args.agent_id) + compose_profile = env_overrides.get("COMPOSE_PROFILE") or "lean" + + os.makedirs(compiled_dir, exist_ok=True) + overlay = write_plugin_volume_overlay( + agent_volume_overlay_path(oci_env_path, args.agent_id), + plugin_paths, + ) + + values = { + "OCI_AGENT_ID": args.agent_id, + "COMPOSE_PROFILE": compose_profile, + "DEV_SOURCE_PATH": ":".join(plugins), + "COMPOSE_PROJECT_NAME": project_name, + "API_PORT": str(api_port), + "SRC_DIR": compiled_dir, + "COMPOSE_BINARY": AGENT_COMPOSE_BINARY, + "API_HOST": "localhost", + "API_PROTOCOL": "http", + # Pulp refuses to start without this; --env can override. + "PULP_SECRET_KEY": "dummy", + "OCI_AGENT_HOST_SRC_DIR": host_src_dir, + "OCI_AGENT_PLUGIN_PATHS": encode_plugin_paths(plugin_paths), + "OCI_AGENT_VOLUME_OVERLAY": overlay, + } + + # Apply user --env overrides (locked keys already rejected). + values.update(env_overrides) + + write_env_file(env_path, values) + print(f"Created agent {args.agent_id}") + print(f" env: {env_path}") + print(f" project: {project_name}") + print(f" port: {values['API_PORT']}") + print(f" src: {compiled_dir}") + print(f" profile: {values['COMPOSE_PROFILE']}") + print(f" compose: {AGENT_COMPOSE_BINARY}") + print(f" plugins: {':'.join(plugins)}") + for plugin in plugins: + print(f" {plugin}: {plugin_paths[plugin]}") + for key in sorted(env_overrides.keys()): + print(f" {key}={env_overrides[key]}") + print(f"Start with: oci-env agent up {args.agent_id}") + + +def agent_up(args): + oci_env_path = get_oci_env_path(args.is_verbose) + env_path, _ = load_agent_env(oci_env_path, args.agent_id) + client = compose_for_agent(args.is_verbose, env_path) + + up_cmd = ["up", "-d"] + if args.build: + up_cmd.append("--build") + rc = client.compose_command(up_cmd, interactive=True) + if rc != 0: + exit(rc) + + client.poll(args.attempts, args.wait) + api = "{}://{}:{}/".format( + client.config["API_PROTOCOL"], + client.config["API_HOST"], + client.config["API_PORT"], + ) + print(f"Agent {args.agent_id} is up at {api}") + + +def agent_down(args): + oci_env_path = get_oci_env_path(args.is_verbose) + env_path, _ = load_agent_env(oci_env_path, args.agent_id) + try: + client = compose_for_agent(args.is_verbose, env_path) + rc = client.compose_command(["down"], interactive=True) + except FileNotFoundError as exc: + exit_with_error(f"compose binary unavailable: {exc}") + exit(rc) + + +def agent_destroy(args): + oci_env_path = get_oci_env_path(args.is_verbose) + env_path, env_data = load_agent_env(oci_env_path, args.agent_id) + compiled_dir = agent_compiled_dir(oci_env_path, args.agent_id) + + try: + client = compose_for_agent(args.is_verbose, env_path) + rc = client.compose_command(["down", "--volumes"], interactive=True) + if rc != 0: + logger.warning( + f"compose down --volumes returned {rc}; continuing with agent cleanup" + ) + except FileNotFoundError as exc: + logger.warning(f"compose binary unavailable ({exc}); continuing with agent cleanup") + except SystemExit: + logger.warning("compose client setup failed; continuing with agent cleanup") + + # Remove the entire agent compiled directory (env, overlay, and any files + # written by parse_profiles). Never deletes bind-mount targets. + if os.path.isdir(compiled_dir): + shutil.rmtree(compiled_dir) + logger.info(f"Removed agent compiled directory {compiled_dir}") + elif os.path.isfile(env_path): + os.remove(env_path) + + print(f"Destroyed agent {args.agent_id}") + + +def agent_ls(args): + oci_env_path = get_oci_env_path(args.is_verbose) + ids = list_agent_ids(oci_env_path) + if not ids: + print("No agent environments found.") + return + + print(f"{'ID':<20} {'PORT':<8} {'STATUS':<8} {'PROJECT':<24} SRC_DIR") + for agent_id in ids: + _, data = load_agent_env(oci_env_path, agent_id) + status = agent_status(oci_env_path, agent_id, data, args.is_verbose) + print( + f"{agent_id:<20} {data.get('API_PORT', '-'):<8} {status:<8} " + f"{data.get('COMPOSE_PROJECT_NAME', '-'):<24} {data.get('SRC_DIR', '-')}" + ) + + +def agent_test(args): + """ + Run tests against an agent environment. + + Functional tests execute inside the agent container, so they use that + environment's live API (API_PORT/ creds from the agent env file). + """ + oci_env_path = get_oci_env_path(args.is_verbose) + env_path, env_data = load_agent_env(oci_env_path, args.agent_id) + client = compose_for_agent(args.is_verbose, env_path) + + # Expose connection hints for tooling that reads the process environment. + os.environ.setdefault("OCI_AGENT_ID", args.agent_id) + os.environ.setdefault("PULP_API_HOST", env_data.get("API_HOST", "localhost")) + os.environ.setdefault("PULP_API_PORT", str(env_data.get("API_PORT", ""))) + os.environ.setdefault( + "PULP_API_ROOT", + os.environ.get("PULP_API_ROOT", "/pulp/"), + ) + os.environ.setdefault( + "DJANGO_SUPERUSER_USERNAME", + env_data.get("DJANGO_SUPERUSER_USERNAME", "admin"), + ) + os.environ.setdefault( + "DJANGO_SUPERUSER_PASSWORD", + env_data.get("DJANGO_SUPERUSER_PASSWORD", "password"), + ) + + from oci_env.commands import test as run_test + + # Build a namespace compatible with commands.test + class _TestArgs: + pass + + test_args = _TestArgs() + test_args.install_deps = args.install_deps + test_args.test = args.test + test_args.plugin = args.plugin + # Drop a leading "--" separator commonly used before pytest paths. + pytest_args = list(args.args or []) + if pytest_args and pytest_args[0] == "--": + pytest_args = pytest_args[1:] + test_args.args = pytest_args + test_args.privileged = args.privileged + test_args.is_verbose = args.is_verbose + + if not test_args.plugin: + # Default to the first DEV_SOURCE_PATH plugin for agent workflows. + plugins = [p for p in env_data.get("DEV_SOURCE_PATH", "").split(":") if p] + if len(plugins) == 1: + test_args.plugin = plugins[0] + else: + exit_with_error( + "Specify -p PLUGIN for agent test when DEV_SOURCE_PATH has multiple plugins" + ) + + print( + f"Running {test_args.test} tests for {test_args.plugin} on agent {args.agent_id} " + f"(API {env_data.get('API_PROTOCOL', 'http')}://{env_data.get('API_HOST', 'localhost')}:" + f"{env_data.get('API_PORT')})" + ) + run_test(test_args, client) + + +def agent_generate_client(args): + oci_env_path = get_oci_env_path(args.is_verbose) + env_path, _ = load_agent_env(oci_env_path, args.agent_id) + client = compose_for_agent(args.is_verbose, env_path) + + from oci_env.commands import generate_client + + class _Args: + pass + + gargs = _Args() + gargs.plugin = args.plugin + gargs.language = args.language + gargs.install_client = args.install_client + gargs.is_verbose = args.is_verbose + generate_client(gargs, client) + + +def agent_dispatch(args): + actions = { + "create": agent_create, + "up": agent_up, + "down": agent_down, + "destroy": agent_destroy, + "ls": agent_ls, + "test": agent_test, + "generate-client": agent_generate_client, + } + action = getattr(args, "agent_action", None) + if action not in actions: + exit_with_error(f"Unknown agent action: {action}") + actions[action](args) diff --git a/client/oci_env/main.py b/client/oci_env/main.py index 63ae9a2..96aaef4 100644 --- a/client/oci_env/main.py +++ b/client/oci_env/main.py @@ -14,6 +14,7 @@ phelper ) +from oci_env.agent import agent_dispatch from oci_env.utils import ( Compose ) @@ -55,6 +56,7 @@ def get_parser(): parse_profile_command(subparsers) parse_poll_command(subparsers) parse_phelper_commands(subparsers) + parse_agent_command(subparsers) return parser @@ -162,6 +164,118 @@ def parse_phelper_commands(subparsers): parser = subparsers.add_parser('pdbreset', help='Reset the Pulp database.') parser.set_defaults(func=phelper, action="dbreset") + +def parse_agent_command(subparsers): + parser = subparsers.add_parser( + "agent", + help="Create and manage lean isolated environments for parallel agent work.", + ) + agent_sub = parser.add_subparsers(dest="agent_action", required=True) + + create = agent_sub.add_parser("create", help="Allocate a lean agent environment.") + create.add_argument("agent_id", help="Unique agent id (used in project name and env file).") + create.add_argument( + "--plugins", + default="pulpcore", + help=( + "Colon-separated PLUGIN[=PATH] entries for DEV_SOURCE_PATH (default: pulpcore). " + "PATH may be a Cursor worktree or any plugin checkout; omit PATH to use " + "/." + ), + ) + create.add_argument( + "--host-src-dir", + default=None, + help=( + "Host directory used to resolve plugins listed without an explicit path " + "(default: SRC_DIR from compose.env or parent of oci_env)." + ), + ) + create.add_argument("--port", type=int, default=None, help="API port (default: allocate from 5100-5199).") + create.add_argument( + "--env", + action="append", + default=[], + dest="env_vars", + metavar="KEY=VALUE", + help=( + "Set a variable in the agent env file (repeatable). " + "Examples: --env COMPOSE_PROFILE=lean:local_fixtures --env LEAN_MEM_LIMIT=1g." + ), + ) + create.set_defaults(func=agent_dispatch, agent_action="create") + + up = agent_sub.add_parser("up", help="Start an agent environment and wait for the API.") + up.add_argument("agent_id", help="Agent id to start.") + up.add_argument("--build", action="store_true", help="Build images before starting.") + up.add_argument("--attempts", type=int, default=30, help="Poll attempts for API readiness.") + up.add_argument("--wait", type=int, default=10, help="Seconds between poll attempts.") + up.set_defaults(func=agent_dispatch, agent_action="up") + + down = agent_sub.add_parser("down", help="Stop an agent environment (keeps volumes).") + down.add_argument("agent_id", help="Agent id to stop.") + down.set_defaults(func=agent_dispatch, agent_action="down") + + destroy = agent_sub.add_parser("destroy", help="Stop an agent environment and remove volumes/env.") + destroy.add_argument("agent_id", help="Agent id to destroy.") + destroy.set_defaults(func=agent_dispatch, agent_action="destroy") + + ls = agent_sub.add_parser("ls", help="List agent environments.") + ls.set_defaults(func=agent_dispatch, agent_action="ls") + + test_cmd = agent_sub.add_parser("test", help="Run tests against an agent environment's live API.") + test_cmd.add_argument("agent_id", help="Agent id to test.") + test_cmd.add_argument( + "test", + nargs="?", + default="functional", + choices=["functional", "unit", "lint", "performance"], + help="Test suite (default: functional).", + ) + test_cmd.add_argument( + "-i", + action="store_true", + dest="install_deps", + help="Deprecated no-op. Test dependencies are always installed.", + ) + test_cmd.add_argument( + "-p", + type=str, + default="", + dest="plugin", + help="Plugin to test (defaults to the sole DEV_SOURCE_PATH plugin).", + ) + test_cmd.add_argument("args", nargs=argparse.REMAINDER, help="Arguments to pass to pytest.") + test_cmd.add_argument("--privileged", action="store_true", dest="privileged") + test_cmd.set_defaults(func=agent_dispatch, agent_action="test") + + gen = agent_sub.add_parser( + "generate-client", + help="Generate (and optionally install) API clients for an agent environment.", + ) + gen.add_argument("agent_id", help="Agent id.") + gen.add_argument( + "plugin", + nargs="?", + default=None, + help="Plugin to generate a client for (default: all in DEV_SOURCE_PATH).", + ) + gen.add_argument( + "-l", + "--language", + default="python", + choices=["python", "ruby"], + help="Client language.", + ) + gen.add_argument( + "-i", + action="store_true", + dest="install_client", + help="Deprecated no-op. Clients are always installed into the agent container.", + ) + gen.set_defaults(func=agent_dispatch, agent_action="generate-client") + + def main(): parser = get_parser() args = parser.parse_args() @@ -170,9 +284,22 @@ def main(): parser.print_help() exit() + # Agent create/ls manage env files themselves and must not require compose.env. + if getattr(args, "func", None) is agent_dispatch: + try: + args.func(args) + except KeyboardInterrupt: + print() + exit(1) + return + client = Compose(args.is_verbose, args.env_file) try: args.func(args, client) except KeyboardInterrupt: print() exit(1) + + +if __name__ == "__main__": + main() diff --git a/docs/dev/guides/agent-envs.md b/docs/dev/guides/agent-envs.md new file mode 100644 index 0000000..85399c4 --- /dev/null +++ b/docs/dev/guides/agent-envs.md @@ -0,0 +1,145 @@ +# Lean multi-agent environments + +Use this guide when several agents (or developers) need isolated Pulp stacks on one host to +run **scoped functional tests against a live API** without sharing mutable state. + +## Model + +Each agent environment is a full Pulp stack (API, content app, worker, Postgres, Redis) in its +own compose project with its own volumes and API port. Agents do **not** share Postgres or a +writable Python environment, so each can migrate, switch git revisions, and install or remove +dependencies independently. + +What is shared safely: + +- The immutable base image (`localhost/oci_env/pulp:base`) after a single host build +- Optional shared fixtures (`local_fixtures`) if every agent points at the same fixtures URL + +The [`lean`](../reference/profiles/lean.md) profile keeps each stack small: one worker and CPU/memory caps. + +Each agent owns a single directory under `.compiled/`: + +```text +.compiled/agent_/ + compose.env # agent env file + plugin_volumes_compose.yaml # bind mounts for plugin checkouts + … # compose files written here on `up` (parse_profiles) +``` + +`SRC_DIR` for the agent is that directory. Plugin checkouts are always bind-mounted into +`/src/` via the volume overlay (never by sharing the host `SRC_DIR` directly). + +## Prerequisites + +1. Install the `oci-env` client (`pip install -e client` from this repo). +2. Build the base image once on the host: + + ```bash + oci-env compose build + ``` + +3. Agent environments always use **podman**. They do **not** copy settings from the host + `compose.env`. `PULP_SECRET_KEY` defaults to `dummy`; pass other Pulp/Django/profile + settings with `--env KEY=VALUE`. + +4. Have plugin checkouts under a common host directory (default: parent of `oci_env`), for example: + + ```text + ~/devel/ + ├── oci_env + ├── pulpcore # includes pulp_file + └── pulp_rpm + ``` + + `pulp_file` ships inside the `pulpcore` checkout — do not list it in `--plugins`. + +## Typical agent loop + +```bash +# 1. Create a lean env (resolves plugins under the default host SRC_DIR) +oci-env agent create pr-123 --plugins pulpcore + +# Or point at an AI-tool / git worktree for plugins under test +oci-env agent create pr-123 \ + --plugins pulpcore=~/.cursor/worktrees/pulpcore/my-wt + +# 2. Start and wait for the live API +oci-env agent up pr-123 + +# 3. Generate/install clients for plugins under test +oci-env agent generate-client pr-123 + +# 4. Run narrowly scoped functional tests against that agent's API +oci-env agent test pr-123 -p pulpcore -- -k test_crud_repos + +# 5. Tear down containers, volumes, and the agent compiled directory +oci-env agent destroy pr-123 +``` + +List allocated agents: + +```bash +oci-env agent ls +``` + +## Create options + +| Flag | Purpose | +|------|---------| +| `--plugins pulpcore[=PATH]:…` | Colon-separated `PLUGIN[=PATH]` list (`DEV_SOURCE_PATH`; default `pulpcore`) | +| `--host-src-dir PATH` | Parent used when a plugin has no `=PATH` (default: host `SRC_DIR` or parent of `oci_env`) | +| `--port 5105` | Pin `API_PORT` (default: allocate from 5100–5199) | +| `--env KEY=VALUE` | Set any other agent env var (repeatable), e.g. `COMPOSE_PROFILE`, `LEAN_MEM_LIMIT` | + +Defaults baked into the agent env file: `COMPOSE_PROFILE=lean`, `COMPOSE_PROJECT_NAME=agent_`, +`COMPOSE_BINARY=podman`, `API_HOST=localhost`, `API_PROTOCOL=http`, `PULP_SECRET_KEY=dummy`, +`SRC_DIR=.compiled/agent_`. Lean profile defaults still supply `PULP_WORKERS=1` and resource +caps unless you override them with `--env`. + +### Plugin paths + +`--plugins` accepts an optional path per plugin. Omit the path to resolve +`/`: + +```bash +# Both plugins from the default host SRC_DIR (e.g. ~/devel/pulpcore, ~/devel/pulp_rpm) +oci-env agent create pr-123 --plugins pulpcore:pulp_rpm + +# Explicit worktree for pulpcore; pulp_rpm from host SRC_DIR +oci-env agent create pr-123 \ + --plugins pulpcore=~/.cursor/worktrees/pulpcore/my-wt:pulp_rpm +``` + +`create` always writes a compose volume overlay that bind-mounts each real checkout at +`/src/` (nested under the base `{SRC_DIR}:/src` mount). + +Destroy removes the entire `.compiled/agent_/` directory; it never deletes the plugin +checkouts that were bind-mounted. + +## Parallelism tips + +- Build images once; agent `up` should not rebuild unless you pass `--build`. +- Keep `DEV_SOURCE_PATH` limited to plugins under test — that dominates startup cost after workers. +- Prefer scoped pytest paths/markers; full suites multiply host load. +- Destroy agents when finished so ports and volumes are reclaimed. +- Default credentials are `admin` / `password` unless overridden with `--env`. + +## How tests hit the live API + +`oci-env agent test` loads that agent's env file (`API_PORT`, credentials, `SRC_DIR`, project name) +and runs the existing `oci-env test` path inside **that** container. Functional tests therefore +exercise the agent's live API, not a shared default on port 5001. + +The functional test runner prints the API URL and auth user before invoking pytest. + +## Manual env files + +Agent commands write env files to `.compiled/agent_/compose.env`. You can still use the generic +multi-env flow with `-e` if you prefer hand-written files; see +[Run multiple environments](create-multiple-envs.md). + +## When not to use lean agents + +Interactive work that needs UI, Kafka, OpenTelemetry, MinIO, or other sidecars should keep using +the normal profiles. Add those profiles with `--env COMPOSE_PROFILE=lean:…` only when required — +they increase resource use and reduce how many agents fit on one machine. diff --git a/docs/dev/guides/create-multiple-envs.md b/docs/dev/guides/create-multiple-envs.md index 6d1c74e..517bfd6 100644 --- a/docs/dev/guides/create-multiple-envs.md +++ b/docs/dev/guides/create-multiple-envs.md @@ -2,6 +2,10 @@ You can running multiple environments simultaneously. +For parallel coding agents that need lean, short-lived stacks with automatic port allocation and +git worktrees, prefer [`oci-env agent`](agent-envs.md) with the `lean` profile instead of +hand-writing env files. + ## Create an `.env` file You may place it in the root of `oci_env` dir: diff --git a/docs/dev/reference/profiles/lean.md b/docs/dev/reference/profiles/lean.md new file mode 100644 index 0000000..a1f8b8f --- /dev/null +++ b/docs/dev/reference/profiles/lean.md @@ -0,0 +1,14 @@ +# lean + +Density-focused profile for parallel or short-lived Pulp environments. + +See the profile README (`oci-env profile docs lean`) and the +[agent environments guide](../../guides/agent-envs.md). + +## Defaults + +| Variable | Default | +|----------|---------| +| `PULP_WORKERS` | `1` | +| `LEAN_MEM_LIMIT` | `2g` | +| `LEAN_CPUS` | `1.0` | diff --git a/profiles/lean/README.md b/profiles/lean/README.md new file mode 100644 index 0000000..b05e5bc --- /dev/null +++ b/profiles/lean/README.md @@ -0,0 +1,56 @@ +# lean + +A density-focused profile for short-lived or parallel Pulp environments (for example multiple +coding agents running scoped functional tests). + +## What it does + +- Sets `PULP_WORKERS=1` so each environment runs a single task worker +- Caps the `pulp` container with `mem_limit` and `cpus` so parallel stacks do not starve the host + +This profile does **not** add sidecars (UI, Kafka, OpenTelemetry, MinIO, etc.). Combine it with +other profiles only when those services are required. + +## Usage + +```bash +COMPOSE_PROFILE=lean +DEV_SOURCE_PATH=pulpcore +PULP_WORKERS=1 +API_PORT=5101 +COMPOSE_PROJECT_NAME=agent_example +``` + +Prefer the `oci-env agent` commands to allocate ports/project names and manage lifecycle: + +```bash +oci-env agent create example --plugins pulpcore +oci-env agent up example +oci-env agent test example -p pulpcore functional -k test_crud_repos +oci-env agent destroy example +``` + +When the code already lives in an AI tool worktree (Cursor `/worktree`, etc.), point at it: + +```bash +oci-env agent create example \ + --plugins pulpcore=/path/to/external/worktree +``` + +## Extra variables + +- `PULP_WORKERS` + - Description: Number of pulpcore workers to start + - Default: `1` +- `LEAN_MEM_LIMIT` + - Description: Memory limit for the `pulp` container (`mem_limit`) + - Default: `2g` +- `LEAN_CPUS` + - Description: CPU limit for the `pulp` container (`cpus`) + - Default: `1.0` + +## Notes + +- Build the base image once per host (`oci-env compose build`); agent environments reuse it +- Keep `DEV_SOURCE_PATH` minimal (only plugins under test) for faster startup +- Full interactive profiles (galaxy UI, otel, …) remain available for human workflows; use `lean` for parallel agent density diff --git a/profiles/lean/compose.yaml b/profiles/lean/compose.yaml new file mode 100644 index 0000000..6370b91 --- /dev/null +++ b/profiles/lean/compose.yaml @@ -0,0 +1,4 @@ +services: + pulp: + mem_limit: "{LEAN_MEM_LIMIT}" + cpus: {LEAN_CPUS} diff --git a/profiles/lean/profile_default_config.env b/profiles/lean/profile_default_config.env new file mode 100644 index 0000000..89d8955 --- /dev/null +++ b/profiles/lean/profile_default_config.env @@ -0,0 +1,3 @@ +PULP_WORKERS=1 +LEAN_MEM_LIMIT=2g +LEAN_CPUS=1.0 diff --git a/profiles/lean/pulp_config.env b/profiles/lean/pulp_config.env new file mode 100644 index 0000000..e578c13 --- /dev/null +++ b/profiles/lean/pulp_config.env @@ -0,0 +1 @@ +PULP_WORKERS={PULP_WORKERS}