diff --git a/.github/workflows/xtest.yml b/.github/workflows/xtest.yml index e2ffead28..f1b5e5fbe 100644 --- a/.github/workflows/xtest.yml +++ b/.github/workflows/xtest.yml @@ -678,6 +678,102 @@ jobs: root-key: ${{ steps.km-check.outputs.root_key }} dpop-challenge-enabled: ${{ inputs.dpop-challenge || false }} + # ####### START ERS-POSTGRES + MULTI-STRATEGY PLATFORM ########## + # Mirrors the multi-KAS pattern: a second `platform` process on port + # 8090 running in multi-strategy ERS mode (SQL provider against the + # ers-postgres container). Tests that don't reference the ers-ms + # fixtures are unaffected. See test_ers_multistrategy.py for the + # regression test that consumes this instance. + # These three steps are best-effort: while opentdf/platform#3645 and + # #3543 are still open, they will fail on any platform tag that lacks + # both. `continue-on-error: true` keeps the rest of the xct matrix + # (test_abac, test_pqc, test_dpop, ...) running. The multi-strategy + # test itself skips at fixture setup when localhost:8090 isn't + # reachable, so a failed startup here does not fail the pytest run. + - name: Start ers-postgres for multi-strategy ERS + id: ers-postgres + if: ${{ steps.multikas.outputs.supported == 'true' }} + continue-on-error: true + working-directory: ${{ steps.run-platform.outputs.platform-working-dir }} + run: |- + docker compose --profile ers-test up -d ers-postgres + # Wait for the container to become healthy so the seed step below + # doesn't race the container's init scripts. + for _ in $(seq 1 30); do + if docker exec ers_test_postgres pg_isready -U ers_test_user -d ers_test >/dev/null 2>&1; then + echo "ers-postgres ready" + exit 0 + fi + sleep 1 + done + echo "ers-postgres failed to become ready" + docker compose --profile ers-test logs ers-postgres + exit 1 + + - name: Seed ers-postgres + if: ${{ steps.multikas.outputs.supported == 'true' }} + continue-on-error: true + run: |- + docker exec -i ers_test_postgres psql \ + -U ers_test_user -d ers_test -v ON_ERROR_STOP=1 <<'SQL' + CREATE TABLE IF NOT EXISTS ers_attributes ( + username TEXT PRIMARY KEY, + department TEXT NOT NULL, + active BOOLEAN NOT NULL DEFAULT true + ); + INSERT INTO ers_attributes (username, department) + VALUES ('opentdf', 'finance') + ON CONFLICT (username) DO NOTHING; + SQL + + - name: Start multi-strategy ERS platform (port 8090) + id: platform-ers-ms + if: ${{ steps.multikas.outputs.supported == 'true' }} + continue-on-error: true + run: |- + set -euo pipefail + config_src="$GITHUB_WORKSPACE/otdftests/xtest/platform-configs/opentdf-multistrategy.yaml" + platform_dir="$GITHUB_WORKSPACE/${PLATFORM_WORKING_DIR}" + cp "$config_src" "$platform_dir/opentdf-ers-ms.yaml" + + # Sync root_key with the primary platform's opentdf.yaml so both + # KAS instances can unwrap each other's wrapped DEKs. The template + # ships with a placeholder that yq overwrites here. + primary_root_key="$(yq e '.services.kas.root_key' "$platform_dir/opentdf.yaml")" + if [ -z "$primary_root_key" ] || [ "$primary_root_key" = "null" ]; then + echo "could not read services.kas.root_key from $platform_dir/opentdf.yaml" + exit 1 + fi + yq e -i ".services.kas.root_key = \"$primary_root_key\"" \ + "$platform_dir/opentdf-ers-ms.yaml" + + log_file="$GITHUB_WORKSPACE/platform-ers-ms.log" + echo "log-file=platform-ers-ms.log" >> "$GITHUB_OUTPUT" + + # Run the second platform in the background. Uses the same source + # tree as the default platform so both share the policy DB, KAS + # keys, and cryptoProvider config -- only entityresolution differs. + cd "$platform_dir" + nohup go run ./service start --config-file opentdf-ers-ms.yaml \ + >"$log_file" 2>&1 & + echo $! > "$GITHUB_WORKSPACE/platform-ers-ms.pid" + + # Wait for /healthz. Expected to fail on any platform tag that + # doesn't include both opentdf/platform#3645 and #3543 -- the + # SQL provider panics at startup on #3543-less platforms. + for _ in $(seq 1 60); do + if curl -sf http://localhost:8090/healthz >/dev/null; then + echo "platform-ers-ms ready" + exit 0 + fi + sleep 2 + done + echo "platform-ers-ms failed to become ready; last 200 log lines:" + tail -n 200 "$log_file" || true + exit 1 + env: + PLATFORM_WORKING_DIR: ${{ steps.run-platform.outputs.platform-working-dir }} + - name: Run attribute based configuration tests if: ${{ steps.multikas.outputs.supported == 'true' }} run: |- @@ -689,13 +785,15 @@ jobs: --sdks-encrypt "${ENCRYPT_SDK}" \ --focus "$FOCUS_SDK" \ $skip_flag \ - test_abac.py test_pqc.py test_dpop.py + test_abac.py test_pqc.py test_dpop.py test_ers_multistrategy.py working-directory: otdftests/xtest env: PLATFORM_DIR: "../../${{ steps.run-platform.outputs.platform-working-dir }}" PLATFORM_TAG: ${{ matrix.platform-tag }} OTDFCTL_HEADS: ${{ steps.configure-go.outputs.heads }} PLATFORM_LOG_FILE: "../../${{ steps.run-platform.outputs.platform-log-file }}" + PLATFORMURL_ERS_MS: "http://localhost:8090" + PLATFORM_ERS_MS_LOG_FILE: "../../platform-ers-ms.log" KAS_ALPHA_LOG_FILE: "../../${{ steps.kas-alpha.outputs.log-file }}" KAS_BETA_LOG_FILE: "../../${{ steps.kas-beta.outputs.log-file }}" KAS_GAMMA_LOG_FILE: "../../${{ steps.kas-gamma.outputs.log-file }}" diff --git a/otdf-local/AGENTS.md b/otdf-local/AGENTS.md index 0c116ef4b..292a778cb 100644 --- a/otdf-local/AGENTS.md +++ b/otdf-local/AGENTS.md @@ -18,8 +18,30 @@ uv run pytest --sdks go -v Auto-configured by otdf-local: - Keycloak: 8888, Postgres: 5432, Platform: 8080 +- Multi-strategy ERS platform: 8090 (backed by ers-postgres on 5433) - KAS: alpha=8181, beta=8282, gamma=8383, delta=8484, km1=8585, km2=8686 +## Multi-strategy ERS platform + +`otdf-local up` boots a **second `platform` process** (`platform-ers-ms`) +on port 8090 alongside the default Keycloak-ERS platform. It runs with +`entityresolution: mode: multi-strategy` and a SQL provider pointed at the +`ers-postgres` container (docker compose profile `ers-test`, port 5433). +Both platforms share the same policy DB, KAS keys, and cryptoProvider +config — only the entity-resolution block differs. This mirrors the +multi-KAS pattern: extra infrastructure is always up; tests that don't +reference the ers-ms fixtures are unaffected. + +- Template config: `xtest/platform-configs/opentdf-multistrategy.yaml`. +- Generated config: `xtest/tmp/config/opentdf-ers-ms.yaml`. +- Log file: `xtest/tmp/logs/platform-ers-ms.log`. +- Env vars exported by `otdf-local env`: `PLATFORMURL_ERS_MS`, `PLATFORM_ERS_MS_LOG_FILE`. +- Restart just this instance: `uv run otdf-local restart platform-ers-ms`. + +The seed row for the multi-strategy SQL provider (`INSERT INTO +ers_attributes VALUES ('opentdf', 'finance')`) is applied during the +`provision` step and is idempotent, so `otdf-local restart` is safe. + ## Restart Procedures ### Full Environment Restart diff --git a/otdf-local/src/otdf_local/cli.py b/otdf-local/src/otdf_local/cli.py index d8e3597ff..4962cc11d 100644 --- a/otdf-local/src/otdf_local/cli.py +++ b/otdf-local/src/otdf_local/cli.py @@ -20,6 +20,7 @@ ProvisionResult, get_docker_service, get_kas_manager, + get_platform_ers_ms_service, get_platform_service, get_provisioner, ) @@ -185,6 +186,45 @@ def up( else: print_success("Provisioning complete") + print_info("Seeding ers-postgres for multi-strategy ERS...") + seed_result = provisioner.provision_ers_ms_seed() + if not seed_result: + _show_provision_error(seed_result, "ers-postgres seed") + print_warning( + "ers-postgres seed failed - multi-strategy ERS tests will fail" + ) + else: + print_success("ers-postgres seeded") + + # Step 2b: Start multi-strategy ERS platform (second platform instance). + # Mirrors additional-KAS pattern: always up (when its template config is + # available), dedicated port, per-instance fixtures in xtest. Existing + # tests that don't reference the ers-ms fixtures are unaffected. + if start_platform: + template_path = settings.platform_ers_ms_template_config + if not template_path.exists(): + print_warning( + f"Multi-strategy platform template not found at {template_path}; " + "skipping ers-ms platform startup." + ) + else: + print_info("Starting multi-strategy ERS platform...") + platform_ers_ms = get_platform_ers_ms_service(settings) + if not platform_ers_ms.start(): + print_error("Failed to start multi-strategy ERS platform") + raise typer.Exit(1) + + with status_spinner("Waiting for multi-strategy ERS platform..."): + try: + wait_for_health( + f"http://localhost:{Ports.PLATFORM_ERS_MS}/healthz", + timeout=120, + service_name="Platform (ERS-MS)", + ) + except WaitTimeoutError as e: + print_warning(str(e)) + print_success("Multi-strategy ERS platform is ready") + # Step 4: Start KAS instances if start_kas: print_info("Starting KAS instances...") @@ -227,6 +267,11 @@ def down( kas_manager = get_kas_manager(settings) kas_manager.stop_all() + # Stop multi-strategy ERS platform + print_info("Stopping multi-strategy ERS platform...") + platform_ers_ms = get_platform_ers_ms_service(settings) + platform_ers_ms.stop() + # Stop Platform print_info("Stopping Platform...") platform = get_platform_service(settings) @@ -266,6 +311,7 @@ def list_services( all_info = [] all_info.extend(docker.get_all_info()) all_info.append(platform.get_info()) + all_info.append(get_platform_ers_ms_service(settings).get_info()) all_info.extend(kas_manager.get_all_info()) # Filter if not showing all @@ -370,6 +416,7 @@ def logs( # Add all services to aggregator aggregator.add_service("platform") + aggregator.add_service("platform-ers-ms") for kas_name in Ports.all_kas_names(): aggregator.add_service(f"kas-{kas_name}") @@ -493,6 +540,13 @@ def restart( print_success("Platform restarted") return + if service == "platform-ers-ms": + print_info("Restarting multi-strategy ERS platform...") + platform_ers_ms = get_platform_ers_ms_service(settings) + platform_ers_ms.restart() + print_success("Multi-strategy ERS platform restarted") + return + if service.startswith("kas-"): kas_name = service[4:] # Remove "kas-" prefix if kas_name not in Ports.all_kas_names(): @@ -523,7 +577,8 @@ def restart( print_error(f"Unknown service: {service}") print_info( - "Valid services: docker, platform, kas-alpha, kas-beta, kas-gamma, kas-delta, kas-km1, kas-km2" + "Valid services: docker, platform, platform-ers-ms, " + "kas-alpha, kas-beta, kas-gamma, kas-delta, kas-km1, kas-km2" ) raise typer.Exit(1) @@ -558,6 +613,7 @@ def env( # Platform configuration env_vars["PLATFORMURL"] = settings.platform_url + env_vars["PLATFORMURL_ERS_MS"] = f"http://localhost:{Ports.PLATFORM_ERS_MS}" env_vars["PLATFORM_DIR"] = str(settings.platform_dir.resolve()) # Schema file for manifest validation @@ -570,6 +626,10 @@ def env( if platform_log.exists(): env_vars["PLATFORM_LOG_FILE"] = str(platform_log.resolve()) + platform_ers_ms_log = settings.platform_ers_ms_log_path + if platform_ers_ms_log.exists(): + env_vars["PLATFORM_ERS_MS_LOG_FILE"] = str(platform_ers_ms_log.resolve()) + # KAS log files kas_env_mapping = { "alpha": "KAS_ALPHA_LOG_FILE", diff --git a/otdf-local/src/otdf_local/config/ports.py b/otdf-local/src/otdf_local/config/ports.py index 21d193358..1aafd1659 100644 --- a/otdf-local/src/otdf_local/config/ports.py +++ b/otdf-local/src/otdf_local/config/ports.py @@ -14,6 +14,10 @@ class Ports: # Platform PLATFORM: int = 8080 + PLATFORM_ERS_MS: int = 8090 + + # ERS test Postgres (docker profile 'ers-test'), used by multi-strategy platform + ERS_POSTGRES: int = 5433 # KAS instances KAS_ALPHA: int = 8181 diff --git a/otdf-local/src/otdf_local/config/settings.py b/otdf-local/src/otdf_local/config/settings.py index 96a4c20e8..27432da6f 100644 --- a/otdf-local/src/otdf_local/config/settings.py +++ b/otdf-local/src/otdf_local/config/settings.py @@ -125,6 +125,21 @@ def kas_template_config(self) -> Path: """KAS config template path.""" return self.platform_dir / "opentdf-kas-mode.yaml" + @property + def platform_ers_ms_template_config(self) -> Path: + """Multi-strategy ERS platform config template path (in xtest repo).""" + return self.xtest_root / "platform-configs" / "opentdf-multistrategy.yaml" + + @property + def platform_ers_ms_config(self) -> Path: + """Generated multi-strategy ERS platform config path.""" + return self.config_dir / "opentdf-ers-ms.yaml" + + @property + def platform_ers_ms_log_path(self) -> Path: + """Log file path for the multi-strategy ERS platform instance.""" + return self.logs_dir / "platform-ers-ms.log" + @property def docker_compose_file(self) -> Path: """Docker compose file path.""" diff --git a/otdf-local/src/otdf_local/services/__init__.py b/otdf-local/src/otdf_local/services/__init__.py index c3d9dabe1..ff756880c 100644 --- a/otdf-local/src/otdf_local/services/__init__.py +++ b/otdf-local/src/otdf_local/services/__init__.py @@ -4,6 +4,10 @@ from otdf_local.services.docker import DockerService, get_docker_service from otdf_local.services.kas import KASManager, KASService, get_kas_manager from otdf_local.services.platform import PlatformService, get_platform_service +from otdf_local.services.platform_ers_ms import ( + PlatformERSMultiStrategyService, + get_platform_ers_ms_service, +) from otdf_local.services.provisioner import ( Provisioner, ProvisionResult, @@ -21,6 +25,8 @@ "get_kas_manager", "PlatformService", "get_platform_service", + "PlatformERSMultiStrategyService", + "get_platform_ers_ms_service", "Provisioner", "ProvisionResult", "get_provisioner", diff --git a/otdf-local/src/otdf_local/services/docker.py b/otdf-local/src/otdf_local/services/docker.py index 911b42e3c..d5d52daae 100644 --- a/otdf-local/src/otdf_local/services/docker.py +++ b/otdf-local/src/otdf_local/services/docker.py @@ -43,7 +43,37 @@ def start(self) -> bool: text=True, cwd=self._compose_file.parent, ) - return result.returncode == 0 + if result.returncode != 0: + return False + + # Additionally boot the ers-test profile so multi-strategy ERS has + # its Postgres backend available. Best-effort: don't fail the core + # Docker startup for users who aren't exercising multi-strategy ERS + # (a missing image, private-registry auth issue, or older checkout + # that lacks the profile should not block Keycloak/Postgres). + ers = subprocess.run( + [ + "docker", + "compose", + "-f", + str(self._compose_file), + "--profile", + "ers-test", + "up", + "-d", + "ers-postgres", + ], + capture_output=True, + text=True, + cwd=self._compose_file.parent, + ) + if ers.returncode != 0: + print( + "warning: failed to start ers-postgres " + "(multi-strategy ERS tests will be skipped): " + f"{ers.stderr.strip()}" + ) + return True def stop(self) -> bool: """Stop Docker compose services.""" @@ -59,8 +89,12 @@ def stop(self) -> bool: return result.returncode == 0 def is_running(self) -> bool: - """Check if Docker services are running.""" - # Check if both Keycloak and PostgreSQL ports are open + """Check if the core Docker services are running. + + ers-postgres is optional (profile 'ers-test'); its status is + reported separately in get_all_info() so tests unrelated to + multi-strategy ERS aren't blocked by its absence. + """ keycloak_up = check_port("localhost", Ports.KEYCLOAK) postgres_up = check_port("localhost", Ports.POSTGRES) return keycloak_up and postgres_up @@ -114,6 +148,7 @@ def get_all_info(self) -> list[ServiceInfo]: """Get info for all Docker services.""" keycloak_running = check_port("localhost", Ports.KEYCLOAK) postgres_running = check_port("localhost", Ports.POSTGRES) + ers_postgres_running = check_port("localhost", Ports.ERS_POSTGRES) keycloak_healthy = None if keycloak_running: @@ -137,6 +172,13 @@ def get_all_info(self) -> list[ServiceInfo]: running=postgres_running, healthy=None, # No HTTP health check for postgres ), + ServiceInfo( + name="ers-postgres", + port=Ports.ERS_POSTGRES, + service_type=ServiceType.DOCKER, + running=ers_postgres_running, + healthy=None, + ), ] diff --git a/otdf-local/src/otdf_local/services/platform_ers_ms.py b/otdf-local/src/otdf_local/services/platform_ers_ms.py new file mode 100644 index 000000000..c8b844714 --- /dev/null +++ b/otdf-local/src/otdf_local/services/platform_ers_ms.py @@ -0,0 +1,150 @@ +"""Multi-strategy ERS platform instance. + +Runs a second `platform` process alongside the default one, configured to +resolve entities via the multi-strategy ERS with a SQL provider pointed at +the ers-postgres container. Exists so xtest can exercise the multi-strategy +code path against the same shared policy DB without disturbing the default +Keycloak-ERS platform. + +The pattern mirrors KASService: always up on `otdf-local up`, dedicated +port, log file, and health check. Tests that don't reference the ers-ms +fixtures are unaffected. +""" + +from pathlib import Path + +from otdf_local.config.features import PlatformFeatures +from otdf_local.config.ports import Ports +from otdf_local.config.settings import Settings +from otdf_local.health.checks import check_http_health, check_port +from otdf_local.process.manager import ( + ManagedProcess, + ProcessManager, + kill_process_on_port, +) +from otdf_local.services.base import Service, ServiceInfo, ServiceType +from otdf_local.utils.yaml import copy_yaml_with_updates, get_nested, load_yaml + + +class PlatformERSMultiStrategyService(Service): + """Second platform instance running in multi-strategy ERS mode.""" + + def __init__( + self, + settings: Settings, + process_manager: ProcessManager | None = None, + ) -> None: + super().__init__(settings) + self._process_manager = process_manager or ProcessManager() + self._process: ManagedProcess | None = None + + @property + def name(self) -> str: + return "platform-ers-ms" + + @property + def port(self) -> int: + return Ports.PLATFORM_ERS_MS + + @property + def service_type(self) -> ServiceType: + return ServiceType.SUBPROCESS + + @property + def health_url(self) -> str: + return f"http://localhost:{self.port}/healthz" + + def _generate_config(self) -> Path: + template_path = self.settings.platform_ers_ms_template_config + if not template_path.exists(): + raise FileNotFoundError( + f"Multi-strategy platform template not found at {template_path}. " + "Expected xtest/platform-configs/opentdf-multistrategy.yaml." + ) + + features = PlatformFeatures.detect(self.settings.platform_dir) + logger_output = "stderr" if features.supports("logger_stderr") else "stdout" + + # Sync the primary platform's root_key so both KAS instances can + # unwrap each other's wrapped DEKs. Mirrors KASService's approach. + primary_config = load_yaml(self.settings.platform_config) + root_key = get_nested(primary_config, "services.kas.root_key", "") + + updates = { + "logger.level": "debug", + "logger.type": "json", + "logger.output": logger_output, + "server.port": self.port, + "services.kas.registered_kas_uri": f"http://localhost:{self.port}", + "services.kas.root_key": root_key, + } + dest = self.settings.platform_ers_ms_config + copy_yaml_with_updates(template_path, dest, updates) + return dest + + def start(self) -> bool: + self.settings.ensure_directories() + kill_process_on_port(self.port) + + config_path = self._generate_config() + + cmd = [ + "go", + "run", + "./service", + "start", + "--config-file", + str(config_path), + ] + log_file = self.settings.platform_ers_ms_log_path + + self._process = self._process_manager.start( + name=self.name, + cmd=cmd, + cwd=self.settings.platform_dir, + log_file=log_file, + env={"OPENTDF_LOG_LEVEL": "info"}, + ) + return self._process is not None + + def stop(self) -> bool: + if self._process: + self._process.stop() + self._process = None + kill_process_on_port(self.port) + return True + + def is_running(self) -> bool: + if self._process and self._process.running: + return True + return check_port("localhost", self.port) + + def is_healthy(self) -> bool | None: + if not self.is_running(): + return None + return check_http_health(self.health_url) + + def get_info(self) -> ServiceInfo: + info = super().get_info() + if self._process: + info.pid = self._process.pid + return info + + +_ers_ms_process_manager: ProcessManager | None = None + + +def get_platform_ers_ms_service( + settings: Settings | None = None, +) -> PlatformERSMultiStrategyService: + """Get a PlatformERSMultiStrategyService with shared process manager.""" + global _ers_ms_process_manager + if _ers_ms_process_manager is None: + _ers_ms_process_manager = ProcessManager() + + if settings is None: + from otdf_local.config.settings import get_settings + + settings = get_settings() + + return PlatformERSMultiStrategyService(settings, _ers_ms_process_manager) diff --git a/otdf-local/src/otdf_local/services/provisioner.py b/otdf-local/src/otdf_local/services/provisioner.py index cb620ca07..7054f6750 100644 --- a/otdf-local/src/otdf_local/services/provisioner.py +++ b/otdf-local/src/otdf_local/services/provisioner.py @@ -5,6 +5,23 @@ from otdf_local.config.settings import Settings +# Seed for the multi-strategy ERS test fixtures. Idempotent so +# `otdf-local restart` doesn't fail on repeat runs. Read by the +# xtest_sql_call1_by_azp / xtest_sql_call2_by_username strategies +# defined in xtest/platform-configs/opentdf-multistrategy.yaml. +_ERS_MS_SEED_SQL = """ +CREATE TABLE IF NOT EXISTS ers_attributes ( + username TEXT PRIMARY KEY, + department TEXT NOT NULL, + active BOOLEAN NOT NULL DEFAULT true +); +INSERT INTO ers_attributes (username, department) +VALUES ('opentdf', 'finance') +ON CONFLICT (username) DO UPDATE +SET department = EXCLUDED.department, + active = true; +""" + @dataclass class ProvisionResult: @@ -31,15 +48,18 @@ def provision_all(self) -> ProvisionResult: """Run all provisioning steps.""" keycloak_result = self.provision_keycloak() fixtures_result = self.provision_fixtures() + ers_ms_result = self.provision_ers_ms_seed() - # If both succeeded, return success - if keycloak_result and fixtures_result: + # If all succeeded, return success + if keycloak_result and fixtures_result and ers_ms_result: return ProvisionResult(success=True) # Otherwise, return failure with first error if not keycloak_result: return keycloak_result - return fixtures_result + if not fixtures_result: + return fixtures_result + return ers_ms_result def provision_keycloak(self) -> ProvisionResult: """Provision Keycloak with required configuration. @@ -61,6 +81,72 @@ def provision_fixtures(self) -> ProvisionResult: """ return self._provision_("fixtures") + def provision_ers_ms_seed(self) -> ProvisionResult: + """Seed the ers-postgres database used by the multi-strategy ERS platform. + + Executes CREATE TABLE + INSERT via `docker exec ers_test_postgres psql`. + Idempotent so `otdf-local restart` doesn't fail on repeat runs. + + No-op when the ers_test_postgres container isn't running — otdf-local + is used by many contributors who don't need multi-strategy ERS, and + a missing container is not a provisioning failure for them. + """ + # Skip cleanly if the container is not running (users who aren't + # exercising multi-strategy ERS don't need this seed). + check_running = subprocess.run( + [ + "docker", + "ps", + "-q", + "-f", + "name=ers_test_postgres", + "-f", + "status=running", + ], + capture_output=True, + text=True, + ) + if not check_running.stdout.strip(): + return ProvisionResult( + success=True, + stdout="ers_test_postgres container not running; skipping seed.", + ) + + cmd = [ + "docker", + "exec", + "-i", + "ers_test_postgres", + "psql", + "-U", + "ers_test_user", + "-d", + "ers_test", + "-v", + "ON_ERROR_STOP=1", + ] + result = subprocess.run( + cmd, + input=_ERS_MS_SEED_SQL, + capture_output=True, + text=True, + ) + if result.returncode != 0: + error_lines = result.stderr.strip().split("\n") + return ProvisionResult( + success=False, + error_message=error_lines[-1] if error_lines else "seed failed", + stdout=result.stdout, + stderr=result.stderr, + return_code=result.returncode, + ) + return ProvisionResult( + success=True, + stdout=result.stdout, + stderr=result.stderr, + return_code=result.returncode, + ) + def _provision_(self, mode: str) -> ProvisionResult: """Execute a provisioning operation. diff --git a/xtest/conftest.py b/xtest/conftest.py index 4f39176be..386abd8ad 100644 --- a/xtest/conftest.py +++ b/xtest/conftest.py @@ -52,6 +52,7 @@ def pytest_report_header() -> list[str]: "fixtures.keys", "fixtures.audit", "fixtures.encryption", + "fixtures.ers_multistrategy", ] diff --git a/xtest/fixtures/audit.py b/xtest/fixtures/audit.py index 2d0783bfc..95f09962d 100644 --- a/xtest/fixtures/audit.py +++ b/xtest/fixtures/audit.py @@ -139,6 +139,7 @@ def kas_log_files(audit_log_config: AuditLogConfig) -> dict[str, Path] | None: Environment Variables: PLATFORM_LOG_FILE: Main KAS/platform log + PLATFORM_ERS_MS_LOG_FILE: Multi-strategy ERS platform log (port 8090) KAS_ALPHA_LOG_FILE, KAS_BETA_LOG_FILE, etc: Additional KAS logs """ log_files = {} @@ -148,6 +149,13 @@ def kas_log_files(audit_log_config: AuditLogConfig) -> dict[str, Path] | None: if platform_log: log_files["kas"] = Path(platform_log) + # Multi-strategy ERS platform runs on 8090 and emits its own audit + # events; tests using audit_logs.assert_rewrap_success against the + # ers-ms KAS need this log tailed too. + ers_ms_log = os.getenv("PLATFORM_ERS_MS_LOG_FILE") + if ers_ms_log: + log_files["platform-ers-ms"] = Path(ers_ms_log) + # Check for additional KAS logs kas_mapping = { "KAS_ALPHA_LOG_FILE": "kas-alpha", diff --git a/xtest/fixtures/ers_multistrategy.py b/xtest/fixtures/ers_multistrategy.py new file mode 100644 index 000000000..fd427237f --- /dev/null +++ b/xtest/fixtures/ers_multistrategy.py @@ -0,0 +1,143 @@ +"""Multi-strategy ERS fixtures. + +Fixtures for tests that target the second platform instance (`platform-ers-ms` +on port 8090) which runs with `entityresolution: mode: multi-strategy` and a +SQL provider. otdf-local boots this instance alongside the default +Keycloak-ERS platform; CI does the equivalent via a start-additional-platform +step. + +Tests that don't reference these fixtures are unaffected — mirroring the +kas_entry_alpha / kas_entry_beta / ... pattern in fixtures.kas. +""" + +import os +import secrets +import string +from urllib.request import urlopen + +import pytest + +import abac +import tdfs +from otdfctl import OpentdfCommandLineTool + + +@pytest.fixture(scope="session") +def platform_url_ers_ms() -> str: + """URL for the multi-strategy ERS platform instance.""" + return os.getenv("PLATFORMURL_ERS_MS", "http://localhost:8090") + + +@pytest.fixture(scope="session") +def _require_ers_ms_platform(platform_url_ers_ms: str) -> None: + """Skip tests that depend on the ers-ms platform when it isn't running. + + The multi-strategy platform requires both opentdf/platform#3645 and + opentdf/platform#3543 to be present. Until both land upstream, CI will + start the ers-ms platform on a best-effort basis and this fixture is + what keeps the test from producing a hard failure that masks unrelated + xtest results. + """ + healthz = f"{platform_url_ers_ms}/healthz" + try: + with urlopen(healthz, timeout=3.0) as resp: # noqa: S310 -- localhost health probe + if resp.status == 200: + return + pytest.skip( + f"ers-ms platform at {platform_url_ers_ms} not healthy (HTTP {resp.status}); " + "requires opentdf/platform#3645 and #3543 to be present" + ) + except OSError as e: + # OSError covers URLError, TimeoutError, and socket errors — + # both are subclasses of OSError in Python 3.11+. + pytest.skip( + f"ers-ms platform at {platform_url_ers_ms} not reachable ({e}); " + "requires opentdf/platform#3645 and #3543 to be present" + ) + + +@pytest.fixture(scope="session") +def kas_url_ers_ms(platform_url_ers_ms: str) -> str: + """KAS URL colocated with the multi-strategy ERS platform.""" + return f"{platform_url_ers_ms}/kas" + + +@pytest.fixture(scope="module") +def kas_entry_ers_ms( + otdfctl: OpentdfCommandLineTool, + cached_kas_keys: abac.PublicKey, + kas_url_ers_ms: str, + _require_ers_ms_platform: None, +) -> abac.KasEntry: + """KAS registry entry for the multi-strategy-ERS platform's KAS.""" + return otdfctl.kas_registry_create_if_not_present(kas_url_ers_ms, cached_kas_keys) + + +@pytest.fixture(scope="module") +def scs_department_finance( + otdfctl: OpentdfCommandLineTool, +) -> abac.SubjectConditionSet: + """SCS that matches subjects whose flattened claims include department=finance. + + The multi-strategy SQL provider maps `azp` (or `userName`) -> a row in + ers_attributes and emits `department` in the output_mapping. The PDP + flattens the returned EntityRepresentation into dot-notation claims, so + `.department` selects the value emitted by the provider. + """ + return otdfctl.scs_create( + [ + abac.SubjectSet( + condition_groups=[ + abac.ConditionGroup( + boolean_operator=abac.ConditionBooleanTypeEnum.AND, + conditions=[ + abac.Condition( + subject_external_selector_value=".department", + operator=abac.SubjectMappingOperatorEnum.IN, + subject_external_values=["finance"], + ) + ], + ) + ] + ) + ] + ) + + +@pytest.fixture(scope="module") +def attribute_ers_ms_finance_grant( + otdfctl: OpentdfCommandLineTool, + kas_entry_ers_ms: abac.KasEntry, + kas_public_key_r1: abac.KasPublicKey, + scs_department_finance: abac.SubjectConditionSet, + _require_ers_ms_platform: None, +) -> abac.AttributeValue: + """Attribute value `department/finance` granted to the ers-ms KAS. + + The KAS URL baked into the TDF's keyAccess entry is the ers-ms KAS + (localhost:8090), so decrypt naturally routes rewrap through the + multi-strategy platform. Access is gated by the .department=finance + subject condition. + """ + suffix = "".join(secrets.choice(string.ascii_lowercase) for _ in range(8)) + ns_name = f"ersms-{suffix}.com" + ns = otdfctl.namespace_create(ns_name) + + attr = otdfctl.attribute_create( + ns, "department", abac.AttributeRule.ANY_OF, ["finance", "legal"] + ) + assert attr.values + finance = next(v for v in attr.values if v.value == "finance") + + otdfctl.scs_map(scs_department_finance, finance) + + pfs = tdfs.get_platform_features() + if "key_management" not in pfs.features: + otdfctl.grant_assign_value(kas_entry_ers_ms, finance) + else: + kas_key = otdfctl.kas_registry_create_public_key_only( + kas_entry_ers_ms, kas_public_key_r1 + ) + otdfctl.key_assign_value(kas_key, finance) + + return finance diff --git a/xtest/platform-configs/opentdf-multistrategy.yaml b/xtest/platform-configs/opentdf-multistrategy.yaml new file mode 100644 index 000000000..2e29f42ca --- /dev/null +++ b/xtest/platform-configs/opentdf-multistrategy.yaml @@ -0,0 +1,165 @@ +logger: + level: debug + type: text + output: stderr +audit: +services: + kas: + registered_kas_uri: http://localhost:8090 + preview: + # EC keys are registered below, so EC TDF wrapping must be enabled + # for tests that route EC-wrapped TDFs through this KAS. + ec_tdf_enabled: true + hybrid_tdf_enabled: false + key_management: false + # NOTE: root_key here is a placeholder. PlatformERSMultiStrategyService + # overwrites it at config-generation time with the primary platform's + # root_key (read from opentdf-dev.yaml) so both KAS instances can unwrap + # each other's wrapped DEKs. Do not treat this literal as a real key. + root_key: PLACEHOLDER_OVERWRITTEN_BY_OTDF_LOCAL_AT_STARTUP + keyring: + - kid: e1 + alg: ec:secp256r1 + - kid: e1 + alg: ec:secp256r1 + legacy: true + - kid: r1 + alg: rsa:2048 + - kid: r1 + alg: rsa:2048 + legacy: true + entityresolution: + # The dispatcher in service/entityresolution/v2/entity_resolution.go reads + # `mode:` (not `type:`); multi-strategy fields live flat under + # entityresolution, not nested under `config:`. The stale example in + # platform/opentdf-ers-test.yaml uses the wrong shape — do not copy it. + mode: multi-strategy + failure_strategy: continue + providers: + sql_postgres: + type: sql + connection: + driver: postgres + host: localhost + port: 5433 + database: ers_test + username: ers_test_user + password: ers_test_pass + ssl_mode: disable + max_open_connections: 10 + max_idle_connections: 5 + connection_max_lifetime: 1h + query_timeout: 30s + mapping_strategies: + # Call 1 (CreateEntityChainsFromTokens): key on the OIDC-standard + # 'azp' claim (authorized party = client_id). Every xtest token + # carries azp=opentdf, so no Keycloak mapper changes are needed. + # Output includes username so the emitted Entity has UserName set, + # which drives call 2. + - name: xtest_sql_call1_by_azp + provider: sql_postgres + entity_type: subject + conditions: + jwt_claims: + - claim: azp + operator: exists + values: [] + input_mapping: + - jwt_claim: azp + provider_field: username + output_mapping: + - source_claim: username + claim_name: username + - source_claim: department + claim_name: department + query: | + SELECT username, department FROM ers_attributes + WHERE username = $1 AND active = true + # Call 2 (ResolveEntities on the chain-Entity produced by call 1): + # the platform protojson-marshals Entity_UserName to a claims map + # containing the proto-cased key 'userName'. This is the strategy + # that triggers the []string metadata bug on unfixed platforms — + # its success path writes attempted_strategies into result.Metadata, + # which structpb.NewStruct rejects, causing the entity to be + # silently dropped and rewrap to return PermissionDenied. + - name: xtest_sql_call2_by_username + provider: sql_postgres + entity_type: subject + conditions: + jwt_claims: + - claim: userName + operator: exists + values: [] + input_mapping: + - jwt_claim: userName + provider_field: username + output_mapping: + - source_claim: username + claim_name: username + - source_claim: department + claim_name: department + query: | + SELECT username, department FROM ers_attributes + WHERE username = $1 AND active = true +server: + public_hostname: localhost + tls: + enabled: false + cert: ./keys/platform.crt + key: ./keys/platform-key.pem + auth: + enabled: true + dpop: + enforce: false + audience: "http://localhost:8080" + issuer: http://localhost:8888/auth/realms/opentdf + policy: + username_claim: + groups_claim: + client_id_claim: + extension: | + g, opentdf-admin, role:admin + g, opentdf-standard, role:standard + csv: + model: + cors: + enabled: true + allowedorigins: + - "*" + allowedmethods: + - GET + - POST + - PATCH + - PUT + - DELETE + - OPTIONS + allowedheaders: + - Accept + - Accept-Encoding + - Authorization + - Connect-Protocol-Version + - Content-Length + - Content-Type + - Dpop + - X-CSRF-Token + - X-Requested-With + - X-Rewrap-Additional-Context + exposedheaders: + - Link + allowcredentials: true + maxage: 3600 + grpc: + reflectionEnabled: true + cryptoProvider: + type: standard + standard: + keys: + - kid: r1 + alg: rsa:2048 + private: kas-private.pem + cert: kas-cert.pem + - kid: e1 + alg: ec:secp256r1 + private: kas-ec-private.pem + cert: kas-ec-cert.pem + port: 8090 diff --git a/xtest/test_ers_multistrategy.py b/xtest/test_ers_multistrategy.py new file mode 100644 index 000000000..5e093c396 --- /dev/null +++ b/xtest/test_ers_multistrategy.py @@ -0,0 +1,83 @@ +"""Regression test for opentdf/platform#3645. + +Without the []string -> []any coercion in multi-strategy/service.go, the v2 +ResolveEntities handler drops the resolved entity via `continue` after +structpb.NewStruct rejects []string in result.Metadata["attempted_strategies"]. +KAS then sees an empty EntityRepresentations, PDP has no subject entity to +evaluate, and rewrap returns PermissionDenied. + +This test targets a dedicated `platform-ers-ms` instance (localhost:8090) +running in multi-strategy ERS mode with a SQL provider. otdf-local boots +this instance alongside the default Keycloak-ERS platform. Existing xtests +never touch these fixtures and are unaffected. + +Bug trigger: encrypt with an attribute whose KAS grant points at the ers-ms +KAS (8090). The TDF manifest's keyAccess.url becomes 8090/kas, so decrypt +naturally routes rewrap through the ers-ms platform. The multi-strategy +SQL provider matches the azp=opentdf claim, queries ers_attributes for +username='opentdf', and returns department=finance. The subject condition +`.department = finance` gates the attribute, so PDP grants and rewrap +succeeds -- IFF result.Metadata serializes correctly. +""" + +import filecmp + +import pytest + +import tdfs +from abac import AttributeValue +from audit_logs import AuditLogAsserter +from fixtures.encryption import EncryptFactory + + +def test_multistrategy_sql_rewrap_succeeds( + attribute_ers_ms_finance_grant: AttributeValue, + encrypt_sdk: tdfs.SDK, + decrypt_sdk: tdfs.SDK, + kas_url_ers_ms: str, + in_focus: set[tdfs.SDK], + audit_logs: AuditLogAsserter, + encrypted_tdf: EncryptFactory, + pt_file, +): + """End-to-end rewrap through the multi-strategy ERS platform. + + Passes on any platform ref that has the []string -> []any coercion fix + for `attempted_strategies` in service/entityresolution/multi-strategy/ + service.go. Fails on any ref that doesn't -- decrypt returns 403 and + the ers-ms platform log contains `proto: invalid type: []string`. + """ + if not in_focus & {encrypt_sdk, decrypt_sdk}: + pytest.skip("Not in focus") + tdfs.skip_if_unsupported(encrypt_sdk, "autoconfigure") + + finance_fqn = attribute_ers_ms_finance_grant.fqn + assert finance_fqn is not None + + ct_file = encrypted_tdf( + encrypt_sdk, + attr_values=[finance_fqn], + target_mode=tdfs.select_target_version(encrypt_sdk, decrypt_sdk), + ) + + # Sanity: the TDF must route rewrap through the ers-ms KAS -- otherwise + # the test would exercise the default Keycloak-ERS platform and prove + # nothing about multi-strategy. + manifest = tdfs.manifest(ct_file) + assert len(manifest.encryptionInformation.keyAccess) == 1 + assert manifest.encryptionInformation.keyAccess[0].url == kas_url_ers_ms, ( + f"expected KAS URL {kas_url_ers_ms} in manifest, got " + f"{manifest.encryptionInformation.keyAccess[0].url}" + ) + + mark = audit_logs.mark("before_ers_ms_decrypt") + + rt_file = encrypted_tdf.rt_file(ct_file, decrypt_sdk) + decrypt_sdk.decrypt(ct_file, rt_file, "ztdf") + assert filecmp.cmp(pt_file, rt_file) + + audit_logs.assert_rewrap_success( + attr_fqns=[finance_fqn], + min_count=1, + since_mark=mark, + )