From 1ac14c6b1e5d6f07b0d9575bcf416ec4d7894f90 Mon Sep 17 00:00:00 2001 From: Ning Zhou Date: Mon, 27 Jul 2026 17:18:42 +0800 Subject: [PATCH] feat(platform): verify local cross-cluster metadata recovery --- .github/workflows/ci.yml | 9 +- ...a-fabric-cross-cluster-recovery.local.yaml | 35 + .../metadata_fabric_cross_cluster_recovery.py | 1097 +++++++++++++++++ .../metadata_fabric_recovery_rehearsal.py | 188 ++- data_agent/platform_truth.py | 17 + ..._metadata_fabric_cross_cluster_recovery.py | 289 +++++ ...test_metadata_fabric_recovery_rehearsal.py | 43 + data_agent/test_platform_truth.py | 5 + ...ocal-metadata-fabric-foundation-sandbox.md | 2 + ...ocal-metadata-fabric-recovery-rehearsal.md | 14 +- ...local-locked-metadata-backup-repository.md | 12 +- ...0-local-cross-cluster-metadata-recovery.md | 97 ++ ...a-fabric-backup-repository-2026-07-27.json | 128 +- ...ric-cross-cluster-recovery-2026-07-27.json | 278 +++++ ...-fabric-recovery-rehearsal-2026-07-27.json | 80 +- docs/roadmap-ar0-platform-truth-2026-07-24.md | 9 +- docs/roadmap.md | 6 +- docs/system-of-record-matrix-2026-07-24.md | 18 +- .../metadata-fabric-cross-cluster-recovery.sh | 22 + 19 files changed, 2161 insertions(+), 188 deletions(-) create mode 100644 config/metadata-fabric-cross-cluster-recovery.local.yaml create mode 100644 data_agent/metadata_fabric_cross_cluster_recovery.py create mode 100644 data_agent/test_metadata_fabric_cross_cluster_recovery.py create mode 100644 docs/architecture-decisions/adr-040-local-cross-cluster-metadata-recovery.md create mode 100644 docs/evidence/metadata-fabric-cross-cluster-recovery-2026-07-27.json create mode 100755 scripts/metadata-fabric-cross-cluster-recovery.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f5a95115..ae04c3cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,7 +5,10 @@ on: branches: [main, develop] pull_request: # Keep the active lineage reviewable until ADR-035 promotes it to main. - branches: [main, feat/v12-extensible-platform] + branches: + - main + - feat/v12-extensible-platform + - feat/ar1-metadata-fabric-durable-backup-contract env: PYTHON_VERSION: "3.13" @@ -82,6 +85,9 @@ jobs: - name: Validate metadata fabric backup repository run: python -m data_agent.metadata_fabric_backup_repository validate + - name: Validate metadata fabric cross-cluster recovery + run: python -m data_agent.metadata_fabric_cross_cluster_recovery validate + - name: Validate DolphinScheduler adapter boundary run: python -m data_agent.dolphinscheduler_adapter validate @@ -119,6 +125,7 @@ jobs: data_agent/test_metadata_fabric_bridge.py \ data_agent/test_metadata_fabric_live_evidence.py \ data_agent/test_metadata_fabric_backup_repository.py \ + data_agent/test_metadata_fabric_cross_cluster_recovery.py \ data_agent/test_metadata_fabric_recovery_rehearsal.py \ data_agent/test_metadata_fabric_sandbox.py \ data_agent/test_platform_authorization.py \ diff --git a/config/metadata-fabric-cross-cluster-recovery.local.yaml b/config/metadata-fabric-cross-cluster-recovery.local.yaml new file mode 100644 index 00000000..d59d257b --- /dev/null +++ b/config/metadata-fabric-cross-cluster-recovery.local.yaml @@ -0,0 +1,35 @@ +schema: gda.metadata_fabric_cross_cluster_recovery_profile.v1 +environment: local_docker_desktop +source_cluster: + context: docker-desktop +recovery_cluster: + context: kind-gda-metadata-recovery + node_image: kindest/node:v1.35.5 +repository: + runtime: docker_host_container + failure_domain: outside_source_and_recovery_kubernetes_clusters + minio_image: minio/minio:RELEASE.2025-04-22T22-12-26Z + mc_image: minio/mc:RELEASE.2025-04-16T18-13-26Z + bucket: gda-metadata-fabric-cross-cluster-backups + versioning: Enabled + object_lock: + enabled: true + mode: COMPLIANCE + retention_days: 1 + identities: + writer_delete_denied: true + reader_write_denied: true +claims: + local_cross_cluster_recovery_verified: false + production_backup_target_verified: false + production_retention_verified: false + production_kms_verified: false + production_tls_verified: false + production_cross_cluster_recovery_verified: false + source_cluster_loss_verified: false + rpo_slo_verified: false + rto_slo_verified: false + oidc_verified: false + network_policy_enforcement_verified: false + writes_to_gda_enabled: false + production_ready: false diff --git a/data_agent/metadata_fabric_cross_cluster_recovery.py b/data_agent/metadata_fabric_cross_cluster_recovery.py new file mode 100644 index 00000000..4e41c231 --- /dev/null +++ b/data_agent/metadata_fabric_cross_cluster_recovery.py @@ -0,0 +1,1097 @@ +"""Verify Metadata Fabric recovery across two local Kubernetes clusters. + +The source remains Docker Desktop Kubernetes. Recovery runs in a distinct kind +cluster, while a locked MinIO repository runs directly on the Docker host with +separate writer and reader identities. The clusters still share one Docker +Desktop host, so this module never claims production durability, source-host +loss recovery, KMS, TLS, RPO/RTO, or production readiness. +""" + +from __future__ import annotations + +import argparse +import base64 +import hashlib +import json +import os +import secrets +import shutil +import subprocess +import sys +import tempfile +import time +from collections.abc import Mapping +from datetime import UTC, datetime +from pathlib import Path +from typing import Any, Callable + +from . import metadata_fabric_backup_repository as repository +from . import metadata_fabric_recovery_rehearsal as recovery + + +CONTRACT_SCHEMA = "gda.metadata_fabric_cross_cluster_recovery_contract.v1" +OBSERVATION_SCHEMA = "gda.metadata_fabric_cross_cluster_recovery_observation.v1" +EVIDENCE_SCHEMA = "gda.metadata_fabric_cross_cluster_recovery_evidence.v1" +PROFILE_SCHEMA = "gda.metadata_fabric_cross_cluster_recovery_profile.v1" +SOURCE_CONTEXT = "docker-desktop" +RECOVERY_CONTEXT = "kind-gda-metadata-recovery" +KIND_NODE_IMAGE = "kindest/node:v1.35.5" +MINIO_IMAGE = repository.MINIO_IMAGE +MC_IMAGE = "minio/mc:RELEASE.2025-04-16T18-13-26Z" +BUCKET = "gda-metadata-fabric-cross-cluster-backups" +RETENTION_MODE = "COMPLIANCE" +RETENTION_DAYS = 1 +FAILURE_DOMAIN = "outside_source_and_recovery_kubernetes_clusters" + +REPO_ROOT = Path(__file__).resolve().parent.parent +DEFAULT_PROFILE_PATH = ( + REPO_ROOT / "config/metadata-fabric-cross-cluster-recovery.local.yaml" +) +DEFAULT_WRAPPER = REPO_ROOT / "scripts/metadata-fabric-cross-cluster-recovery.sh" + + +class MetadataFabricCrossClusterRecoveryError(RuntimeError): + """The cross-cluster recovery contract or rehearsal failed closed.""" + + +def _mapping(value: Any) -> Mapping[str, Any]: + return value if isinstance(value, Mapping) else {} + + +def _profile_errors(profile: Mapping[str, Any]) -> list[str]: + errors: list[str] = [] + source = _mapping(profile.get("source_cluster")) + target = _mapping(profile.get("recovery_cluster")) + backup = _mapping(profile.get("repository")) + lock = _mapping(backup.get("object_lock")) + identities = _mapping(backup.get("identities")) + claims = _mapping(profile.get("claims")) + if ( + profile.get("schema") != PROFILE_SCHEMA + or profile.get("environment") != "local_docker_desktop" + ): + errors.append("cross-cluster profile schema or environment does not match") + if source.get("context") != SOURCE_CONTEXT: + errors.append("source cluster context must remain docker-desktop") + if target.get("context") != RECOVERY_CONTEXT: + errors.append("recovery cluster context does not match") + if source.get("context") == target.get("context"): + errors.append("source and recovery contexts must differ") + if target.get("node_image") != KIND_NODE_IMAGE: + errors.append("recovery kind node image must remain pinned") + if backup.get("runtime") != "docker_host_container": + errors.append("repository must run outside both Kubernetes clusters") + if backup.get("failure_domain") != FAILURE_DOMAIN: + errors.append("repository failure-domain declaration does not match") + if backup.get("minio_image") != MINIO_IMAGE or backup.get("mc_image") != MC_IMAGE: + errors.append("repository runtime images must remain pinned") + if backup.get("bucket") != BUCKET or backup.get("versioning") != "Enabled": + errors.append("repository bucket identity or versioning does not match") + if ( + lock.get("enabled") is not True + or lock.get("mode") != RETENTION_MODE + or lock.get("retention_days") != RETENTION_DAYS + ): + errors.append("local cross-cluster repository must use COMPLIANCE/1 day") + if identities.get("writer_delete_denied") is not True: + errors.append("repository writer must be denied delete access") + if identities.get("reader_write_denied") is not True: + errors.append("repository reader must be denied write access") + for claim in ( + "local_cross_cluster_recovery_verified", + "production_backup_target_verified", + "production_retention_verified", + "production_kms_verified", + "production_tls_verified", + "production_cross_cluster_recovery_verified", + "source_cluster_loss_verified", + "rpo_slo_verified", + "rto_slo_verified", + "oidc_verified", + "network_policy_enforcement_verified", + "writes_to_gda_enabled", + "production_ready", + ): + if claims.get(claim) is not False: + errors.append(f"unverified profile claim must remain false: {claim}") + return errors + + +def build_cross_cluster_contract_report( + profile_path: Path | None = None, + wrapper_path: Path | None = None, +) -> dict[str, Any]: + """Validate the local dual-cluster profile and production claim boundary.""" + profile_file = (profile_path or DEFAULT_PROFILE_PATH).resolve() + wrapper = (wrapper_path or DEFAULT_WRAPPER).resolve() + errors: list[str] = [] + try: + profile = repository._load_yaml_object(profile_file) + errors.extend(_profile_errors(profile)) + except (OSError, TypeError) as exc: + errors.append(f"cross-cluster profile is invalid: {type(exc).__name__}") + + recovery_contract = recovery.build_recovery_contract_report() + if recovery_contract.get("static_contract_verified") is not True: + errors.append("metadata recovery contract is invalid") + try: + production_policy = repository._load_yaml_object( + repository.DEFAULT_POLICY_PATH + ) + errors.extend(repository._production_policy_errors(production_policy)) + except (OSError, TypeError) as exc: + errors.append(f"production backup policy is invalid: {type(exc).__name__}") + try: + wrapper_text = wrapper.read_text(encoding="utf-8") + for marker in ( + "set -euo pipefail", + "metadata_fabric_cross_cluster_recovery", + ): + if marker not in wrapper_text: + errors.append(f"cross-cluster wrapper is missing safety marker: {marker}") + except OSError as exc: + errors.append(f"cross-cluster wrapper is invalid: {type(exc).__name__}") + + files: dict[str, dict[str, str]] = {} + for path in ( + Path(__file__).resolve(), + Path(recovery.__file__).resolve(), + Path(repository.__file__).resolve(), + profile_file, + repository.DEFAULT_POLICY_PATH.resolve(), + wrapper, + ): + if path.is_file(): + try: + relative = path.relative_to(REPO_ROOT).as_posix() + except ValueError: + relative = path.name + files[relative] = { + "path": relative, + "sha256": recovery._file_sha256(path), + } + + stable = { + "schema": CONTRACT_SCHEMA, + "source_context": SOURCE_CONTEXT, + "recovery_context": RECOVERY_CONTEXT, + "kind_node_image": KIND_NODE_IMAGE, + "repository_failure_domain": FAILURE_DOMAIN, + "repository_bucket": BUCKET, + "artifact_inventory": repository.EXPECTED_ARTIFACTS, + "local_static_contract_verified": not errors, + "local_cross_cluster_recovery_verified": False, + "production_cross_cluster_recovery_verified": False, + "files": files, + "errors": errors, + } + return {**stable, "contract_fingerprint": recovery._canonical_sha256(stable)} + + +class _DockerRuntime: + def __init__(self, docker: str = "docker") -> None: + self.docker = docker + + def run( + self, + args: list[str], + *, + timeout: int = 180, + label: str, + ) -> bytes: + try: + completed = subprocess.run( + [self.docker, *args], + capture_output=True, + check=False, + timeout=timeout, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise MetadataFabricCrossClusterRecoveryError( + f"{label} was unavailable" + ) from exc + if completed.returncode != 0: + raise MetadataFabricCrossClusterRecoveryError(f"{label} failed") + return completed.stdout + + def exists(self, kind: str, name: str) -> bool: + try: + completed = subprocess.run( + [self.docker, kind, "inspect", name], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + timeout=30, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise MetadataFabricCrossClusterRecoveryError( + f"Docker {kind} inspection failed" + ) from exc + return completed.returncode == 0 + + +def _write_json_private(path: Path, payload: Mapping[str, Any]) -> None: + recovery._write_private( + path, + (json.dumps(payload, ensure_ascii=True, sort_keys=True) + "\n").encode( + "utf-8" + ), + ) + + +def _access_denied(action: Callable[[], Any]) -> bool: + try: + action() + except Exception as exc: + response = _mapping(getattr(exc, "response", None)) + error = _mapping(response.get("Error")) + return error.get("Code") in { + "AccessDenied", + "InvalidRequest", + "MethodNotAllowed", + } + return False + + +class _ExternalRepositoryRuntime: + def __init__(self, temp_root: Path, docker: str = "docker") -> None: + suffix = secrets.token_hex(8) + self.temp_root = temp_root + self.runtime = _DockerRuntime(docker) + self.container_name = f"gda-metadata-cross-cluster-minio-{suffix}" + self.volume_name = f"gda-metadata-cross-cluster-backup-{suffix}" + self.port = repository._free_local_port() + self.root_user = "gdaroot" + secrets.token_hex(8) + self.root_password = secrets.token_urlsafe(36) + self.writer_user = "gdawriter" + secrets.token_hex(8) + self.writer_password = secrets.token_urlsafe(36) + self.reader_user = "gdareader" + secrets.token_hex(8) + self.reader_password = secrets.token_urlsafe(36) + self.container_started = False + self.volume_created = False + + @property + def endpoint(self) -> str: + return f"http://127.0.0.1:{self.port}" + + def _wait_for_repository(self, client: Any) -> None: + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + try: + client.list_buckets() + return + except Exception: + time.sleep(0.5) + raise MetadataFabricCrossClusterRecoveryError( + "external backup repository did not become ready" + ) + + def _initialize(self, client: Any) -> dict[str, Any]: + try: + client.create_bucket(Bucket=BUCKET, ObjectLockEnabledForBucket=True) + client.put_bucket_versioning( + Bucket=BUCKET, + VersioningConfiguration={"Status": "Enabled"}, + ) + client.put_object_lock_configuration( + Bucket=BUCKET, + ObjectLockConfiguration={ + "ObjectLockEnabled": "Enabled", + "Rule": { + "DefaultRetention": { + "Mode": RETENTION_MODE, + "Days": RETENTION_DAYS, + } + }, + }, + ) + versioning = client.get_bucket_versioning(Bucket=BUCKET) + lock = client.get_object_lock_configuration(Bucket=BUCKET) + except Exception as exc: + raise MetadataFabricCrossClusterRecoveryError( + "external backup repository initialization failed" + ) from exc + lock_config = _mapping(lock.get("ObjectLockConfiguration")) + retention = _mapping(_mapping(lock_config.get("Rule")).get("DefaultRetention")) + controls = { + "versioning_status": versioning.get("Status"), + "object_lock_enabled": lock_config.get("ObjectLockEnabled") == "Enabled", + "default_retention_mode": retention.get("Mode"), + "default_retention_days": retention.get("Days"), + } + if controls != { + "versioning_status": "Enabled", + "object_lock_enabled": True, + "default_retention_mode": RETENTION_MODE, + "default_retention_days": RETENTION_DAYS, + }: + raise MetadataFabricCrossClusterRecoveryError( + "external repository controls do not match" + ) + return controls + + def _provision_identities(self) -> None: + policies = self.temp_root / "policies" + policies.mkdir(mode=0o700) + bucket_arn = f"arn:aws:s3:::{BUCKET}" + common_bucket = { + "Effect": "Allow", + "Action": ["s3:GetBucketLocation", "s3:ListBucket"], + "Resource": [bucket_arn], + } + writer_policy = { + "Version": "2012-10-17", + "Statement": [ + common_bucket, + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:GetObjectRetention", + "s3:GetObjectVersion", + "s3:PutObject", + ], + "Resource": [f"{bucket_arn}/metadata-fabric/*"], + }, + ], + } + reader_policy = { + "Version": "2012-10-17", + "Statement": [ + common_bucket, + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:GetObjectRetention", + "s3:GetObjectVersion", + ], + "Resource": [f"{bucket_arn}/metadata-fabric/*"], + }, + ], + } + _write_json_private(policies / "writer.json", writer_policy) + _write_json_private(policies / "reader.json", reader_policy) + env_path = self.temp_root / "mc-runtime.env" + recovery._write_private( + env_path, + ( + f"MC_HOST_admin=http://{self.root_user}:{self.root_password}@127.0.0.1:9000\n" + f"GDA_WRITER_USER={self.writer_user}\n" + f"GDA_WRITER_PASSWORD={self.writer_password}\n" + f"GDA_READER_USER={self.reader_user}\n" + f"GDA_READER_PASSWORD={self.reader_password}\n" + ).encode("ascii"), + ) + script = "\n".join( + ( + "mc admin policy create admin gda-metadata-writer /policies/writer.json", + "mc admin policy create admin gda-metadata-reader /policies/reader.json", + 'mc admin user add admin "$GDA_WRITER_USER" "$GDA_WRITER_PASSWORD"', + 'mc admin user add admin "$GDA_READER_USER" "$GDA_READER_PASSWORD"', + 'mc admin policy attach admin gda-metadata-writer --user "$GDA_WRITER_USER"', + 'mc admin policy attach admin gda-metadata-reader --user "$GDA_READER_USER"', + ) + ) + self.runtime.run( + [ + "run", + "--rm", + "--network", + f"container:{self.container_name}", + "--env-file", + str(env_path), + "--mount", + f"type=bind,src={policies},dst=/policies,readonly", + "--entrypoint", + "/bin/sh", + MC_IMAGE, + "-ceu", + script, + ], + timeout=120, + label="provision external repository identities", + ) + env_path.unlink() + + def start(self) -> tuple[Any, Any, Any, dict[str, Any]]: + self.runtime.run( + [ + "volume", + "create", + "--label", + "gda.openai.com/ephemeral-owner=metadata-cross-cluster-rehearsal", + self.volume_name, + ], + label="create external repository volume", + ) + self.volume_created = True + self.runtime.run( + [ + "run", + "--rm", + "--mount", + f"type=volume,src={self.volume_name},dst=/data", + "busybox:1.36", + "chown", + "1000:1000", + "/data", + ], + label="prepare external repository volume ownership", + ) + env_path = self.temp_root / "minio-runtime.env" + recovery._write_private( + env_path, + ( + f"MINIO_ROOT_USER={self.root_user}\n" + f"MINIO_ROOT_PASSWORD={self.root_password}\n" + "MINIO_BROWSER=off\n" + "HOME=/home/minio\n" + ).encode("ascii"), + ) + self.runtime.run( + [ + "run", + "--rm", + "--detach", + "--name", + self.container_name, + "--label", + "gda.openai.com/ephemeral-owner=metadata-cross-cluster-rehearsal", + "--publish", + f"127.0.0.1:{self.port}:9000", + "--env-file", + str(env_path), + "--user", + "1000:1000", + "--read-only", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges:true", + "--tmpfs", + "/tmp:rw,noexec,nosuid,nodev,size=1g", + "--tmpfs", + "/home/minio:rw,noexec,nosuid,nodev,size=128m", + "--mount", + f"type=volume,src={self.volume_name},dst=/data", + MINIO_IMAGE, + "server", + "/data", + "--address", + ":9000", + ], + label="start external backup repository", + ) + self.container_started = True + env_path.unlink() + admin = repository._build_s3_client( + self.endpoint, self.root_user, self.root_password + ) + self._wait_for_repository(admin) + controls = self._initialize(admin) + self._provision_identities() + writer = repository._build_s3_client( + self.endpoint, self.writer_user, self.writer_password + ) + reader = repository._build_s3_client( + self.endpoint, self.reader_user, self.reader_password + ) + return admin, writer, reader, controls + + def cleanup(self) -> dict[str, bool]: + if self.runtime.exists("container", self.container_name): + self.runtime.run( + ["rm", "--force", self.container_name], + label="remove external repository container", + ) + container_removed = not self.runtime.exists("container", self.container_name) + if self.runtime.exists("volume", self.volume_name): + self.runtime.run( + ["volume", "rm", self.volume_name], + label="remove external repository volume", + ) + volume_removed = not self.runtime.exists("volume", self.volume_name) + return { + "external_repository_container_removed": container_removed, + "external_repository_volume_removed": volume_removed, + } + + +class _ExternalRepositoryRoundTrip: + def __init__( + self, + admin: Any, + writer: Any, + reader: Any, + controls: Mapping[str, Any], + recovery_point: str, + ) -> None: + self.admin = admin + self.writer = writer + self.reader = reader + self.controls = dict(controls) + self.recovery_point = recovery_point + + def __call__( + self, + artifact_paths: Mapping[str, Path], + artifacts: Mapping[str, Mapping[str, Any]], + ) -> Mapping[str, Any]: + uploaded: dict[str, dict[str, Any]] = {} + try: + for name in sorted(repository.EXPECTED_ARTIFACTS): + path = artifact_paths[name] + body = path.read_bytes() + sha256 = hashlib.sha256(body).hexdigest() + suffix = "tar.gz" if name == "opensearch" else "dump" + object_path = ( + f"metadata-fabric/{self.recovery_point}/{name}.{suffix}" + ) + response = self.writer.put_object( + Bucket=BUCKET, + Key=object_path, + Body=body, + ContentLength=len(body), + ContentMD5=base64.b64encode( + hashlib.md5(body, usedforsecurity=False).digest() + ).decode("ascii"), + Metadata={ + "gda-sha256": sha256, + "gda-format": str(artifacts[name]["format"]), + }, + ) + version_id = response.get("VersionId") + if not version_id: + raise MetadataFabricCrossClusterRecoveryError( + f"external repository did not version {name}" + ) + head = self.reader.head_object( + Bucket=BUCKET, + Key=object_path, + VersionId=version_id, + ) + metadata = _mapping(head.get("Metadata")) + if metadata.get("gda-sha256") != sha256: + raise MetadataFabricCrossClusterRecoveryError( + f"external repository checksum differs for {name}" + ) + writer_delete_denied = _access_denied( + lambda: self.writer.delete_object( + Bucket=BUCKET, + Key=object_path, + VersionId=version_id, + ) + ) + compliance_delete_blocked = _access_denied( + lambda: self.admin.delete_object( + Bucket=BUCKET, + Key=object_path, + VersionId=version_id, + ) + ) + if not writer_delete_denied or not compliance_delete_blocked: + raise MetadataFabricCrossClusterRecoveryError( + f"external repository deletion controls failed for {name}" + ) + retained_until = head.get("ObjectLockRetainUntilDate") + uploaded[name] = { + "object_path": object_path, + "version_id": version_id, + "format": artifacts[name]["format"], + "sha256": sha256, + "bytes": len(body), + "retention_mode": head.get("ObjectLockMode"), + "retained_until": ( + retained_until.isoformat() + if hasattr(retained_until, "isoformat") + else str(retained_until or "") + ), + "writer_delete_denied": writer_delete_denied, + "compliance_delete_blocked": compliance_delete_blocked, + } + + reader_write_denied = _access_denied( + lambda: self.reader.put_object( + Bucket=BUCKET, + Key=f"metadata-fabric/{self.recovery_point}/reader-write-probe", + Body=b"denied", + ContentLength=6, + ) + ) + if not reader_write_denied: + raise MetadataFabricCrossClusterRecoveryError( + "external recovery reader was allowed to write" + ) + for path in artifact_paths.values(): + path.unlink() + removed_before_download = all( + not path.exists() for path in artifact_paths.values() + ) + if not removed_before_download: + raise MetadataFabricCrossClusterRecoveryError( + "local artifacts remained before external repository download" + ) + for name, item in uploaded.items(): + response = self.reader.get_object( + Bucket=BUCKET, + Key=item["object_path"], + VersionId=item["version_id"], + ) + body = response["Body"].read() + if hashlib.sha256(body).hexdigest() != item["sha256"]: + raise MetadataFabricCrossClusterRecoveryError( + f"external repository download differs for {name}" + ) + recovery._write_private(artifact_paths[name], body) + except MetadataFabricCrossClusterRecoveryError: + raise + except Exception as exc: + raise MetadataFabricCrossClusterRecoveryError( + "external repository artifact round-trip failed" + ) from exc + + return { + "provider": "minio_s3_compatible", + "repository_runtime": "docker_host_container", + "failure_domain": FAILURE_DOMAIN, + "bucket": BUCKET, + **self.controls, + "artifact_objects": uploaded, + "identity_controls": { + "separate_writer_reader": True, + "writer_delete_denied": True, + "reader_write_denied": reader_write_denied, + }, + "local_artifacts_removed_before_download": removed_before_download, + "round_trip_verified": True, + "transport_tls_verified": False, + "kms_encryption_verified": False, + "production_durability_verified": False, + } + + +def _repository_errors( + round_trip: Mapping[str, Any], *, verification_time: datetime +) -> list[str]: + errors: list[str] = [] + artifacts = _mapping(round_trip.get("artifact_objects")) + identities = _mapping(round_trip.get("identity_controls")) + if round_trip.get("provider") != "minio_s3_compatible": + errors.append("external repository provider does not match") + if round_trip.get("repository_runtime") != "docker_host_container": + errors.append("repository did not run outside Kubernetes") + if round_trip.get("failure_domain") != FAILURE_DOMAIN: + errors.append("repository failure domain does not match") + if round_trip.get("bucket") != BUCKET: + errors.append("cross-cluster repository bucket does not match") + if round_trip.get("versioning_status") != "Enabled": + errors.append("cross-cluster repository versioning was not enabled") + if round_trip.get("object_lock_enabled") is not True: + errors.append("cross-cluster repository Object Lock was not enabled") + if ( + round_trip.get("default_retention_mode") != RETENTION_MODE + or round_trip.get("default_retention_days") != RETENTION_DAYS + ): + errors.append("cross-cluster repository retention does not match") + if round_trip.get("local_artifacts_removed_before_download") is not True: + errors.append("local artifacts were not removed before external download") + if round_trip.get("round_trip_verified") is not True: + errors.append("external repository round-trip was not verified") + if identities.get("separate_writer_reader") is not True: + errors.append("external repository writer and reader were not separated") + if identities.get("writer_delete_denied") is not True: + errors.append("external repository writer delete was not denied") + if identities.get("reader_write_denied") is not True: + errors.append("external repository reader write was not denied") + for claim in ( + "transport_tls_verified", + "kms_encryption_verified", + "production_durability_verified", + ): + if round_trip.get(claim) is not False: + errors.append(f"external repository may not claim {claim}") + if set(artifacts) != set(repository.EXPECTED_ARTIFACTS): + errors.append("external repository artifact inventory does not match") + version_ids: set[str] = set() + for name, expected_format in repository.EXPECTED_ARTIFACTS.items(): + item = _mapping(artifacts.get(name)) + if item.get("format") != expected_format: + errors.append(f"external repository artifact format differs: {name}") + if not repository._valid_fingerprint(item.get("sha256")): + errors.append(f"external repository artifact checksum is invalid: {name}") + if not isinstance(item.get("bytes"), int) or item.get("bytes", 0) <= 0: + errors.append(f"external repository artifact is empty: {name}") + object_path = str(item.get("object_path") or "") + if not object_path.startswith("metadata-fabric/") or ".." in object_path: + errors.append(f"external repository artifact path is invalid: {name}") + if not item.get("version_id"): + errors.append(f"external repository artifact has no version: {name}") + elif str(item.get("version_id")) in version_ids: + errors.append(f"external repository artifact version is not unique: {name}") + else: + version_ids.add(str(item.get("version_id"))) + if item.get("retention_mode") != RETENTION_MODE: + errors.append(f"external repository artifact is not COMPLIANCE locked: {name}") + try: + retained_until = datetime.fromisoformat(str(item.get("retained_until"))) + if ( + retained_until.tzinfo is None + or retained_until.utcoffset() is None + or retained_until <= verification_time + ): + raise ValueError + except ValueError: + errors.append( + f"external repository artifact retention is not in the future: {name}" + ) + if item.get("writer_delete_denied") is not True: + errors.append(f"external repository writer delete control failed: {name}") + if item.get("compliance_delete_blocked") is not True: + errors.append(f"external repository COMPLIANCE delete was not blocked: {name}") + return errors + + +def _observation_errors( + observation: Mapping[str, Any], *, now: datetime, max_age_seconds: float +) -> list[str]: + errors: list[str] = [] + if recovery._sensitive_paths(observation): + errors.append("cross-cluster observation contains credential-bearing fields") + if observation.get("schema") != OBSERVATION_SCHEMA: + errors.append("cross-cluster observation schema does not match") + try: + observed_at = datetime.fromisoformat(str(observation.get("observed_at"))) + if observed_at.tzinfo is None or observed_at.utcoffset() is None: + raise ValueError + age = (now - observed_at).total_seconds() + if age < -30 or age > max_age_seconds: + errors.append("cross-cluster observation is outside the freshness window") + except ValueError: + errors.append("cross-cluster observation timestamp is invalid") + contract = _mapping(observation.get("contract")) + if contract.get("local_static_contract_verified") is not True: + errors.append("cross-cluster static contract was not verified") + if not repository._valid_fingerprint(contract.get("contract_fingerprint")): + errors.append("cross-cluster contract fingerprint is invalid") + + repository_runtime = _mapping(observation.get("repository")) + if repository_runtime.get("runtime") != "docker_host_container": + errors.append("outer repository runtime does not match") + if repository_runtime.get("failure_domain") != FAILURE_DOMAIN: + errors.append("outer repository failure domain does not match") + if repository_runtime.get("minio_image") != MINIO_IMAGE: + errors.append("outer repository MinIO image does not match") + if repository_runtime.get("mc_image") != MC_IMAGE: + errors.append("outer repository client image does not match") + + recovery_evidence = _mapping(observation.get("recovery_evidence")) + nested_errors = recovery.verify_evidence_integrity(recovery_evidence) + errors.extend(f"nested recovery evidence: {item}" for item in nested_errors) + if recovery_evidence.get("local_cross_cluster_recovery_verified") is not True: + errors.append("nested recovery did not verify distinct local clusters") + if recovery_evidence.get("production_cross_cluster_recovery_verified") is not False: + errors.append("nested recovery overclaims production cross-cluster recovery") + nested_observation = _mapping(recovery_evidence.get("observation")) + cluster = _mapping(nested_observation.get("cluster")) + if cluster.get("context") != SOURCE_CONTEXT: + errors.append("cross-cluster source context does not match") + if cluster.get("recovery_context") != RECOVERY_CONTEXT: + errors.append("cross-cluster recovery context does not match") + if not cluster.get("uid") or not cluster.get("recovery_uid"): + errors.append("cross-cluster identities are unavailable") + elif cluster.get("uid") == cluster.get("recovery_uid"): + errors.append("source and recovery cluster identities are not isolated") + + round_trip = _mapping(nested_observation.get("repository_round_trip")) + errors.extend(_repository_errors(round_trip, verification_time=now)) + source_artifacts = _mapping(nested_observation.get("artifacts")) + repository_artifacts = _mapping(round_trip.get("artifact_objects")) + for name in repository.EXPECTED_ARTIFACTS: + source = _mapping(source_artifacts.get(name)) + stored = _mapping(repository_artifacts.get(name)) + if stored.get("sha256") != source.get("sha256") or stored.get( + "bytes" + ) != source.get("bytes"): + errors.append(f"external repository content differs from recovery: {name}") + + runtime = _mapping(observation.get("runtime_checks")) + for key in ( + "external_repository_container_removed", + "external_repository_volume_removed", + "runtime_credentials_removed", + "recovery_cluster_preserved", + ): + if runtime.get(key) is not True: + errors.append(f"cross-cluster runtime check did not pass: {key}") + return errors + + +def build_cross_cluster_evidence( + observation: Mapping[str, Any], + *, + now: datetime | None = None, + max_age_seconds: float = 3600, +) -> dict[str, Any]: + """Build fail-closed evidence for the host-bounded dual-cluster recovery.""" + current = now or datetime.now(UTC) + if current.tzinfo is None or current.utcoffset() is None: + raise MetadataFabricCrossClusterRecoveryError( + "verification time must be timezone-aware" + ) + errors = _observation_errors( + observation, now=current, max_age_seconds=max_age_seconds + ) + verified = not errors + stable = { + "schema": EVIDENCE_SCHEMA, + "environment": "local_same_host_distinct_kubernetes_clusters", + "source_context": SOURCE_CONTEXT, + "recovery_context": RECOVERY_CONTEXT, + "observation_fingerprint": recovery._canonical_sha256(observation), + "checks": { + "static_contract": "passed" if verified else "blocked", + "cluster_identity_isolation": "passed" if verified else "blocked", + "repository_failure_domain": "passed" if verified else "blocked", + "writer_reader_separation": "passed" if verified else "blocked", + "compliance_retention": "passed" if verified else "blocked", + "artifact_round_trip": "passed" if verified else "blocked", + "cross_cluster_restore": "passed" if verified else "blocked", + "ephemeral_cleanup": "passed" if verified else "blocked", + "production_boundaries": "passed", + }, + "errors": errors, + "cross_cluster_recovery_scope": ( + "local_same_host_distinct_kubernetes_clusters_external_s3_repository" + ), + "local_cross_cluster_recovery_verified": verified, + "local_external_repository_verified": verified, + "local_writer_reader_identity_separation_verified": verified, + "cross_cluster_recovery_verified": False, + "production_cross_cluster_recovery_verified": False, + "production_backup_target_verified": False, + "production_retention_verified": False, + "production_kms_verified": False, + "production_tls_verified": False, + "source_cluster_loss_verified": False, + "cross_region_recovery_verified": False, + "rpo_slo_verified": False, + "rto_slo_verified": False, + "oidc_verified": False, + "network_policy_enforcement_verified": False, + "writes_to_gda_enabled": False, + "production_ready": False, + "observation": observation, + } + return { + **stable, + "generated_at": current.isoformat(), + "status": "local_cross_cluster_recovery_verified" if verified else "blocked", + "evidence_fingerprint": recovery._canonical_sha256(stable), + } + + +def verify_evidence_integrity(report: Mapping[str, Any]) -> list[str]: + errors: list[str] = [] + if recovery._sensitive_paths(report): + errors.append("cross-cluster evidence contains credential-bearing fields") + if report.get("schema") != EVIDENCE_SCHEMA: + errors.append("cross-cluster evidence schema does not match") + stable = { + key: value + for key, value in report.items() + if key not in {"generated_at", "status", "evidence_fingerprint"} + } + if report.get("evidence_fingerprint") != recovery._canonical_sha256(stable): + errors.append("cross-cluster evidence fingerprint does not match") + for claim in ( + "cross_cluster_recovery_verified", + "production_cross_cluster_recovery_verified", + "production_backup_target_verified", + "production_retention_verified", + "production_kms_verified", + "production_tls_verified", + "source_cluster_loss_verified", + "cross_region_recovery_verified", + "rpo_slo_verified", + "rto_slo_verified", + "oidc_verified", + "network_policy_enforcement_verified", + "writes_to_gda_enabled", + "production_ready", + ): + if report.get(claim) is not False: + errors.append(f"cross-cluster evidence may not claim {claim}") + return errors + + +def run_live_cross_cluster_recovery( + *, + kubectl: str = "kubectl", + docker: str = "docker", + source_context: str = SOURCE_CONTEXT, + recovery_context: str = RECOVERY_CONTEXT, +) -> dict[str, Any]: + """Run the locked repository round-trip into a distinct local cluster.""" + if source_context != SOURCE_CONTEXT or recovery_context != RECOVERY_CONTEXT: + raise MetadataFabricCrossClusterRecoveryError( + "live contexts do not match the bounded local profile" + ) + started = datetime.now(UTC) + contract = build_cross_cluster_contract_report() + if contract["local_static_contract_verified"] is not True: + raise MetadataFabricCrossClusterRecoveryError( + "cross-cluster static contract is invalid" + ) + source_runner = recovery._CommandRunner(kubectl, source_context) + target_runner = recovery._CommandRunner(kubectl, recovery_context) + source_uid = recovery._cluster_uid(source_runner) + recovery_uid = recovery._cluster_uid(target_runner) + if not source_uid or not recovery_uid or source_uid == recovery_uid: + raise MetadataFabricCrossClusterRecoveryError( + "source and recovery clusters are not independently identified" + ) + + temp_root = Path(tempfile.mkdtemp(prefix="gda-metadata-cross-cluster-")) + os.chmod(temp_root, 0o700) + external = _ExternalRepositoryRuntime(temp_root, docker) + recovery_observation: dict[str, Any] = {} + cleanup = { + "external_repository_container_removed": False, + "external_repository_volume_removed": False, + } + failure: Exception | None = None + try: + admin, writer, reader, controls = external.start() + recovery_point = "rp-" + started.strftime("%Y%m%d%H%M%SZ").lower() + recovery_observation = recovery.run_live_recovery_rehearsal( + kubectl=kubectl, + source_context=source_context, + recovery_context=recovery_context, + artifact_round_trip=_ExternalRepositoryRoundTrip( + admin, + writer, + reader, + controls, + recovery_point, + ), + ) + except Exception as exc: + failure = exc + finally: + try: + cleanup = external.cleanup() + except Exception as exc: + failure = failure or exc + shutil.rmtree(temp_root, ignore_errors=True) + runtime_credentials_removed = not temp_root.exists() + + if failure is not None: + if isinstance(failure, MetadataFabricCrossClusterRecoveryError): + raise failure + if isinstance(failure, recovery.MetadataFabricRecoveryError): + raise MetadataFabricCrossClusterRecoveryError( + "cross-cluster metadata recovery failed" + ) from failure + raise MetadataFabricCrossClusterRecoveryError( + "live cross-cluster rehearsal failed" + ) from failure + + completed = datetime.now(UTC) + recovery_evidence = recovery.build_recovery_evidence( + recovery_observation, now=completed + ) + observation = { + "schema": OBSERVATION_SCHEMA, + "observed_at": completed.isoformat(), + "started_at": started.isoformat(), + "duration_seconds": round((completed - started).total_seconds(), 3), + "contract": { + "local_static_contract_verified": contract[ + "local_static_contract_verified" + ], + "contract_fingerprint": contract["contract_fingerprint"], + }, + "repository": { + "runtime": "docker_host_container", + "failure_domain": FAILURE_DOMAIN, + "minio_image": MINIO_IMAGE, + "mc_image": MC_IMAGE, + }, + "recovery_evidence": recovery_evidence, + "runtime_checks": { + **cleanup, + "runtime_credentials_removed": runtime_credentials_removed, + "recovery_cluster_preserved": ( + recovery._cluster_uid(target_runner) == recovery_uid + ), + }, + } + return build_cross_cluster_evidence(observation, now=completed) + + +def _load_json_object(path: Path) -> dict[str, Any]: + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise MetadataFabricCrossClusterRecoveryError("JSON input must be an object") + return payload + + +def _write_report(report: Mapping[str, Any], output: Path | None) -> None: + rendered = json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + if output is None: + print(rendered, end="") + return + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(rendered, encoding="utf-8") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + subparsers.add_parser("validate") + run_parser = subparsers.add_parser("run") + run_parser.add_argument("--kubectl", default="kubectl") + run_parser.add_argument("--docker", default="docker") + run_parser.add_argument("--source-context", default=SOURCE_CONTEXT) + run_parser.add_argument("--recovery-context", default=RECOVERY_CONTEXT) + run_parser.add_argument("--output", type=Path) + run_parser.add_argument("--recovery-output", type=Path) + verify_parser = subparsers.add_parser("verify") + verify_parser.add_argument("--input", type=Path, required=True) + args = parser.parse_args(argv) + + try: + if args.command == "validate": + report = build_cross_cluster_contract_report() + _write_report(report, None) + return 0 if report["local_static_contract_verified"] else 1 + if args.command == "run": + report = run_live_cross_cluster_recovery( + kubectl=args.kubectl, + docker=args.docker, + source_context=args.source_context, + recovery_context=args.recovery_context, + ) + _write_report(report, args.output) + if args.recovery_output is not None: + nested = _mapping(_mapping(report.get("observation")).get("recovery_evidence")) + _write_report(nested, args.recovery_output) + return 0 if report["local_cross_cluster_recovery_verified"] else 1 + report = _load_json_object(args.input) + errors = verify_evidence_integrity(report) + _write_report({"verified": not errors, "errors": errors}, None) + return 0 if not errors else 1 + except ( + OSError, + ValueError, + json.JSONDecodeError, + MetadataFabricCrossClusterRecoveryError, + KeyboardInterrupt, + ) as exc: + print(f"metadata cross-cluster recovery: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/data_agent/metadata_fabric_recovery_rehearsal.py b/data_agent/metadata_fabric_recovery_rehearsal.py index 10485b49..4c6c93c7 100644 --- a/data_agent/metadata_fabric_recovery_rehearsal.py +++ b/data_agent/metadata_fabric_recovery_rehearsal.py @@ -458,8 +458,18 @@ def _observation_errors( errors.append("static recovery contract fingerprint is invalid") cluster = _mapping(observation.get("cluster")) - if cluster.get("context") != "docker-desktop" or not cluster.get("uid"): + source_context = cluster.get("context") + source_cluster_uid = cluster.get("uid") + recovery_context = cluster.get("recovery_context", source_context) + recovery_cluster_uid = cluster.get("recovery_uid", source_cluster_uid) + if source_context != "docker-desktop" or not source_cluster_uid: errors.append("rehearsal did not run on the bounded Docker Desktop cluster") + if not recovery_context or not recovery_cluster_uid: + errors.append("recovery cluster identity is unavailable") + if (source_context == recovery_context) != ( + source_cluster_uid == recovery_cluster_uid + ): + errors.append("source and recovery cluster context/identity isolation disagrees") source_ns = _mapping(cluster.get("source_namespace")) recovery_ns = _mapping(cluster.get("recovery_namespace")) if source_ns.get("name") != SOURCE_NAMESPACE or not source_ns.get("uid"): @@ -537,9 +547,27 @@ def build_recovery_evidence( observation, now=current, max_age_seconds=max_age_seconds ) verified = not errors + cluster = _mapping(observation.get("cluster")) + source_context = cluster.get("context") + source_cluster_uid = cluster.get("uid") + recovery_context = cluster.get("recovery_context", source_context) + recovery_cluster_uid = cluster.get("recovery_uid", source_cluster_uid) + cross_cluster = ( + source_context != recovery_context + and source_cluster_uid != recovery_cluster_uid + ) + scope = ( + "local_distinct_kubernetes_clusters_new_namespace_and_pvcs" + if cross_cluster + else "local_same_cluster_new_namespace_and_pvcs" + ) stable = { "schema": EVIDENCE_SCHEMA, - "environment": "local_isolated_recovery_rehearsal", + "environment": ( + "local_cross_cluster_recovery_rehearsal" + if cross_cluster + else "local_isolated_recovery_rehearsal" + ), "source_namespace": SOURCE_NAMESPACE, "recovery_namespace": RECOVERY_NAMESPACE, "observation_fingerprint": _canonical_sha256(observation), @@ -553,10 +581,12 @@ def build_recovery_evidence( "production_boundaries": "passed", }, "errors": errors, - "backup_restore_scope": "local_same_cluster_new_namespace_and_pvcs", + "backup_restore_scope": scope, "backup_restore_verified": verified, "local_backup_restore_verified": verified, + "local_cross_cluster_recovery_verified": verified and cross_cluster, "production_backup_restore_verified": False, + "production_cross_cluster_recovery_verified": False, "rpo_slo_verified": False, "rto_slo_verified": False, "cross_cluster_recovery_verified": False, @@ -571,7 +601,13 @@ def build_recovery_evidence( return { **stable, "generated_at": current.isoformat(), - "status": "local_backup_restore_verified" if verified else "blocked", + "status": ( + "local_cross_cluster_backup_restore_verified" + if verified and cross_cluster + else "local_backup_restore_verified" + if verified + else "blocked" + ), "evidence_fingerprint": _canonical_sha256(stable), } @@ -591,14 +627,23 @@ def verify_evidence_integrity(report: Mapping[str, Any]) -> list[str]: errors.append("evidence fingerprint does not match its stable content") if report.get("production_backup_restore_verified") is not False: errors.append("evidence may not claim production backup/restore") + if report.get("production_cross_cluster_recovery_verified") is not False: + errors.append("evidence may not claim production cross-cluster recovery") if report.get("production_ready") is not False: errors.append("evidence may not claim production readiness") return errors class _CommandRunner: - def __init__(self, kubectl: str = "kubectl") -> None: + def __init__(self, kubectl: str = "kubectl", context: str | None = None) -> None: self.kubectl = kubectl + self.context = context + + def kubectl_args(self, args: list[str]) -> list[str]: + command = [self.kubectl] + if self.context is not None: + command.extend(["--context", self.context]) + return [*command, *args] def run( self, @@ -633,7 +678,7 @@ def kubectl_run( label: str, ) -> bytes: return self.run( - [self.kubectl, *args], + self.kubectl_args(args), input_bytes=input_bytes, timeout=timeout, label=label, @@ -651,7 +696,9 @@ def kubectl_json(self, args: list[str], *, label: str) -> dict[str, Any]: def namespace_exists(self, namespace: str) -> bool: try: completed = subprocess.run( - [self.kubectl, "get", "namespace", namespace, "-o", "name"], + self.kubectl_args( + ["get", "namespace", namespace, "-o", "name"] + ), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False, @@ -1103,10 +1150,18 @@ def _restore_source_services( def run_live_recovery_rehearsal( *, kubectl: str = "kubectl", + source_context: str = "docker-desktop", + recovery_context: str | None = None, artifact_round_trip: ArtifactRoundTrip | None = None, ) -> dict[str, Any]: """Execute the bounded local rehearsal and return an allowlisted observation.""" - runner = _CommandRunner(kubectl) + target_context = recovery_context or source_context + source_runner = _CommandRunner(kubectl, source_context) + recovery_runner = ( + source_runner + if target_context == source_context + else _CommandRunner(kubectl, target_context) + ) started = datetime.now(UTC) contract = build_recovery_contract_report() if contract["static_contract_verified"] is not True: @@ -1114,13 +1169,9 @@ def run_live_recovery_rehearsal( sandbox = build_sandbox_report() if sandbox["static_contract_verified"] is not True: raise MetadataFabricRecoveryError("source sandbox contract is invalid") - context = _decode( - runner.kubectl_run(["config", "current-context"], label="read cluster context"), - "cluster context", - ).strip() - if context != "docker-desktop": + if source_context != "docker-desktop": raise MetadataFabricRecoveryError("recovery rehearsal requires docker-desktop") - if runner.namespace_exists(RECOVERY_NAMESPACE): + if recovery_runner.namespace_exists(RECOVERY_NAMESPACE): raise MetadataFabricRecoveryError("recovery Namespace already exists; refusing cleanup ambiguity") temp_root = Path(tempfile.mkdtemp(prefix="gda-metadata-recovery-")) @@ -1130,8 +1181,15 @@ def run_live_recovery_rehearsal( recovered_markers: dict[str, dict[str, Any]] = {} recovery_pvcs: dict[str, Any] = {} recovery_identity: dict[str, Any] = {} - source_identity = _namespace_identity(runner, SOURCE_NAMESPACE) - cluster_uid = _cluster_uid(runner) + source_identity = _namespace_identity(source_runner, SOURCE_NAMESPACE) + source_cluster_uid = _cluster_uid(source_runner) + recovery_cluster_uid = _cluster_uid(recovery_runner) + if not source_cluster_uid or not recovery_cluster_uid: + raise MetadataFabricRecoveryError("source or recovery cluster identity is unavailable") + if (source_context == target_context) != ( + source_cluster_uid == recovery_cluster_uid + ): + raise MetadataFabricRecoveryError("cluster context and identity isolation disagree") original_replicas: dict[str, int] = {} source_quiesced = False source_snapshot_staging_initially_empty = False @@ -1147,17 +1205,17 @@ def run_live_recovery_rehearsal( repository_round_trip: dict[str, Any] | None = None try: - runner.kubectl_run( + source_runner.kubectl_run( ["apply", "--dry-run=server", "-k", str(REPO_ROOT / "k8s/metadata-fabric-sandbox")], timeout=180, label="server validate source snapshot configuration", ) - runner.kubectl_run( + source_runner.kubectl_run( ["apply", "-k", str(REPO_ROOT / "k8s/metadata-fabric-sandbox")], timeout=180, label="apply source snapshot configuration", ) - runner.kubectl_run( + source_runner.kubectl_run( [ "-n", SOURCE_NAMESPACE, @@ -1175,7 +1233,7 @@ def run_live_recovery_rehearsal( ("metadata-gravitino", "statefulset"), ): value = _decode( - runner.kubectl_run( + source_runner.kubectl_run( [ "-n", SOURCE_NAMESPACE, @@ -1192,11 +1250,11 @@ def run_live_recovery_rehearsal( if original_replicas[name] != 1: raise MetadataFabricRecoveryError(f"{name} must have exactly one source replica") - runner.kubectl_run( + source_runner.kubectl_run( ["-n", SOURCE_NAMESPACE, "scale", "deployment/openmetadata", "--replicas=0"], label="quiesce OpenMetadata", ) - runner.kubectl_run( + source_runner.kubectl_run( [ "-n", SOURCE_NAMESPACE, @@ -1207,13 +1265,13 @@ def run_live_recovery_rehearsal( label="quiesce Gravitino", ) _wait_for_no_pods( - runner, + source_runner, SOURCE_NAMESPACE, "app.kubernetes.io/name=openmetadata", label="wait for OpenMetadata quiescence", ) _wait_for_no_pods( - runner, + source_runner, SOURCE_NAMESPACE, "app.kubernetes.io/name=metadata-gravitino", label="wait for Gravitino quiescence", @@ -1223,13 +1281,13 @@ def run_live_recovery_rehearsal( for name, target in POSTGRES_TARGETS.items(): source_pod = str(target["source_pod"]) source_markers[name] = _postgres_marker( - runner, SOURCE_NAMESPACE, source_pod, label=f"source {name}" + source_runner, SOURCE_NAMESPACE, source_pod, label=f"source {name}" ) backup_path = temp_root / f"{name}.dump" _write_private( backup_path, _postgres_dump( - runner, + source_runner, SOURCE_NAMESPACE, source_pod, label=f"backup {name}", @@ -1238,13 +1296,13 @@ def run_live_recovery_rehearsal( artifacts[name] = _artifact(backup_path, str(target["format"])) source_os_marker, source_index_names = _opensearch_marker( - runner, + source_runner, SOURCE_NAMESPACE, "metadata-opensearch-0", label="source opensearch", ) source_markers["opensearch"] = source_os_marker - initial_staging = runner.kubectl_run( + initial_staging = source_runner.kubectl_run( [ "-n", SOURCE_NAMESPACE, @@ -1264,7 +1322,7 @@ def run_live_recovery_rehearsal( raise MetadataFabricRecoveryError("source snapshot staging is not empty") snapshot_name = "gda-" + started.strftime("%Y%m%d%H%M%SZ").lower() repository = _opensearch_request( - runner, + source_runner, SOURCE_NAMESPACE, "metadata-opensearch-0", "PUT", @@ -1276,7 +1334,7 @@ def run_live_recovery_rehearsal( raise MetadataFabricRecoveryError("source snapshot repository was not acknowledged") source_repository_created = True snapshot = _opensearch_request( - runner, + source_runner, SOURCE_NAMESPACE, "metadata-opensearch-0", "PUT", @@ -1297,7 +1355,7 @@ def run_live_recovery_rehearsal( os_backup_path = temp_root / "opensearch-snapshot.tar.gz" _write_private( os_backup_path, - runner.kubectl_run( + source_runner.kubectl_run( [ "-n", SOURCE_NAMESPACE, @@ -1342,29 +1400,35 @@ def run_live_recovery_rehearsal( f"repository round-trip changed {name} content" ) - runner.kubectl_run( + recovery_runner.kubectl_run( ["apply", "-f", str(DEFAULT_RECOVERY_MANIFEST_DIR / "namespace.yaml")], label="create recovery Namespace", ) recovery_created = True _create_runtime_credential( - runner, RECOVERY_NAMESPACE, "recovery-openmetadata-postgresql", temp_root + recovery_runner, + RECOVERY_NAMESPACE, + "recovery-openmetadata-postgresql", + temp_root, ) _create_runtime_credential( - runner, RECOVERY_NAMESPACE, "recovery-gravitino-postgresql", temp_root + recovery_runner, + RECOVERY_NAMESPACE, + "recovery-gravitino-postgresql", + temp_root, ) - runner.kubectl_run( + recovery_runner.kubectl_run( ["apply", "--dry-run=server", "-k", str(DEFAULT_RECOVERY_MANIFEST_DIR)], timeout=180, label="server validate recovery workloads", ) - runner.kubectl_run( + recovery_runner.kubectl_run( ["apply", "-k", str(DEFAULT_RECOVERY_MANIFEST_DIR)], timeout=180, label="apply recovery workloads", ) for statefulset in RECOVERY_STATEFULSETS: - runner.kubectl_run( + recovery_runner.kubectl_run( [ "-n", RECOVERY_NAMESPACE, @@ -1376,26 +1440,26 @@ def run_live_recovery_rehearsal( timeout=930, label=f"wait for {statefulset}", ) - recovery_identity = _namespace_identity(runner, RECOVERY_NAMESPACE) - recovery_pvcs = _pvc_identities(runner) + recovery_identity = _namespace_identity(recovery_runner, RECOVERY_NAMESPACE) + recovery_pvcs = _pvc_identities(recovery_runner) for name, target in POSTGRES_TARGETS.items(): backup_path = temp_root / f"{name}.dump" _postgres_restore( - runner, + recovery_runner, RECOVERY_NAMESPACE, str(target["recovery_pod"]), backup_path.read_bytes(), label=f"restore {name}", ) recovered_markers[name] = _postgres_marker( - runner, + recovery_runner, RECOVERY_NAMESPACE, str(target["recovery_pod"]), label=f"recovered {name}", ) - runner.kubectl_run( + recovery_runner.kubectl_run( [ "-n", RECOVERY_NAMESPACE, @@ -1419,7 +1483,7 @@ def run_live_recovery_rehearsal( label="import OpenSearch snapshot artifact", ) recovery_repository = _opensearch_request( - runner, + recovery_runner, RECOVERY_NAMESPACE, "recovery-opensearch-0", "PUT", @@ -1430,7 +1494,7 @@ def run_live_recovery_rehearsal( if recovery_repository.get("acknowledged") is not True: raise MetadataFabricRecoveryError("recovery snapshot repository was not acknowledged") _, recovery_initial_names = _opensearch_marker( - runner, + recovery_runner, RECOVERY_NAMESPACE, "recovery-opensearch-0", label="initial recovery opensearch", @@ -1438,7 +1502,7 @@ def run_live_recovery_rehearsal( if recovery_initial_names: encoded_indices = quote(",".join(recovery_initial_names), safe=",.-_") cleared = _opensearch_request( - runner, + recovery_runner, RECOVERY_NAMESPACE, "recovery-opensearch-0", "DELETE", @@ -1450,7 +1514,7 @@ def run_live_recovery_rehearsal( "recovery search target cleanup was not acknowledged" ) _, remaining_initial_names = _opensearch_marker( - runner, + recovery_runner, RECOVERY_NAMESPACE, "recovery-opensearch-0", label="cleared recovery opensearch", @@ -1459,7 +1523,7 @@ def run_live_recovery_rehearsal( if not recovery_search_target_cleared: raise MetadataFabricRecoveryError("recovery search target is not empty") restored = _opensearch_request( - runner, + recovery_runner, RECOVERY_NAMESPACE, "recovery-opensearch-0", "POST", @@ -1472,7 +1536,7 @@ def run_live_recovery_rehearsal( if _mapping(restored_result.get("shards")).get("failed") != 0: raise MetadataFabricRecoveryError("OpenSearch restore reported failed shards") _opensearch_request( - runner, + recovery_runner, RECOVERY_NAMESPACE, "recovery-opensearch-0", "GET", @@ -1481,7 +1545,7 @@ def run_live_recovery_rehearsal( label="wait for recovered OpenSearch health", ) recovered_os_marker, _ = _opensearch_marker( - runner, + recovery_runner, RECOVERY_NAMESPACE, "recovery-opensearch-0", expected_names=source_index_names, @@ -1495,7 +1559,7 @@ def run_live_recovery_rehearsal( if source_repository_created: if snapshot_name is not None: _opensearch_request( - runner, + source_runner, SOURCE_NAMESPACE, "metadata-opensearch-0", "DELETE", @@ -1503,7 +1567,7 @@ def run_live_recovery_rehearsal( label="remove source rehearsal snapshot", ) deleted_repository = _opensearch_request( - runner, + source_runner, SOURCE_NAMESPACE, "metadata-opensearch-0", "DELETE", @@ -1514,7 +1578,7 @@ def run_live_recovery_rehearsal( raise MetadataFabricRecoveryError( "source snapshot repository cleanup was not acknowledged" ) - runner.kubectl_run( + source_runner.kubectl_run( [ "-n", SOURCE_NAMESPACE, @@ -1538,13 +1602,13 @@ def run_live_recovery_rehearsal( try: if original_replicas: source_services_restored = _restore_source_services( - runner, original_replicas + source_runner, original_replicas ) except Exception as exc: failure = failure or exc try: if recovery_created: - runner.kubectl_run( + recovery_runner.kubectl_run( [ "delete", "namespace", @@ -1555,7 +1619,7 @@ def run_live_recovery_rehearsal( timeout=630, label="remove recovery Namespace", ) - recovery_namespace_removed = not runner.namespace_exists( + recovery_namespace_removed = not recovery_runner.namespace_exists( RECOVERY_NAMESPACE ) except Exception as exc: @@ -1579,9 +1643,11 @@ def run_live_recovery_rehearsal( "contract_fingerprint": contract["contract_fingerprint"], }, "cluster": { - "context": context, - "uid": cluster_uid, + "context": source_context, + "uid": source_cluster_uid, "source_namespace": source_identity, + "recovery_context": target_context, + "recovery_uid": recovery_cluster_uid, "recovery_namespace": recovery_identity, }, "recovery_pvcs": recovery_pvcs, @@ -1626,6 +1692,8 @@ def main(argv: list[str] | None = None) -> int: subparsers.add_parser("validate") run_parser = subparsers.add_parser("run") run_parser.add_argument("--kubectl", default="kubectl") + run_parser.add_argument("--source-context", default="docker-desktop") + run_parser.add_argument("--recovery-context") run_parser.add_argument("--output", type=Path) verify_parser = subparsers.add_parser("verify") verify_parser.add_argument("--input", type=Path, required=True) @@ -1637,7 +1705,11 @@ def main(argv: list[str] | None = None) -> int: _write_report(report, None) return 0 if report["static_contract_verified"] else 1 if args.command == "run": - observation = run_live_recovery_rehearsal(kubectl=args.kubectl) + observation = run_live_recovery_rehearsal( + kubectl=args.kubectl, + source_context=args.source_context, + recovery_context=args.recovery_context, + ) report = build_recovery_evidence(observation) _write_report(report, args.output) return 0 if report["local_backup_restore_verified"] else 1 diff --git a/data_agent/platform_truth.py b/data_agent/platform_truth.py index a1c446df..415e2214 100644 --- a/data_agent/platform_truth.py +++ b/data_agent/platform_truth.py @@ -494,6 +494,23 @@ def _config( ), "Production backup controller and independent recovery reader", ), + RuntimeSpec( + "metadata_cross_cluster_recovery_rehearsal", + "recovery_rehearsal", + "governed", + "evidence_durable", + "host-external versioned S3 observation + committed cross-cluster evidence", + "sre", + "local_verification_only", + ("data_agent/metadata_fabric_cross_cluster_recovery.py",), + ( + ( + "data_agent/metadata_fabric_cross_cluster_recovery.py", + "class _ExternalRepositoryRuntime", + ), + ), + "Production backup controller and failure-domain-isolated recovery reader", + ), RuntimeSpec( "datalake_monitor", "monitor_loop", diff --git a/data_agent/test_metadata_fabric_cross_cluster_recovery.py b/data_agent/test_metadata_fabric_cross_cluster_recovery.py new file mode 100644 index 00000000..07b0a5c4 --- /dev/null +++ b/data_agent/test_metadata_fabric_cross_cluster_recovery.py @@ -0,0 +1,289 @@ +import hashlib +import json +from copy import deepcopy +from datetime import UTC, datetime +from pathlib import Path + +import yaml + +from data_agent import metadata_fabric_cross_cluster_recovery as cross_cluster +from data_agent import metadata_fabric_recovery_rehearsal as recovery + + +NOW = datetime(2026, 7, 27, 9, 0, tzinfo=UTC) + + +class _Denied(Exception): + response = {"Error": {"Code": "AccessDenied"}} + + +class _Body: + def __init__(self, payload: bytes) -> None: + self.payload = payload + + def read(self) -> bytes: + return self.payload + + +class _FakeS3: + def __init__(self, role: str, objects: dict) -> None: + self.role = role + self.objects = objects + + def put_object(self, **kwargs): + if self.role == "reader": + raise _Denied + key = str(kwargs["Key"]) + body = bytes(kwargs["Body"]) + version = f"version-{len(self.objects) + 1}" + self.objects[(key, version)] = body + return {"VersionId": version} + + def head_object(self, **kwargs): + body = self.objects[(str(kwargs["Key"]), str(kwargs["VersionId"]))] + return { + "Metadata": {"gda-sha256": hashlib.sha256(body).hexdigest()}, + "ObjectLockMode": cross_cluster.RETENTION_MODE, + "ObjectLockRetainUntilDate": NOW, + "ContentLength": len(body), + } + + def delete_object(self, **kwargs): + raise _Denied + + def get_object(self, **kwargs): + body = self.objects[(str(kwargs["Key"]), str(kwargs["VersionId"]))] + return {"Body": _Body(body)} + + +def _cross_cluster_observation() -> dict: + evidence_path = ( + Path(__file__).resolve().parent.parent + / "docs/evidence/metadata-fabric-recovery-rehearsal-2026-07-27.json" + ) + committed = json.loads(evidence_path.read_text(encoding="utf-8")) + recovered = deepcopy(committed["observation"]) + recovered["observed_at"] = NOW.isoformat() + recovered["started_at"] = NOW.isoformat() + recovered["duration_seconds"] = 120.0 + recovered["cluster"]["recovery_context"] = cross_cluster.RECOVERY_CONTEXT + recovered["cluster"]["recovery_uid"] = "recovery-cluster-uid" + objects = { + name: { + "object_path": f"metadata-fabric/rp-test/{name}.backup", + "version_id": f"version-{index}", + "format": artifact["format"], + "sha256": artifact["sha256"], + "bytes": artifact["bytes"], + "retention_mode": cross_cluster.RETENTION_MODE, + "retained_until": "2026-07-28T09:00:00+00:00", + "writer_delete_denied": True, + "compliance_delete_blocked": True, + } + for index, (name, artifact) in enumerate( + recovered["artifacts"].items(), start=1 + ) + } + recovered["repository_round_trip"] = { + "provider": "minio_s3_compatible", + "repository_runtime": "docker_host_container", + "failure_domain": cross_cluster.FAILURE_DOMAIN, + "bucket": cross_cluster.BUCKET, + "versioning_status": "Enabled", + "object_lock_enabled": True, + "default_retention_mode": cross_cluster.RETENTION_MODE, + "default_retention_days": cross_cluster.RETENTION_DAYS, + "artifact_objects": objects, + "identity_controls": { + "separate_writer_reader": True, + "writer_delete_denied": True, + "reader_write_denied": True, + }, + "local_artifacts_removed_before_download": True, + "round_trip_verified": True, + "transport_tls_verified": False, + "kms_encryption_verified": False, + "production_durability_verified": False, + } + recovery_evidence = recovery.build_recovery_evidence(recovered, now=NOW) + return { + "schema": cross_cluster.OBSERVATION_SCHEMA, + "observed_at": NOW.isoformat(), + "started_at": NOW.isoformat(), + "duration_seconds": 150.0, + "contract": { + "local_static_contract_verified": True, + "contract_fingerprint": "f" * 64, + }, + "repository": { + "runtime": "docker_host_container", + "failure_domain": cross_cluster.FAILURE_DOMAIN, + "minio_image": cross_cluster.MINIO_IMAGE, + "mc_image": cross_cluster.MC_IMAGE, + }, + "recovery_evidence": recovery_evidence, + "runtime_checks": { + "external_repository_container_removed": True, + "external_repository_volume_removed": True, + "runtime_credentials_removed": True, + "recovery_cluster_preserved": True, + }, + } + + +def test_static_cross_cluster_contract_is_valid_and_portable(): + report = cross_cluster.build_cross_cluster_contract_report() + + assert report["local_static_contract_verified"] is True + assert report["errors"] == [] + assert report["source_context"] != report["recovery_context"] + assert report["production_cross_cluster_recovery_verified"] is False + assert all(not Path(item["path"]).is_absolute() for item in report["files"].values()) + + +def test_static_contract_rejects_shared_context_and_production_overclaim(tmp_path): + profile = yaml.safe_load( + cross_cluster.DEFAULT_PROFILE_PATH.read_text(encoding="utf-8") + ) + profile["recovery_cluster"]["context"] = cross_cluster.SOURCE_CONTEXT + profile["claims"]["production_ready"] = True + target = tmp_path / "profile.yaml" + target.write_text(yaml.safe_dump(profile), encoding="utf-8") + + report = cross_cluster.build_cross_cluster_contract_report(profile_path=target) + + assert report["local_static_contract_verified"] is False + rendered = "\n".join(report["errors"]) + assert "recovery cluster context" in rendered + assert "contexts must differ" in rendered + assert "production_ready" in rendered + + +def test_valid_observation_proves_only_host_bounded_cross_cluster_recovery(): + report = cross_cluster.build_cross_cluster_evidence( + _cross_cluster_observation(), now=NOW + ) + + assert report["status"] == "local_cross_cluster_recovery_verified" + assert report["local_cross_cluster_recovery_verified"] is True + assert report["local_external_repository_verified"] is True + assert report["local_writer_reader_identity_separation_verified"] is True + assert report["cross_cluster_recovery_verified"] is False + assert report["production_cross_cluster_recovery_verified"] is False + assert report["source_cluster_loss_verified"] is False + assert report["production_ready"] is False + assert cross_cluster.verify_evidence_integrity(report) == [] + + +def test_cross_cluster_verifier_blocks_shared_cluster_and_identity_drift(): + observation = _cross_cluster_observation() + nested = observation["recovery_evidence"] + nested_observation = nested["observation"] + nested_observation["cluster"]["recovery_uid"] = nested_observation["cluster"]["uid"] + nested_observation["repository_round_trip"]["identity_controls"][ + "reader_write_denied" + ] = False + observation["recovery_evidence"] = recovery.build_recovery_evidence( + nested_observation, now=NOW + ) + + report = cross_cluster.build_cross_cluster_evidence(observation, now=NOW) + + assert report["status"] == "blocked" + rendered = "\n".join(report["errors"]) + assert "distinct local clusters" in rendered + assert "reader write was not denied" in rendered + + +def test_cross_cluster_verifier_blocks_repository_provenance_and_retention_drift(): + observation = _cross_cluster_observation() + observation["repository"]["minio_image"] = "minio/minio:latest" + round_trip = observation["recovery_evidence"]["observation"][ + "repository_round_trip" + ] + round_trip["transport_tls_verified"] = True + round_trip["artifact_objects"]["openmetadata_postgresql"][ + "retained_until" + ] = "2026-07-27T08:59:59+00:00" + nested_observation = observation["recovery_evidence"]["observation"] + observation["recovery_evidence"] = recovery.build_recovery_evidence( + nested_observation, now=NOW + ) + + report = cross_cluster.build_cross_cluster_evidence(observation, now=NOW) + + assert report["status"] == "blocked" + rendered = "\n".join(report["errors"]) + assert "MinIO image does not match" in rendered + assert "may not claim transport_tls_verified" in rendered + assert "retention is not in the future" in rendered + + +def test_external_round_trip_uses_writer_and_reader_and_removes_local_files(tmp_path): + objects: dict = {} + admin = _FakeS3("admin", objects) + writer = _FakeS3("writer", objects) + reader = _FakeS3("reader", objects) + paths: dict[str, Path] = {} + artifacts: dict[str, dict] = {} + for index, (name, format_name) in enumerate( + cross_cluster.repository.EXPECTED_ARTIFACTS.items(), start=1 + ): + path = tmp_path / f"{name}.backup" + payload = f"artifact-{index}".encode("ascii") + path.write_bytes(payload) + paths[name] = path + artifacts[name] = { + "format": format_name, + "sha256": hashlib.sha256(payload).hexdigest(), + "bytes": len(payload), + } + + round_trip = cross_cluster._ExternalRepositoryRoundTrip( + admin, + writer, + reader, + { + "versioning_status": "Enabled", + "object_lock_enabled": True, + "default_retention_mode": cross_cluster.RETENTION_MODE, + "default_retention_days": cross_cluster.RETENTION_DAYS, + }, + "rp-test", + ) + report = round_trip(paths, artifacts) + + assert report["local_artifacts_removed_before_download"] is True + assert report["identity_controls"] == { + "separate_writer_reader": True, + "writer_delete_denied": True, + "reader_write_denied": True, + } + assert all(path.is_file() for path in paths.values()) + assert all( + hashlib.sha256(paths[name].read_bytes()).hexdigest() + == artifacts[name]["sha256"] + for name in paths + ) + + +def test_committed_cross_cluster_evidence_is_integral_and_current(): + evidence_path = ( + Path(__file__).resolve().parent.parent + / "docs/evidence/metadata-fabric-cross-cluster-recovery-2026-07-27.json" + ) + report = json.loads(evidence_path.read_text(encoding="utf-8")) + nested = report["observation"]["recovery_evidence"] + + assert cross_cluster.verify_evidence_integrity(report) == [] + assert report["observation"]["contract"]["contract_fingerprint"] == ( + cross_cluster.build_cross_cluster_contract_report()["contract_fingerprint"] + ) + assert recovery.verify_evidence_integrity(nested) == [] + assert nested["observation"]["contract"]["contract_fingerprint"] == ( + recovery.build_recovery_contract_report()["contract_fingerprint"] + ) + assert report["local_cross_cluster_recovery_verified"] is True + assert report["production_cross_cluster_recovery_verified"] is False + assert report["source_cluster_loss_verified"] is False + assert report["production_ready"] is False diff --git a/data_agent/test_metadata_fabric_recovery_rehearsal.py b/data_agent/test_metadata_fabric_recovery_rehearsal.py index e3b0c10f..6062f057 100644 --- a/data_agent/test_metadata_fabric_recovery_rehearsal.py +++ b/data_agent/test_metadata_fabric_recovery_rehearsal.py @@ -124,6 +124,18 @@ def test_static_recovery_contract_is_valid_and_portable(): ) +def test_command_runner_binds_every_kubectl_call_to_its_context(): + runner = recovery._CommandRunner("/usr/local/bin/kubectl", "kind-recovery") + + assert runner.kubectl_args(["get", "nodes"]) == [ + "/usr/local/bin/kubectl", + "--context", + "kind-recovery", + "get", + "nodes", + ] + + def test_static_contract_rejects_api_token_mount(tmp_path): target = tmp_path / "recovery" shutil.copytree(recovery.DEFAULT_RECOVERY_MANIFEST_DIR, target) @@ -158,6 +170,37 @@ def test_valid_observation_proves_only_local_recovery(): assert recovery.verify_evidence_integrity(report) == [] +def test_distinct_cluster_observation_proves_only_local_cross_cluster_recovery(): + observation = _observation() + observation["cluster"]["recovery_context"] = "kind-gda-metadata-recovery" + observation["cluster"]["recovery_uid"] = ( + "00000000-0000-4000-8000-000000000004" + ) + + report = recovery.build_recovery_evidence(observation, now=NOW) + + assert report["status"] == "local_cross_cluster_backup_restore_verified" + assert report["backup_restore_scope"] == ( + "local_distinct_kubernetes_clusters_new_namespace_and_pvcs" + ) + assert report["local_cross_cluster_recovery_verified"] is True + assert report["cross_cluster_recovery_verified"] is False + assert report["production_cross_cluster_recovery_verified"] is False + assert report["production_ready"] is False + assert recovery.verify_evidence_integrity(report) == [] + + +def test_recovery_verifier_blocks_context_uid_isolation_disagreement(): + observation = _observation() + observation["cluster"]["recovery_context"] = "kind-gda-metadata-recovery" + observation["cluster"]["recovery_uid"] = observation["cluster"]["uid"] + + report = recovery.build_recovery_evidence(observation, now=NOW) + + assert report["status"] == "blocked" + assert any("context/identity isolation disagrees" in item for item in report["errors"]) + + def test_recovery_verifier_blocks_database_drift_and_shared_search_identity(): observation = _observation() observation["recovered_markers"]["openmetadata_postgresql"][ diff --git a/data_agent/test_platform_truth.py b/data_agent/test_platform_truth.py index 3304f9a9..18a03d5b 100644 --- a/data_agent/test_platform_truth.py +++ b/data_agent/test_platform_truth.py @@ -188,6 +188,11 @@ def test_repository_source_access_and_runtime_baselines_match(): and item["production_role"] == "local_verification_only" for item in static_report["runtime"]["inventory"] ) + assert any( + item["runtime_id"] == "metadata_cross_cluster_recovery_rehearsal" + and item["production_role"] == "local_verification_only" + for item in static_report["runtime"]["inventory"] + ) def test_runtime_report_detects_unregistered_background_mechanism(tmp_path): diff --git a/docs/architecture-decisions/adr-037-local-metadata-fabric-foundation-sandbox.md b/docs/architecture-decisions/adr-037-local-metadata-fabric-foundation-sandbox.md index 723236ea..50a812b8 100644 --- a/docs/architecture-decisions/adr-037-local-metadata-fabric-foundation-sandbox.md +++ b/docs/architecture-decisions/adr-037-local-metadata-fabric-foundation-sandbox.md @@ -106,3 +106,5 @@ Pod 重建后 PVC/schema/index 连续只证明本地持久卷被重新挂载和 2026-07-27 更新： [ADR-038](adr-038-local-metadata-fabric-recovery-rehearsal.md) 已完成 M2b-1 本地同集群新 namespace/PVC 的三存储恢复演练；生产 backup target/retention、PITR、RPO/RTO 与跨集群恢复仍属于上述 next gate。 2026-07-27 后续更新： [ADR-039](adr-039-local-locked-metadata-backup-repository.md) 已完成 M2b-2 本地隔离 S3-compatible repository 的 versioning/Object Lock、删除本地 artifact、按 version 下载并恢复；外部生产 bucket、COMPLIANCE retention、TLS/KMS、独立 workload identity 与跨集群恢复仍属于 next gate。 + +2026-07-27 M2b-3 更新：[ADR-040](adr-040-local-cross-cluster-metadata-recovery.md) 已在 `docker-desktop` 与独立 kind cluster 之间，从两个 Kubernetes cluster 外的 Docker-host MinIO 完成 `COMPLIANCE/1 day`、独立 writer/reader 和三存储恢复。该证据仍共享同一 Docker Desktop 主机；生产 bucket/KMS/TLS/workload identity、source-host loss 与 RPO/RTO 仍属于 next gate。 diff --git a/docs/architecture-decisions/adr-038-local-metadata-fabric-recovery-rehearsal.md b/docs/architecture-decisions/adr-038-local-metadata-fabric-recovery-rehearsal.md index ab635538..99877ff7 100644 --- a/docs/architecture-decisions/adr-038-local-metadata-fabric-recovery-rehearsal.md +++ b/docs/architecture-decisions/adr-038-local-metadata-fabric-recovery-rehearsal.md @@ -6,7 +6,7 @@ **Decision owners**: Metadata Platform, SRE, Security, Platform Architecture -**Related decisions**: [ADR-006](adr-006-openmetadata-governance-and-active-metadata-platform.md) · [ADR-037](adr-037-local-metadata-fabric-foundation-sandbox.md) · [ADR-039](adr-039-local-locked-metadata-backup-repository.md) +**Related decisions**: [ADR-006](adr-006-openmetadata-governance-and-active-metadata-platform.md) · [ADR-037](adr-037-local-metadata-fabric-foundation-sandbox.md) · [ADR-039](adr-039-local-locked-metadata-backup-repository.md) · [ADR-040](adr-040-local-cross-cluster-metadata-recovery.md) **Evidence**: [metadata-fabric-recovery-rehearsal-2026-07-27.json](../evidence/metadata-fabric-recovery-rehearsal-2026-07-27.json) @@ -74,15 +74,15 @@ OpenSearch source/recovered 必须完全匹配 version、索引总数/名称和 2026-07-27 在 ADR-039 repository round-trip 接入后的复测结果： -- 恢复子流程耗时 `115.137` 秒；该观测值不是 RTO SLO； -- OpenMetadata PostgreSQL backup `1,209,521` bytes，恢复 176 张表； +- 恢复子流程耗时 `111.123` 秒；该观测值不是 RTO SLO； +- OpenMetadata PostgreSQL backup `1,298,657` bytes，恢复 176 张表； - Gravitino PostgreSQL backup `188,109` bytes，恢复 39 张表； -- OpenSearch snapshot archive `387,684` bytes，恢复 79 个索引； +- OpenSearch snapshot archive `396,797` bytes，恢复 79 个索引； - 两个 PostgreSQL 的 table/row/sequence/extension fingerprints 全部一致； - OpenSearch index/document fingerprints 一致，source/recovery cluster UUID 不同； - 恢复 PVC 为 2 Gi、2 Gi、8 Gi，均为新的 Bound identity； - source provider 恢复 Ready，恢复 namespace、本地 artifact 与 source snapshot staging 全部清理； -- evidence fingerprint 为 `3cf46cc83a8feaa4142893a06f84e9008a7d805a64fb6708d176ca976a4dbd62`。 +- evidence fingerprint 为 `da1214294045f8b0abe2e2775b81ef33967eac9ab0e97055ae80212ac0c08a4b`。 Evidence 不包含 Secret，且由 required CI 的静态 validator、单元负例和 evidence integrity test 约束。 @@ -90,6 +90,6 @@ Evidence 不包含 Secret，且由 required CI 的静态 validator、单元负 **Positive**：M2b 第一次拥有覆盖 OpenMetadata PostgreSQL、Gravitino PostgreSQL 和 OpenSearch 的真实逻辑备份、隔离恢复、内容一致性和 cleanup 闭环；升级演练不再建立在“原 PVC 还能挂载”的假设上。 -**Negative**：演练会短暂停止两个 provider；默认 runner 仍允许仅使用本机临时 artifact。ADR-039 增加的 repository callback 只验证同集群本地 MinIO round-trip，仍没有验证云端 durability、TLS/KMS、生产 retention、PITR、跨集群调度和 RPO/RTO。 +**Negative**：演练会短暂停止两个 provider；默认 runner 仍允许仅使用本机临时 artifact。ADR-039 增加的 repository callback 只验证同集群本地 MinIO round-trip；ADR-040 虽复用该 runner 完成本机双集群恢复，仍没有验证云端 durability、TLS/KMS、生产 retention、PITR、source-host loss 和 RPO/RTO。 -**Next gate**：M2b 继续完成真实外部 backup account/bucket、COMPLIANCE retention、TLS/KMS、独立 writer/reader identity 与跨集群恢复，再完成 OIDC/workload identity、NetworkPolicy enforcement、upgrade/rollback、metrics/OTel、registry provenance 和 owner/runbook；之后才进入 M3。 +**Next gate**：M2b 继续完成 source host/cluster 外的生产 backup account/bucket、COMPLIANCE retention、TLS/KMS、workload identity、source-loss recovery 与 RPO/RTO，再完成 OIDC、NetworkPolicy enforcement、upgrade/rollback、metrics/OTel、registry provenance 和 owner/runbook；之后才进入 M3。 diff --git a/docs/architecture-decisions/adr-039-local-locked-metadata-backup-repository.md b/docs/architecture-decisions/adr-039-local-locked-metadata-backup-repository.md index 1cefda75..c808dde6 100644 --- a/docs/architecture-decisions/adr-039-local-locked-metadata-backup-repository.md +++ b/docs/architecture-decisions/adr-039-local-locked-metadata-backup-repository.md @@ -6,7 +6,7 @@ **Decision owners**: Metadata Platform, SRE, Security, Platform Architecture -**Related decisions**: [ADR-019](adr-019-configuration-and-runtime-truth.md) · [ADR-037](adr-037-local-metadata-fabric-foundation-sandbox.md) · [ADR-038](adr-038-local-metadata-fabric-recovery-rehearsal.md) +**Related decisions**: [ADR-019](adr-019-configuration-and-runtime-truth.md) · [ADR-037](adr-037-local-metadata-fabric-foundation-sandbox.md) · [ADR-038](adr-038-local-metadata-fabric-recovery-rehearsal.md) · [ADR-040](adr-040-local-cross-cluster-metadata-recovery.md) **Evidence**: [metadata-fabric-backup-repository-2026-07-27.json](../evidence/metadata-fabric-backup-repository-2026-07-27.json) @@ -63,14 +63,14 @@ ADR-038 runner 增加可选 artifact callback，不改变默认恢复路径。M2 2026-07-27 的真实本地演练结果： -- 总耗时 `143.107` 秒，repository-backed recovery 子流程 `115.137` 秒；二者都不是 RTO SLO； -- 独立 repository PVC 为 8 Gi，UID `bd4056b1-0782-4c2f-99ca-4c06ee8021f9`； +- 总耗时 `137.914` 秒，repository-backed recovery 子流程 `111.123` 秒；二者都不是 RTO SLO； +- 独立 repository PVC 为 8 Gi，UID `22204802-c426-4840-8045-ad03a53c88e4`； - bucket versioning 与 Object Lock 均启用；三个对象均获得唯一 version ID、`GOVERNANCE` retention 和次日 retain-until； - 三个 retained version 的无 bypass 删除全部被拒绝； - 本地 artifact 全部删除后重新下载，恢复 OpenMetadata 176 张表、Gravitino 39 张表和 OpenSearch 79 个索引； - source 五个 Pod 恢复 Ready，repository/recovery namespace、临时 PVC、credential 与 port-forward 全部清理； -- repository evidence fingerprint 为 `07834430fb140e147624f4ab3c93e6d7907648e91a149cb19db178ea6085d1ed`； -- recovery evidence fingerprint 为 `3cf46cc83a8feaa4142893a06f84e9008a7d805a64fb6708d176ca976a4dbd62`。 +- repository evidence fingerprint 为 `2897f9e6aaae21fb366da0b72edea5cf072d5b2c1aeac0807d263bd0a5f5f133`； +- recovery evidence fingerprint 为 `da1214294045f8b0abe2e2775b81ef33967eac9ab0e97055ae80212ac0c08a4b`。 ## Claim Boundary @@ -95,4 +95,4 @@ ADR-038 runner 增加可选 artifact callback，不改变默认恢复路径。M2 **Negative**：本地 MinIO 与 source 仍在同一 kind cluster；root 管理员和 namespace/PVC 删除仍能绕过 API retention；没有 TLS、KMS、workload identity、跨 account 或第二 cluster。 -**Next gate**：在独立 account/project 建立真实 S3-compatible bucket、KMS 与 writer/reader identity，验证 COMPLIANCE retention、独立 read-after-write、source cluster loss 后的跨集群恢复，再冻结 RPO/RTO。若云/provider 不支持等价 Object Lock、KMS 或 identity，必须阻断该 DeploymentProfile，不能降级为普通 versioned bucket。 +**Next gate**：[ADR-040](adr-040-local-cross-cluster-metadata-recovery.md) 已将本地 repository 移出两个 Kubernetes cluster，并验证本机双集群恢复、`COMPLIANCE/1 day` 与独立 MinIO writer/reader。下一步仍须在 source host/cluster 外的独立 account/project 建立真实 S3-compatible bucket、KMS、TLS 与 workload identity，验证 source loss 后恢复并冻结 RPO/RTO。若云/provider 不支持等价 Object Lock、KMS、identity 或独立故障域，必须阻断该 DeploymentProfile，不能降级为普通 versioned bucket。 diff --git a/docs/architecture-decisions/adr-040-local-cross-cluster-metadata-recovery.md b/docs/architecture-decisions/adr-040-local-cross-cluster-metadata-recovery.md new file mode 100644 index 00000000..979254d6 --- /dev/null +++ b/docs/architecture-decisions/adr-040-local-cross-cluster-metadata-recovery.md @@ -0,0 +1,97 @@ +# ADR-040: Local Cross-Cluster Metadata Recovery + +**Status**: Accepted + +**Date**: 2026-07-27 + +**Decision owners**: Metadata Platform, SRE, Security, Platform Architecture + +**Related decisions**: [ADR-019](adr-019-configuration-and-runtime-truth.md) · [ADR-038](adr-038-local-metadata-fabric-recovery-rehearsal.md) · [ADR-039](adr-039-local-locked-metadata-backup-repository.md) + +**Evidence**: [metadata-fabric-cross-cluster-recovery-2026-07-27.json](../evidence/metadata-fabric-cross-cluster-recovery-2026-07-27.json) + +## Context + +ADR-039 证明三份 metadata backup artifact 可以离开 runner 临时目录，在同一 Kubernetes 集群内的独立 MinIO/PVC 中获得 version 与 Object Lock 后重新下载和恢复，但 source 与 repository 仍共享 cluster lifecycle。下一步需要验证 recovery workload 能否在具有独立 API、namespace、node runtime 与 cluster UID 的第二个 Kubernetes cluster 中，从 Kubernetes 之外的 repository 恢复。 + +本地只有一台 Docker Desktop 主机，没有获准使用的云 backup account、KMS key、OIDC/workload identity 或异地主机。因此本决策只能证明同一 Docker host 内的双集群恢复，不能证明 source-host loss、生产 durability、跨区域 DR 或 RPO/RTO。 + +## Options Considered + +| 方案 | 优点 | 代价/风险 | 结论 | +|---|---|---|---| +| 继续在 source cluster 新建 namespace | 最少资源，复用 ADR-038 | 不是独立 cluster failure domain | 拒绝作为 M2b-3 证据 | +| 在 source cluster 内运行 vcluster | API 表面独立，启动快 | 仍共享 source cluster node、CNI、storage 与 lifecycle | 拒绝作为故障域证明 | +| Docker Desktop kubeadm 单节点 cluster | 与桌面产品集成 | 只能保留一个 Docker Desktop cluster，无法同时作为 source 与 recovery | 不采用 | +| 保留 Docker Desktop source，另建固定版本 kind cluster | 两个 context 和 cluster UID 可同时验证；可销毁重建 recovery cluster | 仍共享 Docker daemon、主机、电源和网络 | 采用，声明严格限定在本机 | + +## Decision + +### 1. 双集群边界 + +source 固定为 `docker-desktop`，recovery 固定为 `kind-gda-metadata-recovery`，kind node image 固定为 `kindest/node:v1.35.5`。每个 `kubectl` 调用都显式绑定 context，不依赖全局 current-context。runner 在开始前读取两个 `kube-system` namespace UID；context 是否相同必须与 cluster UID 是否相同一致，且 M2b-3 要求二者都不同。 + +source provider 只在有界备份窗口内 quiesce。三份 artifact 恢复到第二集群的 `gda-metadata-recovery-rehearsal` namespace 和新 PVC，source namespace/PVC 不迁移，source cluster 也不被删除。因此 `source_cluster_loss_verified=false`。 + +### 2. Kubernetes 外 repository + +固定版本 MinIO 作为 Docker-host container 运行，data 使用独立临时 Docker volume，不属于 source 或 recovery Kubernetes cluster。bucket 固定启用 versioning 和 `COMPLIANCE/1 day` Object Lock。 + +runner 动态创建三种本地 S3 identity：管理员只用于初始化和验证 COMPLIANCE 删除阻断；writer 可写入和读取但没有删除权限；reader 只读且写入 probe 必须被拒绝。三份 artifact 必须获得唯一 version ID、未来 retain-until、匹配的 SHA-256/bytes；writer 删除和管理员删除 retained version 都必须失败。本地 artifact 删除后只能按 version ID 由 reader 下载，再进入第二集群恢复。 + +这些是本地 MinIO 用户，不是云 workload identity；repository 使用 loopback HTTP，没有 TLS 或 KMS，因此相应生产声明保持 `false`。 + +### 3. 恢复与 cleanup + +复用 ADR-038 的 PostgreSQL custom dump、OpenSearch native snapshot、内容 marker 和新 PVC 验证，但 source 与 recovery 使用独立 command runner。恢复后必须同时满足： + +1. OpenMetadata PostgreSQL、Gravitino PostgreSQL 和 OpenSearch 内容 marker 与 source 一致； +2. source provider 全部恢复 Ready； +3. recovery namespace/PVC 删除； +4. Docker-host MinIO container、volume 和运行期凭据删除； +5. recovery cluster UID 保持可访问且未被 rehearsal 清理。 + +`platform_truth.RUNTIME_INVENTORY` 将该 runtime 登记为 `local_verification_only`。它不是 scheduler、生产 backup controller 或运行状态权威；唯一持久结果是 committed、可校验且绑定当前合同指纹的 evidence。 + +## Verification + +2026-07-27 的真实本地演练结果： + +- 总耗时 `122.791` 秒；该观测值不是 RTO SLO； +- source cluster UID `c3c9b...445ab`，recovery cluster UID `04ddf...94858`； +- OpenMetadata PostgreSQL artifact `1,269,181` bytes，恢复 176 张表； +- Gravitino PostgreSQL artifact `188,109` bytes，恢复 39 张表； +- OpenSearch artifact `393,375` bytes，恢复 79 个索引； +- bucket versioning=`Enabled`、Object Lock=`Enabled`、retention=`COMPLIANCE/1 day`；writer delete、reader write 和管理员删除 retained version 均被拒绝； +- source 五个 Pod 恢复 Ready；recovery namespace、Docker container/volume 和运行期凭据全部清理；recovery cluster 保持 Ready； +- contract fingerprint `352f59b10d845b288a465b59386e7a15f42aa706559fe337db5ed1dcb7234815`； +- evidence fingerprint `9eaf8cec9ec2d3763260c271c0c27c1c3251717820d38aadfef0bec7d7a574a8`。 + +required CI 同时校验 evidence 自身完整性、嵌套 recovery evidence 与当前合同指纹绑定。 + +## Claim Boundary + +允许声明： + +- `local_cross_cluster_recovery_verified=true`； +- scope 仅为 `local_same_host_distinct_kubernetes_clusters_external_s3_repository`； +- `local_external_repository_verified=true`； +- `local_writer_reader_identity_separation_verified=true`。 + +仍固定为 `false`： + +- `cross_cluster_recovery_verified` 与 `production_cross_cluster_recovery_verified`； +- `production_backup_target_verified`、`production_retention_verified`； +- `production_kms_verified`、`production_tls_verified`； +- `source_cluster_loss_verified`、`cross_region_recovery_verified`； +- `rpo_slo_verified`、`rto_slo_verified`； +- `oidc_verified`、`network_policy_enforcement_verified`； +- `writes_to_gda_enabled`、`production_ready`。 + +## Consequences + +**Positive**：恢复链第一次跨越两个独立 Kubernetes API/cluster UID；repository 也移出两个 Kubernetes lifecycle，并以 COMPLIANCE retention 和独立 writer/reader 权限完成真实恢复。 + +**Negative**：两个 cluster 和 repository 仍共享一台 Docker Desktop 主机与 Docker daemon；删除 Docker volume、主机故障或管理员越权仍可物理破坏数据。静态 MinIO identity、loopback HTTP、一天 retention 和手工准备 kind cluster 都不能进入生产 profile。 + +**Next gate**：在 source host/cluster 之外的独立 account/project 建立生产 bucket、KMS、TLS 与 workload identity，执行 source cluster/host unavailable 条件下的恢复并冻结 RPO/RTO；随后完成 OIDC、NetworkPolicy enforcement、upgrade/rollback、registry provenance、metrics/OTel 和 owner/runbook。任何 provider 不满足 COMPLIANCE Object Lock、KMS、identity 或独立故障域时必须阻断该 DeploymentProfile。 diff --git a/docs/evidence/metadata-fabric-backup-repository-2026-07-27.json b/docs/evidence/metadata-fabric-backup-repository-2026-07-27.json index a5514bca..75ab2ce9 100644 --- a/docs/evidence/metadata-fabric-backup-repository-2026-07-27.json +++ b/docs/evidence/metadata-fabric-backup-repository-2026-07-27.json @@ -15,8 +15,8 @@ "cross_region_recovery_verified": false, "environment": "local_isolated_s3_backup_repository_rehearsal", "errors": [], - "evidence_fingerprint": "07834430fb140e147624f4ab3c93e6d7907648e91a149cb19db178ea6085d1ed", - "generated_at": "2026-07-27T07:37:44.190592+00:00", + "evidence_fingerprint": "2897f9e6aaae21fb366da0b72edea5cf072d5b2c1aeac0807d263bd0a5f5f133", + "generated_at": "2026-07-27T09:07:24.757688+00:00", "local_repository_round_trip_verified": true, "network_policy_enforcement_verified": false, "observation": { @@ -24,7 +24,7 @@ "context": "docker-desktop", "repository_namespace": { "name": "gda-metadata-backup-repository", - "uid": "b3f62f00-871f-43ee-bde6-e37c4c17cc71" + "uid": "3141bd3a-4157-4078-a45f-b7d06cd112d3" }, "source_namespace": { "name": "gda-metadata-sandbox", @@ -33,11 +33,11 @@ "uid": "c3c9bbab-2b36-4359-a7ac-e3de194445ab" }, "contract": { - "contract_fingerprint": "fc7578b2c71aacfeb0e78f4ccc1858fc7e020631e5436068f26ae0501888d928", + "contract_fingerprint": "ee011bd0c5f986f50c68a3d8f98899ef300fcd1363223dcc1485f7d20a5f9a45", "local_static_contract_verified": true }, - "duration_seconds": 143.107, - "observed_at": "2026-07-27T07:37:44.190592+00:00", + "duration_seconds": 137.914, + "observed_at": "2026-07-27T09:07:24.757688+00:00", "recovery_evidence": { "backup_restore_scope": "local_same_cluster_new_namespace_and_pvcs", "backup_restore_verified": true, @@ -54,34 +54,37 @@ "cross_region_recovery_verified": false, "environment": "local_isolated_recovery_rehearsal", "errors": [], - "evidence_fingerprint": "3cf46cc83a8feaa4142893a06f84e9008a7d805a64fb6708d176ca976a4dbd62", - "generated_at": "2026-07-27T07:37:28.806368+00:00", + "evidence_fingerprint": "da1214294045f8b0abe2e2775b81ef33967eac9ab0e97055ae80212ac0c08a4b", + "generated_at": "2026-07-27T09:07:09.480972+00:00", "local_backup_restore_verified": true, + "local_cross_cluster_recovery_verified": false, "network_policy_enforcement_verified": false, "observation": { "artifacts": { "gravitino_postgresql": { "bytes": 188109, "format": "postgresql_custom_dump_v1", - "sha256": "b56fae7ba17166690df1ac52814734d10aaab6d036e467141601aa13913493fa" + "sha256": "6d6013aa01872c335eb07ea3733eae49acbdab5b6f9d241144663bf21585d2f1" }, "openmetadata_postgresql": { - "bytes": 1209521, + "bytes": 1298657, "format": "postgresql_custom_dump_v1", - "sha256": "2f6de329c6ebff68227a27958040c29d41396c9f1a61421eaf0630ac344a7e26" + "sha256": "7aff5a3b62ddfae6ccc46cc1dcf13b72c326085b08a6f3a78a78c87609eaacad" }, "opensearch": { - "bytes": 387684, + "bytes": 396797, "format": "opensearch_fs_snapshot_tar_gzip_v1", - "sha256": "7732ffc33827b9ffc4ed5e3f3744e4f54b1a73a2d9e0444fe7b34e16c8f37da3" + "sha256": "b36dec1ca16827ab5c363bb98a7f994d211bc139d2ad84b2498fdf3bcdfdc59d" } }, "cluster": { "context": "docker-desktop", + "recovery_context": "docker-desktop", "recovery_namespace": { "name": "gda-metadata-recovery-rehearsal", - "uid": "d9236d7c-8c9c-45af-a46b-7148b47b4db0" + "uid": "58c6389c-3f89-4f83-a392-02d9f3c679a7" }, + "recovery_uid": "c3c9bbab-2b36-4359-a7ac-e3de194445ab", "source_namespace": { "name": "gda-metadata-sandbox", "uid": "812c9c5c-9ce2-409e-a6a8-a3f24785aa0c" @@ -89,11 +92,11 @@ "uid": "c3c9bbab-2b36-4359-a7ac-e3de194445ab" }, "contract": { - "contract_fingerprint": "83e3f86542076987b36cb8b99fb01c2db7e92c15624164f9766ff0a6fe26640a", + "contract_fingerprint": "afb168b7586113eda95b874c10351f1e4f5eb33c6e661e1bfedfda7103ee6121", "static_contract_verified": true }, - "duration_seconds": 115.137, - "observed_at": "2026-07-27T07:37:28.805987+00:00", + "duration_seconds": 111.123, + "observed_at": "2026-07-27T09:07:09.480617+00:00", "recovered_markers": { "gravitino_postgresql": { "extension_fingerprint": "6a0a92045f06baabcf22b103007285ec0bcb00cdbf6f6f30b34b0a41a73c6c2f", @@ -107,14 +110,14 @@ "openmetadata_postgresql": { "extension_fingerprint": "5425606415e192199fdbef130acb53a732064cb692916f5278b9e8eebc5b2f59", "image_version": "16.10-bookworm", - "row_count_fingerprint": "dbb8f15b9df8780d3be128ef2a9ddad7b292980e811a81a65349e831e64b40f9", - "sequence_state_fingerprint": "769c215213fe71957ea945bd85ad0db889cda410f47443173f19457a17af5ac4", + "row_count_fingerprint": "97d270fe6fb399e6fc3a4788ba8da13d647772857c8b61a60d6695c30cd04cb9", + "sequence_state_fingerprint": "089f6120440918b8ea1502c0372ee8f687c5284aa17b26bf8dc826c5b7500d39", "server_version": "16.10 (Debian 16.10-1.pgdg12+1)", "table_count": 176, "table_name_fingerprint": "9b56cb8952f4680c1972aeafb461b0ef8c204326432ed8bab6226e2eedbb1167" }, "opensearch": { - "cluster_uuid": "9o2bzn8fS5mfJ0nEHkGQlQ", + "cluster_uuid": "sY84Z4TpSHyS5uywqpyt9Q", "document_count_fingerprint": "b9a9da89701697d1778735a3cd1196422f624049847009a593c56e5fdfe7f109", "index_count": 79, "index_name_fingerprint": "5dd3252013d217ea4e014951fbd687ef912dac847ee8d385adf6bd561070f00a", @@ -126,20 +129,20 @@ "recovery-gravitino-data-recovery-gravitino-postgresql-0": { "capacity": "2Gi", "phase": "Bound", - "uid": "9ba5f905-6f4d-4763-b9ba-6092207036d0", - "volume_name": "pvc-9ba5f905-6f4d-4763-b9ba-6092207036d0" + "uid": "abe89775-0d29-4186-b032-75f8c5c77ce5", + "volume_name": "pvc-abe89775-0d29-4186-b032-75f8c5c77ce5" }, "recovery-openmetadata-data-recovery-openmetadata-postgresql-0": { "capacity": "2Gi", "phase": "Bound", - "uid": "6d1b0cae-869e-4923-bd66-315b29abf887", - "volume_name": "pvc-6d1b0cae-869e-4923-bd66-315b29abf887" + "uid": "6383ada0-c77b-4614-b389-ffc099625f96", + "volume_name": "pvc-6383ada0-c77b-4614-b389-ffc099625f96" }, "recovery-opensearch-data-recovery-opensearch-0": { "capacity": "8Gi", "phase": "Bound", - "uid": "c3d5585e-89f6-45ee-b43f-f2a155204916", - "volume_name": "pvc-c3d5585e-89f6-45ee-b43f-f2a155204916" + "uid": "ee77c740-cfd2-4f21-9b11-4129b0e2b389", + "volume_name": "pvc-ee77c740-cfd2-4f21-9b11-4129b0e2b389" } }, "repository_round_trip": { @@ -147,32 +150,32 @@ "gravitino_postgresql": { "bytes": 188109, "format": "postgresql_custom_dump_v1", - "object_path": "metadata-fabric/rp-20260727073521z/gravitino_postgresql.dump", - "retained_until": "2026-07-28T07:36:07.228000+00:00", + "object_path": "metadata-fabric/rp-20260727090506z/gravitino_postgresql.dump", + "retained_until": "2026-07-28T09:05:51.903000+00:00", "retained_version_delete_blocked": true, "retention_mode": "GOVERNANCE", - "sha256": "b56fae7ba17166690df1ac52814734d10aaab6d036e467141601aa13913493fa", - "version_id": "ea0d2e75-185d-4f44-8ed9-74655b0423db" + "sha256": "6d6013aa01872c335eb07ea3733eae49acbdab5b6f9d241144663bf21585d2f1", + "version_id": "ce7ccab6-6706-42ad-a3c1-de284563fd97" }, "openmetadata_postgresql": { - "bytes": 1209521, + "bytes": 1298657, "format": "postgresql_custom_dump_v1", - "object_path": "metadata-fabric/rp-20260727073521z/openmetadata_postgresql.dump", - "retained_until": "2026-07-28T07:36:07.241000+00:00", + "object_path": "metadata-fabric/rp-20260727090506z/openmetadata_postgresql.dump", + "retained_until": "2026-07-28T09:05:51.917000+00:00", "retained_version_delete_blocked": true, "retention_mode": "GOVERNANCE", - "sha256": "2f6de329c6ebff68227a27958040c29d41396c9f1a61421eaf0630ac344a7e26", - "version_id": "b6ce20d7-6e95-4612-9740-d0bd5682ff81" + "sha256": "7aff5a3b62ddfae6ccc46cc1dcf13b72c326085b08a6f3a78a78c87609eaacad", + "version_id": "0ebe97e1-c692-4a84-b565-3b0d5426737c" }, "opensearch": { - "bytes": 387684, + "bytes": 396797, "format": "opensearch_fs_snapshot_tar_gzip_v1", - "object_path": "metadata-fabric/rp-20260727073521z/opensearch.tar.gz", - "retained_until": "2026-07-28T07:36:07.256000+00:00", + "object_path": "metadata-fabric/rp-20260727090506z/opensearch.tar.gz", + "retained_until": "2026-07-28T09:05:51.930000+00:00", "retained_version_delete_blocked": true, "retention_mode": "GOVERNANCE", - "sha256": "7732ffc33827b9ffc4ed5e3f3744e4f54b1a73a2d9e0444fe7b34e16c8f37da3", - "version_id": "a9591f1d-c984-4323-925d-0c4ff0b0de04" + "sha256": "b36dec1ca16827ab5c363bb98a7f994d211bc139d2ad84b2498fdf3bcdfdc59d", + "version_id": "f3d0f897-29a6-4317-a233-d2a93c7594d7" } }, "bucket": "gda-metadata-fabric-backups", @@ -210,8 +213,8 @@ "openmetadata_postgresql": { "extension_fingerprint": "5425606415e192199fdbef130acb53a732064cb692916f5278b9e8eebc5b2f59", "image_version": "16.10-bookworm", - "row_count_fingerprint": "dbb8f15b9df8780d3be128ef2a9ddad7b292980e811a81a65349e831e64b40f9", - "sequence_state_fingerprint": "769c215213fe71957ea945bd85ad0db889cda410f47443173f19457a17af5ac4", + "row_count_fingerprint": "97d270fe6fb399e6fc3a4788ba8da13d647772857c8b61a60d6695c30cd04cb9", + "sequence_state_fingerprint": "089f6120440918b8ea1502c0372ee8f687c5284aa17b26bf8dc826c5b7500d39", "server_version": "16.10 (Debian 16.10-1.pgdg12+1)", "table_count": 176, "table_name_fingerprint": "9b56cb8952f4680c1972aeafb461b0ef8c204326432ed8bab6226e2eedbb1167" @@ -225,11 +228,12 @@ "version": "3.3.2" } }, - "started_at": "2026-07-27T07:35:33.668759+00:00" + "started_at": "2026-07-27T09:05:18.357484+00:00" }, - "observation_fingerprint": "938c0f9842cf0c6218ad4ade675b4e773b538270a605ca33d9215859a0e5f6a1", + "observation_fingerprint": "06a6d5bba92a3f75b5b25cf3ea2c7f91d321454eaf4b5f5608416f468cff0dc1", "oidc_verified": false, "production_backup_restore_verified": false, + "production_cross_cluster_recovery_verified": false, "production_ready": false, "recovery_namespace": "gda-metadata-recovery-rehearsal", "rpo_slo_verified": false, @@ -244,40 +248,40 @@ "capacity": "8Gi", "name": "repository-data-metadata-backup-minio-0", "phase": "Bound", - "uid": "bd4056b1-0782-4c2f-99ca-4c06ee8021f9", - "volume_name": "pvc-bd4056b1-0782-4c2f-99ca-4c06ee8021f9" + "uid": "22204802-c426-4840-8045-ad03a53c88e4", + "volume_name": "pvc-22204802-c426-4840-8045-ad03a53c88e4" }, "repository_round_trip": { "artifact_objects": { "gravitino_postgresql": { "bytes": 188109, "format": "postgresql_custom_dump_v1", - "object_path": "metadata-fabric/rp-20260727073521z/gravitino_postgresql.dump", - "retained_until": "2026-07-28T07:36:07.228000+00:00", + "object_path": "metadata-fabric/rp-20260727090506z/gravitino_postgresql.dump", + "retained_until": "2026-07-28T09:05:51.903000+00:00", "retained_version_delete_blocked": true, "retention_mode": "GOVERNANCE", - "sha256": "b56fae7ba17166690df1ac52814734d10aaab6d036e467141601aa13913493fa", - "version_id": "ea0d2e75-185d-4f44-8ed9-74655b0423db" + "sha256": "6d6013aa01872c335eb07ea3733eae49acbdab5b6f9d241144663bf21585d2f1", + "version_id": "ce7ccab6-6706-42ad-a3c1-de284563fd97" }, "openmetadata_postgresql": { - "bytes": 1209521, + "bytes": 1298657, "format": "postgresql_custom_dump_v1", - "object_path": "metadata-fabric/rp-20260727073521z/openmetadata_postgresql.dump", - "retained_until": "2026-07-28T07:36:07.241000+00:00", + "object_path": "metadata-fabric/rp-20260727090506z/openmetadata_postgresql.dump", + "retained_until": "2026-07-28T09:05:51.917000+00:00", "retained_version_delete_blocked": true, "retention_mode": "GOVERNANCE", - "sha256": "2f6de329c6ebff68227a27958040c29d41396c9f1a61421eaf0630ac344a7e26", - "version_id": "b6ce20d7-6e95-4612-9740-d0bd5682ff81" + "sha256": "7aff5a3b62ddfae6ccc46cc1dcf13b72c326085b08a6f3a78a78c87609eaacad", + "version_id": "0ebe97e1-c692-4a84-b565-3b0d5426737c" }, "opensearch": { - "bytes": 387684, + "bytes": 396797, "format": "opensearch_fs_snapshot_tar_gzip_v1", - "object_path": "metadata-fabric/rp-20260727073521z/opensearch.tar.gz", - "retained_until": "2026-07-28T07:36:07.256000+00:00", + "object_path": "metadata-fabric/rp-20260727090506z/opensearch.tar.gz", + "retained_until": "2026-07-28T09:05:51.930000+00:00", "retained_version_delete_blocked": true, "retention_mode": "GOVERNANCE", - "sha256": "7732ffc33827b9ffc4ed5e3f3744e4f54b1a73a2d9e0444fe7b34e16c8f37da3", - "version_id": "a9591f1d-c984-4323-925d-0c4ff0b0de04" + "sha256": "b36dec1ca16827ab5c363bb98a7f994d211bc139d2ad84b2498fdf3bcdfdc59d", + "version_id": "f3d0f897-29a6-4317-a233-d2a93c7594d7" } }, "bucket": "gda-metadata-fabric-backups", @@ -298,9 +302,9 @@ "runtime_credentials_removed": true }, "schema": "gda.metadata_fabric_backup_repository_observation.v1", - "started_at": "2026-07-27T07:35:21.083519+00:00" + "started_at": "2026-07-27T09:05:06.843622+00:00" }, - "observation_fingerprint": "afe2a882c83d774bcf4f19ffa8eace946de3e3c54a7a2b3416cc66c6aecbd1d9", + "observation_fingerprint": "d844f9afe2b8590190f2aac437201f15f9d3d8fca3b8266dea68a8fa3db2a4a4", "oidc_verified": false, "production_backup_target_verified": false, "production_kms_verified": false, diff --git a/docs/evidence/metadata-fabric-cross-cluster-recovery-2026-07-27.json b/docs/evidence/metadata-fabric-cross-cluster-recovery-2026-07-27.json new file mode 100644 index 00000000..c040f75f --- /dev/null +++ b/docs/evidence/metadata-fabric-cross-cluster-recovery-2026-07-27.json @@ -0,0 +1,278 @@ +{ + "checks": { + "artifact_round_trip": "passed", + "cluster_identity_isolation": "passed", + "compliance_retention": "passed", + "cross_cluster_restore": "passed", + "ephemeral_cleanup": "passed", + "production_boundaries": "passed", + "repository_failure_domain": "passed", + "static_contract": "passed", + "writer_reader_separation": "passed" + }, + "cross_cluster_recovery_scope": "local_same_host_distinct_kubernetes_clusters_external_s3_repository", + "cross_cluster_recovery_verified": false, + "cross_region_recovery_verified": false, + "environment": "local_same_host_distinct_kubernetes_clusters", + "errors": [], + "evidence_fingerprint": "9eaf8cec9ec2d3763260c271c0c27c1c3251717820d38aadfef0bec7d7a574a8", + "generated_at": "2026-07-27T09:02:45.066081+00:00", + "local_cross_cluster_recovery_verified": true, + "local_external_repository_verified": true, + "local_writer_reader_identity_separation_verified": true, + "network_policy_enforcement_verified": false, + "observation": { + "contract": { + "contract_fingerprint": "352f59b10d845b288a465b59386e7a15f42aa706559fe337db5ed1dcb7234815", + "local_static_contract_verified": true + }, + "duration_seconds": 122.791, + "observed_at": "2026-07-27T09:02:45.066081+00:00", + "recovery_evidence": { + "backup_restore_scope": "local_distinct_kubernetes_clusters_new_namespace_and_pvcs", + "backup_restore_verified": true, + "checks": { + "artifact_integrity": "passed", + "ephemeral_cleanup": "passed", + "opensearch_restore": "passed", + "postgresql_restore": "passed", + "production_boundaries": "passed", + "source_availability_restored": "passed", + "static_contract": "passed" + }, + "cross_cluster_recovery_verified": false, + "cross_region_recovery_verified": false, + "environment": "local_cross_cluster_recovery_rehearsal", + "errors": [], + "evidence_fingerprint": "723da40ef6d31b9cdfcfb4dff4e59a0431910ea8f5446c020e7292200d7a7980", + "generated_at": "2026-07-27T09:02:45.066081+00:00", + "local_backup_restore_verified": true, + "local_cross_cluster_recovery_verified": true, + "network_policy_enforcement_verified": false, + "observation": { + "artifacts": { + "gravitino_postgresql": { + "bytes": 188109, + "format": "postgresql_custom_dump_v1", + "sha256": "19201a61bbe0e2c2f14c99b720089ea7f679502a7b1a065c92069e5b3c28a9dc" + }, + "openmetadata_postgresql": { + "bytes": 1269181, + "format": "postgresql_custom_dump_v1", + "sha256": "0fca6a84ba0bc60a6a3cb9bb972f90f7efe0102a5b5d02d1b024879fe98ac1db" + }, + "opensearch": { + "bytes": 393375, + "format": "opensearch_fs_snapshot_tar_gzip_v1", + "sha256": "21850ce3c2887fb7c16bc836af0204e1de55fffa3898fb600597e4e244984f46" + } + }, + "cluster": { + "context": "docker-desktop", + "recovery_context": "kind-gda-metadata-recovery", + "recovery_namespace": { + "name": "gda-metadata-recovery-rehearsal", + "uid": "06d4bef0-f7e2-47bc-ad63-abf1cd708ed4" + }, + "recovery_uid": "04ddf8f4-6d62-4e6d-bbd7-2eecb3f94858", + "source_namespace": { + "name": "gda-metadata-sandbox", + "uid": "812c9c5c-9ce2-409e-a6a8-a3f24785aa0c" + }, + "uid": "c3c9bbab-2b36-4359-a7ac-e3de194445ab" + }, + "contract": { + "contract_fingerprint": "afb168b7586113eda95b874c10351f1e4f5eb33c6e661e1bfedfda7103ee6121", + "static_contract_verified": true + }, + "duration_seconds": 120.483, + "observed_at": "2026-07-27T09:02:44.714256+00:00", + "recovered_markers": { + "gravitino_postgresql": { + "extension_fingerprint": "6a0a92045f06baabcf22b103007285ec0bcb00cdbf6f6f30b34b0a41a73c6c2f", + "image_version": "16.10-bookworm", + "row_count_fingerprint": "9af129a1e57abefe0e5ada16bd7d93d7d6f614def8814e8c853b87c539e6e9a6", + "sequence_state_fingerprint": "51fe857234adbc1e7462e24022f0b3ba383ff0b87b4b833d2c99f1674b60c9a4", + "server_version": "16.10 (Debian 16.10-1.pgdg12+1)", + "table_count": 39, + "table_name_fingerprint": "7db0d765156ca83f8cfab3c48b0acd40eefb04c8bad3170c2d7cf6079cbfe9fe" + }, + "openmetadata_postgresql": { + "extension_fingerprint": "5425606415e192199fdbef130acb53a732064cb692916f5278b9e8eebc5b2f59", + "image_version": "16.10-bookworm", + "row_count_fingerprint": "6678a25f70e036ed1a8c65a02c134db85622e6ca650b330bea76757fbca1e572", + "sequence_state_fingerprint": "c89d39c21646c56a9f523757c3aefe41cfe651516c16786e1ffffe53e2df502b", + "server_version": "16.10 (Debian 16.10-1.pgdg12+1)", + "table_count": 176, + "table_name_fingerprint": "9b56cb8952f4680c1972aeafb461b0ef8c204326432ed8bab6226e2eedbb1167" + }, + "opensearch": { + "cluster_uuid": "27gwc3rgTWeouD96vZNeMQ", + "document_count_fingerprint": "b9a9da89701697d1778735a3cd1196422f624049847009a593c56e5fdfe7f109", + "index_count": 79, + "index_name_fingerprint": "5dd3252013d217ea4e014951fbd687ef912dac847ee8d385adf6bd561070f00a", + "total_index_count": 79, + "version": "3.3.2" + } + }, + "recovery_pvcs": { + "recovery-gravitino-data-recovery-gravitino-postgresql-0": { + "capacity": "2Gi", + "phase": "Bound", + "uid": "9f7da7d4-8dab-4b70-bed3-05866fd38df4", + "volume_name": "pvc-9f7da7d4-8dab-4b70-bed3-05866fd38df4" + }, + "recovery-openmetadata-data-recovery-openmetadata-postgresql-0": { + "capacity": "2Gi", + "phase": "Bound", + "uid": "4d26a324-e3e0-4ece-b8c7-1e3626dfcb04", + "volume_name": "pvc-4d26a324-e3e0-4ece-b8c7-1e3626dfcb04" + }, + "recovery-opensearch-data-recovery-opensearch-0": { + "capacity": "8Gi", + "phase": "Bound", + "uid": "7134be6c-65d2-49d2-8c4b-60acb469e9af", + "volume_name": "pvc-7134be6c-65d2-49d2-8c4b-60acb469e9af" + } + }, + "repository_round_trip": { + "artifact_objects": { + "gravitino_postgresql": { + "bytes": 188109, + "compliance_delete_blocked": true, + "format": "postgresql_custom_dump_v1", + "object_path": "metadata-fabric/rp-20260727090042z/gravitino_postgresql.dump", + "retained_until": "2026-07-28T09:01:19.845000+00:00", + "retention_mode": "COMPLIANCE", + "sha256": "19201a61bbe0e2c2f14c99b720089ea7f679502a7b1a065c92069e5b3c28a9dc", + "version_id": "c3685a8e-e4ce-43bc-b03f-9c93ff47e460", + "writer_delete_denied": true + }, + "openmetadata_postgresql": { + "bytes": 1269181, + "compliance_delete_blocked": true, + "format": "postgresql_custom_dump_v1", + "object_path": "metadata-fabric/rp-20260727090042z/openmetadata_postgresql.dump", + "retained_until": "2026-07-28T09:01:19.860000+00:00", + "retention_mode": "COMPLIANCE", + "sha256": "0fca6a84ba0bc60a6a3cb9bb972f90f7efe0102a5b5d02d1b024879fe98ac1db", + "version_id": "2f03c0c3-3885-4b25-a3c7-aa32959e57e0", + "writer_delete_denied": true + }, + "opensearch": { + "bytes": 393375, + "compliance_delete_blocked": true, + "format": "opensearch_fs_snapshot_tar_gzip_v1", + "object_path": "metadata-fabric/rp-20260727090042z/opensearch.tar.gz", + "retained_until": "2026-07-28T09:01:19.871000+00:00", + "retention_mode": "COMPLIANCE", + "sha256": "21850ce3c2887fb7c16bc836af0204e1de55fffa3898fb600597e4e244984f46", + "version_id": "0a14208f-1766-48c0-b7f8-129c31180b2b", + "writer_delete_denied": true + } + }, + "bucket": "gda-metadata-fabric-cross-cluster-backups", + "default_retention_days": 1, + "default_retention_mode": "COMPLIANCE", + "failure_domain": "outside_source_and_recovery_kubernetes_clusters", + "identity_controls": { + "reader_write_denied": true, + "separate_writer_reader": true, + "writer_delete_denied": true + }, + "kms_encryption_verified": false, + "local_artifacts_removed_before_download": true, + "object_lock_enabled": true, + "production_durability_verified": false, + "provider": "minio_s3_compatible", + "repository_runtime": "docker_host_container", + "round_trip_verified": true, + "transport_tls_verified": false, + "versioning_status": "Enabled" + }, + "runtime_checks": { + "local_artifacts_removed": true, + "recovery_namespace_removed": true, + "recovery_search_target_cleared": true, + "source_quiesced": true, + "source_services_restored": true, + "source_snapshot_staging_cleaned": true, + "source_snapshot_staging_initially_empty": true + }, + "schema": "gda.metadata_fabric_recovery_observation.v1", + "source_markers": { + "gravitino_postgresql": { + "extension_fingerprint": "6a0a92045f06baabcf22b103007285ec0bcb00cdbf6f6f30b34b0a41a73c6c2f", + "image_version": "16.10-bookworm", + "row_count_fingerprint": "9af129a1e57abefe0e5ada16bd7d93d7d6f614def8814e8c853b87c539e6e9a6", + "sequence_state_fingerprint": "51fe857234adbc1e7462e24022f0b3ba383ff0b87b4b833d2c99f1674b60c9a4", + "server_version": "16.10 (Debian 16.10-1.pgdg12+1)", + "table_count": 39, + "table_name_fingerprint": "7db0d765156ca83f8cfab3c48b0acd40eefb04c8bad3170c2d7cf6079cbfe9fe" + }, + "openmetadata_postgresql": { + "extension_fingerprint": "5425606415e192199fdbef130acb53a732064cb692916f5278b9e8eebc5b2f59", + "image_version": "16.10-bookworm", + "row_count_fingerprint": "6678a25f70e036ed1a8c65a02c134db85622e6ca650b330bea76757fbca1e572", + "sequence_state_fingerprint": "c89d39c21646c56a9f523757c3aefe41cfe651516c16786e1ffffe53e2df502b", + "server_version": "16.10 (Debian 16.10-1.pgdg12+1)", + "table_count": 176, + "table_name_fingerprint": "9b56cb8952f4680c1972aeafb461b0ef8c204326432ed8bab6226e2eedbb1167" + }, + "opensearch": { + "cluster_uuid": "_h-TPGNqQq65KEOkhWpTyg", + "document_count_fingerprint": "b9a9da89701697d1778735a3cd1196422f624049847009a593c56e5fdfe7f109", + "index_count": 79, + "index_name_fingerprint": "5dd3252013d217ea4e014951fbd687ef912dac847ee8d385adf6bd561070f00a", + "total_index_count": 79, + "version": "3.3.2" + } + }, + "started_at": "2026-07-27T09:00:44.231026+00:00" + }, + "observation_fingerprint": "76af7dd7f858cc762f5f85a08e2fcc82df9d3d0a2be3c205d71686cbd70c7388", + "oidc_verified": false, + "production_backup_restore_verified": false, + "production_cross_cluster_recovery_verified": false, + "production_ready": false, + "recovery_namespace": "gda-metadata-recovery-rehearsal", + "rpo_slo_verified": false, + "rto_slo_verified": false, + "schema": "gda.metadata_fabric_recovery_evidence.v1", + "source_namespace": "gda-metadata-sandbox", + "status": "local_cross_cluster_backup_restore_verified", + "upgrade_verified": false, + "writes_to_gda_enabled": false + }, + "repository": { + "failure_domain": "outside_source_and_recovery_kubernetes_clusters", + "mc_image": "minio/mc:RELEASE.2025-04-16T18-13-26Z", + "minio_image": "minio/minio:RELEASE.2025-04-22T22-12-26Z", + "runtime": "docker_host_container" + }, + "runtime_checks": { + "external_repository_container_removed": true, + "external_repository_volume_removed": true, + "recovery_cluster_preserved": true, + "runtime_credentials_removed": true + }, + "schema": "gda.metadata_fabric_cross_cluster_recovery_observation.v1", + "started_at": "2026-07-27T09:00:42.274984+00:00" + }, + "observation_fingerprint": "d86e4e2f5d1906a683e778c80ff8908bc9eb49b1cd222e6966edee5b2dd9c17b", + "oidc_verified": false, + "production_backup_target_verified": false, + "production_cross_cluster_recovery_verified": false, + "production_kms_verified": false, + "production_ready": false, + "production_retention_verified": false, + "production_tls_verified": false, + "recovery_context": "kind-gda-metadata-recovery", + "rpo_slo_verified": false, + "rto_slo_verified": false, + "schema": "gda.metadata_fabric_cross_cluster_recovery_evidence.v1", + "source_cluster_loss_verified": false, + "source_context": "docker-desktop", + "status": "local_cross_cluster_recovery_verified", + "writes_to_gda_enabled": false +} diff --git a/docs/evidence/metadata-fabric-recovery-rehearsal-2026-07-27.json b/docs/evidence/metadata-fabric-recovery-rehearsal-2026-07-27.json index 64e8f413..7b210819 100644 --- a/docs/evidence/metadata-fabric-recovery-rehearsal-2026-07-27.json +++ b/docs/evidence/metadata-fabric-recovery-rehearsal-2026-07-27.json @@ -14,34 +14,37 @@ "cross_region_recovery_verified": false, "environment": "local_isolated_recovery_rehearsal", "errors": [], - "evidence_fingerprint": "3cf46cc83a8feaa4142893a06f84e9008a7d805a64fb6708d176ca976a4dbd62", - "generated_at": "2026-07-27T07:37:28.806368+00:00", + "evidence_fingerprint": "da1214294045f8b0abe2e2775b81ef33967eac9ab0e97055ae80212ac0c08a4b", + "generated_at": "2026-07-27T09:07:09.480972+00:00", "local_backup_restore_verified": true, + "local_cross_cluster_recovery_verified": false, "network_policy_enforcement_verified": false, "observation": { "artifacts": { "gravitino_postgresql": { "bytes": 188109, "format": "postgresql_custom_dump_v1", - "sha256": "b56fae7ba17166690df1ac52814734d10aaab6d036e467141601aa13913493fa" + "sha256": "6d6013aa01872c335eb07ea3733eae49acbdab5b6f9d241144663bf21585d2f1" }, "openmetadata_postgresql": { - "bytes": 1209521, + "bytes": 1298657, "format": "postgresql_custom_dump_v1", - "sha256": "2f6de329c6ebff68227a27958040c29d41396c9f1a61421eaf0630ac344a7e26" + "sha256": "7aff5a3b62ddfae6ccc46cc1dcf13b72c326085b08a6f3a78a78c87609eaacad" }, "opensearch": { - "bytes": 387684, + "bytes": 396797, "format": "opensearch_fs_snapshot_tar_gzip_v1", - "sha256": "7732ffc33827b9ffc4ed5e3f3744e4f54b1a73a2d9e0444fe7b34e16c8f37da3" + "sha256": "b36dec1ca16827ab5c363bb98a7f994d211bc139d2ad84b2498fdf3bcdfdc59d" } }, "cluster": { "context": "docker-desktop", + "recovery_context": "docker-desktop", "recovery_namespace": { "name": "gda-metadata-recovery-rehearsal", - "uid": "d9236d7c-8c9c-45af-a46b-7148b47b4db0" + "uid": "58c6389c-3f89-4f83-a392-02d9f3c679a7" }, + "recovery_uid": "c3c9bbab-2b36-4359-a7ac-e3de194445ab", "source_namespace": { "name": "gda-metadata-sandbox", "uid": "812c9c5c-9ce2-409e-a6a8-a3f24785aa0c" @@ -49,11 +52,11 @@ "uid": "c3c9bbab-2b36-4359-a7ac-e3de194445ab" }, "contract": { - "contract_fingerprint": "83e3f86542076987b36cb8b99fb01c2db7e92c15624164f9766ff0a6fe26640a", + "contract_fingerprint": "afb168b7586113eda95b874c10351f1e4f5eb33c6e661e1bfedfda7103ee6121", "static_contract_verified": true }, - "duration_seconds": 115.137, - "observed_at": "2026-07-27T07:37:28.805987+00:00", + "duration_seconds": 111.123, + "observed_at": "2026-07-27T09:07:09.480617+00:00", "recovered_markers": { "gravitino_postgresql": { "extension_fingerprint": "6a0a92045f06baabcf22b103007285ec0bcb00cdbf6f6f30b34b0a41a73c6c2f", @@ -67,14 +70,14 @@ "openmetadata_postgresql": { "extension_fingerprint": "5425606415e192199fdbef130acb53a732064cb692916f5278b9e8eebc5b2f59", "image_version": "16.10-bookworm", - "row_count_fingerprint": "dbb8f15b9df8780d3be128ef2a9ddad7b292980e811a81a65349e831e64b40f9", - "sequence_state_fingerprint": "769c215213fe71957ea945bd85ad0db889cda410f47443173f19457a17af5ac4", + "row_count_fingerprint": "97d270fe6fb399e6fc3a4788ba8da13d647772857c8b61a60d6695c30cd04cb9", + "sequence_state_fingerprint": "089f6120440918b8ea1502c0372ee8f687c5284aa17b26bf8dc826c5b7500d39", "server_version": "16.10 (Debian 16.10-1.pgdg12+1)", "table_count": 176, "table_name_fingerprint": "9b56cb8952f4680c1972aeafb461b0ef8c204326432ed8bab6226e2eedbb1167" }, "opensearch": { - "cluster_uuid": "9o2bzn8fS5mfJ0nEHkGQlQ", + "cluster_uuid": "sY84Z4TpSHyS5uywqpyt9Q", "document_count_fingerprint": "b9a9da89701697d1778735a3cd1196422f624049847009a593c56e5fdfe7f109", "index_count": 79, "index_name_fingerprint": "5dd3252013d217ea4e014951fbd687ef912dac847ee8d385adf6bd561070f00a", @@ -86,20 +89,20 @@ "recovery-gravitino-data-recovery-gravitino-postgresql-0": { "capacity": "2Gi", "phase": "Bound", - "uid": "9ba5f905-6f4d-4763-b9ba-6092207036d0", - "volume_name": "pvc-9ba5f905-6f4d-4763-b9ba-6092207036d0" + "uid": "abe89775-0d29-4186-b032-75f8c5c77ce5", + "volume_name": "pvc-abe89775-0d29-4186-b032-75f8c5c77ce5" }, "recovery-openmetadata-data-recovery-openmetadata-postgresql-0": { "capacity": "2Gi", "phase": "Bound", - "uid": "6d1b0cae-869e-4923-bd66-315b29abf887", - "volume_name": "pvc-6d1b0cae-869e-4923-bd66-315b29abf887" + "uid": "6383ada0-c77b-4614-b389-ffc099625f96", + "volume_name": "pvc-6383ada0-c77b-4614-b389-ffc099625f96" }, "recovery-opensearch-data-recovery-opensearch-0": { "capacity": "8Gi", "phase": "Bound", - "uid": "c3d5585e-89f6-45ee-b43f-f2a155204916", - "volume_name": "pvc-c3d5585e-89f6-45ee-b43f-f2a155204916" + "uid": "ee77c740-cfd2-4f21-9b11-4129b0e2b389", + "volume_name": "pvc-ee77c740-cfd2-4f21-9b11-4129b0e2b389" } }, "repository_round_trip": { @@ -107,32 +110,32 @@ "gravitino_postgresql": { "bytes": 188109, "format": "postgresql_custom_dump_v1", - "object_path": "metadata-fabric/rp-20260727073521z/gravitino_postgresql.dump", - "retained_until": "2026-07-28T07:36:07.228000+00:00", + "object_path": "metadata-fabric/rp-20260727090506z/gravitino_postgresql.dump", + "retained_until": "2026-07-28T09:05:51.903000+00:00", "retained_version_delete_blocked": true, "retention_mode": "GOVERNANCE", - "sha256": "b56fae7ba17166690df1ac52814734d10aaab6d036e467141601aa13913493fa", - "version_id": "ea0d2e75-185d-4f44-8ed9-74655b0423db" + "sha256": "6d6013aa01872c335eb07ea3733eae49acbdab5b6f9d241144663bf21585d2f1", + "version_id": "ce7ccab6-6706-42ad-a3c1-de284563fd97" }, "openmetadata_postgresql": { - "bytes": 1209521, + "bytes": 1298657, "format": "postgresql_custom_dump_v1", - "object_path": "metadata-fabric/rp-20260727073521z/openmetadata_postgresql.dump", - "retained_until": "2026-07-28T07:36:07.241000+00:00", + "object_path": "metadata-fabric/rp-20260727090506z/openmetadata_postgresql.dump", + "retained_until": "2026-07-28T09:05:51.917000+00:00", "retained_version_delete_blocked": true, "retention_mode": "GOVERNANCE", - "sha256": "2f6de329c6ebff68227a27958040c29d41396c9f1a61421eaf0630ac344a7e26", - "version_id": "b6ce20d7-6e95-4612-9740-d0bd5682ff81" + "sha256": "7aff5a3b62ddfae6ccc46cc1dcf13b72c326085b08a6f3a78a78c87609eaacad", + "version_id": "0ebe97e1-c692-4a84-b565-3b0d5426737c" }, "opensearch": { - "bytes": 387684, + "bytes": 396797, "format": "opensearch_fs_snapshot_tar_gzip_v1", - "object_path": "metadata-fabric/rp-20260727073521z/opensearch.tar.gz", - "retained_until": "2026-07-28T07:36:07.256000+00:00", + "object_path": "metadata-fabric/rp-20260727090506z/opensearch.tar.gz", + "retained_until": "2026-07-28T09:05:51.930000+00:00", "retained_version_delete_blocked": true, "retention_mode": "GOVERNANCE", - "sha256": "7732ffc33827b9ffc4ed5e3f3744e4f54b1a73a2d9e0444fe7b34e16c8f37da3", - "version_id": "a9591f1d-c984-4323-925d-0c4ff0b0de04" + "sha256": "b36dec1ca16827ab5c363bb98a7f994d211bc139d2ad84b2498fdf3bcdfdc59d", + "version_id": "f3d0f897-29a6-4317-a233-d2a93c7594d7" } }, "bucket": "gda-metadata-fabric-backups", @@ -170,8 +173,8 @@ "openmetadata_postgresql": { "extension_fingerprint": "5425606415e192199fdbef130acb53a732064cb692916f5278b9e8eebc5b2f59", "image_version": "16.10-bookworm", - "row_count_fingerprint": "dbb8f15b9df8780d3be128ef2a9ddad7b292980e811a81a65349e831e64b40f9", - "sequence_state_fingerprint": "769c215213fe71957ea945bd85ad0db889cda410f47443173f19457a17af5ac4", + "row_count_fingerprint": "97d270fe6fb399e6fc3a4788ba8da13d647772857c8b61a60d6695c30cd04cb9", + "sequence_state_fingerprint": "089f6120440918b8ea1502c0372ee8f687c5284aa17b26bf8dc826c5b7500d39", "server_version": "16.10 (Debian 16.10-1.pgdg12+1)", "table_count": 176, "table_name_fingerprint": "9b56cb8952f4680c1972aeafb461b0ef8c204326432ed8bab6226e2eedbb1167" @@ -185,11 +188,12 @@ "version": "3.3.2" } }, - "started_at": "2026-07-27T07:35:33.668759+00:00" + "started_at": "2026-07-27T09:05:18.357484+00:00" }, - "observation_fingerprint": "938c0f9842cf0c6218ad4ade675b4e773b538270a605ca33d9215859a0e5f6a1", + "observation_fingerprint": "06a6d5bba92a3f75b5b25cf3ea2c7f91d321454eaf4b5f5608416f468cff0dc1", "oidc_verified": false, "production_backup_restore_verified": false, + "production_cross_cluster_recovery_verified": false, "production_ready": false, "recovery_namespace": "gda-metadata-recovery-rehearsal", "rpo_slo_verified": false, diff --git a/docs/roadmap-ar0-platform-truth-2026-07-24.md b/docs/roadmap-ar0-platform-truth-2026-07-24.md index e6218582..fed8896d 100644 --- a/docs/roadmap-ar0-platform-truth-2026-07-24.md +++ b/docs/roadmap-ar0-platform-truth-2026-07-24.md @@ -199,7 +199,7 @@ Temporal 继续保持目标组件状态，不在这一包并行接入。OpenMeta 当前完成仅指本地合同、授权 evidence、outbox/callback 代码、数据库成功终局门、托管 worker 代码、默认关闭的部署模板及离线 activation/release preflight、candidate/registry/provenance/artifact-release/live observation evidence gate、合成 golden slice、定向测试、真实 PostgreSQL 16 事务边界和 canonical mainline 治理。`candidate_validated`、`registry_subject_bound`、本地合成 `provenance_verified`、`ready_for_activation`、`ready_for_staging_apply`、`verified_for_staging_apply` 和本地 live collection 都不等于真实镜像已 attested 或 staging 已部署；真实 IAM/OIDC 与 service token 生命周期、首次 GHCR publish/verify、真实 provenance artifact verify、registry-backed live staging revision、worker/callback 扩容运行、golden slice staging 运行链、受保护 release/live evidence provenance、独立 DolphinScheduler metadata PostgreSQL 和真实数据终局证据仍属于 4.7 后续切片。 -### 4.8 Metadata Fabric Bridge M1 + M2a + M2b-1 + M2b-2（只读、foundation、本地恢复与 repository round-trip 已验证） +### 4.8 Metadata Fabric Bridge M1 + M2a + M2b-1/M2b-2/M2b-3（本机双集群恢复已验证） 第八块回到 AR-1 的 metadata control plane，以 [ADR-036](architecture-decisions/adr-036-read-only-metadata-fabric-bridge-contract.md) 固定 OpenMetadata + Gravitino + GDA Control Ledger 的首条 table slice： @@ -211,10 +211,11 @@ Temporal 继续保持目标组件状态，不在这一包并行接入。OpenMeta 6. 地类图斑合成 ResourceVersion 的 binding/reconciliation fingerprint 已冻结，24 个定向测试与 required CI validator 覆盖正向 replay、双租户和 authority/security 负例。 7. [ADR-037](architecture-decisions/adr-037-local-metadata-fabric-foundation-sandbox.md) 已在两节点 ARM64 Docker Desktop Kubernetes 中运行 OpenMetadata `1.13.1`、Gravitino `1.3.0`、两个独立 PostgreSQL 和 OpenSearch；三块 PVC、五个专用 ServiceAccount、ClusterIP-only Service、无 token mount 与外部 Secret 边界均进入静态合同； 8. Secret-free live collector 已验证固定 provider version/revision、176/39 张 PostgreSQL 表、78 个 OpenSearch 索引、三块 PVC identity 和五个 Pod 的受控替换；最终 evidence fingerprint 为 `ac21ee50ba3c1f27f949420cc7e4483963714b6b955bb0157eca1dd39cf102c3`。 -9. [ADR-038](architecture-decisions/adr-038-local-metadata-fabric-recovery-rehearsal.md) 已将两个 PostgreSQL custom dump 和 OpenSearch native snapshot 恢复到独立临时 namespace 的三块新 PVC，验证 176/39 张表的 table/row/sequence/extension fingerprints 与 79 个索引的 name/document fingerprints，恢复 source availability 并清理临时资源；repository-backed 复测 evidence fingerprint 为 `3cf46cc83a8feaa4142893a06f84e9008a7d805a64fb6708d176ca976a4dbd62`。 -10. [ADR-039](architecture-decisions/adr-039-local-locked-metadata-backup-repository.md) 已在独立 namespace/PVC 的 MinIO bucket 启用 versioning 与 Object Lock，将三份真实 artifact 上传并取得 version ID/retention，确认 retained version 删除被拒绝，删除本地副本后按 version 下载并完成 ADR-038 恢复；repository evidence fingerprint 为 `07834430fb140e147624f4ab3c93e6d7907648e91a149cb19db178ea6085d1ed`。 +9. [ADR-038](architecture-decisions/adr-038-local-metadata-fabric-recovery-rehearsal.md) 已将两个 PostgreSQL custom dump 和 OpenSearch native snapshot 恢复到独立临时 namespace 的三块新 PVC，验证 176/39 张表的 table/row/sequence/extension fingerprints 与 79 个索引的 name/document fingerprints，恢复 source availability 并清理临时资源；repository-backed 复测 evidence fingerprint 为 `da1214294045f8b0abe2e2775b81ef33967eac9ab0e97055ae80212ac0c08a4b`。 +10. [ADR-039](architecture-decisions/adr-039-local-locked-metadata-backup-repository.md) 已在独立 namespace/PVC 的 MinIO bucket 启用 versioning 与 Object Lock，将三份真实 artifact 上传并取得 version ID/retention，确认 retained version 删除被拒绝，删除本地副本后按 version 下载并完成 ADR-038 恢复；repository evidence fingerprint 为 `2897f9e6aaae21fb366da0b72edea5cf072d5b2c1aeac0807d263bd0a5f5f133`。 +11. [ADR-040](architecture-decisions/adr-040-local-cross-cluster-metadata-recovery.md) 已将 source 固定为 `docker-desktop`、recovery 固定为独立 `kind-gda-metadata-recovery`，从两个 Kubernetes cluster 外的 Docker-host MinIO 以 `COMPLIANCE/1 day`、独立 writer/reader、按 version 下载完成三存储恢复；evidence fingerprint 为 `9eaf8cec9ec2d3763260c271c0c27c1c3251717820d38aadfef0bec7d7a574a8`，且只证明同一 Docker Desktop 主机内的双集群恢复。 -此处 M1 只证明静态合同和只读 HTTP 边界；M2a 只证明本地 live foundation 与 PVC 重挂载连续性；M2b-1 只将 `backup_restore_verified` 推进到 `local_same_cluster_new_namespace_and_pvcs`；M2b-2 只将 `backup_repository_verified` 推进到 `local_same_cluster_isolated_s3_compatible_repository`。它没有写入或连接真实 GDA Resource，`production_provider_verified`、`production_table_catalog_provider_verified`、`production_backup_restore_verified`、`production_backup_target_verified`、`production_retention_verified`、`network_policy_enforcement_verified`、`oidc_verified`、`upgrade_verified`、`writes_to_gda_enabled` 和 `production_ready` 仍为 `false`。外部 backup account/bucket、COMPLIANCE retention、TLS/KMS、独立 writer/reader identity、跨集群恢复及其后的升级/registry/M3 仍是退出门。 +此处 M1 只证明静态合同和只读 HTTP 边界；M2a 只证明本地 live foundation 与 PVC 重挂载连续性；M2b-1/M2b-2 分别限定在同集群新 PVC 和同集群隔离 repository；M2b-3 的 `local_cross_cluster_recovery_verified=true` 只限定在 `local_same_host_distinct_kubernetes_clusters_external_s3_repository`。它没有写入或连接真实 GDA Resource，`production_provider_verified`、`production_table_catalog_provider_verified`、`production_backup_restore_verified`、`production_backup_target_verified`、`production_retention_verified`、`production_cross_cluster_recovery_verified`、`source_cluster_loss_verified`、`network_policy_enforcement_verified`、`oidc_verified`、`upgrade_verified`、`writes_to_gda_enabled` 和 `production_ready` 仍为 `false`。source host/cluster 外的生产 bucket、KMS/TLS/workload identity、source-loss recovery、RPO/RTO 及其后的升级/registry/M3 仍是退出门。 ## 5. 重新评估条件 diff --git a/docs/roadmap.md b/docs/roadmap.md index 6d518930..770cc82c 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -417,7 +417,7 @@ AR-0 Architecture / Schema / Runtime Truth - Web、DolphinScheduler、OpenMetadata、Gravitino、worker 和 bridge 重启后不丢版本、产品、血缘、PlatformRun correlation 或审批状态。 - DataOps 状态可从 DolphinScheduler/OpenMetadata/Gravitino/GDA evidence 恢复；任何 release/deployment 都能从事件、Artifact、评测、策略、incident 和 rollback pointer 重放。Temporal durable recovery 在 AR-5/AR-7 单独验收。 -当前 Metadata Fabric 证据边界：M1 只读 bridge 合同已验证；ADR-037 的 M2a 本地 foundation 已验证五个 Pod/PVC 连续性；ADR-038 的 M2b-1 已验证两份 PostgreSQL dump 和 OpenSearch snapshot 到新 namespace/PVC 的内容恢复；ADR-039 的 M2b-2 又在独立 MinIO/PVC 启用 versioning/Object Lock，将三份真实 artifact 上传、锁定、删除本地副本、按 version 下载并完成恢复。`backup_restore_verified=true` 只适用于本地同集群新 namespace/PVC，`backup_repository_verified=true` 只适用于本地同集群隔离 S3-compatible repository；二者都不证明外部生产 bucket、COMPLIANCE retention、TLS/KMS、独立 workload identity、PITR、RPO/RTO、跨集群恢复、OIDC、upgrade、NetworkPolicy enforcement 或 production ready。 +当前 Metadata Fabric 证据边界：M1 只读 bridge 合同已验证；ADR-037 的 M2a 本地 foundation 已验证五个 Pod/PVC 连续性；ADR-038 的 M2b-1 已验证两份 PostgreSQL dump 和 OpenSearch snapshot 到新 namespace/PVC 的内容恢复；ADR-039 的 M2b-2 又在独立 MinIO/PVC 启用 versioning/Object Lock，将三份真实 artifact 上传、锁定、删除本地副本、按 version 下载并完成恢复；ADR-040 的 M2b-3 已从 `docker-desktop` 恢复到独立 kind cluster，并将 `COMPLIANCE/1 day` repository 置于两个 Kubernetes cluster 之外的 Docker host container，验证独立 writer/reader 权限。`local_cross_cluster_recovery_verified=true` 的 scope 仅为同一 Docker Desktop 主机内的两个 Kubernetes cluster；它不证明 source-host loss、外部生产 bucket、生产 retention、TLS/KMS、workload identity、PITR、RPO/RTO、生产跨集群/跨区域恢复、OIDC、upgrade、NetworkPolicy enforcement 或 production ready。 ### AR-2 — Source, Ingestion and Geospatial Lakehouse Vertical Slice（P0） @@ -684,7 +684,7 @@ Golden checks 至少覆盖： 1. 导出所有目标环境 schema/config fingerprint，修复重复 migration ID、checksum 和 fail-open runner。 2. 完成部署、存储、bucket、registry、scheduler/job、API/GIS endpoint、图层/样式/缓存、provider/Gateway、数据资产、消费者和权限事实盘点；部署 OpenMetadata/Gravitino/DolphinScheduler/Temporal sandbox，冻结 owner、version、OIDC、backup/restore 和升级责任。 3. 冻结 ResourceURN、ResourceVersion、PlatformDefinition/PlatformRun/FrameworkAttemptObservation/Artifact/LineageEvent、SubjectContext 与 storage/table/compute provider 最小合同。 -4. 分阶段实现 `gda-metadata-fabric-bridge`：M1 只读 mapping/reconciliation、M2a 本地 foundation/重启连续性、M2b-1 本地三存储恢复及 M2b-2 隔离 versioned/Object-Locked repository round-trip 已验证。下一步完成 M2b 外部生产 bucket、COMPLIANCE retention、TLS/KMS、独立 writer/reader identity 与跨集群恢复，再推进 OIDC、upgrade/rollback、NetworkPolicy enforcement、registry provenance、metrics/OTel 和 owner/runbook；之后才进入 M3 ingestion/OpenLineage/conformance。 +4. 分阶段实现 `gda-metadata-fabric-bridge`：M1 只读 mapping/reconciliation、M2a 本地 foundation/重启连续性、M2b-1 本地三存储恢复、M2b-2 隔离 versioned/Object-Locked repository round-trip，以及 M2b-3 本机双集群 + Kubernetes 外 COMPLIANCE repository + 独立 writer/reader 已验证。下一步完成 source host/cluster 外的生产 bucket、KMS/TLS/workload identity、source-loss recovery 与 RPO/RTO，再推进 OIDC、upgrade/rollback、NetworkPolicy enforcement、registry provenance、metrics/OTel 和 owner/runbook；之后才进入 M3 ingestion/OpenLineage/conformance。 5. 实现 `gda-orchestration-gateway`、DolphinScheduler process/task/schedule/complement/worker-group、Spark/Flink provider task adapter 和故障注入；不再开发新的 lease/queue/scheduler。 6. 冻结首条地类图斑数据、标准版本、敏感级别、owner、SLO 和 golden result。 7. 冻结 Default Lakehouse、Cloud Managed、Lightweight Integrated profiles；以统一 Run 完成默认 MinIO/Iceberg/Spark/Flink、轻量 PostGIS/DuckDB 和 Azure 代表 adapter 的 conformance smoke。 @@ -726,7 +726,7 @@ AR-4 parity/control gate 退出前暂停以下主线扩张： | 阶段 | 状态 | 下一证据 | |---|---|---| | AR-0 Architecture/Schema/Runtime Truth Freeze | `in_progress` | 全环境 schema/config fingerprint、迁移 fail-closed、事实清单、storage/compute/GIS serving provider profile/capability、ADR-017 benchmark、owner/SLO 和首条数据/服务验收集冻结 | -| AR-1 Unified Metadata + Orchestration Control Planes | `in_progress` | controlled gateway、DolphinScheduler adapter、Metadata Fabric M1/M2a、M2b-1 本地恢复及 M2b-2 versioned/Object-Locked repository round-trip 已验证；下一证据是外部生产 bucket/COMPLIANCE retention/TLS/KMS/跨集群恢复、OIDC、NetworkPolicy enforcement、升级回滚/registry provenance，以及受控 ingestion/replay 与无双写验收 | +| AR-1 Unified Metadata + Orchestration Control Planes | `in_progress` | controlled gateway、DolphinScheduler adapter、Metadata Fabric M1/M2a、M2b-1/M2b-2 及 M2b-3 本机双集群恢复已验证；下一证据是 source host/cluster 外的生产 bucket、KMS/TLS/workload identity、source-loss recovery、RPO/RTO、OIDC、NetworkPolicy enforcement、升级回滚/registry provenance，以及受控 ingestion/replay 与无双写验收 | | AR-2 Source/Ingestion + Geospatial Lakehouse Vertical Slice | `planned` | 三类代表源、`DriveTransfer` 云盘客户端和大文件恢复通过统一控制面；默认湖仓、轻量存算一体及 Azure 代表 adapter 通过 provider conformance 与 Raw -> ADS 验收 | | AR-3 Data Product Engineering + Governance Workbench | `planned` | Blueprint、模型、Visual/SQL/Notebook、DataOps CI/CD、质量/安全/审批共用 definition 和产品生命周期 | | AR-4 Asset/GIS Service/Spatial Experience Operations | `planned` | Service Control Plane、Features/Tiles/MVT/COG/STAC/export 及条件 legacy OGC/3D/EDR provider、Gateway/权限/缓存、原子切换/回滚、Discover/Operate/Govern 和无 LLM 多入口通过 conformance/parity/control gate | diff --git a/docs/system-of-record-matrix-2026-07-24.md b/docs/system-of-record-matrix-2026-07-24.md index c37814e0..de4bce7c 100644 --- a/docs/system-of-record-matrix-2026-07-24.md +++ b/docs/system-of-record-matrix-2026-07-24.md @@ -2,9 +2,9 @@ 日期：2026-07-27 -阶段：AR-0 `in_progress`；AR-1 gateway、成功终局 evidence gate、DolphinScheduler adapter sandbox POC、Metadata Fabric M1/M2a、M2b-1 本地恢复与 M2b-2 隔离 repository round-trip 已验证，生产切换仍 `in_progress` +阶段：AR-0 `in_progress`；AR-1 gateway、成功终局 evidence gate、DolphinScheduler adapter sandbox POC、Metadata Fabric M1/M2a、M2b-1/M2b-2 与 M2b-3 本机双集群恢复已验证，生产切换仍 `in_progress` -适用分支：`feat/ar1-metadata-fabric-durable-backup-contract` +适用分支：`feat/ar1-metadata-fabric-cross-cluster-recovery` ## 判定规则 @@ -20,13 +20,13 @@ | SQL schema 历史 | PostgreSQL `schema_migrations`，以完整 migration ID + checksum 为权威 | migration CLI 的 JSON 报告 | 保持现有 ledger；任何 drift fail closed | Data Platform | AR-0，已验证 | | 部署配置策略 | Compose/K8s/进程环境；`platform_truth.CONFIG_SPECS` 定义关键类型与策略；DolphinScheduler worker 有默认零副本、外部 ConfigMap/Secret 驱动的 Kustomize 模板、静态 validator 和 staging activation preflight | `.env` 仅补默认；脱敏 snapshot、Secret key attestation、未扩容 Deployment 和 `ready_for_activation` 都是观测/模板 | 版本化 DeploymentProfile + secret reference；部署环境始终优先；模板或 preflight 通过都不等于环境已启用 | Platform/SRE/Security | AR-0，部分实现；worker 模板/preflight 本地已验证 | | 环境发布与晋级 | 本地 candidate/registry/provenance/release/live 合同已绑定 publisher、verifier、OCI 和 manifest identity；canonical `main@0182406`、archive refs、三组 active ruleset 与 `staging-provenance` protected environment 已建立，但尚无成功 publisher/verifier 或 deployment | 旧 mainline、feature branch、CI artifact、JSON、离线 report 和合成 `verified_for_staging_apply` 都不能单独成为发布权威；publisher SHA、verifier SHA 与 branch lineage 必须分别验证 | 由受保护 environment 的 DeploymentRevision 绑定 OCI、provenance artifact、release manifest 与全部 live verdict | Platform/SRE/Security/Repository Owner | AR-1 mainline 治理已恢复 -> 首次 GHCR publish/verify -> 真实 staging | -| 后台运行时清单 | `platform_truth.RUNTIME_INVENTORY` 是代码层登记；`gda_control` 已有受控 PlatformRun 写入口；DolphinScheduler managed worker 已登记但尚无生产调用方；M2b-2 的前台 bounded port-forward 已登记为 `local_verification_only` recovery rehearsal，不是 scheduler、worker 或状态权威 | AST primitive report、worker status JSON、FrameworkAttemptObservation、DolphinScheduler instance state、本地 recovery evidence | PlatformRun ledger 唯一登记最终状态；framework/provider attempt 只能回报观测；本地演练进程与 evidence 不得变成生产控制器 | Platform Architecture | AR-1 adapter/worker 本地已验证；metadata recovery runner 仅本地验证 -> staging 控制链待接入 | +| 后台运行时清单 | `platform_truth.RUNTIME_INVENTORY` 是代码层登记；`gda_control` 已有受控 PlatformRun 写入口；DolphinScheduler managed worker 已登记但尚无生产调用方；M2b-2 port-forward 与 M2b-3 Docker-host repository/双 context runner 均登记为 `local_verification_only`，不是 scheduler、worker 或状态权威 | AST primitive report、worker status JSON、FrameworkAttemptObservation、DolphinScheduler instance state、本地 recovery evidence | PlatformRun ledger 唯一登记最终状态；framework/provider attempt 只能回报观测；本地演练进程与 evidence 不得变成生产控制器 | Platform Architecture | AR-1 adapter/worker 本地已验证；metadata recovery runner 仅本地验证 -> staging 控制链待接入 | | 原始文件/对象 | 当前 local uploads、S3/MinIO/OBS 均可能被直接写入，权威边界未统一 | 临时上传、下载缓存、预览文件 | Landing object 以 immutable URI + checksum + retention 为权威；本地 scratch 可删除 | Data Platform | AR-2 | | 湖仓表与 snapshot | Iceberg/STAC/S3A 有局部实现，尚无通用发布权威 | STAC item、GeoParquet export | Iceberg catalog snapshot 是分析表版本权威；对象是物理内容，STAC 是发现投影 | Data Platform | AR-2 | | 在线空间数据 | PostGIS 业务表是当前编辑/查询事实，部分临时表混入 | Martin MVT、API JSON、导出文件 | 已批准 DataProductVersion 物化到 PostGIS；不能由瓦片或临时表反向定义产品版本 | GIS/Data Platform | AR-2 -> AR-4 | | 数据资产身份与版本 | `gda_control.resource/resource_version` 已实现 identity、hash、predecessor、tenant FK 和幂等 gateway 写入；`agent_data_assets`、`agent_asset_versions` 仍是兼容写路径 | UI catalog、search index、STAC | GDA ledger 管身份与版本绑定；旧行只有在 tenant、authority identity、checksum 和 version evidence 完整时才可形成 eligible plan；OpenMetadata 管治理目录，Gravitino 管技术对象映射 | Metadata Platform | AR-1 gateway 已验证 -> 生产切换待验收 | -| 技术元数据 | M1 已冻结 Gravitino table ref/reconciliation；M2a 已运行 Gravitino `1.3.0` + 独立 PostgreSQL；M2b-1 已匹配新 PVC 中 39 张表 fingerprints；M2b-2 已将真实 dump 置入独立 versioned/Object-Locked repository，删除本地副本后按 version 下载恢复，但未创建 production catalog | harvester 结果、合成 Gravitino response、本地 sandbox/recovery/repository observation | 源系统技术对象是原始证据；Gravitino 映射并联邦，不能覆盖业务 ResourceVersion；backup object 只承载恢复 artifact | Metadata Platform | AR-1 M1 + 本地 M2a/M2b-1/M2b-2 已验证 -> 外部生产恢复/M3 conformance 待执行 | -| 治理目录 | M1 已冻结 OpenMetadata table ref/reconciliation；M2a 已运行 OpenMetadata `1.13.1` + 独立 PostgreSQL/OpenSearch；M2b-1 已匹配新 PVC 中 176 张表与 79 个索引；M2b-2 已验证三份真实 artifact 的 version/retention/download/restore，但未接入真实 GDA Resource | 搜索/页面视图、合成 OpenMetadata response、本地 sandbox/recovery/repository observation | OpenMetadata 为 owner/glossary/classification/quality discoverability 权威；GDA ledger 保留审批证据；backup repository 不是目录或资源版本权威 | Governance | AR-1 M1 + 本地 M2a/M2b-1/M2b-2 已验证 -> 外部生产恢复/M3 ingestion 待执行 | +| 技术元数据 | M1 已冻结 Gravitino table ref/reconciliation；M2a 已运行 Gravitino `1.3.0` + 独立 PostgreSQL；M2b-1/M2b-2 已完成本地新 PVC 与锁定 repository 恢复；M2b-3 已将 Gravitino dump 从 Docker-host repository 恢复到独立 kind cluster，但未创建 production catalog | harvester 结果、合成 Gravitino response、本地 sandbox/recovery/repository observation | 源系统技术对象是原始证据；Gravitino 映射并联邦，不能覆盖业务 ResourceVersion；backup object 只承载恢复 artifact | Metadata Platform | AR-1 M1 + 本地 M2a/M2b-1/M2b-2/M2b-3 已验证 -> 外部生产恢复/M3 conformance 待执行 | +| 治理目录 | M1 已冻结 OpenMetadata table ref/reconciliation；M2a 已运行 OpenMetadata `1.13.1` + 独立 PostgreSQL/OpenSearch；M2b-1/M2b-2 已完成本地新 PVC 与锁定 repository 恢复；M2b-3 已将 OpenMetadata PostgreSQL/OpenSearch artifact 恢复到独立 kind cluster，但未接入真实 GDA Resource | 搜索/页面视图、合成 OpenMetadata response、本地 sandbox/recovery/repository observation | OpenMetadata 为 owner/glossary/classification/quality discoverability 权威；GDA ledger 保留审批证据；backup repository 不是目录或资源版本权威 | Governance | AR-1 M1 + 本地 M2a/M2b-1/M2b-2/M2b-3 已验证 -> 外部生产恢复/M3 ingestion 待执行 | | 血缘 | `gda_control.lineage_event` 已实现 immutable version edge 和幂等 gateway ingest；`agent_asset_lineage` 旧记录仍是可变 asset edge | OpenMetadata lineage graph、UI DAG | 只有 source/target ResourceVersion 与 event checksum 证据完整的旧记录可形成 eligible plan；目录图只作可重建投影 | Data Platform | AR-1 gateway 已验证 -> adapter 待接入 | | Definition | `gda_control.platform_definition_version` 已绑定 definition ResourceVersion、完整逻辑 hash 和原子 gateway registration；3.4.2 adapter 可编译、创建并上线 provider DAG；binding 已以 append-only `execution_plan` Artifact 持久化并可按 tenant + artifact UUID 读取，旧 workflow/template/YAML 仍在写入 | 编辑器状态、DolphinScheduler DAG/definition | 旧 workflow 必须规范化并完整 hash 后才可形成 PlatformDefinitionVersion；provider binding 作为 ExecutionPlanArtifact/evidence，不可反写 definition | DataOps | AR-1 binding persistence 代码已验证 -> staging 调用链待验收 | | Run 最终状态 | `gda_control.platform_run/event` 已实现受控 submit/read/CAS；通用 transition 已禁止 `succeeded`，专用数据库 finalizer 只接受精确 workload、DolphinScheduler success observation、内容匹配 output、独立 passed QualityResult/evidence 和 input-to-output lineage；adapter standalone API path 已验证，但端到端 staging 尚未完成，legacy 路径继续运行 | Redis progress、日志、DolphinScheduler state、attempt observation | 旧 run 到 PlatformRun 永久 prohibited；已有 PlatformRun correlation 时才可转为 observation；provider 终态只进入 `reconciling`，ledger 经证据门唯一裁决成功 | DataOps/AgentOps | AR-1 success authority 本地/PostgreSQL 已验证 -> staging/生产切换待验收 | @@ -45,7 +45,7 @@ 1. Redis、进程内 task、WebSocket、MVT、STAC、搜索索引和 UI 状态都不能成为最终事实源。 2. 一个 provider 返回 success 只形成 attempt observation；通用 transition 不能写 `succeeded`，必须由数据库 finalizer 验证 output hash、独立 passed QualityResult/evidence 和 input-to-output lineage。 -3. OpenMetadata、Gravitino、DolphinScheduler 和 Temporal 在完整真实 POC/退出门前不得写成当前生产权威；OpenMetadata/Gravitino 的 M2a foundation、M2b-1 同集群恢复与 M2b-2 同集群 repository round-trip 只证明本地 live/PVC/逻辑恢复/S3 API 行为，不改变此边界。 +3. OpenMetadata、Gravitino、DolphinScheduler 和 Temporal 在完整真实 POC/退出门前不得写成当前生产权威；OpenMetadata/Gravitino 的 M2a foundation、M2b-1/M2b-2 同集群恢复及 M2b-3 同主机双集群恢复只证明本地 live/PVC/逻辑恢复/S3 API 行为，不改变此边界。 4. 新增 registry、metadata table、queue、scheduler 或后台任务前，必须先更新本矩阵、runtime inventory、owner 和迁移/恢复策略。 5. 自然资源地类图斑纵向链是首个验证载体：其 input checksum、标准版本、DataProductVersion、QualityResult、LineageEvent、RunRef 和 serving revision 必须贯通后，才能宣称控制面闭环。 6. `gda_control` 已有测试验证的 gateway role/API，但没有生产业务调用方；gateway 可用不等于生产运行链已经切换。 @@ -56,7 +56,7 @@ 11. `platform_command_outbox` 只拥有投递状态；callback 只触发 reconcile，不能把 provider payload 直接写成 PlatformRun 状态或平台终局。 12. QualityResult evaluator 必须是 workload，且成功终局中的 evaluator 不能等于 Run workload；该代码级职责分离不替代生产 IAM。 13. `candidate_validated`、`registry_subject_bound`、GitHub provenance action 成功、CI artifact、离线 preflight、未独立 attested 的 live observation JSON 或人工批准都不能单独授权 production；缺少同一 source revision 的 OCI subject 独立验证、registry/live revision/identity/health/golden-slice 绑定及受保护 provenance 时，promotion 必须失败。 -14. Metadata Fabric M1 只允许 OpenMetadata/Gravitino GET；M2a/M2b-1/M2b-2 只运行本地 foundation/recovery/repository 且 `writes_to_gda_enabled=false`。`backup_restore_verified=true` 只适用于 `local_same_cluster_new_namespace_and_pvcs`，`backup_repository_verified=true` 只适用于 `local_same_cluster_isolated_s3_compatible_repository`；这些 evidence 都不等于 production provider、外部 backup target、COMPLIANCE retention、TLS/KMS、跨集群恢复、OIDC、upgrade 或写权威。 +14. Metadata Fabric M1 只允许 OpenMetadata/Gravitino GET；M2a/M2b-1/M2b-2/M2b-3 只运行本地 foundation/recovery/repository 且 `writes_to_gda_enabled=false`。M2b-3 的 `local_cross_cluster_recovery_verified=true` 只适用于同一 Docker Desktop 主机内的两个 Kubernetes cluster 与 Docker-host repository；这些 evidence 都不等于 production provider、外部生产 backup target、生产 retention、TLS/KMS/workload identity、source-host loss、生产跨集群/跨区域恢复、OIDC、upgrade 或写权威。 ## 已建立的 AR-0/AR-1 entry 证据 @@ -73,7 +73,7 @@ - staging candidate evidence 已在本地绑定 Git SHA、本地 image ID、97/97 schema fingerprint、严格脱敏配置、runtime inventory 和 JUnit 汇总；管理员/普通角色 ledger 一致，candidate 仍固定 `staging_deployed=false`、`production_promotion_allowed=false`。GitHub Runner 和真实 staging 尚未运行该链。 - GHCR publication contract 已固定单次 application image build、OCI revision/source label、远端 raw manifest `sha256`、candidate-to-subject binding 和 GitHub OIDC provenance；独立 verifier 已区分 publisher/verifier revision，固定证书 repository/workflow/ref/digest/issuer/runner 策略并对验证 evidence 再 attested；release gate 已验证该 evidence artifact 身份并从中唯一派生 manifest image。canonical mainline、protected environment、reviewer 和 Actions 权限已配置；一次意外 publisher run 在依赖安装阶段取消，尚无真实 published/verified subject、provenance artifact 或 verified release。 - live staging collector 已对 Docker Desktop 集群完成只读实采：candidate、collection freshness、97/97 应用角色 schema、runtime baseline 和 health/readiness 通过；App/Outbox 已改为直接读取 migration ledger 并禁用 token automount，重采确认 token 隔离通过。tagged 本地镜像、缺 source/candidate/platform 注解、非 strict staging profile 及缺真实 golden-slice 仍正确阻断；合成完整 evidence 可验证 live 绑定，但因缺受保护 provenance/attestation 仍固定禁止 production promotion。 -- Metadata Fabric M1 已冻结 OpenMetadata/Gravitino 只读 binding/reconciliation；M2a 已在两节点 ARM64 Docker Desktop Kubernetes 验证五个 Pod、三块 PVC、176/39 张表和 78 个索引的连续性，fingerprint `ac21ee50ba3c1f27f949420cc7e4483963714b6b955bb0157eca1dd39cf102c3`。M2b-1 已将两份 PostgreSQL dump 和 OpenSearch snapshot 恢复到新 namespace/PVC，repository-backed 复测 evidence fingerprint `3cf46cc83a8feaa4142893a06f84e9008a7d805a64fb6708d176ca976a4dbd62`。M2b-2 又在独立 MinIO/PVC 启用 versioning/Object Lock，三份真实 artifact 获得 version/retention、删除被拒绝，本地副本删除后按 version 下载并完成恢复；repository evidence fingerprint `07834430fb140e147624f4ab3c93e6d7907648e91a149cb19db178ea6085d1ed`。生产 target/retention/TLS/KMS、RPO/RTO、跨集群、OIDC 和真实 ingestion flags 仍为 false。 +- Metadata Fabric M1 已冻结 OpenMetadata/Gravitino 只读 binding/reconciliation；M2a 已在两节点 ARM64 Docker Desktop Kubernetes 验证五个 Pod、三块 PVC、176/39 张表和 78 个索引连续性。M2b-1/M2b-2 已验证同集群新 PVC 和 versioned/Object-Locked repository round-trip。M2b-3 又从 `docker-desktop` 经 Docker-host `COMPLIANCE/1 day` MinIO、独立 writer/reader，将三份 artifact 恢复到独立 kind cluster；两个 cluster UID 不同且 cleanup 完成，evidence fingerprint `9eaf8cec9ec2d3763260c271c0c27c1c3251717820d38aadfef0bec7d7a574a8`。生产 target/retention/TLS/KMS/workload identity、source-host loss、RPO/RTO、生产跨集群/跨区域、OIDC 和真实 ingestion flags 仍为 false。 ## 下一验收证据 @@ -82,5 +82,5 @@ - staging 的 migration role、应用 login membership、连接池 role/tenant 复位、双租户 API 和 success finalization 运行产物； - DolphinScheduler adapter 的真实 IAM/OIDC、service token provisioning/轮换、provider 最小权限、binding artifact staging 接入、managed outbox worker/provider callback 实际扩容部署、唯一 worker ID、status/lease 故障恢复和无双写证据； - 首条真实图斑链对 golden slice 的 output hash、独立质量结果/evidence、血缘、发布 revision 和 rollback 演练； -- OpenMetadata/Gravitino M2b 的外部 backup account/bucket、COMPLIANCE retention、TLS/KMS、独立 writer/reader identity、PITR/跨集群恢复、RPO/RTO、OIDC、NetworkPolicy enforcement、upgrade/rollback、registry provenance、metrics/OTel 和 owner/runbook，以及 M3 ingestion/OpenLineage/无双写/conformance；M1 fixture、M2a PVC、M2b-1 同集群恢复与 M2b-2 本地 repository evidence 均不计入生产退出门； +- OpenMetadata/Gravitino M2b 的 source host/cluster 外生产 backup account/bucket、KMS/TLS/workload identity、PITR/source-loss recovery、RPO/RTO、OIDC、NetworkPolicy enforcement、upgrade/rollback、registry provenance、metrics/OTel 和 owner/runbook，以及 M3 ingestion/OpenLineage/无双写/conformance；M1 fixture、M2a PVC、M2b-1/M2b-2 同集群 evidence 与 M2b-3 同主机双集群 evidence 均不计入生产退出门； - DolphinScheduler/Temporal sandbox 的独立数据库、备份恢复、身份、版本和升级责任证明；DolphinScheduler standalone/H2 不计入此退出门。 diff --git a/scripts/metadata-fabric-cross-cluster-recovery.sh b/scripts/metadata-fabric-cross-cluster-recovery.sh new file mode 100755 index 00000000..20704609 --- /dev/null +++ b/scripts/metadata-fabric-cross-cluster-recovery.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +COMMON_GIT_DIR="$(git -C "$ROOT" rev-parse --path-format=absolute --git-common-dir 2>/dev/null || true)" +SHARED_ROOT="" +if [ -n "$COMMON_GIT_DIR" ]; then + SHARED_ROOT="$(cd "$COMMON_GIT_DIR/.." && pwd)" +fi + +if [ -n "${PYTHON:-}" ]; then + : +elif [ -x "$ROOT/.venv/bin/python" ]; then + PYTHON="$ROOT/.venv/bin/python" +elif [ -n "$SHARED_ROOT" ] && [ -x "$SHARED_ROOT/.venv/bin/python" ]; then + PYTHON="$SHARED_ROOT/.venv/bin/python" +else + PYTHON="python" +fi + +cd "$ROOT" +exec "$PYTHON" -m data_agent.metadata_fabric_cross_cluster_recovery "$@"