diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 4f39093beb3..669b6457123 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -408,6 +408,43 @@ jobs: run: | go tool -modfile=tools/task/go.mod task test-sandbox + test-fuzz: + needs: + - cleanups + + # Wide rotating seed window with drift on: too slow for every PR, so nightly only and + # not part of test-result. The committed acceptance test still checks no-panic per PR. + if: ${{ github.event_name == 'schedule' }} + name: "task test-fuzz" + runs-on: + group: databricks-protected-runner-group-large + labels: linux-ubuntu-latest-large + + defaults: + run: + shell: bash + + permissions: + id-token: write + contents: read + + steps: + - name: Checkout repository and submodules + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup build environment + uses: ./.github/actions/setup-build-environment + with: + cache-key: test-fuzz + + - name: Run tests + env: + FUZZ_SEED_COUNT: "25" + run: | + # Non-overlapping windows, so CI explores new configs every run. + export FUZZ_SEED_START=$(( GITHUB_RUN_NUMBER * FUZZ_SEED_COUNT )) + go tool -modfile=tools/task/go.mod task test-fuzz + # This job groups the result of all the above test jobs. # It is a required check, so it blocks auto-merge and the merge queue. # diff --git a/Taskfile.yml b/Taskfile.yml index 3c7026d9818..579095fad85 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -739,6 +739,23 @@ tasks: --packages ./acceptance/... \ -- -timeout=${LOCAL_TIMEOUT:-60m} -run "TestAccept/cmd/sandbox" + test-fuzz: + desc: Run schema fuzz invariant tests (random configs, direct engine) + # No `sources:` fingerprint: the window depends on FUZZ_* env vars Task can't see. + cmds: + - | + # Wider window than the committed run, with drift on; a repro narrows it via + # FUZZ_SEED_START/COUNT. + export FUZZ_SEED_COUNT="${FUZZ_SEED_COUNT:-200}" + export FUZZ_CHECK_DRIFT="${FUZZ_CHECK_DRIFT:-1}" + # -count=1: only the script reads FUZZ_*, so the test cache would serve another window's + # result as this one's. + {{.GO_TOOL}} gotestsum \ + --format ${GOTESTSUM_FORMAT:-pkgname-and-test-fails} \ + --no-summary=skipped \ + --packages ./acceptance/... \ + -- -count=1 -timeout=${LOCAL_TIMEOUT:-30m} -run "TestAccept/bundle/fuzz" + # --- Integration tests --- integration: diff --git a/acceptance/bin/emit_fuzz_config.py b/acceptance/bin/emit_fuzz_config.py new file mode 100755 index 00000000000..28509eabdd6 --- /dev/null +++ b/acceptance/bin/emit_fuzz_config.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +""" +Emit a fuzz databricks.yml on stdout for the current seed, picking the strategy from FUZZ_MODE so +the invariant scripts don't each duplicate the branch: + + generate - build from scratch by walking `bundle schema` (gen_fuzz_config.py). + mutate - perturb a curated invariant config (mutate_fuzz_config.py). + +Reads FUZZ_SEED, FUZZ_SCHEMA, FUZZ_MODE, UNIQUE_NAME and INVARIANT_DIR, which fuzz/script and the +invariant prologue set. +""" + +import json +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from envsubst import substitute_variables +from gen_fuzz_config import gen_config, to_yaml +from mutate_fuzz_config import load_yaml, mutate + +# Curated single-resource configs that deploy standalone (only $UNIQUE_NAME, no init script). All +# are in the invariant INPUT_CONFIG matrix, so they stay deploy-verified. +MUTATE_BASES = [ + "catalog", + "external_location", + "job", + "model", + "model_serving_endpoint", + "pipeline", + "registered_model", + "schema", + "secret_scope", + "sql_warehouse", + "volume", +] + + +def generate(seed): + with open(os.environ["FUZZ_SCHEMA"]) as f: + schema = json.load(f) + return to_yaml(gen_config(schema, seed, os.environ["UNIQUE_NAME"])) + + +def mutate_base(seed): + name = MUTATE_BASES[seed % len(MUTATE_BASES)] + path = os.path.join(os.environ["INVARIANT_DIR"], "configs", name + ".yml.tmpl") + unique = os.environ["UNIQUE_NAME"] + with open(path) as f: + rendered = substitute_variables(f.read()) + config = load_yaml(rendered) + # The schema lets mutate inject valid optional fields, not just perturb existing ones. + with open(os.environ["FUZZ_SCHEMA"]) as f: + schema = json.load(f) + return to_yaml(mutate(config, seed, schema=schema, unique=unique)) + + +def main(): + seed = int(os.environ["FUZZ_SEED"]) + mode = os.environ["FUZZ_MODE"] + if mode == "generate": + sys.stdout.write(generate(seed)) + elif mode == "mutate": + sys.stdout.write(mutate_base(seed)) + else: + sys.exit(f"emit_fuzz_config: unknown FUZZ_MODE {mode!r}") + + +if __name__ == "__main__": + main() diff --git a/acceptance/bin/gen_fuzz_config.py b/acceptance/bin/gen_fuzz_config.py new file mode 100755 index 00000000000..fadd5483845 --- /dev/null +++ b/acceptance/bin/gen_fuzz_config.py @@ -0,0 +1,431 @@ +""" +Generate a random bundle config from the bundle JSON schema. + +Walks `databricks bundle schema` (resolving $ref, picking concrete oneOf/anyOf branches) and emits +one random resource, seeded by the caller. Free-form scalars are sometimes replaced with dangerous +values (DANGEROUS_STRINGS/INTS) to probe input handling. The harness drops configs the CLI +rejects, so output may be structurally random but invalid. + +Used as a library by emit_fuzz_config.py and mutate_fuzz_config.py. +""" + +import json +import os +import random +import re +import sys + +# Depth past which optional properties are no longer emitted, to keep configs from exploding. +MAX_DEPTH = 6 + +# Hard cap on the walk. MAX_DEPTH gates optional properties only; required ones and map values +# recurse regardless, so a required-only cycle (task -> for_each_task -> task) would exhaust the +# stack. Fail loudly: that is a schema or generator bug, not something to silently truncate. +MAX_RECURSION = 30 + +# The ${...} interpolation branch the schema wraps every field in (see +# bundle/internal/schema/main.go addInterpolationPatterns); we emit concrete values. +INTERPOLATION_MARKER = "\\$\\{" + +# Keep in sync with libs/jsonschema.Type. gen_scalar exits on anything else. +SCALAR_TYPES = {"boolean", "integer", "number", "string"} + +# Cross-resource refs must resolve on every workspace. "main"/"default" are the standard seeded +# catalog/schema; a random name deploys on the fake server but real UC rejects it +# (CATALOG_DOES_NOT_EXIST), dropping the config. +DEFAULT_CATALOG = "main" +DEFAULT_SCHEMA = "default" + +# "account users" exists on every workspace; each securable gets one privilege UC accepts. A random +# principal or inapplicable privilege deploys on the fake server but fails on UC. +DEFAULT_PRINCIPAL = "account users" +GRANT_PRIVILEGE = { + "catalogs": "USE_CATALOG", + "schemas": "USE_SCHEMA", + "volumes": "READ_VOLUME", + "registered_models": "EXECUTE", + "external_locations": "READ_FILES", + "vector_search_indexes": "SELECT", +} + +# Permissions can't be variable refs; each entry needs a concrete principal and a level valid for +# the resource type. +DEFAULT_PERMISSION_GROUP = "users" +PERMISSION_LEVEL = { + "alerts": "CAN_MANAGE", + "apps": "CAN_USE", + "clusters": "CAN_ATTACH_TO", + "dashboards": "CAN_READ", + "database_instances": "CAN_USE", + "experiments": "CAN_READ", + "genie_spaces": "CAN_READ", + "jobs": "CAN_VIEW", + "model_serving_endpoints": "CAN_VIEW", + "models": "CAN_READ", + "pipelines": "CAN_VIEW", + "postgres_projects": "CAN_USE", + "secret_scopes": "READ", + "sql_warehouses": "CAN_VIEW", + "vector_search_endpoints": "CAN_USE", +} + +# Fields the backend computes; emitting them causes false drift after migrate. Mirrors +# output_only/backend_defaults in dresources/resources.yml. Blocked by name everywhere, so writable +# exceptions (an external volume's storage_location) need a curated config. +SKIP_PROPERTY_NAMES = frozenset( + { + "browse_only", + "created_at", + "created_by", + "creator_name", + # Backend-assigned; the CLI rejects an etag set in bundle config. + "etag", + "full_name", + "metastore_id", + "owner", + "storage_location", + "updated_at", + "updated_by", + } +) + +# Fields these resources need to deploy but that the schema's required[] omits (or can't express in +# YAML). Values come from the *_BY_RESOURCE tables below. +RESOURCE_REQUIRED_FIELDS = { + "registered_models": frozenset({"catalog_name", "name", "schema_name"}), + "dashboards": frozenset({"display_name", "file_path", "warehouse_id"}), + "alerts": frozenset({"display_name", "file_path", "warehouse_id"}), + "apps": frozenset({"name", "source_code_path"}), + "genie_spaces": frozenset({"serialized_space", "title", "warehouse_id"}), +} + +# Fields that conflict with the set we emit. Dashboards/Genie spaces take their body from file_path +# XOR an inline serialized_* field; emitting both is rejected. +RESOURCE_SKIP_FIELDS = { + "dashboards": frozenset({"serialized_dashboard"}), + "genie_spaces": frozenset({"file_path"}), + "apps": frozenset({"git_repository", "git_source"}), +} + +# Resources allowing only a fixed field set in YAML. Alerts read their spec from the +# .dbalert.json at file_path; the CLI rejects other fields (load_dbalert_files.go). +RESOURCE_FIELD_ALLOWLIST = { + "alerts": frozenset({"display_name", "file_path", "lifecycle", "permissions", "warehouse_id"}), +} + +# Serialized-body fixtures copied into each seed dir from invariant/data; the extension selects the +# parser. +FILE_PATH_BY_RESOURCE = { + "dashboards": "./dashboard.lvdash.json", + "alerts": "./alert.dbalert.json", +} + +# A local directory holding app source, also copied in from data/. +APP_SOURCE_CODE_PATH = "./app" + +# An absolute workspace path is treated as already-remote, skipping the local-notebook +# existence/extension check a bare token would fail. +NOTEBOOK_PATH = "/Shared/notebook" + +# parent_path is a workspace folder; pin it to a valid one. The CLI re-adds the /Workspace prefix +# on read, so a mismatched value plans a spurious recreate. +PARENT_PATH = "/Workspace/Shared" + +# String in the schema but parsed as protobuf.Duration at load (suspend_timeout_duration, ttl); a +# bare token fails to parse. +DURATION_VALUE = "3600s" + +# Dangerous/near-range-end probes for free-form scalars. The CLI must reject or round-trip these +# without panicking; mutate_fuzz_config reuses them. +DANGEROUS_STRINGS = [ + "", + " ", + "a" * 300, + "line1\nline2", + "tab\there", + "\U0001f680-unicode-\u00e9", + "quote\"and'apostrophe", + "${resources.jobs.does_not_exist.id}", + "../../etc/passwd", +] +DANGEROUS_INTS = [ + 2**31 - 1, + 2**31, + -(2**31), + 2**63 - 1, + -(2**63), + -1, +] + +# Inject a dangerous value only sometimes, so the config usually still deploys and exercises the +# invariant, not just the reject path. +DANGEROUS_PROB = 0.15 + + +class Generator: + def __init__(self, schema, rng, unique): + self.root = schema + self.rng = rng + self.unique = unique + # Top-level resource type, set before generating its element so grants/permissions can pick + # a value valid for that securable. + self.rtype = None + + def resolve(self, schema): + # Follow $ref chains ("#/$defs/.../resources.Job"), indexing $defs by path segment. + while isinstance(schema, dict) and "$ref" in schema: + cur = self.root["$defs"] + for part in schema["$ref"].split("/")[2:]: + cur = cur[part] + schema = cur + return schema + + def is_interpolation(self, branch): + return branch.get("type") == "string" and INTERPOLATION_MARKER in branch.get("pattern", "") + + def choose_branch(self, branches): + # Prefer concrete branches over the ${...} alternatives. + concrete = [b for b in branches if not self.is_interpolation(b)] + return self.rng.choice(concrete or branches) + + def field_behaviors(self, schema): + if not isinstance(schema, dict): + return [] + resolved = self.resolve(schema) + behaviors = list(schema.get("x-databricks-field-behaviors", [])) + if resolved is not schema: + behaviors.extend(resolved.get("x-databricks-field-behaviors", [])) + return behaviors + + def should_skip_property(self, prop_name, prop_schema): + if prop_name in SKIP_PROPERTY_NAMES: + return True + if prop_name in RESOURCE_SKIP_FIELDS.get(self.rtype, ()): + return True + resolved = self.resolve(prop_schema) + if "OUTPUT_ONLY" in self.field_behaviors(prop_schema): + return True + if resolved.get("readOnly"): + return True + return False + + def gen(self, schema, depth, name=""): + if depth > MAX_RECURSION: + sys.exit(f"gen_fuzz_config: schema walk exceeded {MAX_RECURSION} levels at {name!r}") + + # A Genie space body is free-form but the backend rejects unknown keys, so emit the minimal + # accepted body instead of a random object. + if name == "serialized_space": + return {"version": 1} + + schema = self.resolve(schema) + if not isinstance(schema, dict) or not schema: + return self.gen_scalar({"type": "string"}, name) + + if name == "grants": + return self.gen_grants() + if name == "permissions": + return self.gen_permissions() + + if "const" in schema: + return schema["const"] + if schema.get("enum"): + return self.rng.choice(schema["enum"]) + + for key in ("oneOf", "anyOf"): + if schema.get(key): + return self.gen(self.choose_branch(schema[key]), depth, name) + + t = schema.get("type") + if t == "object" or "properties" in schema or self.is_map(schema): + return self.gen_object(schema, depth) + if t == "array": + return self.gen_array(schema, depth, name) + return self.gen_scalar(schema, name) + + def is_map(self, schema): + return isinstance(schema.get("additionalProperties"), dict) and not schema.get("properties") + + def gen_object(self, schema, depth): + props = schema.get("properties", {}) + required = set(schema.get("required", [])) + allowlist = None + if depth == 0 and self.rtype: + required |= RESOURCE_REQUIRED_FIELDS.get(self.rtype, set()) + allowlist = RESOURCE_FIELD_ALLOWLIST.get(self.rtype) + result = {} + + for prop_name, prop_schema in props.items(): + # A restricted resource (e.g. alerts) rejects any field outside its allow-list, even a + # schema-required one it reads from the file instead. + if allowlist is not None and prop_name not in allowlist: + continue + if self.should_skip_property(prop_name, prop_schema): + continue + # Emit optional fields less often deeper down to keep configs from exploding. + keep = prop_name in required or (depth < MAX_DEPTH and self.rng.random() < 0.35) + if not keep: + continue + value = self.gen(prop_schema, depth + 1, prop_name) + # Drop an object whose every property was skipped: `{}` carries no information and some + # fields reject it outright. + if value is None or value == {}: + continue + result[prop_name] = value + + # Map type: synthesize a few random keys, e.g. resources. or string maps like tags. + if self.is_map(schema): + for _ in range(self.rng.randint(1, 2)): + key = self.token() + result[key] = self.gen(schema["additionalProperties"], depth + 1, key) + + return result + + def gen_array(self, schema, depth, name): + items = schema.get("items") + if not items or depth >= MAX_DEPTH: + return [] + return [self.gen(items, depth + 1, name) for _ in range(self.rng.randint(1, 3))] + + def gen_grants(self): + # One known-good grant for the securable. No valid privilege means no grants node: UC + # rejects a wrong one, and an empty one only reproduces the known drift bugs. + privilege = GRANT_PRIVILEGE.get(self.rtype) + if privilege is None: + return None + return [{"principal": DEFAULT_PRINCIPAL, "privileges": [privilege]}] + + def gen_permissions(self): + # As gen_grants: no valid level means no permissions node, rather than a random principal, a + # ${...} ref, or an empty list. + level = PERMISSION_LEVEL.get(self.rtype) + if level is None: + return None + return [{"level": level, "group_name": DEFAULT_PERMISSION_GROUP}] + + def gen_scalar(self, schema, name): + t = schema.get("type") + if t == "boolean": + # Cleanup must be able to destroy the bundle. + if name == "prevent_destroy": + return False + return self.rng.choice([True, False]) + if t == "integer": + # In hours, but UC accepts only a window of 0 or 7-30 days (0 or 168-720 hours). + if name == "custom_max_retention_hours": + return self.rng.choice([0, self.rng.randint(168, 720)]) + if self.rng.random() < DANGEROUS_PROB: + return self.rng.choice(DANGEROUS_INTS) + return self.rng.choice([0, 1, self.rng.randint(2, 1000)]) + if t == "number": + return round(self.rng.uniform(0, 1000), 2) + # Fail loud on an unknown type; a missing type is "any" and falls through to string. + if t is not None and t not in SCALAR_TYPES: + sys.exit(f"gen_fuzz_config: unhandled schema type {t!r}") + # Pin cross-resource refs and typed-string fields to accepted values; a random token fails + # format/existence validation and drops the config. + if name == "catalog_name": + return DEFAULT_CATALOG + if name == "schema_name": + return DEFAULT_SCHEMA + if name == "warehouse_id": + # Always set by the acceptance harness (see acceptance_test.go); an empty one here + # would silently reject every warehouse-backed seed instead of failing. + return os.environ["TEST_DEFAULT_WAREHOUSE_ID"] + if name == "notebook_path": + return NOTEBOOK_PATH + if name == "source_code_path": + return APP_SOURCE_CODE_PATH + if name == "parent_path": + return PARENT_PATH + if name == "file_path": + return FILE_PATH_BY_RESOURCE.get(self.rtype, self.token()) + if name.endswith("_duration") or name == "ttl": + return DURATION_VALUE + if name == "name" and self.rtype == "vector_search_indexes": + # UC requires the full catalog.schema.table name; each part is alphanumeric+_. + table = re.sub(r"[^0-9a-zA-Z_]", "_", f"fuzz_index_{self.unique}") + return f"{DEFAULT_CATALOG}.{DEFAULT_SCHEMA}.{table}" + if name in ("name", "display_name"): + return f"fuzz-{name}-{self.unique}" + # Free-form string with no pinned meaning (description, comment, tag): safe to probe + # dangerous input here, unlike the pinned fields above. + if self.rng.random() < DANGEROUS_PROB: + return self.rng.choice(DANGEROUS_STRINGS) + return self.token() + + def token(self): + return "fuzz_" + "".join(self.rng.choice("abcdefghijklmnopqrstuvwxyz0123456789") for _ in range(8)) + + +def resource_types(gen): + # resources is oneOf[{ object with one property per resource type }]. + resources = gen.resolve(gen.root["properties"]["resources"]) + obj = next(b for b in resources["oneOf"] if b.get("type") == "object") + return obj["properties"] + + +def resource_element(gen, type_schema): + # Each type is a map; the element schema is the object branch's additionalProperties. + map_schema = gen.resolve(type_schema) + obj = next(b for b in map_schema["oneOf"] if b.get("type") == "object") + return obj["additionalProperties"] + + +def gen_config(schema, seed, unique, allowed=frozenset()): + gen = Generator(schema, random.Random(seed), unique) + + types = resource_types(gen) + candidates = [t for t in types if not allowed or t in allowed] + if not candidates: + sys.exit(f"no resource types to generate from (allowed={sorted(allowed)})") + + rtype = gen.rng.choice(sorted(candidates)) + gen.rtype = rtype + instance = gen.gen(resource_element(gen, types[rtype]), 0) + + return { + # Same name shape as the curated configs, so targets that derive workspace paths from the + # bundle name work unchanged. + "bundle": {"name": f"test-bundle-{unique}"}, + "resources": {rtype: {f"fuzz_{rtype}_{seed}": instance}}, + } + + +def to_yaml(obj, indent=0, list_item=False): + pad = " " * indent + if isinstance(obj, dict): + if not obj: + return f"{pad}{{}}\n" if not list_item else f"{pad}- {{}}\n" + out = "" + first = True + for k, v in obj.items(): + prefix = pad + "- " if list_item and first else (pad + " " if list_item else pad) + child_indent = indent + 2 if list_item else indent + 1 + if isinstance(v, (dict, list)) and v: + out += f"{prefix}{k}:\n" + to_yaml(v, child_indent) + else: + out += f"{prefix}{k}: {dump_scalar(v)}\n" + first = False + return out + if isinstance(obj, list): + if not obj: + return f"{pad}- []\n" if list_item else f"{pad}[]\n" + # A list inside a list: the marker needs its own line, else the two flatten into one. + if list_item: + return f"{pad}-\n" + to_yaml(obj, indent + 1) + out = "" + for item in obj: + if isinstance(item, (dict, list)): + out += to_yaml(item, indent, list_item=True) + else: + out += f"{pad}- {dump_scalar(item)}\n" + return out + return f"{pad}{dump_scalar(obj)}\n" + + +def dump_scalar(v): + # ensure_ascii=False keeps non-ASCII as literal UTF-8. The default escapes astral chars (e.g. + # the rocket probe) into surrogate pairs that YAML rejects, killing the config at parse time + # before it reaches bundle logic. Control chars stay escaped by json.dumps (YAML ok). + return json.dumps(v, ensure_ascii=False) diff --git a/acceptance/bin/gen_fuzz_config_check.py b/acceptance/bin/gen_fuzz_config_check.py new file mode 100755 index 00000000000..6e0db97b938 --- /dev/null +++ b/acceptance/bin/gen_fuzz_config_check.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +""" +Contract check for gen_fuzz_config.to_yaml: every scalar is on its own line as +`key: `, which mutate_fuzz_config's line-based loader relies on. Prints each case's +YAML (diffed by the harness) and exits non-zero on a violation. +""" + +import json +import os +import re +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from gen_fuzz_config import DANGEROUS_STRINGS, SKIP_PROPERTY_NAMES, gen_config, to_yaml + +# Tricky shapes: strings with ':' and '"', nested maps, lists of dicts, lists in lists, +# empty containers. +CASES = [ + {"comment": "value: with a colon", "description": 'quote " and : colon'}, + {"resources": {"jobs": {"j": {"name": "n", "tags": {"team": "jobs"}}}}}, + {"tasks": [{"description": "d", "timeout_seconds": 3600}, {"comment": "c"}]}, + {"nums": [0, 1, 2], "flag": True, "ratio": 1.5, "empty_map": {}, "empty_list": []}, + {"matrix": [[1, 2], [], {}]}, +] + +HEADER = re.compile(r"[\w.\-]+:$") # non-empty container: `key:` +SCALAR = re.compile(r"[\w.\-]+: (.+)$") # `key: ` + + +def check_line(line): + rest = line.lstrip(" ") + if rest == "-": + return # nested container marker; value is on following lines + rest = rest.removeprefix("- ") + if HEADER.fullmatch(rest): + return # container header; value is on following lines + m = SCALAR.fullmatch(rest) + if m: + json.loads(m.group(1)) # value must be single-line JSON + return + json.loads(rest) # bare list scalar: `- ` + + +def main(): + failed = False + for case in CASES: + text = to_yaml(case) + sys.stdout.write(text) + for line in text.splitlines(): + if not line.strip(): + continue + try: + check_line(line) + except ValueError: + sys.stderr.write(f"contract violation: not `key: `: {line!r}\n") + failed = True + + # Each DANGEROUS_STRINGS probe must still serialize to a single `key: ` line. + for i, val in enumerate(DANGEROUS_STRINGS): + line = to_yaml({"description": val}).rstrip("\n") + if "\n" in line or not SCALAR.fullmatch(line): + sys.stderr.write(f"DANGEROUS_STRINGS[{i}] broke the one-line scalar contract: {line!r}\n") + failed = True + + schema_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../bundle/schema/jsonschema.json") + with open(schema_path) as f: + schema = json.load(f) + seed24 = gen_config(schema, seed=24, unique="check", allowed={"registered_models"}) + rm = seed24["resources"]["registered_models"]["fuzz_registered_models_24"] + if SKIP_PROPERTY_NAMES & set(rm): + sys.stderr.write( + f"seed 24 registered_models emitted output-only fields: {sorted(SKIP_PROPERTY_NAMES & set(rm))}\n" + ) + failed = True + for field in ("name", "catalog_name", "schema_name"): + if field not in rm: + sys.stderr.write(f"seed 24 registered_models missing {field}\n") + failed = True + + if failed: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/acceptance/bin/mutate_fuzz_config.py b/acceptance/bin/mutate_fuzz_config.py new file mode 100755 index 00000000000..f373693ea84 --- /dev/null +++ b/acceptance/bin/mutate_fuzz_config.py @@ -0,0 +1,257 @@ +""" +Mutate a known-good bundle config by deleting, perturbing, and adding random fields. + +Complements gen_fuzz_config.py: instead of building from the schema, this perturbs a curated +invariant config that already deploys, so it reaches a much higher deploy rate. + +Two mutation kinds, chosen per step: + +- destructive (always): delete a field or replace it with a token, a dangerous value, or an empty + container. Stays within the base's fields, so it finds only reject/panic bugs. +- additive (with a schema): inject a valid optional field the base omits, valued by the schema + generator. This is what reaches reconcile/drift bugs. + +The harness only asserts no-panic on fuzzed configs, so an invalid mutation is fine: the CLI must +reject it cleanly, not crash. + +Used as a library by emit_fuzz_config.py. +""" + +import os +import random +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from gen_fuzz_config import DANGEROUS_INTS, DANGEROUS_STRINGS, Generator, resource_element, resource_types + +DANGEROUS = DANGEROUS_STRINGS + DANGEROUS_INTS + +# Chance a step injects a field rather than perturbing one. Biased high: injection is the path to +# drift bugs, and destructive coverage is already dense. +ADD_PROB = 0.6 + + +def tokenize(text): + # (indent, content) per non-blank, non-comment line. Only full-line comments are stripped; the + # curated bases have no trailing "#" in values. + out = [] + for raw in text.splitlines(): + stripped = raw.lstrip(" ") + if not stripped or stripped.startswith("#"): + continue + out.append((len(raw) - len(stripped), stripped.rstrip())) + return out + + +def scalar(text): + if text in ("", "null", "~"): + return None + # to_yaml emits empty containers in flow form; read them back so load -> emit -> load holds. + if text == "[]": + return [] + if text == "{}": + return {} + # Populated flow style is the one shape this loader cannot represent: "[id]" would read back as + # the string "[id]", turning a list into a scalar, and load -> emit -> load stays a fixed point, + # so the round-trip check cannot see it either. Exit so a new MUTATE_BASES entry fails the + # selftest rather than silently costing coverage. + if text[0] in "[{": + sys.exit(f"mutate_fuzz_config: flow-style value is not supported: {text!r}") + if text == "true": + return True + if text == "false": + return False + try: + return int(text) + except ValueError: + pass + try: + return float(text) + except ValueError: + pass + if len(text) >= 2 and text[0] == text[-1] and text[0] in "\"'": + return text[1:-1] + return text + + +def parse_block(tokens, i, indent): + if i >= len(tokens): + return {}, i + first = tokens[i][1] + if first.startswith("- ") or first == "-": + return parse_seq(tokens, i, indent) + if ": " in first or first.endswith(":"): + return parse_map(tokens, i, indent) + # Bare scalar: the whole block is a single value (e.g. a list scalar item). + return scalar(first), i + 1 + + +def parse_map(tokens, i, indent): + result = {} + while i < len(tokens) and tokens[i][0] == indent: + content = tokens[i][1] + if content.startswith("- "): + break + if ": " in content: + key, _, rest = content.partition(": ") + result[key.strip()] = scalar(rest) + i += 1 + elif content.endswith(":"): + key = content[:-1].strip() + i += 1 + if i < len(tokens) and tokens[i][0] > indent: + value, i = parse_block(tokens, i, tokens[i][0]) + else: + value = None + result[key] = value + else: + break + return result, i + + +def parse_seq(tokens, i, indent): + result = [] + while i < len(tokens) and tokens[i][0] == indent and (tokens[i][1].startswith("- ") or tokens[i][1] == "-"): + after = tokens[i][1][2:] if tokens[i][1].startswith("- ") else "" + child_indent = indent + 2 + # The item is its own block: the inline remainder (re-indented to child_indent) plus any + # deeper continuation lines that belong to it. + item = [] + if after: + item.append((child_indent, after)) + i += 1 + while i < len(tokens) and tokens[i][0] >= child_indent: + item.append(tokens[i]) + i += 1 + result.append(parse_block(item, 0, child_indent)[0] if item else None) + return result, i + + +def load_yaml(text): + tokens = tokenize(text) + value, _ = parse_block(tokens, 0, 0) + return value + + +def collect(node, out): + # (container, key) per child, so a mutation can delete or replace it in place. + if isinstance(node, dict): + for k, v in node.items(): + out.append((node, k)) + collect(v, out) + elif isinstance(node, list): + for idx, v in enumerate(node): + out.append((node, idx)) + collect(v, out) + + +def token(rng): + return "fuzz_" + "".join(rng.choice("abcdefghijklmnopqrstuvwxyz0123456789") for _ in range(8)) + + +def mutate_once(rng, roots): + refs = [] + for root in roots: + collect(root, refs) + if not refs: + return + container, key = rng.choice(refs) + op = rng.choice(["delete", "scalar", "dangerous", "empty"]) + if op == "delete": + del container[key] + elif op == "scalar": + container[key] = token(rng) + elif op == "dangerous": + container[key] = rng.choice(DANGEROUS) + else: + container[key] = rng.choice([{}, [], None]) + + +def collect_insertions(gen, node, schema, rtype, out): + # Record every writable optional field absent from an object, walking node and schema together + # so nested objects are candidates too. + schema = gen.resolve(schema) + if not isinstance(schema, dict): + return + + branches = schema.get("oneOf") or schema.get("anyOf") + if branches: + # Pick the branch matching the node we have, not a random one. + picked = None + for branch in branches: + resolved = gen.resolve(branch) + if isinstance(node, dict) and ( + resolved.get("type") == "object" or "properties" in resolved or gen.is_map(resolved) + ): + picked = resolved + break + if isinstance(node, list) and resolved.get("type") == "array": + picked = resolved + break + if picked is None: + return + schema = picked + + if isinstance(node, dict): + props = schema.get("properties", {}) + for name, prop_schema in props.items(): + if name not in node and not gen.should_skip_property(name, prop_schema): + out.append((node, name, prop_schema, rtype)) + for key, value in node.items(): + if key in props and isinstance(value, (dict, list)): + collect_insertions(gen, value, props[key], rtype, out) + if gen.is_map(schema): + for value in node.values(): + if isinstance(value, (dict, list)): + collect_insertions(gen, value, schema["additionalProperties"], rtype, out) + elif isinstance(node, list): + items = schema.get("items") + if items: + for value in node: + if isinstance(value, (dict, list)): + collect_insertions(gen, value, items, rtype, out) + + +def add_field(gen, rng, config): + # Inject one valid optional field, absent from the base, into a random insertion point. + types = resource_types(gen) + points = [] + for rtype, instances in config.get("resources", {}).items(): + if rtype not in types or not isinstance(instances, dict): + continue + element = resource_element(gen, types[rtype]) + for instance in instances.values(): + if isinstance(instance, dict): + gen.rtype = rtype + collect_insertions(gen, instance, element, rtype, points) + if not points: + return + node, name, prop_schema, rtype = rng.choice(points) + # rtype drives grants/permissions/typed-string generation. + gen.rtype = rtype + value = gen.gen(prop_schema, 1, name) + if value is not None: + node[name] = value + + +def mutate(config, seed, schema=None, unique="fuzz"): + # Without a schema only the destructive mutations run; the selftest uses that path to print + # configs that don't churn as the schema grows. + rng = random.Random(seed) + gen = Generator(schema, rng, unique) if schema is not None else None + + # Mutate only inside resource instances: keep the bundle/name and resources skeleton so there + # is always something to deploy, while every instance field is fair game. + roots = [] + for instances in config.get("resources", {}).values(): + if isinstance(instances, dict): + roots.extend(v for v in instances.values() if isinstance(v, (dict, list))) + + for _ in range(rng.randint(1, 3)): + if gen is not None and rng.random() < ADD_PROB: + add_field(gen, rng, config) + else: + mutate_once(rng, roots) + + return config diff --git a/acceptance/bin/mutate_fuzz_config_check.py b/acceptance/bin/mutate_fuzz_config_check.py new file mode 100755 index 00000000000..25f22915d6b --- /dev/null +++ b/acceptance/bin/mutate_fuzz_config_check.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +""" +Contract check for mutate_fuzz_config's minimal YAML loader and mutation engine +(the harness diffs stdout; a non-zero exit marks a violation on stderr): + +- The loader round-trips every curated base: load -> to_yaml -> load is a fixed point, so + a base the loader can't represent is caught here, not as a confusing fuzz failure. +- Mutation is deterministic for a fixed seed (reproducible repros). + +It also prints a few mutated configs so an algorithm change shows up as an output diff. +""" + +import json +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from emit_fuzz_config import MUTATE_BASES +from envsubst import substitute_variables +from gen_fuzz_config import SKIP_PROPERTY_NAMES, to_yaml +from mutate_fuzz_config import load_yaml, mutate + +CONFIGS = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "bundle", "invariant", "configs") +SCHEMA = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "bundle", "schema", "jsonschema.json") + + +def render(name): + with open(os.path.join(CONFIGS, name + ".yml.tmpl")) as f: + return substitute_variables(f.read()) + + +def instance(config): + # The curated bases are single-resource; return that one resource instance. + (instances,) = config["resources"].values() + (value,) = instances.values() + return value + + +def main(): + # Fixed so the printed configs are stable regardless of the harness's unique name. + os.environ["UNIQUE_NAME"] = "check" + failed = False + + for name in MUTATE_BASES: + text = render(name) + parsed = load_yaml(text) + if not isinstance(parsed, dict) or "resources" not in parsed: + sys.stderr.write(f"{name}: base did not parse to a config with resources\n") + failed = True + continue + if load_yaml(to_yaml(parsed)) != parsed: + sys.stderr.write(f"{name}: loader is not a round-trip fixed point\n") + failed = True + + for seed in range(5): + a = to_yaml(mutate(load_yaml(render("volume")), seed)) + b = to_yaml(mutate(load_yaml(render("volume")), seed)) + if a != b: + sys.stderr.write(f"seed {seed}: mutation is not deterministic\n") + failed = True + + for seed in range(3): + sys.stdout.write(f"=== volume seed={seed} ===\n") + sys.stdout.write(to_yaml(mutate(load_yaml(render("volume")), seed))) + + # Assert-only (no stdout) so this doesn't churn as the schema grows. The registered_model + # base sets no optional fields, so any added field must have been injected. + with open(SCHEMA) as f: + schema = json.load(f) + + for seed in range(5): + a = to_yaml(mutate(load_yaml(render("registered_model")), seed, schema=schema, unique="check")) + b = to_yaml(mutate(load_yaml(render("registered_model")), seed, schema=schema, unique="check")) + if a != b: + sys.stderr.write(f"seed {seed}: schema-aware mutation is not deterministic\n") + failed = True + + base_fields = set(instance(load_yaml(render("registered_model")))) + injected = False + for seed in range(30): + fields = set(instance(mutate(load_yaml(render("registered_model")), seed, schema=schema, unique="check"))) + added = fields - base_fields + if added: + injected = True + # Injecting an output-only field would manufacture false drift. + leaked = SKIP_PROPERTY_NAMES & added + if leaked: + sys.stderr.write(f"seed {seed}: injected output-only field(s): {sorted(leaked)}\n") + failed = True + if not injected: + sys.stderr.write("schema-aware mutation never injected an optional field\n") + failed = True + + if failed: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/acceptance/bin/run_fuzz.py b/acceptance/bin/run_fuzz.py new file mode 100755 index 00000000000..0060ba67fa3 --- /dev/null +++ b/acceptance/bin/run_fuzz.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +""" +Seed loop for the invariant fuzzer. Runs one seed per iteration by calling the seed_body bash +function that acceptance/bundle/fuzz/script exports, which sources the invariant target picked by +FUZZ_TARGET, and classifies each outcome: + + deployed - the config deployed and the invariant held + rejected - the CLI refused the config before deploying it; the common case, not a bug + gap - the config needs a route the testserver does not model + hang - the seed outlived FUZZ_SEED_TIMEOUT + bug - a panic, an internal error, a generator failure, or a broken invariant + +Every seed adds a line to LOG.summary. A bug or a hang also writes a ready-to-run repro to +LOG.repro and exits non-zero. Nothing is written to stdout: the committed run asserts empty output. + +FUZZ_TARGET and FUZZ_MODE come from the test.toml matrix; FUZZ_SEED_START, FUZZ_SEED_COUNT, +FUZZ_SEED_TIMEOUT and FUZZ_TIME_BUDGET are optional knobs the caller sets (see task test-fuzz). +FUZZ_CHECK_DRIFT is read only to name the oracle in the repro; script.prepare acts on it. +""" + +import os +import shutil +import signal +import subprocess +import sys +import time +from collections import Counter +from pathlib import Path + +# Per-seed cap: a seed past this budget is stuck, not slow. Set FUZZ_SEED_TIMEOUT=0 to disable. +SEED_TIMEOUT = float(os.environ.get("FUZZ_SEED_TIMEOUT", "180")) + +# Overall budget (seconds): stop starting new seeds past it and exit cleanly, so a slow-but- +# progressing variant isn't force-killed at the per-script Timeout and read as a failure. 900s +# leaves margin under the 20m test.toml Timeout. Set FUZZ_TIME_BUDGET=0 to disable. +BUDGET = float(os.environ.get("FUZZ_TIME_BUDGET", "900")) + +# Grace period between SIGQUIT and the SIGKILL backstop. +QUIT_GRACE = 10 + +TARGET = os.environ["FUZZ_TARGET"] +MODE = os.environ["FUZZ_MODE"] + +# Which no-drift oracle script.prepare installed. Part of the repro because the two disagree: the +# committed run leaves this at 0 and gets the plan-determinism diff, while task test-fuzz defaults +# it to 1 and gets the exact check. +CHECK_DRIFT = os.environ.get("FUZZ_CHECK_DRIFT", "0") + +POSIX = os.name == "posix" + +# Resolved rather than passed as a bare name: on Windows, CreateProcess searches System32 first, +# where "bash" is the WSL launcher stub, which exits non-zero with no distribution installed and +# makes every seed read as rejected. shutil.which searches PATH, finding the bash we run under. +BASH = shutil.which("bash") + + +def read(path): + """Log contents as bytes; a fuzzed config can put arbitrary bytes in there. Empty if absent.""" + return path.read_bytes() if path.exists() else b"" + + +def concat_logs(seed_dir): + return b"".join(read(p) for p in sorted(seed_dir.glob("LOG.*"))) + + +def kill_seed(proc): + if not POSIX: + # Windows has neither SIGQUIT nor the process group below, so this is all it can do. + proc.kill() + return + # SIGQUIT first for Go's goroutine dump, then SIGKILL as a backstop. + os.killpg(proc.pid, signal.SIGQUIT) + try: + proc.wait(timeout=QUIT_GRACE) + except subprocess.TimeoutExpired: + os.killpg(proc.pid, signal.SIGKILL) + + +def run_seed(seed_dir, seed): + """Run one seed in a fresh bash. Returns its exit code and whether it had to be killed.""" + with open(seed_dir / "LOG.check", "wb") as log: + proc = subprocess.Popen( + [BASH, "-euo", "pipefail", "-c", 'seed_body "$@"', "_", str(seed_dir), str(seed)], + stdout=log, + stderr=subprocess.STDOUT, + # Own process group, so killing a hung seed also takes down the CLI it is waiting on. + start_new_session=POSIX, + ) + try: + return proc.wait(timeout=SEED_TIMEOUT or None), False + except subprocess.TimeoutExpired: + kill_seed(proc) + return proc.wait(), True + + +def oracle_verdict(seed_dir): + """The no-drift oracle's own verdict, if it reached one. Empty if it never ran or was happy.""" + # Both oracles report a violation in a form only they produce, so a seed that broke the + # invariant is still recognisable when it also touched a route the testserver lacks. + if b"Unexpected action=" in read(seed_dir / "LOG.check"): + # verify_no_drift.py, the exact check shared with the curated invariant targets. + return "planned a change after deploy" + if read(seed_dir / "LOG.plan.determinism.diff").strip(): + # The plan-determinism diff script.prepare substitutes when FUZZ_CHECK_DRIFT is 0. + return "planned differently on two consecutive runs" + return "" + + +def classify(seed_dir): + """Classify a seed that exited non-zero. Returns its kind and, for a failure, the reason.""" + # The generator only writes to stderr when it fails: our bug, not a rejected config. + gen_err = read(seed_dir / "LOG.gen.err") + if gen_err: + first_line = gen_err.splitlines()[0].decode(errors="replace") + return "bug", f"could not be generated: {first_line}" + + logs = concat_logs(seed_dir) + + # A panic or internal error anywhere is a bug even if the CLI then rejects the config. + if b"panic:" in logs or b"internal error" in logs: + return "bug", "panicked or hit an internal error" + + # Before the gap marker: the cleanup destroy runs on every seed, so an unmodeled delete route + # puts that marker in the logs of seeds whose invariant genuinely failed. + verdict = oracle_verdict(seed_dir) + if verdict: + return "bug", verdict + + # Marker body of the catch-all stubs in fuzz/test.toml: the route is one the testserver does + # not model. Checked before the marker below, else an unmodeled route reached during plan or + # destroy reads as a drift bug. + if b"TESTSERVER_GAP" in logs: + return "gap", "" + + # Failing after INPUT_CONFIG_OK means the config deployed but drifted (or destroy failed); + # failing before it with no panic just means the config was rejected. + if b"INPUT_CONFIG_OK" in read(seed_dir / "LOG.check"): + return "bug", "broke the invariant" + + return "rejected", "" + + +def record(kind, seed): + """One machine-readable line per seed. To a file, not stdout, so empty output still holds.""" + with open("LOG.summary", "a") as f: + f.write(f"{kind} seed={seed} target={TARGET} mode={MODE}\n") + + +def fail(seed, kind, reason, prefix=""): + record(kind, seed) + # The repro goes to a file because the harness rewrites env-var values in stdout. + Path("LOG.repro").write_text( + f"fuzz: seed {seed} {reason}, reproduce with: {prefix}FUZZ_SEED_START={seed} " + f"FUZZ_SEED_COUNT=1 FUZZ_TARGET={TARGET} FUZZ_MODE={MODE} " + f"FUZZ_CHECK_DRIFT={CHECK_DRIFT} task test-fuzz\n" + ) + sys.exit(1) + + +def totals(): + """Per-variant tally for triage. Reached only on a clean run; a bug or hang exits above.""" + summary = Path("LOG.summary") + if not summary.exists(): + return Counter() + + # Count before appending the header, else it would count itself. + kinds = Counter(line.split()[0] for line in summary.read_text().splitlines()) + with summary.open("a") as f: + f.write("--- totals ---\n") + for kind, n in sorted(kinds.items()): + f.write(f"{n} {kind}\n") + return kinds + + +def main(): + start = time.monotonic() + seed_start = int(os.environ.get("FUZZ_SEED_START", "0")) + count = int(os.environ.get("FUZZ_SEED_COUNT", "5")) + + for offset in range(count): + # A clean stop, not a failure, so log to a file. + if BUDGET and time.monotonic() - start >= BUDGET: + Path("LOG.budget").write_text( + f"fuzz: stopping after {offset}/{count} seeds; hit FUZZ_TIME_BUDGET={BUDGET:g}s\n" + ) + break + + seed = seed_start + offset + seed_dir = Path(f"seed-{seed}") + seed_dir.mkdir(exist_ok=True) + + returncode, killed = run_seed(seed_dir, seed) + if returncode == 0: + record("deployed", seed) + continue + + # A seed that had to be killed hung, which is distinct from a drift bug. + if killed: + fail(seed, "hang", f"hung (>{SEED_TIMEOUT:g}s)", "FUZZ_SEED_TIMEOUT=0 ") + + kind, reason = classify(seed_dir) + if reason: + fail(seed, kind, reason) + record(kind, seed) + + kinds = totals() + + # Nothing deploying is not a pass: it means the schema, generator or fixtures are broken, which + # otherwise looks just like the CLI correctly rejecting random input. A single-seed replay is + # exempt, where one rejected config is a normal outcome. + if count > 1 and not kinds["deployed"]: + sys.exit("fuzz: no seed deployed; the schema, generator or fixtures are broken") + + +if __name__ == "__main__": + main() diff --git a/acceptance/bundle/fuzz/README.md b/acceptance/bundle/fuzz/README.md new file mode 100644 index 00000000000..88c54247f30 --- /dev/null +++ b/acceptance/bundle/fuzz/README.md @@ -0,0 +1,40 @@ +This is a harness over the invariant tests in ../invariant rather than an invariant itself: it runs +generated configs through a real invariant target script. script only sets up the per-seed +environment; acceptance/bin/run_fuzz.py drives the seed loop and classifies each outcome as +deployed / rejected / gap / hang / bug. Both the target and the way configs are built are matrixed +in test.toml: + +`FUZZ_TARGET` picks the invariant, and each one is also a curated invariant test that runs over the +`INPUT_CONFIG` matrix: + +- `no_drift` -- deploy, then no drift +- `migrate` -- Terraform deploy, migrate to direct, then no drift +- `delete_idempotent` -- deploy, delete by emptying the config, then re-run the delete on restored state +- `destroy_idempotent` -- deploy, destroy, then destroy again on restored state + +`FUZZ_MODE` picks how the config is built: + +- `generate` -- build a random resource by walking the live `databricks bundle schema` +- `mutate` -- perturb one of the curated configs (see MUTATE_BASES in emit_fuzz_config.py) + +Free-form scalars are occasionally replaced with dangerous / near-range-end values (empty, +whitespace, over-long, control characters, int32/int64 boundaries) to probe the CLI's input +handling. + +The invariant helpers come from ../invariant/script.prepare, which script.prepare sources directly +because test.toml and script.prepare only merge along the directory chain. For the same reason the +server stubs and ignore patterns this test needs are copied into test.toml. + +A generated config can reach an API route the testserver does not model, which is a coverage gap +rather than a missing stub. test.toml answers those with a per-method catch-all stub returning a +`TESTSERVER_GAP` marker, so the seed is recorded as a gap instead of failing the run, and the seed's +log names the route the CLI could not reach. + +Since the schema comes from the CLI under test, an unrelated struct change can shift a +seed onto a new config. A failure is a real CLI bug (panic, internal error, or drift), +not flakiness; the failing seed's `LOG.repro` prints a ready-to-run repro, of the form +`FUZZ_SEED_START= FUZZ_SEED_COUNT=1 FUZZ_TARGET=no_drift FUZZ_MODE=generate FUZZ_CHECK_DRIFT=0 task test-fuzz`. + +`FUZZ_CHECK_DRIFT` is part of the repro because it selects the oracle: at `0` (the committed run) +`invariant_verify_no_drift` is replaced with a plan-determinism diff, and at `1` (`task test-fuzz` +and the nightly) the exact check from ../invariant runs unchanged. diff --git a/acceptance/bundle/fuzz/out.test.toml b/acceptance/bundle/fuzz/out.test.toml new file mode 100644 index 00000000000..2d8512338e6 --- /dev/null +++ b/acceptance/bundle/fuzz/out.test.toml @@ -0,0 +1,10 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.FUZZ_MODE = ["generate", "mutate"] +EnvMatrix.FUZZ_TARGET = [ + "no_drift", + "migrate", + "delete_idempotent", + "destroy_idempotent" +] diff --git a/acceptance/bundle/fuzz/output.txt b/acceptance/bundle/fuzz/output.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/fuzz/script b/acceptance/bundle/fuzz/script new file mode 100644 index 00000000000..4a471000377 --- /dev/null +++ b/acceptance/bundle/fuzz/script @@ -0,0 +1,28 @@ +# Invariant fuzzing: generate a random config per seed and run a real invariant target from +# ../invariant, which reaches the generator through the helper overrides in script.prepare. +# run_fuzz.py owns the seed loop and the outcome classification; see README.md. + +# no_drift/script reads READPLAN via readplanarg; the fuzzer skips the saved-plan matrix. +export READPLAN="" + +# Emit the schema from the CLI under test so the generator always matches it. +$CLI bundle schema > schema.json 2>LOG.schema.err +cat LOG.schema.err | contains.py '!panic:' '!internal error' > /dev/null + +# run_fuzz.py calls this once per seed. It has to be a bash function so the target script it +# sources sees the invariant helpers that script.prepare defined. +seed_body() { + cd "$1" + # Seeds share one long-lived workspace, so scope the unique name to the seed; otherwise state + # a seed leaves behind reads back as drift in the next one. Scoped here rather than in the + # generator so targets that derive workspace paths from $UNIQUE_NAME see it too. + export UNIQUE_NAME="$UNIQUE_NAME-$2" + export FUZZ_SEED="$2" + export FUZZ_SCHEMA="../schema.json" + source "$INVARIANT_DIR/$FUZZ_TARGET/script" +} + +# run_fuzz.py spawns a fresh bash per seed, so export seed_body and the helpers it calls (trace). +export -f $(compgen -A function) + +run_fuzz.py diff --git a/acceptance/bundle/fuzz/script.prepare b/acceptance/bundle/fuzz/script.prepare new file mode 100644 index 00000000000..d7bdfef131e --- /dev/null +++ b/acceptance/bundle/fuzz/script.prepare @@ -0,0 +1,46 @@ +# Fuzz overrides of the shared invariant helpers. This test lives outside the invariant subtree, +# so the harness does not concatenate them; source them explicitly, before the overrides below. +# The target scripts stay unaware of fuzzing. +export INVARIANT_DIR="$TESTDIR/../invariant" +source "$INVARIANT_DIR/script.prepare" + +# The config comes from the generator rather than configs/$INPUT_CONFIG, which the fuzzer leaves +# unset. validate runs here as an isolated panic surface, and rejects an invalid config before the +# target's deploy. +invariant_render() { + # Stage the fixtures the generator's file_path/source_code_path fields point at. + cp -r "$INVARIANT_DIR/data/." . &> LOG.cp + + emit_fuzz_config.py > databricks.yml 2>LOG.gen.err + cp databricks.yml LOG.config + + trace $CLI bundle validate &> LOG.validate + cat LOG.validate | contains.py '!panic:' '!internal error' > /dev/null +} + +# A generated config can deploy and still legitimately differ from the fake server, which does +# not round-trip every field, so the exact check false-positives. Swap in the one oracle that +# does not depend on server fidelity: planning is deterministic, so two consecutive plans of the +# same state must be byte-identical. Nightly runs set FUZZ_CHECK_DRIFT=1 to keep the exact check. +# +# Tested against 0 rather than for emptiness so run_fuzz.py's repro can select this oracle with an +# explicit FUZZ_CHECK_DRIFT=0; an empty value is re-defaulted to 1 by the task's +# ${FUZZ_CHECK_DRIFT:-1}. +if [ "${FUZZ_CHECK_DRIFT:-0}" = 0 ]; then + invariant_verify_no_drift() { + # Compare only when both plans succeed -- a plan that fails on an unmodeled read is a gap + # and can differ run to run. + set +e + $CLI bundle plan -o json > LOG.plan1.json 2>LOG.plan1.err + local plan1_rc=$? + $CLI bundle plan -o json > LOG.plan2.json 2>LOG.plan2.err + local plan2_rc=$? + set -e + cat LOG.plan1.err | contains.py '!panic:' '!internal error' > /dev/null + cat LOG.plan2.err | contains.py '!panic:' '!internal error' > /dev/null + if [ "$plan1_rc" -eq 0 ] && [ "$plan2_rc" -eq 0 ]; then + # diff exits non-zero on any difference; under set -e that fails the seed as a bug. + diff LOG.plan1.json LOG.plan2.json > LOG.plan.determinism.diff + fi + } +fi diff --git a/acceptance/bundle/fuzz/test.toml b/acceptance/bundle/fuzz/test.toml new file mode 100644 index 00000000000..c16f239dc18 --- /dev/null +++ b/acceptance/bundle/fuzz/test.toml @@ -0,0 +1,86 @@ +# Local only: against a real workspace each seed's deploy/migrate/plan/destroy round trip +# takes minutes and trips SEED_TIMEOUT, and the cloud run leaves FUZZ_CHECK_DRIFT unset, so it +# would only re-assert the no-panic property the local run already covers. +Cloud = false + +# Room for the nightly FUZZ_TIME_BUDGET (run_fuzz.py) plus the last seed's tail. +Timeout = '20m' + +# test.toml only merges along the directory chain, so the engine pin, the ignore patterns and the +# per-route [[Server]] stubs below are copied from ../invariant/test.toml, where the targets live. +# A stub added there and not here does not fail this test: the catch-alls at the bottom answer the +# route and its seeds are recorded as gaps. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +Ignore = [ + ".databricks", + ".venv", + "databricks.yml", + "plan.json", + "*.py", + "*.json", + "*.err", + "app", + # Snapshot of pre-delete state the idempotency targets keep; may linger if a seed fails. + ".databricks.backup", +] + +# The idempotency targets assert that a delete or destroy re-run succeeds, which holds regardless of +# how faithfully the fake server round-trips fields, so they keep their real oracle under fuzzing. +# +# ../invariant/continue_293 is left out: it deploys with the pinned v0.293.0 binary first, and that +# version does not know many current fields and resource types, so it would reject most seeds before +# the current CLI ever ran and the window would measure v0.293.0's schema instead. +EnvMatrix.FUZZ_TARGET = ["no_drift", "migrate", "delete_idempotent", "destroy_idempotent"] + +# generate = build from the schema; mutate = perturb a curated config. See emit_fuzz_config.py. +EnvMatrix.FUZZ_MODE = ["generate", "mutate"] + +# Generate mode pins name/display_name, so it cannot vary the identifier the delete path reads, +# and ../invariant/delete_idempotent already covers every resource type. Mutate can hit the +# identifier and reshape grants/permissions, so it is the mode worth running here. +EnvMatrixExclude.no_generate_on_delete_idempotent = ["FUZZ_MODE=generate", "FUZZ_TARGET=delete_idempotent"] +EnvMatrixExclude.no_generate_on_destroy_idempotent = ["FUZZ_MODE=generate", "FUZZ_TARGET=destroy_idempotent"] + +# Fake SQL endpoint for local tests +[[Server]] +Pattern = "POST /api/2.0/sql/statements/" +Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' + +[[Server]] +Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" +Response.Body = '{"status": "OK"}' + +# Catch-alls, one per method. A route the testserver does not model is a coverage gap for a config +# nobody wrote by hand, not a missing stub, so these answer it with a marker body run_fuzz.py +# classifies as a gap, instead of letting the unhandled-request check fail the whole run. +# +# They shadow nothing: wildcard patterns go to ServeMux, which matches most-specific-first, and +# exact paths are looked up before the mux is consulted (see the Router type doc). +# +# No HEAD entry: ServeMux matches a GET pattern for HEAD too, so "HEAD /{path...}" conflicts with +# every GET wildcard and panics at registration. The GET catch-all answers HEAD anyway. +[[Server]] +Pattern = "GET /{path...}" +Response.StatusCode = 501 +Response.Body = '{"message": "TESTSERVER_GAP"}' + +[[Server]] +Pattern = "POST /{path...}" +Response.StatusCode = 501 +Response.Body = '{"message": "TESTSERVER_GAP"}' + +[[Server]] +Pattern = "PUT /{path...}" +Response.StatusCode = 501 +Response.Body = '{"message": "TESTSERVER_GAP"}' + +[[Server]] +Pattern = "PATCH /{path...}" +Response.StatusCode = 501 +Response.Body = '{"message": "TESTSERVER_GAP"}' + +[[Server]] +Pattern = "DELETE /{path...}" +Response.StatusCode = 501 +Response.Body = '{"message": "TESTSERVER_GAP"}' diff --git a/acceptance/bundle/invariant/migrate/script b/acceptance/bundle/invariant/migrate/script index 4cd19571951..a3829ed039f 100644 --- a/acceptance/bundle/invariant/migrate/script +++ b/acceptance/bundle/invariant/migrate/script @@ -13,7 +13,7 @@ MIGRATE_ARGS="" # (order-sensitive), terraform plan reports positional drift when the bundle config # specifies depends_on in a different order than the provider's sorted state. # This is a false positive -- the logical dependencies are identical. -if [[ "$INPUT_CONFIG" == "job_with_depends_on.yml.tmpl" ]]; then +if [[ "${INPUT_CONFIG:-}" == "job_with_depends_on.yml.tmpl" ]]; then MIGRATE_ARGS="--noplancheck" fi diff --git a/acceptance/bundle/invariant/script.prepare b/acceptance/bundle/invariant/script.prepare index 54f2a3dbf25..01f56bac96d 100644 --- a/acceptance/bundle/invariant/script.prepare +++ b/acceptance/bundle/invariant/script.prepare @@ -1,26 +1,32 @@ # Shared setup for the invariant targets; each script keeps only the invariant it asserts. +# Root of configs/ and data/. For the targets in this subtree $TESTDIR is the target directory, so +# the default resolves here; a caller outside the subtree sets it before sourcing. Exported because +# the fuzzer runs each seed in a fresh bash. +export INVARIANT_DIR="${INVARIANT_DIR:-$TESTDIR/..}" + invariant_cleanup() { trace $CLI bundle destroy --auto-approve &> LOG.destroy cat LOG.destroy | contains.py '!panic:' '!internal error' > /dev/null - CLEANUP_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-cleanup.sh" + # INPUT_CONFIG is unset for a caller that generates its own config, and scripts run under set -u. + CLEANUP_SCRIPT="$INVARIANT_DIR/configs/${INPUT_CONFIG:-}-cleanup.sh" if [ -f "$CLEANUP_SCRIPT" ]; then source "$CLEANUP_SCRIPT" &> LOG.cleanup fi } # Separate from invariant_setup so a caller that generates its own config can override the -# render alone; child script.prepare files are concatenated after this one. +# render alone, by redefining it after sourcing this file. invariant_render() { - cp -r "$TESTDIR/../data/." . &> LOG.cp + cp -r "$INVARIANT_DIR/data/." . &> LOG.cp - INIT_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-init.sh" + INIT_SCRIPT="$INVARIANT_DIR/configs/$INPUT_CONFIG-init.sh" if [ -f "$INIT_SCRIPT" ]; then source "$INIT_SCRIPT" &> LOG.init fi - envsubst < "$TESTDIR/../configs/$INPUT_CONFIG" > databricks.yml + envsubst < "$INVARIANT_DIR/configs/$INPUT_CONFIG" > databricks.yml cp databricks.yml LOG.config } diff --git a/acceptance/selftest/gen_fuzz_config/out.test.toml b/acceptance/selftest/gen_fuzz_config/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/selftest/gen_fuzz_config/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/selftest/gen_fuzz_config/output.txt b/acceptance/selftest/gen_fuzz_config/output.txt new file mode 100644 index 00000000000..ec780260c85 --- /dev/null +++ b/acceptance/selftest/gen_fuzz_config/output.txt @@ -0,0 +1,26 @@ +comment: "value: with a colon" +description: "quote \" and : colon" +resources: + jobs: + j: + name: "n" + tags: + team: "jobs" +tasks: + - description: "d" + timeout_seconds: 3600 + - comment: "c" +nums: + - 0 + - 1 + - 2 +flag: true +ratio: 1.5 +empty_map: {} +empty_list: [] +matrix: + - + - 1 + - 2 + - [] + - {} diff --git a/acceptance/selftest/gen_fuzz_config/script b/acceptance/selftest/gen_fuzz_config/script new file mode 100644 index 00000000000..2737c67674d --- /dev/null +++ b/acceptance/selftest/gen_fuzz_config/script @@ -0,0 +1 @@ +gen_fuzz_config_check.py diff --git a/acceptance/selftest/mutate_fuzz_config/out.test.toml b/acceptance/selftest/mutate_fuzz_config/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/selftest/mutate_fuzz_config/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/selftest/mutate_fuzz_config/output.txt b/acceptance/selftest/mutate_fuzz_config/output.txt new file mode 100644 index 00000000000..4b7289cfa51 --- /dev/null +++ b/acceptance/selftest/mutate_fuzz_config/output.txt @@ -0,0 +1,36 @@ +=== volume seed=0 === +bundle: + name: "test-bundle-check" +resources: + volumes: + foo: + name: "test-volume-check" + catalog_name: "main" + schema_name: [] + grants: + - principal: "account users" +=== volume seed=1 === +bundle: + name: "test-bundle-check" +resources: + volumes: + foo: + name: "test-volume-check" + catalog_name: " " + schema_name: "default" + grants: + - principal: "account users" + privileges: + - "READ_VOLUME" +=== volume seed=2 === +bundle: + name: "test-bundle-check" +resources: + volumes: + foo: + name: "test-volume-check" + schema_name: "default" + grants: + - principal: "account users" + privileges: + - "READ_VOLUME" diff --git a/acceptance/selftest/mutate_fuzz_config/script b/acceptance/selftest/mutate_fuzz_config/script new file mode 100644 index 00000000000..2d57c739261 --- /dev/null +++ b/acceptance/selftest/mutate_fuzz_config/script @@ -0,0 +1 @@ +mutate_fuzz_config_check.py