diff --git a/docs/resources/api_integration.md b/docs/resources/api_integration.md index a931197..cf2d6f2 100644 --- a/docs/resources/api_integration.md +++ b/docs/resources/api_integration.md @@ -8,12 +8,21 @@ description: >- [Snowflake Documentation](https://docs.snowflake.com/en/sql-reference/sql/create-api-integration) | Snowcap CLI label: `api_integration` Manages API integrations in Snowflake, allowing external services to interact with Snowflake resources securely. -This class supports creating, replacing, and checking the existence of API integrations with various configurations. +This class supports creating, replacing, and checking the existence of API integrations across multiple cloud providers +and Git HTTPS providers. +## Supported `api_provider` values + +| `api_provider` | Required fields | Used for | +|---|---|---| +| `AWS_API_GATEWAY`, `AWS_PRIVATE_API_GATEWAY`, `AWS_GOV_API_GATEWAY`, `AWS_GOV_PRIVATE_API_GATEWAY` | `api_aws_role_arn` | External functions calling AWS API Gateway | +| `AZURE_API_MANAGEMENT` | `azure_tenant_id`, `azure_ad_application_id` | External functions calling Azure API Management | +| `GOOGLE_API_GATEWAY` | `google_audience` | External functions calling Google API Gateway | +| `GIT_HTTPS_API` | (none beyond `api_allowed_prefixes`) | Snowflake [Git repositories](git_repository.md) connecting to GitHub/GitLab/Bitbucket/etc. | ## Examples -### YAML +### YAML — AWS API Gateway ```yaml api_integrations: @@ -24,9 +33,19 @@ api_integrations: api_allowed_prefixes: ["/prod/", "/dev/"] api_blocked_prefixes: ["/test/"] api_key: "ABCD1234" - comment: "Example API integration" + comment: "Example AWS API integration" ``` +### YAML — GitHub (used by GitRepository) + +```yaml +api_integrations: + - name: github_api_integration + api_provider: GIT_HTTPS_API + api_allowed_prefixes: ["https://github.com/some-org/"] + enabled: true + comment: "GitHub integration for git repos" +``` ### Python @@ -39,7 +58,7 @@ api_integration = APIIntegration( api_allowed_prefixes=["/prod/", "/dev/"], api_blocked_prefixes=["/test/"], api_key="ABCD1234", - comment="Example API integration" + comment="Example API integration", ) ``` @@ -47,12 +66,29 @@ api_integration = APIIntegration( ## Fields * `name` (string, required) - The unique name of the API integration. -* `api_provider` (string or ApiProvider, required) - The provider of the API service. Defaults to AWS_API_GATEway. -* `api_aws_role_arn` (string, required) - The AWS IAM role ARN associated with the API integration. -* `api_key` (string) - The API key used for authentication. +* `api_provider` (string or ApiProvider, required) - The provider of the API service. See table above for supported values. +* `api_aws_role_arn` (string) - The AWS IAM role ARN. Required for AWS providers; omit for AZURE/GOOGLE/GIT_HTTPS_API. +* `azure_tenant_id` (string) - Azure AD tenant ID. Required for `AZURE_API_MANAGEMENT`. +* `azure_ad_application_id` (string) - Azure AD application registration ID. Required for `AZURE_API_MANAGEMENT`. +* `google_audience` (string) - GCP audience identifier. Required for `GOOGLE_API_GATEWAY`. +* `api_key` (string) - Optional API key used for authentication. * `api_allowed_prefixes` (list) - The list of allowed prefixes for the API endpoints. * `api_blocked_prefixes` (list) - The list of blocked prefixes for the API endpoints. * `enabled` (bool, required) - Specifies if the API integration is enabled. Defaults to TRUE. * `comment` (string) - A comment or description for the API integration. +## Granting on an integration + +Snowflake's `GRANT USAGE ON INTEGRATION ` SQL is valid for any subtype. In YAML you may use either the +concrete subtype (`on: api integration `) — preferred — or the generic umbrella (`on: integration `): + +```yaml +grants: + - priv: USAGE + on: api integration github_api_integration # preferred — explicit subtype + to: some_role + - priv: USAGE + on: integration github_api_integration # also supported (umbrella) + to: another_role +``` diff --git a/docs/resources/git_repository.md b/docs/resources/git_repository.md new file mode 100644 index 0000000..22949c8 --- /dev/null +++ b/docs/resources/git_repository.md @@ -0,0 +1,75 @@ +--- +description: >- + A git repository in Snowflake. +--- + +# GitRepository + +[Snowflake Documentation](https://docs.snowflake.com/en/sql-reference/sql/create-git-repository) | Snowcap CLI label: `git_repository` + +A Git Repository in Snowflake represents an externally hosted Git repository (GitHub, +GitLab, Bitbucket, etc.) that has been registered for use with Snowflake's Git +integration. Once registered, files in the repository can be referenced via stage +syntax in `COPY`, `EXECUTE IMMEDIATE`, and other commands. + +A git repository depends on an [APIIntegration](api_integration.md) whose +`api_allowed_prefixes` covers the repository's `origin` URL, and optionally on a +[Secret](generic_secret.md) (for private repos) referenced via `git_credentials`. + + +## Examples + +### YAML + +```yaml +git_repositories: + - name: some_git_repository + database: some_db + schema: some_schema + origin: https://github.com/some-org/some-repo.git + api_integration: some_api_integration + git_credentials: some_secret + comment: Example git repository +``` + + +### Python + +```python +git_repository = GitRepository( + name="some_git_repository", + database="some_db", + schema="some_schema", + origin="https://github.com/some-org/some-repo.git", + api_integration="some_api_integration", + git_credentials="some_secret", + comment="Example git repository", +) +``` + + +## Fields + +* `name` (string, required) - The name of the git repository. +* `origin` (string, required) - The URL of the externally hosted Git repository + (e.g., `https://github.com/some-org/some-repo.git`). +* `api_integration` (string, required) - The name of the API integration object + Snowflake will use to interact with the repository. The API integration's + `api_allowed_prefixes` must include the `origin` URL. +* `git_credentials` (string) - The name of a [Secret](generic_secret.md) holding + credentials for accessing a private repository. Optional for public repos. +* `comment` (string) - A comment for the git repository. +* `owner` (string or [Role](role.md)) - The owner role of the git repository. + Defaults to `SYSADMIN`. + + +## Grants + +Snowcap supports `READ`, `WRITE`, and `OWNERSHIP` privileges on git repositories: + +```yaml +grants: + - priv: READ + on: git repository some_db.some_schema.some_git_repository + to: some_role +``` diff --git a/mkdocs.yml b/mkdocs.yml index 0ab4523..30170bb 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -167,6 +167,8 @@ nav: - AzureStorageIntegration: resources/azure_storage_integration.md - GCSStorageIntegration: resources/gcs_storage_integration.md - S3StorageIntegration: resources/s3storage_integration.md + - Git: + - GitRepository: resources/git_repository.md - Orchestration: - Alert: resources/alert.md - Task: resources/task.md diff --git a/snowcap/data_provider.py b/snowcap/data_provider.py index 06f36d3..3435269 100644 --- a/snowcap/data_provider.py +++ b/snowcap/data_provider.py @@ -1335,13 +1335,22 @@ def fetch_api_integration(session: SnowflakeConnection, fqn: FQN): properties = _desc_type2_result_to_dict(desc_result, lower_properties=True) owner = _fetch_owner(session, "INTEGRATION", fqn) + # Different api_provider values return different DESC properties: + # AWS_API_GATEWAY family -> api_aws_role_arn + # AZURE_API_MANAGEMENT -> azure_tenant_id, azure_ad_application_id + # GOOGLE_API_GATEWAY -> google_audience + # GIT_HTTPS_API -> none of the above + # Use .get() so missing fields fall back to None instead of crashing. return { "name": _quote_snowflake_identifier(data["name"]), "api_provider": properties["api_provider"], - "api_aws_role_arn": properties["api_aws_role_arn"], + "api_aws_role_arn": properties.get("api_aws_role_arn") or None, + "azure_tenant_id": properties.get("azure_tenant_id") or None, + "azure_ad_application_id": properties.get("azure_ad_application_id") or None, + "google_audience": properties.get("google_audience") or None, "enabled": properties["enabled"], - "api_allowed_prefixes": properties["api_allowed_prefixes"], - "api_blocked_prefixes": properties["api_blocked_prefixes"], + "api_allowed_prefixes": properties.get("api_allowed_prefixes"), + "api_blocked_prefixes": properties.get("api_blocked_prefixes"), "owner": owner, "comment": data["comment"] or None, } @@ -1595,6 +1604,34 @@ def fetch_event_table(session: SnowflakeConnection, fqn: FQN): } +def fetch_integration(session: SnowflakeConnection, fqn: FQN): + """ + Fetch any integration regardless of concrete subtype. + + Backs the generic `ResourceType.INTEGRATION` registered in RESOURCE_SCOPES so + that grants like `on: integration ` parse and resolve (Snowflake's + `GRANT USAGE ON INTEGRATION ` syntax accepts any subtype; this fetcher + returns the minimum SHOW INTEGRATIONS metadata so the grant's required-ref + check succeeds). + """ + show_result = execute(session, "SHOW INTEGRATIONS", cacheable=True) + show_result = _filter_result(show_result, name=fqn.name) + if len(show_result) == 0: + return None + if len(show_result) > 1: + raise Exception(f"Found multiple integrations matching {fqn}") + data = show_result[0] + owner = _fetch_owner(session, "INTEGRATION", fqn) + return { + "name": _quote_snowflake_identifier(data["name"]), + "type": data["type"], + "category": data["category"], + "enabled": data["enabled"] == "true", + "comment": data["comment"] or None, + "owner": owner, + } + + def fetch_external_access_integration(session: SnowflakeConnection, fqn: FQN): integrations = _show_resources(session, "EXTERNAL ACCESS INTEGRATIONS", fqn) if len(integrations) == 0: @@ -2348,6 +2385,23 @@ def fetch_schema(session: SnowflakeConnection, fqn: FQN, include_params: bool = } +def fetch_git_repository(session: SnowflakeConnection, fqn: FQN): + show_result = _show_resources(session, "GIT REPOSITORIES", fqn) + if len(show_result) == 0: + return None + if len(show_result) > 1: + raise Exception(f"Found multiple git repositories matching {fqn}") + data = show_result[0] + return { + "name": _quote_snowflake_identifier(data["name"]), + "origin": data["origin"], + "api_integration": data["api_integration"], + "git_credentials": data.get("git_credentials") or None, + "comment": data["comment"] or None, + "owner": _get_owner_identifier(data), + } + + def fetch_secret(session: SnowflakeConnection, fqn: FQN): show_result = _show_resources(session, "SECRETS", fqn) if len(show_result) == 0: @@ -3311,6 +3365,17 @@ def list_dynamic_tables(session: SnowflakeConnection) -> list[FQN]: return list_schema_scoped_resource(session, "DYNAMIC TABLES") +def list_integrations(session: SnowflakeConnection) -> list[FQN]: + """List every integration in the account (any subtype). + + Backs the generic `ResourceType.INTEGRATION` (umbrella). Concrete subtypes + (API/CATALOG/EXTERNAL_ACCESS/NOTIFICATION/SECURITY/STORAGE) still have their + own list_*_integrations functions for typed manifests. + """ + show_result = execute(session, "SHOW INTEGRATIONS", cacheable=True) + return [FQN(name=resource_name_from_snowflake_metadata(row["name"])) for row in show_result] + + def list_external_access_integrations(session: SnowflakeConnection) -> list[FQN]: return list_account_scoped_resource(session, "EXTERNAL ACCESS INTEGRATIONS") @@ -3830,6 +3895,10 @@ def list_schemas(session: SnowflakeConnection, database=None) -> list[FQN]: raise +def list_git_repositories(session: SnowflakeConnection) -> list[FQN]: + return list_schema_scoped_resource(session, "GIT REPOSITORIES") + + def list_secrets(session: SnowflakeConnection) -> list[FQN]: return list_schema_scoped_resource(session, "SECRETS") diff --git a/snowcap/privs.py b/snowcap/privs.py index db2b37a..7bf8b5b 100644 --- a/snowcap/privs.py +++ b/snowcap/privs.py @@ -278,6 +278,12 @@ class SecretPriv(Priv): USAGE = "USAGE" +class GitRepositoryPriv(Priv): + OWNERSHIP = "OWNERSHIP" + READ = "READ" + WRITE = "WRITE" + + class SequencePriv(Priv): ALL = "ALL" OWNERSHIP = "OWNERSHIP" @@ -389,7 +395,7 @@ class WarehousePriv(Priv): ResourceType.FAILOVER_GROUP: FailoverGroupPriv, ResourceType.FILE_FORMAT: FileFormatPriv, ResourceType.FUNCTION: FunctionPriv, - ResourceType.GIT_REPOSITORY: None, + ResourceType.GIT_REPOSITORY: GitRepositoryPriv, ResourceType.GRANT: None, ResourceType.HYBRID_TABLE: None, ResourceType.ICEBERG_TABLE: IcebergTablePriv, @@ -445,6 +451,7 @@ class WarehousePriv(Priv): ResourceType.FAILOVER_GROUP: AccountPriv.CREATE_FAILOVER_GROUP, ResourceType.FILE_FORMAT: SchemaPriv.CREATE_FILE_FORMAT, ResourceType.FUNCTION: SchemaPriv.CREATE_FUNCTION, + ResourceType.GIT_REPOSITORY: SchemaPriv.CREATE_GIT_REPOSITORY, # ResourceType.GRANT: AccountPriv.CREATE_GRANT, ResourceType.MATERIALIZED_VIEW: SchemaPriv.CREATE_MATERIALIZED_VIEW, ResourceType.NETWORK_POLICY: AccountPriv.CREATE_NETWORK_POLICY, diff --git a/snowcap/resources/__init__.py b/snowcap/resources/__init__.py index a6d7983..0a7688c 100644 --- a/snowcap/resources/__init__.py +++ b/snowcap/resources/__init__.py @@ -16,6 +16,7 @@ from .failover_group import FailoverGroup from .file_format import CSVFileFormat, JSONFileFormat, ParquetFileFormat from .function import JavascriptUDF, PythonUDF +from .git_repository import GitRepository from .grant import DatabaseRoleGrant, Grant, RoleGrant from .hybrid_table import HybridTable from .iceberg_table import SnowflakeIcebergTable @@ -99,6 +100,7 @@ "GCPOutboundNotificationIntegration", "GCSStorageIntegration", "GenericSecret", + "GitRepository", "GlueCatalogIntegration", "Grant", "HybridTable", diff --git a/snowcap/resources/api_integration.py b/snowcap/resources/api_integration.py index 306197c..da48dea 100644 --- a/snowcap/resources/api_integration.py +++ b/snowcap/resources/api_integration.py @@ -9,20 +9,34 @@ class ApiProvider(ParseableEnum): + # AWS API Gateway flavors AWS_API_GATEWAY = "AWS_API_GATEWAY" AWS_PRIVATE_API_GATEWAY = "AWS_PRIVATE_API_GATEWAY" AWS_GOV_API_GATEWAY = "AWS_GOV_API_GATEWAY" AWS_GOV_PRIVATE_API_GATEWAY = "AWS_GOV_PRIVATE_API_GATEWAY" + # Azure / Google + AZURE_API_MANAGEMENT = "AZURE_API_MANAGEMENT" + GOOGLE_API_GATEWAY = "GOOGLE_API_GATEWAY" + # Git integration (used by CREATE GIT REPOSITORY via api_integration=...) + GIT_HTTPS_API = "GIT_HTTPS_API" @dataclass(unsafe_hash=True) class _APIIntegration(ResourceSpec): name: ResourceName api_provider: ApiProvider - api_aws_role_arn: str enabled: bool api_allowed_prefixes: list[str] api_blocked_prefixes: list[str] = None + # AWS-only: AWS IAM role used by Snowflake to call the gateway. + # Other providers (GIT_HTTPS_API, AZURE_API_MANAGEMENT, GOOGLE_API_GATEWAY) + # don't use this field. + api_aws_role_arn: str = None + # Azure-only: tenant ID + AD application ID. + azure_tenant_id: str = None + azure_ad_application_id: str = None + # Google-only: GCP audience. + google_audience: str = None api_key: str = field(default=None, metadata={"fetchable": False}) owner: Role = "ACCOUNTADMIN" comment: str = None @@ -81,6 +95,9 @@ class APIIntegration(NamedResource, Resource): props = Props( api_provider=EnumProp("api_provider", ApiProvider), api_aws_role_arn=StringProp("api_aws_role_arn"), + azure_tenant_id=StringProp("azure_tenant_id"), + azure_ad_application_id=StringProp("azure_ad_application_id"), + google_audience=StringProp("google_audience"), api_key=StringProp("api_key"), api_allowed_prefixes=StringListProp("api_allowed_prefixes", parens=True), api_blocked_prefixes=StringListProp("api_blocked_prefixes", parens=True), @@ -94,10 +111,17 @@ def __init__( self, name: str, api_provider: ApiProvider, - api_aws_role_arn: str, - enabled: bool, - api_allowed_prefixes: list[str], + # NB: api_aws_role_arn was previously a required positional arg. + # It is now optional (default None) so non-AWS api_provider values + # (GIT_HTTPS_API, AZURE_API_MANAGEMENT, GOOGLE_API_GATEWAY) can omit + # it. Positional order preserved to keep existing callers working. + api_aws_role_arn: str = None, + enabled: bool = True, + api_allowed_prefixes: list[str] = None, api_blocked_prefixes: list[str] = None, + azure_tenant_id: str = None, + azure_ad_application_id: str = None, + google_audience: str = None, api_key: str = None, owner: str = "ACCOUNTADMIN", comment: str = None, @@ -108,6 +132,9 @@ def __init__( name=self._name, api_provider=api_provider, api_aws_role_arn=api_aws_role_arn, + azure_tenant_id=azure_tenant_id, + azure_ad_application_id=azure_ad_application_id, + google_audience=google_audience, api_key=api_key, api_allowed_prefixes=api_allowed_prefixes, api_blocked_prefixes=api_blocked_prefixes, diff --git a/snowcap/resources/git_repository.py b/snowcap/resources/git_repository.py new file mode 100644 index 0000000..d14ccef --- /dev/null +++ b/snowcap/resources/git_repository.py @@ -0,0 +1,104 @@ +from dataclasses import dataclass + +from ..enums import ResourceType +from ..props import ( + IdentifierProp, + Props, + StringProp, +) +from ..resource_name import ResourceName +from ..role_ref import RoleRef +from ..scope import SchemaScope +from .resource import NamedResource, Resource, ResourceSpec + + +@dataclass(unsafe_hash=True) +class _GitRepository(ResourceSpec): + name: ResourceName + origin: str + api_integration: str + git_credentials: str = None + comment: str = None + owner: RoleRef = "SYSADMIN" + + +class GitRepository(NamedResource, Resource): + """ + Description: + A Git Repository in Snowflake represents an externally hosted Git + repository (GitHub, GitLab, Bitbucket, etc.) that has been registered + for use with Snowflake's Git integration. Once registered, files in + the repository can be referenced via stage syntax in COPY, EXECUTE + IMMEDIATE, and other commands. + + Snowflake Docs: + https://docs.snowflake.com/en/sql-reference/sql/create-git-repository + + Fields: + name (string, required): The name of the git repository. + origin (string, required): The URL of the externally hosted Git + repository (e.g., "https://github.com/some-org/some-repo.git"). + api_integration (string, required): The name of the API integration + object Snowflake will use to interact with the repository. The + API integration's allowed prefixes must include the origin URL. + git_credentials (string): The name of the secret holding credentials + for accessing a private repository. Optional for public repos. + comment (string): A comment for the git repository. + owner (string or Role): The owner of the git repository. Defaults to + SYSADMIN. + + Python: + + ```python + git_repository = GitRepository( + name="some_git_repository", + origin="https://github.com/some-org/some-repo.git", + api_integration="some_api_integration", + git_credentials="some_secret", + comment="some_comment", + owner="SYSADMIN", + ) + ``` + + Yaml: + + ```yaml + git_repositories: + - name: some_git_repository + origin: https://github.com/some-org/some-repo.git + api_integration: some_api_integration + git_credentials: some_secret + comment: some_comment + owner: SYSADMIN + ``` + """ + + resource_type = ResourceType.GIT_REPOSITORY + props = Props( + origin=StringProp("origin"), + api_integration=IdentifierProp("api_integration"), + git_credentials=IdentifierProp("git_credentials"), + comment=StringProp("comment"), + ) + scope = SchemaScope() + spec = _GitRepository + + def __init__( + self, + name: str, + origin: str, + api_integration: str, + git_credentials: str = None, + comment: str = None, + owner: str = "SYSADMIN", + **kwargs, + ): + super().__init__(name, **kwargs) + self._data: _GitRepository = _GitRepository( + name=self._name, + origin=origin, + api_integration=api_integration, + git_credentials=git_credentials, + comment=comment, + owner=owner, + ) diff --git a/snowcap/resources/grant.py b/snowcap/resources/grant.py index cdd3176..84175bb 100644 --- a/snowcap/resources/grant.py +++ b/snowcap/resources/grant.py @@ -183,15 +183,27 @@ def __init__( else: raise ValueError("You must specify at least one privilege") if isinstance(on, list): - has_many_ons = True + # A list of `on:` items is one of two things: + # + # (1) A single grant whose target is described by a multi-element + # spec, like ["FUTURE", "TABLES", "SCHEMA", "db.schema"] or + # ["ALL", "TABLES", Schema(name=...)]. + # (2) Multiple grants — one per item — where each item is a + # complete `on` spec (e.g. ["warehouse FOO", "warehouse BAR"] + # or ["all schemas in database X", "future schemas in database X"]). + # + # Heuristic: form (1) starts with the keyword FUTURE or ALL as its + # first element. Anything else is form (2). + first = on[0] + first_is_grant_type_keyword = ( + isinstance(first, str) + and first.upper() in (GrantType.FUTURE, GrantType.ALL) + ) for item in on: - if isinstance(item, str): - if item.upper().find(GrantType.FUTURE) == -1 and item.upper().find(GrantType.ALL) == -1: - has_many_ons = False - break - elif isinstance(item, list): + if isinstance(item, list): if item[0].upper() not in (GrantType.FUTURE, GrantType.ALL): raise ValueError("You must specify a valid Grant Type when specifying a list of grants") + has_many_ons = not first_is_grant_type_keyword if has_many_ons: self.rest_of_ons = on[1:] on = on[0] @@ -256,16 +268,41 @@ def __init__( # Split the string while preserving case for the FQN part # "SCHEMA db.schema" -> ["SCHEMA", "db.schema"] # "FUTURE TABLES IN SCHEMA db.schema" -> ["FUTURE", "TABLES", "SCHEMA", "db.schema"] + # "GIT REPOSITORY db.schema.repo" -> ["GIT REPOSITORY", "db.schema.repo"] + # Multi-word resource types (GIT REPOSITORY, API INTEGRATION, + # NETWORK RULE, IMAGE REPOSITORY, etc.) are matched by looking + # ahead to consume consecutive words. + resource_type_values = {e.value for e in ResourceType} + multi_word_types = sorted( + (v for v in resource_type_values if " " in v), + key=lambda v: -len(v.split()), + ) parts = on.split(" ") on_items = [] - for i, part in enumerate(parts): + i = 0 + while i < len(parts): + part = parts[i] if part.upper() == "IN": + i += 1 + continue + # Try to match a multi-word resource type first + # (e.g. "git repository" or "external access integration") + matched_multi = None + for mw in multi_word_types: + mw_words = mw.split() + if i + len(mw_words) <= len(parts): + candidate = " ".join(parts[i : i + len(mw_words)]).upper() + if candidate == mw: + matched_multi = mw + break + if matched_multi is not None: + on_items.append(matched_multi) + i += len(matched_multi.split()) continue - # Uppercase only keyword parts (SCHEMA, TABLE, FUTURE, ALL, etc.) - # but preserve case for the last item which is the FQN - # Note: we normalize underscores to spaces to match ResourceType values + # Single-word matches: ResourceType values or + # GrantType.FUTURE / GrantType.ALL keywords part_normalized = part.upper().replace("_", " ") - if part_normalized in [e.value for e in ResourceType] or part.upper() in [ + if part_normalized in resource_type_values or part.upper() in [ GrantType.FUTURE, GrantType.ALL, ]: @@ -273,6 +310,7 @@ def __init__( elif part: # This is likely the FQN - preserve case on_items.append(part) + i += 1 if len(on_items) < 2: raise ValueError("You must specify at least three parameters: [grant_type, items_type, object]") elif on_items[0].upper() in [e.value for e in ResourceType]: diff --git a/snowcap/resources/resource.py b/snowcap/resources/resource.py index 86e93a0..033277d 100644 --- a/snowcap/resources/resource.py +++ b/snowcap/resources/resource.py @@ -279,6 +279,14 @@ def get_metadata(cls, field_name: str) -> ResourceSpecMetadata: RESOURCE_SCOPES = { ResourceType.ACCOUNT: OrganizationScope(), + # Generic INTEGRATION — umbrella for API/CATALOG/EXTERNAL_ACCESS/NOTIFICATION/ + # SECURITY/STORAGE integrations. Snowflake's `GRANT USAGE ON INTEGRATION ` + # syntax accepts any subtype; users naturally write `on: integration ` in + # snowcap YAML, so we register an AccountScope here to let those grants parse. + # We don't expose a concrete `Integration` resource class — declarative + # management still goes through the specific subtypes (APIIntegration, + # CatalogIntegration, etc.). + ResourceType.INTEGRATION: AccountScope(), } diff --git a/tests/fixtures/json/api_integration.json b/tests/fixtures/json/api_integration.json index c45f498..25ed61a 100644 --- a/tests/fixtures/json/api_integration.json +++ b/tests/fixtures/json/api_integration.json @@ -4,6 +4,9 @@ "api_provider": "AWS_API_GATEWAY", "api_key": "api-987654321", "api_aws_role_arn": "arn:aws:iam::123456789012:role/my_cloud_account_role", + "azure_tenant_id": null, + "azure_ad_application_id": null, + "google_audience": null, "api_allowed_prefixes": [ "https://xyz.execute-api.us-west-2.amazonaws.com/production" ], @@ -12,4 +15,4 @@ ], "enabled": true, "comment": "This is a test API integration" -} \ No newline at end of file +} diff --git a/tests/fixtures/json/git_repository.json b/tests/fixtures/json/git_repository.json new file mode 100644 index 0000000..dcf81df --- /dev/null +++ b/tests/fixtures/json/git_repository.json @@ -0,0 +1,8 @@ +{ + "name": "my_git_repository", + "origin": "https://github.com/some-org/some-repo.git", + "api_integration": "my_api_integration", + "git_credentials": "my_git_secret", + "comment": "Example git repository", + "owner": "SYSADMIN" +} diff --git a/tests/fixtures/sql/git_repository.sql b/tests/fixtures/sql/git_repository.sql new file mode 100644 index 0000000..b17caca --- /dev/null +++ b/tests/fixtures/sql/git_repository.sql @@ -0,0 +1,5 @@ +CREATE OR REPLACE GIT REPOSITORY my_git_repository + ORIGIN = 'https://github.com/some-org/some-repo.git' + API_INTEGRATION = my_api_integration + GIT_CREDENTIALS = my_git_secret + COMMENT = 'Example git repository'; diff --git a/tests/integration/data_provider/test_fetch_resource.py b/tests/integration/data_provider/test_fetch_resource.py index 99bf5de..a1ef2ea 100644 --- a/tests/integration/data_provider/test_fetch_resource.py +++ b/tests/integration/data_provider/test_fetch_resource.py @@ -300,6 +300,34 @@ def test_fetch_api_integration(cursor, suffix, marked_for_cleanup): assert result == data +def test_fetch_git_repository(cursor, suffix, marked_for_cleanup): + api_integration = res.APIIntegration( + name=f"API_INTEGRATION_FOR_GIT_REPO_{suffix}", + api_provider="AWS_API_GATEWAY", + api_aws_role_arn="arn:aws:iam::123456789012:role/MyRole", + api_allowed_prefixes=["https://github.com/"], + comment="API integration for git repo test", + enabled=True, + owner=TEST_ROLE, + ) + create(cursor, api_integration) + marked_for_cleanup.append(api_integration) + + repo = res.GitRepository( + name=f"GIT_REPOSITORY_EXAMPLE_{suffix}", + origin="https://github.com/some-org/some-repo.git", + api_integration=api_integration.name, + comment="Example git repository (public)", + owner=TEST_ROLE, + ) + create(cursor, repo) + marked_for_cleanup.append(repo) + + result = safe_fetch(cursor, repo.urn) + assert result is not None + assert_resource_dicts_eq_ignore_nulls_and_unfetchable(repo.spec, result, repo.to_dict()) + + def test_fetch_password_secret(cursor, suffix, marked_for_cleanup): secret = res.PasswordSecret( name=f"PASSWORD_SECRET_EXAMPLE_{suffix}", diff --git a/tests/test_grant.py b/tests/test_grant.py index ca396f3..fa71c65 100644 --- a/tests/test_grant.py +++ b/tests/test_grant.py @@ -547,3 +547,74 @@ def test_role_grants_mixed_patterns_in_same_config(self): blueprint_config = collect_blueprint_config(config) # 1 + 2 + 2 = 5 grants total assert len(blueprint_config.resources) == 5 + + +class TestGrantOnList: + """Tests for `on:` list expansion (multiple grants from one entry).""" + + def test_on_list_of_plain_objects_expands_to_multiple_grants(self): + """Test: on: [warehouse FOO, warehouse BAR] expands to 2 grants.""" + grant = res.Grant( + priv="USAGE", + on=["warehouse FOO", "warehouse BAR"], + to="somerole", + ) + assert grant.priv == "USAGE" + assert grant.on == "FOO" + assert grant.on_type == ResourceType.WAREHOUSE + assert grant.rest_of_ons == ["warehouse BAR"] + additional = grant.process_shortcuts() + assert len(additional) == 1 + assert additional[0].on == "BAR" + assert additional[0].on_type == ResourceType.WAREHOUSE + + def test_on_list_of_mixed_resource_types(self): + """Test: on: [warehouse FOO, database BAR] expands to 2 grants of different types.""" + grant = res.Grant( + priv="USAGE", + on=["warehouse FOO", "database BAR"], + to="somerole", + ) + assert grant.on == "FOO" + assert grant.on_type == ResourceType.WAREHOUSE + additional = grant.process_shortcuts() + assert len(additional) == 1 + assert additional[0].on == "BAR" + assert additional[0].on_type == ResourceType.DATABASE + + def test_on_list_of_multiword_types(self): + """Test: on: list of multi-word resource types (git repository) expands.""" + grant = res.Grant( + priv="READ", + on=["git repository D.S.A", "git repository D.S.B"], + to="somerole", + ) + assert grant.on_type == ResourceType.GIT_REPOSITORY + additional = grant.process_shortcuts() + assert len(additional) == 1 + assert additional[0].on_type == ResourceType.GIT_REPOSITORY + assert additional[0].on == "D.S.B" + + def test_on_list_4_element_grant_type_form_unchanged(self): + """Test: existing on: [FUTURE/ALL, items, object_type, object] form still + parses as a single grant (not expanded into multiple).""" + grant = res.Grant( + priv="SELECT", + on=["FUTURE", "TABLES", "SCHEMA", "D.S"], + to="somerole", + ) + assert grant.rest_of_ons == [] + assert grant.on == "D.S" + assert grant.on_type == ResourceType.SCHEMA + + def test_on_list_of_all_future_grants_still_expands(self): + """Test: on: [all schemas in DB X, future schemas in DB X] still expands + (existing behavior preserved).""" + grant = res.Grant( + priv="USAGE", + on=["all schemas in database raw_prd", "future schemas in database raw_prd"], + to="somerole", + ) + assert grant.rest_of_ons == ["future schemas in database raw_prd"] + additional = grant.process_shortcuts() + assert len(additional) == 1 diff --git a/tests/test_resource_types.py b/tests/test_resource_types.py index bc253f8..5aa27a6 100644 --- a/tests/test_resource_types.py +++ b/tests/test_resource_types.py @@ -909,6 +909,36 @@ def test_generic_secret_minimal(self): assert secret.name == "test_secret" +class TestGitRepository: + """Tests for GitRepository resource.""" + + def test_git_repository_minimal(self): + """Test GitRepository with minimal required properties.""" + repo = res.GitRepository( + name="test_repo", + database="test_db", + schema="test_schema", + origin="https://github.com/some-org/some-repo.git", + api_integration="my_api_integration", + ) + assert repo.name == "test_repo" + assert repo.resource_type == ResourceType.GIT_REPOSITORY + + def test_git_repository_with_credentials(self): + """Test GitRepository with optional git_credentials secret.""" + repo = res.GitRepository( + name="test_repo", + database="test_db", + schema="test_schema", + origin="https://github.com/some-org/some-repo.git", + api_integration="my_api_integration", + git_credentials="my_secret", + comment="A private repo", + ) + assert repo._data.git_credentials == "my_secret" + assert repo._data.comment == "A private repo" + + class TestPasswordPolicy: """Tests for PasswordPolicy resource.""" @@ -1040,6 +1070,66 @@ def test_api_integration_minimal(self): assert api.name == "test_api" assert api.resource_type == ResourceType.API_INTEGRATION + def test_api_integration_git_https_no_aws_role(self): + """Non-AWS api_provider (GIT_HTTPS_API) doesn't require api_aws_role_arn.""" + api = res.APIIntegration( + name="github_int", + api_provider="GIT_HTTPS_API", + api_allowed_prefixes=["https://github.com/some-org/"], + enabled=True, + ) + assert api._data.api_aws_role_arn is None + assert api._data.api_provider.value == "GIT_HTTPS_API" + + def test_api_integration_azure(self): + """AZURE_API_MANAGEMENT provider uses azure_tenant_id + azure_ad_application_id.""" + api = res.APIIntegration( + name="azure_int", + api_provider="AZURE_API_MANAGEMENT", + azure_tenant_id="11111111-1111-1111-1111-111111111111", + azure_ad_application_id="22222222-2222-2222-2222-222222222222", + api_allowed_prefixes=["https://example.azure-api.net/"], + enabled=True, + ) + assert api._data.azure_tenant_id is not None + assert api._data.api_aws_role_arn is None + + def test_api_integration_google(self): + """GOOGLE_API_GATEWAY provider uses google_audience.""" + api = res.APIIntegration( + name="gcp_int", + api_provider="GOOGLE_API_GATEWAY", + google_audience="some-audience-arn", + api_allowed_prefixes=["https://example.run.app/"], + enabled=True, + ) + assert api._data.google_audience == "some-audience-arn" + assert api._data.api_aws_role_arn is None + + +class TestGenericIntegrationGrants: + """Tests for ResourceType.INTEGRATION (umbrella) grants.""" + + def test_generic_integration_grant_parses(self): + """`on: integration ` parses to ResourceType.INTEGRATION (no longer KeyError).""" + grant = res.Grant(priv="USAGE", on="integration SOME_INTEGRATION", to="somerole") + assert grant._data.on_type == ResourceType.INTEGRATION + assert grant._data.on == "SOME_INTEGRATION" + + def test_generic_integration_list_grant_expands(self): + """List form of generic integration grants works with the on:list expansion.""" + grant = res.Grant( + priv="USAGE", + on=["integration A", "integration B"], + to="somerole", + ) + assert grant._data.on_type == ResourceType.INTEGRATION + assert grant.rest_of_ons == ["integration B"] + rest = grant.process_shortcuts() + assert len(rest) == 1 + assert rest[0]._data.on_type == ResourceType.INTEGRATION + assert rest[0]._data.on == "B" + class TestShare: """Tests for Share resource."""