Merge pull request #50 from labd/feature/api-gateway-improvements

WIP: Feature/api gateway improvements

Author: tleguijt

CHANGELOG.md

@@ -7,13 +7,14 @@
 - Add support for multiple API endpoints:
     - `base_url` replaced with `endpoints`
     - `has_public_api` replaced with `endpoint`
-- Add new required `frontdoor_id` Terraform variable for components with `endpoint` defined
 - Improved dependencies between components and MACH-managed commercetools configurations
 - Improved git log parsing
 - Add `mach bootstrap` commands:
     - `mach bootstrap config` for creating a new MACH configuration
     - `mach bootstrap component` for creating a new MACH component
 - Updated Terraform commercetools provider to `0.24.1`
+- AWS: Set `auto-deploy` on API gateway stage
+- Azure: Add new required `frontdoor_id` Terraform variable for components with `endpoint` defined
 
 
 **Breaking changes**

src/mach/commands.py

@@ -23,25 +23,12 @@ def terraform_command(f):
         default=None,
         help="YAML file to parse. If not set parse all *.yml files.",
     )
-    @click.option(
-        "--with-sp-login",
-        is_flag=True,
-        default=False,
-        help="If az login with service principal environment variables "
-        "(ARM_CLIENT_ID, ARM_CLIENT_SECRET, ARM_TENANT_ID) should be done.",
-    )
-    @click.option(
-        "--auto-approve",
-        is_flag=True,
-        default=False,
-        help="",
-    )
     @click.option(
         "--output-path",
         default="deployments",
         help="Output path, defaults to `cwd`/deployments.",
     )
-    def new_func(file, with_sp_login: bool, auto_approve: bool, output_path: str):
+    def new_func(file, output_path: str, **kwargs):
         files = get_input_files(file)
 
         try:
@@ -52,9 +39,8 @@ def new_func(file, with_sp_login: bool, auto_approve: bool, output_path: str):
         try:
             result = f(
                 file=file,
-                with_sp_login=with_sp_login,
-                auto_approve=auto_approve,
                 configs=configs,
+                **kwargs,
             )
         except subprocess.CalledProcessError as e:
             click.echo("Failed to run")
@@ -84,13 +70,26 @@ def plan(file, configs, *args, **kwargs):
 
 
 @mach.command()
+@click.option(
+    "--with-sp-login",
+    is_flag=True,
+    default=False,
+    help="If az login with service principal environment variables "
+    "(ARM_CLIENT_ID, ARM_CLIENT_SECRET, ARM_TENANT_ID) should be done.",
+)
+@click.option(
+    "--auto-approve",
+    is_flag=True,
+    default=False,
+    help="",
+)
 @terraform_command
 def apply(file, configs, with_sp_login, auto_approve, *args, **kwargs):
     """Apply the configuration."""
     for config in configs:
         generate_terraform(config)
         apply_terraform(
-            config.deployment_path,
+            config,
             with_sp_login=with_sp_login,
             auto_approve=auto_approve,
         )

src/mach/parse.py

@@ -1,3 +1,4 @@
+from collections import defaultdict
 from os.path import basename, splitext
 from pathlib import Path
 from typing import Dict, List
@@ -10,6 +11,7 @@
     ComponentConfig,
     MachConfig,
     SentryDsn,
+    Site,
     SiteAzureSettings,
 )
 from mach.validate import validate_config
@@ -114,9 +116,25 @@ def resolve_site_configs(config: MachConfig) -> MachConfig:
 
     config = resolve_site_components(config)
 
+    for site in config.sites:
+        resolve_endpoint_components(site)
+
     return config
 
 
+def resolve_endpoint_components(site: Site):
+    endpoint_components = defaultdict(list)
+
+    for c in site.components:
+        if not site.endpoints:
+            continue
+
+        endpoint_components[c.endpoint].append(c)
+
+    for endpoint in site.endpoints:
+        endpoint.components = endpoint_components.get(endpoint.key, [])
+
+
 def resolve_site_components(config: MachConfig) -> MachConfig:
     """If no component info is specified, use global component settings."""
     component_info: Dict[str, ComponentConfig] = {

src/mach/templates/partials/aws.tf

@@ -26,7 +26,7 @@ provider "aws" {
 {% if site.used_endpoints %}
   {% include 'partials/endpoints/aws_domains.tf' %}
 
-  {% for endpoint_name, endpoint_url in site.used_endpoints.items() %}
+  {% for endpoint in site.used_endpoints %}
     {% include 'partials/endpoints/aws_api_gateway.tf' %}
 
   {% endfor %}

src/mach/templates/partials/endpoints/aws_api_gateway.tf

@@ -1,83 +1,75 @@
-resource "aws_acm_certificate" "{{ endpoint_name|slugify }}" {
-  domain_name       = "{{ endpoint_url }}"
+resource "aws_acm_certificate" "{{ endpoint.key|slugify }}" {
+  domain_name       = "{{ endpoint.url }}"
   validation_method = "DNS"
 }
 
-resource "aws_route53_record" "{{ endpoint_name|slugify }}_acm_validation" {
+resource "aws_route53_record" "{{ endpoint.key|slugify }}_acm_validation" {
   zone_id = data.aws_route53_zone.main.zone_id
-  name    = tolist(aws_acm_certificate.{{ endpoint_name|slugify }}.domain_validation_options)[0].resource_record_name
-  type    = tolist(aws_acm_certificate.{{ endpoint_name|slugify }}.domain_validation_options)[0].resource_record_type
+  name    = tolist(aws_acm_certificate.{{ endpoint.key|slugify }}.domain_validation_options)[0].resource_record_name
+  type    = tolist(aws_acm_certificate.{{ endpoint.key|slugify }}.domain_validation_options)[0].resource_record_type
   ttl     = 60
-  records = [tolist(aws_acm_certificate.{{ endpoint_name|slugify }}.domain_validation_options)[0].resource_record_value]
+  records = [tolist(aws_acm_certificate.{{ endpoint.key|slugify }}.domain_validation_options)[0].resource_record_value]
 }
 
 # API Gateway
-resource "aws_apigatewayv2_api" "{{ endpoint_name|slugify }}_gateway" {
-  name                       = "{{ site.identifier }}-api"
+resource "aws_apigatewayv2_api" "{{ endpoint.key|slugify }}_gateway" {
+  name                       = "{{ site.identifier }}-{{ endpoint.key|slugify }}-api"
   protocol_type              = "HTTP"
 }
 
-resource "aws_apigatewayv2_route" "{{ endpoint_name|slugify }}_application" {
-  api_id    = aws_apigatewayv2_api.{{ endpoint_name|slugify }}_gateway.id
+resource "aws_apigatewayv2_route" "{{ endpoint.key|slugify }}_application" {
+  api_id    = aws_apigatewayv2_api.{{ endpoint.key|slugify }}_gateway.id
   route_key = "$default"
 }
 
-resource "aws_apigatewayv2_deployment" "{{ endpoint_name|slugify }}_default" {
-  api_id      = aws_apigatewayv2_api.{{ endpoint_name|slugify }}_gateway.id
+resource "aws_apigatewayv2_deployment" "{{ endpoint.key|slugify }}_default" {
+  api_id      = aws_apigatewayv2_api.{{ endpoint.key|slugify }}_gateway.id
   description = "Stage for default release"
 
-  triggers = {
-    redeployment = sha1(join(",", list(
-      {% for component in site.public_api_components %}
-      module.{{ component.name }}.component_version,
-      {% endfor %}
-    )))
-  }
-
   lifecycle {
     create_before_destroy = true
   }
 
   depends_on = [
-    {% for component in site.public_api_components %}
+    {% for component in endpoint.components %}
     module.{{ component.name }},
     {% endfor %}
   ]
 }
 
-resource "aws_apigatewayv2_stage" "{{ endpoint_name|slugify }}_default" {
+resource "aws_apigatewayv2_stage" "{{ endpoint.key|slugify }}_default" {
   name                  = "$default"
-  api_id                = aws_apigatewayv2_api.{{ endpoint_name|slugify }}_gateway.id
-  deployment_id         = aws_apigatewayv2_deployment.{{ endpoint_name|slugify }}_default.id
-
-  depends_on = [aws_apigatewayv2_deployment.{{ endpoint_name|slugify }}_default]
+  description           = "Stage for default release"
+  api_id                = aws_apigatewayv2_api.{{ endpoint.key|slugify }}_gateway.id
+  deployment_id         = aws_apigatewayv2_deployment.{{ endpoint.key|slugify }}_default.id
+  auto_deploy           = true
 }
 
 # Route53 mappings
-resource "aws_apigatewayv2_domain_name" "{{ endpoint_name|slugify }}" {
-  domain_name = "{{ endpoint_url }}"
+resource "aws_apigatewayv2_domain_name" "{{ endpoint.key|slugify }}" {
+  domain_name = "{{ endpoint.url }}"
 
   domain_name_configuration {
-    certificate_arn = aws_acm_certificate.{{ endpoint_name|slugify }}.arn
+    certificate_arn = aws_acm_certificate.{{ endpoint.key|slugify }}.arn
     endpoint_type   = "REGIONAL"
     security_policy = "TLS_1_2"
   }
 }
 
-resource "aws_route53_record" "{{ endpoint_name|slugify }}" {
-  name    = aws_apigatewayv2_domain_name.{{ endpoint_name|slugify }}.domain_name
+resource "aws_route53_record" "{{ endpoint.key|slugify }}" {
+  name    = aws_apigatewayv2_domain_name.{{ endpoint.key|slugify }}.domain_name
   type    = "A"
   zone_id = data.aws_route53_zone.main.id
 
   alias {
-    name                   = aws_apigatewayv2_domain_name.{{ endpoint_name|slugify }}.domain_name_configuration[0].target_domain_name
-    zone_id                = aws_apigatewayv2_domain_name.{{ endpoint_name|slugify }}.domain_name_configuration[0].hosted_zone_id
+    name                   = aws_apigatewayv2_domain_name.{{ endpoint.key|slugify }}.domain_name_configuration[0].target_domain_name
+    zone_id                = aws_apigatewayv2_domain_name.{{ endpoint.key|slugify }}.domain_name_configuration[0].hosted_zone_id
     evaluate_target_health = false
   }
 }
 
-resource "aws_apigatewayv2_api_mapping" "{{ endpoint_name|slugify }}" {
-  api_id      = aws_apigatewayv2_api.{{ endpoint_name|slugify }}_gateway.id
-  stage       = aws_apigatewayv2_stage.{{ endpoint_name|slugify }}_default.id
-  domain_name = "{{ endpoint_url }}"
+resource "aws_apigatewayv2_api_mapping" "{{ endpoint.key|slugify }}" {
+  api_id      = aws_apigatewayv2_api.{{ endpoint.key|slugify }}_gateway.id
+  stage       = aws_apigatewayv2_stage.{{ endpoint.key|slugify }}_default.id
+  domain_name = "{{ endpoint.url }}"
 }

src/mach/terraform.py

@@ -49,19 +49,28 @@ def plan_terraform(output_dir: Path):
 
 
 def apply_terraform(
-    output_dir: Path, *, with_sp_login: bool = False, auto_approve: bool = False
+    config: MachConfig,
+    *,
+    with_sp_login: bool = False,
+    auto_approve: bool = False,
 ):
     """Terraform apply for all generated sites."""
-    for site_dir in output_dir.iterdir():
-        if site_dir.is_dir():
-            click.echo(f"Applying Terraform for {site_dir.name}")
-            run_terraform("init", site_dir)
-            if with_sp_login:
-                azure_sp_login()
-            cmd = ["apply"]
-            if auto_approve:
-                cmd += ["-auto-approve"]
-            run_terraform(cmd, site_dir)
+    for site in config.sites:
+        site_dir = config.deployment_path / Path(site.identifier)
+        if not site_dir.is_dir():
+            click.echo(f"Could not find site directory {site_dir}")
+            continue
+
+        click.echo(f"Applying Terraform for {site.identifier}")
+        run_terraform("init", site_dir)
+
+        if with_sp_login:
+            azure_sp_login()
+
+        cmd = ["apply"]
+        if auto_approve:
+            cmd += ["-auto-approve"]
+        run_terraform(cmd, site_dir)
 
 
 def azure_sp_login():

src/mach/types.py

@@ -6,6 +6,7 @@
 from dataclasses_json import config, dataclass_json
 from dataclasses_jsonschema import JsonSchemaMixin
 from mach import utils
+from marshmallow import ValidationError, fields
 
 TerraformVariables = Dict[str, Any]
 StoreVariables = Dict[str, TerraformVariables]
@@ -398,13 +399,67 @@ def merge(self, config: AzureConfig):
         self.service_object_ids = self.service_object_ids or config.service_object_ids
 
 
+@dataclass_json
+@dataclass
+class Endpoint:
+    url: str
+    key: str = field(metadata=config(exclude=lambda x: True))
+    components: Optional[List[Component]] = _list()
+
+    @property
+    def contains_defaults(self):
+        """Indicate if this endpoint contains just default values.
+
+        Other then the `url` attribute.
+        If only defaults, we can serialize the endpoints by just
+        rendering the url, not the entire object.
+
+        At this moment, we don't have any additional options, so it's always default.
+        This can be extended in the future.
+        """
+        return True
+
+    def __post_init__(self):
+        """Ensure endpoints have protocol stripped."""
+        self.url = utils.strip_protocol(self.url)
+
+
+class EndpointsField(fields.Dict):
+    def _serialize(self, value, attr, obj, **kwargs):
+        result = {}
+        for endpoint in value:
+            if endpoint.contains_defaults:
+                result[endpoint.key] = endpoint.url
+            else:
+                result[endpoint.key] = endpoint.to_dict()
+
+        return super()._deserialize(result, attr, obj, **kwargs)
+
+    def _deserialize(self, value, attr, data, **kwargs):
+        value = super()._deserialize(value, attr, data, **kwargs)
+        result = []
+        for k, v in value.items():
+            if isinstance(v, str):
+                result.append(Endpoint(key=k, url=v))
+            elif isinstance(v, dict):
+                v["key"] = k
+                result.append(Endpoint.schema(infer_missing=True).load(v))
+            else:
+                raise ValidationError(f"Unexpected value found for endpoint {k}")
+
+        return result
+
+
 @dataclass_json
 @dataclass
 class Site(JsonSchemaMixin):
     """Site definition."""
 
     identifier: str
-    endpoints: Dict[str, str] = _default({})
+    endpoints: Optional[List[Endpoint]] = field(
+        default_factory=list,
+        metadata=config(mm_field=EndpointsField(), exclude=lambda x: not x),
+    )
     commercetools: Optional[CommercetoolsSettings] = _none()
     contentful: Optional[ContentfulSettings] = _none()
     amplience: Optional[AmplienceSettings] = _none()
@@ -417,19 +472,10 @@ class Site(JsonSchemaMixin):
     def public_api_components(self) -> List[Component]:
         return [c for c in self.components if c.endpoint]
 
-    def __post_init__(self):
-        """Ensure endpoints have protocol stripped."""
-        if self.endpoints:
-            self.endpoints = {
-                k: utils.strip_protocol(v) for k, v in self.endpoints.items()
-            }
-
     @property
-    def used_endpoints(self):
+    def used_endpoints(self) -> List[Endpoint]:
         """Return only the endpoints that are actually used by the components."""
-        used_endpoints = {c.endpoint for c in self.components if c.endpoint}
-
-        return {k: v for k, v in self.endpoints.items() if k in used_endpoints}
+        return [ep for ep in self.endpoints if ep.components]
 
 
 @dataclass_json