refactor: Utilize multistage builds to improve caching and compatibility

The main changes here are;
* using multi-stage builds for better caching and smaller images
* explicitly listing the files that should be included / "COPY"able into the containers

For the API, I moved the tests outside the `src` folder.
That way, we do not include the tests in the finished image.
Since we use multi-stage builds, we can install the necessary compilers for running on ARM.
The compilers are not part of the "production" image, and we don't need to worry about them taking up more space in the image layers.

I also created a user (similar to what's done in `nginx`) so that we can run the tests without running the container as root.

When only installing the necessary packages, I noticed that `click` is not explicitly defined, so it wasn't included in `site-packages`.
By explicitly listing it in `pyproject.toml`, that issue is resolved.

Author: sindre-nistad

.pre-commit-config.yaml

@@ -26,7 +26,7 @@ repos:
       - id: mixed-line-ending
         exclude: ^.*\.(lock)$
       - id: detect-private-key
-        exclude: api/src/tests/integration/mock_authentication.py
+        exclude: api/tests/integration/mock_authentication.py
       - id: no-commit-to-branch
         args: [--branch, main, --branch, master]
         stages: [commit-msg]
@@ -80,7 +80,7 @@ repos:
     hooks:
       - id: pytest
         name: pytest-check
-        entry: sh -c "cd ./api/src/ && poetry run pytest ./tests/"
+        entry: sh -c "cd ./api/ && poetry run pytest ./tests/"
         language: system
         pass_filenames: false
         always_run: true

api/.dockerignore

@@ -1,3 +1,11 @@
-.venv
-.pytest_cache
-.mypy_cache
+# Ignore all by default
+*
+!pyproject.toml
+!poetry.lock
+
+!src/**/*.py
+!src/*.py
+!src/init.sh
+
+!tests/**/*.py
+!tests/test_data/

api/Dockerfile

@@ -1,26 +1,76 @@
-FROM --platform=linux/amd64 python:3.12-slim AS base
+# syntax=docker/dockerfile:1
+FROM python:3.13.2-alpine3.21 AS base
+ENV XDG_CACHE_HOME=/var/lib/
+ENV SITE_PACKAGES=/usr/local/lib/python3.13/site-packages
+
+ENV USER="api-user"
+
+RUN adduser \
+    --disabled-password \
+    --home /code \
+    --gecos "" \
+    --uid 1000 \
+    "$USER"
+
 WORKDIR /code
 CMD ["/code/src/init.sh", "api"]
 EXPOSE 5000
 
 ENV PYTHONUNBUFFERED=1
-ENV PYTHONPATH=/code
+ENV PYTHONPATH=/code/src
+
+FROM base AS dependencies
+ENV POETRY_VERSION=1.8.5
+
+ENV TO_BE_REMOVED="/usr/local/share/to-be-removed.txt"
+# Enumerate files that should not be present in dependencies-slim
+# that way, the relevant folders can be COPY-ed without incurring the size cost of already existing items
+RUN <<EOF
+#!/usr/bin/env sh
+set -eu
 
-RUN pip install --upgrade pip && \
-    pip install poetry && \
+find /usr/local/bin -mindepth 1 > "$TO_BE_REMOVED"
+find "$SITE_PACKAGES" -maxdepth 1 -mindepth 1 >> "$TO_BE_REMOVED"
+
+EOF
+
+ENV PATH="/root/.local/bin:$PATH"
+RUN \
+    --mount=type=cache,target="$XDG_CACHE_HOME/pip" \
+    pip install --upgrade pip && \
+    pip install --user "poetry==$POETRY_VERSION" && \
     poetry config virtualenvs.create false
 
-COPY pyproject.toml pyproject.toml
-COPY poetry.lock poetry.lock
+RUN --mount=type=cache,target=/var/cache/apk \
+    apk add  \
+        linux-headers \
+        musl-dev \
+        gcc
+
+COPY --chown="$USER:$USER" \
+    pyproject.toml poetry.lock ./
+RUN --mount=type=cache,target="$XDG_CACHE_HOME/pip" \
+    poetry install --no-dev
+
+FROM dependencies AS dependencies-slim
+RUN <<EOF
+#!/usr/bin/env sh
+set -eu
+
+# Remove all files / directories that are (already) present in the final image
+cat "$TO_BE_REMOVED" | while IFS= read -r path; do
+    rm -rf "$path"
+done
+EOF
 
-FROM base AS development
-RUN poetry install
-WORKDIR /code/src
-COPY src .
+FROM dependencies AS development
+RUN --mount=type=cache,target="$XDG_CACHE_HOME/pip" \
+    poetry install --with dev
+COPY --chown="$USER:$USER" . .
 USER 1000
 
 FROM base AS prod
-RUN poetry install --without dev
-WORKDIR /code/src
-COPY src .
+COPY --from=dependencies-slim "$SITE_PACKAGES" "$SITE_PACKAGES"
+COPY --from=dependencies-slim /usr/local/bin/ /usr/local/bin/
+COPY --chown="$USER:$USER" src src
 USER 1000

api/poetry.lock

@@ -20,6 +20,7 @@ pydantic-extra-types = "^2.10"
 azure-monitor-opentelemetry = "^1.6.2"
 opentelemetry-instrumentation-fastapi = "^0.48b0"
 cryptography = "^44.0.0"
+click = "^8.1.8"
 
 [tool.poetry.group.dev.dependencies]
 pre-commit = ">=3"
@@ -65,7 +66,7 @@ strict = true
 
 [tool.ruff]
 
-src = ["src"]
+src = [".", "src"]
 target-version = "py311"
 line-length = 119
 
@@ -87,7 +88,7 @@ ignore = [
 
 [tool.ruff.lint.per-file-ignores]
 "__init__.py" = ["E402"]  # Ignore `E402` (import violations) in all `__init__.py` files
-"src/tests/*" = ["S101"]  # Allow the use of ´assert´ in tests
+"tests/*" = ["S101"]  # Allow the use of ´assert´ in tests
 
 [tool.codespell]
 skip = "*.lock,*.cjs"
@@ -100,6 +101,9 @@ markers = [
     "integration: mark a test as integration test."
 ]
 testpaths = [
-    "src/tests/unit",
-    "src/tests/integration"
+    "tests/unit",
+    "tests/integration"
+]
+pythonpath = [
+    "src"
 ]

api/pyproject.toml

@@ -1,8 +1,11 @@
-#!/bin/sh
+#!/usr/bin/env sh
 set -eu
 
+ROOT_DIR="$(cd "$(dirname -- "$0")" && pwd -P)"
+readonly ROOT_DIR
+
 if [ "$1" = 'api' ]; then
-  python3 ./app.py run
+  python3 "$ROOT_DIR/app.py" run
 else
   exec "$@"
 fi