Apim 6679 python update

.github/SBOM-README.md

@@ -0,0 +1,58 @@
+# SBOM & Vulnerability Scanning Automation
+
+This repository uses GitHub Actions to automatically generate a Software Bill of Materials (SBOM), scan for vulnerabilities, and produce package inventory reports.
+
+All reports are named with the repository name for easy identification.
+
+## Features
+
+SBOM Generation: Uses Syft to generate an SPDX JSON SBOM.
+SBOM Merging: Merges SBOMs for multiple tools if needed.
+SBOM to CSV: Converts SBOM JSON to a CSV report.
+Vulnerability Scanning: Uses Grype to scan the SBOM for vulnerabilities and outputs a CSV report.
+Package Inventory: Extracts a simple package list (name, type, version) as a CSV.
+Artifacts: All reports are uploaded as workflow artifacts with the repository name in the filename.
+
+## Workflow Overview
+
+The main workflow is defined in .github/workflows/sbom.yml
+
+## Scripts
+
+scripts/create-sbom.sh
+Generates an SBOM for the repo and for specified tools, merging them as needed.
+scripts/update-sbom.py
+Merges additional SBOMs into the main SBOM.
+.github/scripts/sbom_json_to_csv.py
+Converts the SBOM JSON to a detailed CSV report.
+.github/scripts/grype_json_to_csv.py
+Converts Grype’s vulnerability scan JSON output to a CSV report.
+Output columns: REPO, NAME, INSTALLED, FIXED-IN, TYPE, VULNERABILITY, SEVERITY
+.github/scripts/sbom_packages_to_csv.py
+Extracts a simple package inventory from the SBOM.
+Output columns: name, type, version
+
+## Example Reports
+
+Vulnerability Report
+grype-report-[RepoName].csv
+REPO,NAME,INSTALLED,FIXED-IN,TYPE,VULNERABILITY,SEVERITY
+my-repo,Flask,2.1.2,,library,CVE-2022-12345,High
+...
+
+Package Inventory
+sbom-packages-[RepoName].csv
+name,type,version
+Flask,library,2.1.2
+Jinja2,library,3.1.2
+...
+
+## Usage
+
+Push to main branch or run the workflow manually.
+Download artifacts from the workflow run summary.
+
+## Customization
+
+Add more tools to scripts/create-sbom.sh as needed.
+Modify scripts to adjust report formats or add more metadata.

.github/scripts/grype_json_to_csv.py

@@ -0,0 +1,28 @@
+import json
+import csv
+import sys
+
+input_file = sys.argv[1] if len(sys.argv) > 1 else "grype-report.json"
+output_file = sys.argv[2] if len(sys.argv) > 2 else "grype-report.csv"
+
+with open(input_file, "r", encoding="utf-8") as f:
+    data = json.load(f)
+
+columns = ["NAME", "INSTALLED", "FIXED-IN", "TYPE", "VULNERABILITY", "SEVERITY"]
+
+with open(output_file, "w", newline="", encoding="utf-8") as csvfile:
+    writer = csv.DictWriter(csvfile, fieldnames=columns)
+    writer.writeheader()
+    for match in data.get("matches", []):
+        pkg = match.get("artifact", {})
+        vuln = match.get("vulnerability", {})
+        row = {
+            "NAME": pkg.get("name", ""),
+            "INSTALLED": pkg.get("version", ""),
+            "FIXED-IN": vuln.get("fix", {}).get("versions", [""])[0] if vuln.get("fix", {}).get("versions") else "",
+            "TYPE": pkg.get("type", ""),
+            "VULNERABILITY": vuln.get("id", ""),
+            "SEVERITY": vuln.get("severity", ""),
+        }
+        writer.writerow(row)
+print(f"CSV export complete: {output_file}")

.github/scripts/sbom_json_to_csv.py

@@ -0,0 +1,78 @@
+import json
+import csv
+import sys
+# from pathlib import Path
+from tabulate import tabulate
+
+input_file = sys.argv[1] if len(sys.argv) > 1 else "sbom.json"
+output_file = sys.argv[2] if len(sys.argv) > 2 else "sbom.csv"
+
+with open(input_file, "r", encoding="utf-8") as f:
+    sbom = json.load(f)
+
+packages = sbom.get("packages", [])
+
+columns = [
+    "name",
+    "versionInfo",
+    "type",
+    "supplier",
+    "downloadLocation",
+    "licenseConcluded",
+    "licenseDeclared",
+    "externalRefs"
+]
+
+
+def get_type(pkg):
+    spdxid = pkg.get("SPDXID", "")
+    if "-" in spdxid:
+        parts = spdxid.split("-")
+        if len(parts) > 2:
+            return parts[2]
+    refs = pkg.get("externalRefs", [])
+    for ref in refs:
+        if ref.get("referenceType") == "purl":
+            return ref.get("referenceLocator", "").split("/")[0]
+    return ""
+
+
+def get_external_refs(pkg):
+    refs = pkg.get("externalRefs", [])
+    return ";".join([ref.get("referenceLocator", "") for ref in refs])
+
+
+with open(output_file, "w", newline="", encoding="utf-8") as csvfile:
+    writer = csv.DictWriter(csvfile, fieldnames=columns)
+    writer.writeheader()
+    for pkg in packages:
+        row = {
+            "name": pkg.get("name", ""),
+            "versionInfo": pkg.get("versionInfo", ""),
+            "type": get_type(pkg),
+            "supplier": pkg.get("supplier", ""),
+            "downloadLocation": pkg.get("downloadLocation", ""),
+            "licenseConcluded": pkg.get("licenseConcluded", ""),
+            "licenseDeclared": pkg.get("licenseDeclared", ""),
+            "externalRefs": get_external_refs(pkg)
+        }
+        writer.writerow(row)
+
+print(f"CSV export complete: {output_file}")
+
+
+with open("sbom_table.txt", "w", encoding="utf-8") as f:
+    table = []
+    for pkg in packages:
+        row = [
+            pkg.get("name", ""),
+            pkg.get("versionInfo", ""),
+            get_type(pkg),
+            pkg.get("supplier", ""),
+            pkg.get("downloadLocation", ""),
+            pkg.get("licenseConcluded", ""),
+            pkg.get("licenseDeclared", ""),
+            get_external_refs(pkg)
+        ]
+        table.append(row)
+    f.write(tabulate(table, columns, tablefmt="grid"))

.github/scripts/sbom_packages_to_csv.py

@@ -0,0 +1,28 @@
+import json
+import csv
+import sys
+import os
+
+input_file = sys.argv[1] if len(sys.argv) > 1 else "sbom.json"
+repo_name = sys.argv[2] if len(sys.argv) > 2 else os.getenv("GITHUB_REPOSITORY", "unknown-repo").split("/")[-1]
+output_file = f"sbom-packages-{repo_name}.csv"
+
+with open(input_file, "r", encoding="utf-8") as f:
+    sbom = json.load(f)
+
+packages = sbom.get("packages", [])
+
+columns = ["name", "type", "version"]
+
+with open(output_file, "w", newline="", encoding="utf-8") as csvfile:
+    writer = csv.DictWriter(csvfile, fieldnames=columns)
+    writer.writeheader()
+    for pkg in packages:
+        row = {
+            "name": pkg.get("name", ""),
+            "type": pkg.get("type", ""),
+            "version": pkg.get("versionInfo", "")
+        }
+        writer.writerow(row)
+
+print(f"Package list CSV generated: {output_file}")

.github/workflows/publish.yml

@@ -12,10 +12,10 @@ jobs:
               with:
                   fetch-depth: 0  # This causes all history to be fetched, which is required for calculate-version to function
 
-            - name: Install Python 3.8
+            - name: Install Python 3.13
               uses: actions/setup-python@v5
               with:
-                  python-version: 3.8
+                  python-version: 3.13
 
             - name: Install poetry
               run: pip install poetry

.github/workflows/sbom.yml

@@ -0,0 +1,110 @@
+name: SBOM Vulnerability Scanning
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: "Run SBOM check"
+        required: true
+        type: choice
+        options:
+          - yes
+          - no
+
+env:
+  SYFT_VERSION: "1.27.1"
+  TF_VERSION: "1.12.2"
+
+jobs:
+  deploy:
+    name: Software Bill of Materials
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5        
+
+      - name: Setup Python 3.13
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd
+
+      - uses: terraform-linters/setup-tflint@ae78205cfffec9e8d93fd2b3115c7e9d3166d4b6
+        name: Setup TFLint
+
+      - name: Set architecture variable
+        id: os-arch
+        run: |
+          case "${{ runner.arch }}" in
+            X64) ARCH="amd64" ;;
+            ARM64) ARCH="arm64" ;;
+          esac
+          echo "arch=${ARCH}" >> $GITHUB_OUTPUT
+
+      - name: Download and setup Syft
+        run: |
+          DOWNLOAD_URL="https://github.com/anchore/syft/releases/download/v${{ env.SYFT_VERSION }}/syft_${{ env.SYFT_VERSION }}_linux_${{ steps.os-arch.outputs.arch }}.tar.gz"
+          echo "Downloading: ${DOWNLOAD_URL}"
+
+          curl -L -o syft.tar.gz "${DOWNLOAD_URL}"
+          tar -xzf syft.tar.gz
+          chmod +x syft
+
+          # Add to PATH for subsequent steps
+          echo "$(pwd)" >> $GITHUB_PATH       
+
+      - name: Create SBOM
+        run: bash scripts/create-sbom.sh terraform python tflint
+
+      - name: Convert SBOM JSON to CSV
+        run: |
+          pip install --upgrade pip
+          pip install tabulate  
+          REPO_NAME=$(basename $GITHUB_REPOSITORY)
+          python .github/scripts/sbom_json_to_csv.py sbom.json SBOM_${REPO_NAME}.csv
+
+      - name: Upload SBOM CSV as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: sbom-csv
+          path: SBOM_${{ github.event.repository.name }}.csv
+
+      - name: Install Grype
+        run: |
+          curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin
+
+      - name: Scan SBOM for Vulnerabilities (JSON)
+        run: |
+          grype sbom:sbom.json -o json > grype-report.json
+
+
+
+      - name: Convert Grype JSON to CSV
+        run: |
+          pip install --upgrade pip
+          REPO_NAME=$(basename $GITHUB_REPOSITORY)
+          python .github/scripts/grype_json_to_csv.py grype-report.json grype-report-${REPO_NAME}.csv
+
+
+      - name: Upload Vulnerability Report
+        uses: actions/upload-artifact@v4
+        with:
+          name: grype-report
+          path: grype-report-${{ github.event.repository.name }}.csv
+
+      - name: Generate Package Inventory CSV
+        run: |
+          pip install --upgrade pip
+          REPO_NAME=$(basename $GITHUB_REPOSITORY)
+          python .github/scripts/sbom_packages_to_csv.py sbom.json $REPO_NAME
+
+      - name: Upload Package Inventory CSV
+        uses: actions/upload-artifact@v4
+        with:
+          name: sbom-packages
+          path: sbom-packages-${{ github.event.repository.name }}.csv

README.md

@@ -1,6 +1,25 @@
 # api-management-utils
 Scripts and utilities used across API managment platform and services 
 
+
+################################################################################
+###########               Python upgrade to 3.13                     ###########
+###########       Utils Repo has been updated to python 3.13         ###########
+################################################################################
+
+########### We are continuing to support python 3.8/9(which are currently out of support) until January 26th 2026 ###########
+###########          After the deadline your pipelines will fail if you are using python version 3.8/9            ###########
+
+Python upgrade related changes
+###############################
+Projects using Python versions older than 3.13 and extending their pipeline with the utils repository must update their pipelines to ensure compatibility with the latest changes.
+For detailed guidance, please refer to the APIM FAQ page:
+https://nhsd-confluence.digital.nhs.uk/spaces/APM/pages/1226682275/Pipeline+Queries
+
+Note: Projects running Python version 3.13 or later do not need any pipeline modifications.
+
+
+
 ## Scripts
 * `template.py` - cli for basic jinja templating
 * `test_pull_request_deployments.py` - cli for testing utils against other repositories

ansible/collections/ansible_collections/nhsd/apigee/Makefile

@@ -4,10 +4,10 @@ update-schema:
 	@PYTHONPATH=../../../ poetry run python scripts/update_schema.py
 
 unit-test:
-	@poetry run ansible-test units --python=3.8
+	@poetry run ansible-test units --python=3.13
 
 integration-test:
-	@poetry run ansible-test integration --python=3.8
+	@poetry run ansible-test integration --python=3.13
 
 test: unit-test integration-test
 

ansible/collections/ansible_collections/nhsd/apigee/plugins/module_utils/utils.py

@@ -5,6 +5,7 @@
 import typing
 
 from ansible_collections.nhsd.apigee.plugins.module_utils import constants
+from ansible.utils.unsafe_proxy import AnsibleUnsafeText
 
 
 def exclude_keys(dict_, keys_to_ignore):
@@ -19,7 +20,7 @@ def delta(before, after, keys_to_ignore=None):
             before,
             after,
             ignore_order=True,
-            ignore_type_in_groups=[(ansible.utils.unsafe_proxy.AnsibleUnsafeText,str)],
+            ignore_type_in_groups=[(AnsibleUnsafeText,str)],
             exclude_paths=[f"root['{key}']" for key in keys_to_ignore],
         ).to_json()
     )

ansible/collections/ansible_collections/nhsd/apigee/roles/deploy_manifest/tasks/main.yml

@@ -54,7 +54,7 @@
   # since products can only be created if they reference proxies
   # that exist.  This safety check can be removed when the
   # manifest manages proxies too.
-  when: item.name | regex_search('^' + SERVICE_NAME + '-' + PULL_REQUEST | default(APIGEE_ENVIRONMENT))
+  when: (item.name is match('^' ~ SERVICE_NAME ~ '-' ~ (PULL_REQUEST | default(APIGEE_ENVIRONMENT))))
 
 # - name: deploy apigee specs
 #   nhsd.apigee.deploy_spec:

ansible/collections/ansible_collections/nhsd/apigee/tests/integration/targets/test_validate_manifest/tasks/main.yml

@@ -17,7 +17,6 @@
 - name: validate test manifest files
   include_role:
     name: validate_manifest
-    vars_from: "{{ DIST_DIR }}/vars.yml"
   loop: "{{ DIST_DIRS }}"
   loop_control:
     loop_var: DIST_DIR