_python.yml

name: Reusable python linter workflow
on:
  workflow_call:
    inputs:
      # Base configs
      python_versions:
        description: Python versions to use (in the form of a JSON array)
        type: string
        required: true
      ubuntu_version:
        description: Ubuntu version to use
        type: string
        default: latest
        required: false
      working_directory:
        description: Directory that must be run against the linters
        type: string
        required: true
      requirements_path:
        description: Path to the requirements.txt file
        type: string
        required: true
      project_dev_requirements_file:
        description: Path to an additional project dev requirements file
        type: string
        required: false
      install_from:
        description: Directory that must be used to install the packages
        type: string
        required: false
        default: .
      packages_path:
        description: Path to the packages.txt file (APT requirements)
        type: string
        required: false
      env:
        description: Environment variables to set
        type: string
        required: false
        default: >-
          {}
      max_timeout:
        description: Max time that the CI can be run
        type: number
        required: false
        default: 30

      # Formatters
      use_black:
        description: Use black formatter
        type: boolean
        required: false
      use_isort:
        description: Use isort formatter
        type: boolean
        required: false
      use_ruff_formatter:
        description: Use ruff formatter
        type: boolean
        required: false

      # Linters
      use_autoflake:
        description: Use autoflake linter
        type: boolean
        required: false
      use_bandit:
        description: Use bandit linter
        type: boolean
        required: false
      use_flake8:
        description: Use flake8 linter
        type: boolean
        required: false
      use_pylint:
        description: Use pylint linter
        type: boolean
        required: false
      use_ruff_linter:
        description: Use ruff linter
        type: boolean
        required: false

      # Coverage configs
      use_coverage:
        description: Use coverage.py.
        type: boolean
        required: false
      coverage_config_path:
        description: Path to the coverage.py config file
        type: string
        required: false
        default: .coveragerc
      upload_coverage:
        description: Upload coverage.py report to github
        type: boolean
        required: false

      # CodeQL configs
      run_codeql:
        description: Run codeql
        type: boolean
        required: false
      
      # Services
      use_celery:
        description: Create a celery worker
        type: boolean
        required: false
      use_elastic_search:
        description: Use elastic_search service
        type: boolean
        required: false
      use_memcached:
        description: Use memcached service
        type: boolean
        required: false
      use_mongo:
        description: Use mongo service
        type: boolean
        required: false
      use_postgres:
        description: Use postgres service
        type: boolean
        required: false
      use_rabbitmq:
        description: Use rabbitmq service
        type: boolean
        required: false
      use_redis:
        description: Use redis service
        type: boolean
        required: false

      # Services configs
      ## Celery service configs
      celery_app:
        description: Celery app name. Requires use_celery to be true
        type: string
        required: false
      celery_queues:
        description: Celery queues separated by ,. Requires use_celery to be true
        type: string
        required: false
        default: default
      ## Elasticsearch service configs
      elasticsearch_version:
        description: Elasticsearch container version
        type: string
        required: false
        default: latest
      elasticsearch_port:
        description: Elasticsearch container port
        type: string
        required: false
        default: 9200
      ## Memcached service configs
      memcached_version:
        description: Memcached alpine container version
        type: string
        required: false
        default: latest
      ## Mongo service configs
      mongo_version:
        description: Mongo container version
        type: string
        required: false
        default: latest
      ## Postgres service configs
      postgres_db:
        description: Postgres service db. Requires use_postgres to be true
        type: string
        required: false
        default: db
      postgres_user:
        description: Postgres service user. Requires use_postgres to be true
        type: string
        required: false
        default: user
      postgres_password:
        description: Postgres service password. Requires use_postgres to be true
        type: string
        required: false
        default: password
      postgres_version:
        description: Postgres service version. Requires use_postgres to be true
        type: string
        required: false
        default: latest
      ## RabbitMQ service configs
      rabbitmq_version:
        description: RabbitMQ management-alpine container version
        type: string
        required: false
        default: latest
      ## Redis service configs
      redis_version:
        description: Redis alpine container version
        type: string
        required: false
        default: latest
      

      # Django configs
      django_settings_module:
        description: Path to the django settings file
        type: string
        required: false
      check_migrations:
        description: Check if migrations are valid. Require django_settings_module to be set.
        type: boolean
        required: false
      check_requirements_licenses:
        description: Check if requirements have a valid license. Require django_settings_module to be set.
        type: boolean
        required: false
        default: true
      ignore_requirements_licenses_regex:
        description: Regex of repositories of which ignore license
        type: string
        required: false
        default: uWSGI.*|lunardate.*|.*QuokkaClient.*|pyquokka.*
      tags_for_slow_tests:
        description: Tags for tests that will be run only on master/main branch, space separated. Can be used only for django projects.
        default: slow
        type: string
        required: false
      tags_for_manual_tests:
        description: Tags for tests that will not be run on the CI, space separated. Can be used only for django projects.
        default: manual
        type: string
        required: false

      # Misc configs
      custom_command:
        description: String of custom command to run
        type: string
        required: false

      check_docs_directory:
        description: Check docs using rstcheck inside this directory
        type: string
        required: false
      check_dockerfile:
        description: Check dockerfile build. WARNING action total time may increase significantly
        type: string
        required: false

jobs:
  python:
    name: Run python test suite
    runs-on: ubuntu-${{ inputs.ubuntu_version }}
    timeout-minutes: ${{ inputs.max_timeout }}
    strategy:
      matrix:
        python_version: ${{ fromJson(inputs.python_versions) }}
        language: ['python']
    env: ${{ fromJson(inputs.env) }}
    steps:
      - name: Check out latest commit
        uses: actions/checkout@v4

      - name: Set up Python
        id: setup_python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python_version }}

      - name: Inject stuff to environment
        run: |
          if [[ '${{ inputs.django_settings_module}}' != 'false' ]]; then
            echo "PYTHONPATH=${{ github.workspace }}/${{ inputs.working_directory }}" >> $GITHUB_ENV
            echo "DJANGO_SETTINGS_MODULE=${{ inputs.django_settings_module }}" >> $GITHUB_ENV
          fi
          if [[ '${{ inputs.run_codeql}}' != 'false' ]]; then
            echo "CODEQL_PYTHON=$(which python)" >> $GITHUB_ENV
          fi
        shell:
          bash

      - name: Restore APT cache related to PR event
        id: restore_apt_cache_pr
        uses: ./.github/actions/apt_requirements/restore_apt_cache
        with:
          apt_requirements_file_path: ${{ inputs.packages_path }}
      
      - name: Restore APT cache related to target branch
        id: restore_apt_cache_target_branch
        if: steps.restore_apt_cache_pr.outputs.cache-hit != 'true'
        uses: ./.github/actions/apt_requirements/restore_apt_cache
        with:
          apt_requirements_file_path: ${{ inputs.packages_path }}
          git_reference: ${{ github.base_ref }}
      
      - name: Refresh APT repositories
        if: >
          steps.restore_apt_cache_pr.outputs.cache-hit != 'true' &&
          steps.restore_apt_cache_target_branch.outputs.cache-hit != 'true'
        run: |
          sudo apt-get update
        shell: bash

      - name: Install APT requirements
        run: |
          sudo apt-get install -y --no-install-recommends $(tr '\n' ' ' < ${{ inputs.packages_path }})
        shell: bash
      
      - name: Save APT cache related to PR event
        if: >
          steps.restore_apt_cache_pr.outputs.cache-hit != 'true'
        uses: ./.github/actions/apt_requirements/save_apt_cache
        with:
          apt_requirements_file_path: ${{ inputs.packages_path }}

      - name: Create linter requirements file
        uses: ./.github/actions/python_requirements/create_linter_requirements_file
        with:
          install_from: ${{ inputs.install_from }}
          django_settings_module: ${{ inputs.django_settings_module }}
          use_autoflake: ${{ inputs.use_autoflake }}
          use_bandit: ${{ inputs.use_bandit }}
          use_black: ${{ inputs.use_black }}
          use_flake8: ${{ inputs.use_flake8 }}
          use_isort: ${{ inputs.use_isort }}
          use_pylint: ${{ inputs.use_pylint }}
          use_ruff_formatter: ${{ inputs.use_ruff_formatter }}
          use_ruff_linter: ${{ inputs.use_ruff_linter }}

      - name: Create dev requirements file
        uses: ./.github/actions/python_requirements/create_dev_requirements_file
        with:
          install_from: ${{ inputs.install_from }}
          project_dev_requirements_file: ${{ inputs.project_dev_requirements_file }}

      - name: Create docs requirements file
        uses: ./.github/actions/python_requirements/create_docs_requirements_file
        with:
          install_from: ${{ inputs.install_from }}
          check_docs_directory: ${{ inputs.check_docs_directory }}
          django_settings_module: ${{ inputs.django_settings_module }}

      - name: Restore Python virtual environment related to PR event
        id: restore_python_virtual_environment_pr
        uses: ./.github/actions/python_requirements/restore_virtualenv/
        with:
          requirements_paths: "${{ inputs.requirements_path }} requirements-linters.txt requirements-dev.txt requirements-docs.txt"
          python_version: ${{ steps.setup_python.outputs.python-version }}

      - name: Restore Python virtual environment related to target branch
        id: restore_python_virtual_environment_target_branch
        if: steps.restore_python_virtual_environment_pr.outputs.cache-hit != 'true'
        uses: ./.github/actions/python_requirements/restore_virtualenv/
        with:
          requirements_paths: ${{ inputs.requirements_path }}
          git_reference: ${{ github.base_ref }}
          python_version: ${{ steps.setup_python.outputs.python-version }}

      - name: Create Python virtual environment
        if: >
         steps.restore_python_virtual_environment_pr.outputs.cache-hit != 'true' && 
         steps.restore_python_virtual_environment_target_branch.outputs.cache-hit != 'true'
        uses: ./.github/actions/python_requirements/create_virtualenv

      - name: Restore pip cache related to PR event
        id: restore_pip_cache_pr
        if: > 
          steps.restore_python_virtual_environment_pr.outputs.cache-hit != 'true' && 
          steps.restore_python_virtual_environment_target_branch.outputs.cache-hit != 'true'
        uses: ./.github/actions/python_requirements/restore_pip_cache

      - name: Restore pip cache related to target branch
        id: restore_pip_cache_target_branch
        if: >
          steps.restore_python_virtual_environment_pr.outputs.cache-hit != 'true' &&
          steps.restore_python_virtual_environment_target_branch.outputs.cache-hit != 'true' &&
          steps.restore_pip_cache_pr.outputs.cache-hit != 'true'
        uses: ./.github/actions/python_requirements/restore_pip_cache
        with:
          git_reference: ${{ github.base_ref }}
      
      - name: Install project requirements
        if: >
          steps.restore_python_virtual_environment_pr.outputs.cache-hit != 'true' &&
          steps.restore_python_virtual_environment_target_branch.outputs.cache-hit != 'true'
        run: pip install -r ${{ inputs.requirements_path }}
        shell: bash
        working-directory: ${{ inputs.install_from }}

      - name: Install other requirements
        if: >
          steps.restore_python_virtual_environment_pr.outputs.cache-hit != 'true'
        run: |
          pip install -r requirements-dev.txt
          pip install -r requirements-linters.txt
          pip install -r requirements-docs.txt
        shell: bash
        working-directory: ${{ inputs.install_from }}

      - name: Check requirements licenses
        if: >
            inputs.check_requirements_licenses && 
            steps.restore_python_virtual_environment_pr.outputs.cache-hit != 'true'
        id: license_check_report
        continue-on-error: true
        uses: pilosus/action-pip-license-checker@v2
        with:
          requirements: ${{ inputs.install_from }}/${{ inputs.requirements_path }}
          exclude: ${{ inputs.ignore_requirements_licenses_regex }}
          headers: true
          fail: 'StrongCopyleft,NetworkCopyleft,Error'
          fails-only: true

      - name: Print wrong licenses
        if: steps.license_check_report.outcome == 'failure'
        run: |
          echo "License check failed"
          echo "===================="
          echo "${{ steps.license_check_report.outputs.report }}"
          echo "===================="
          exit 1
        shell: bash

      - name: Save Python virtual environment related to PR event
        if: >
          steps.restore_python_virtual_environment_pr.outputs.cache-hit != 'true'
        uses: ./.github/actions/python_requirements/save_virtualenv
        with:
          requirements_paths: "${{ inputs.requirements_path }} requirements-linters.txt requirements-dev.txt requirements-docs.txt"
          python_version: ${{ steps.setup_python.outputs.python-version }}

      - name: Save pip cache related to PR event
        if: >
          steps.restore_python_virtual_environment_pr.outputs.cache-hit != 'true' &&
          steps.restore_pip_cache_pr.outputs.cache-hit != 'true'
        uses: ./.github/actions/python_requirements/save_pip_cache

      - name: Run linters
        uses: ./.github/actions/python_linter
        if: > 
          inputs.use_black ||
          inputs.use_isort ||
          inputs.use_flake8 ||
          inputs.use_pylint ||
          inputs.use_bandit ||
          inputs.use_autoflake ||
          inputs.use_ruff_formatter ||
          inputs.use_ruff_linter
        with:
          working_directory: ${{ inputs.working_directory }}
          use_black: ${{ inputs.use_black }}
          use_isort: ${{ inputs.use_isort }}
          use_flake8: ${{ inputs.use_flake8 }}
          use_pylint: ${{ inputs.use_pylint }}
          use_bandit: ${{ inputs.use_bandit }}
          use_autoflake: ${{ inputs.use_autoflake }}
          use_ruff_formatter: ${{ inputs.use_ruff_formatter }}
          use_ruff_linter: ${{ inputs.use_ruff_linter }}

      - name: Run CodeQL
        if: inputs.run_codeql
        uses: ./.github/actions/codeql
        with:
          language: python
          working_directory: ${{ inputs.working_directory }}

      - name: Build Docs
        if: inputs.check_docs_directory
        run: |
          rstcheck -r  --report-level error --ignore-directives automodule,autoclass .
          python3 -m sphinx -a -T -E -b html -d  _build/doctrees -D language=en . _build/html
        shell: bash
        working-directory: ${{ inputs.check_docs_directory }}

      - name: Start services
        uses: ./.github/actions/services
        if: inputs.use_postgres || inputs.use_elastic_search || inputs.use_memcached ||  inputs.use_redis || inputs.use_rabbitmq || inputs.use_mongo
        with:
          use_mongo: ${{ inputs.use_mongo }}
          use_redis: ${{ inputs.use_redis }}
          use_postgres: ${{ inputs.use_postgres }}
          use_elastic_search: ${{ inputs.use_elastic_search }}
          use_memcached: ${{ inputs.use_memcached }}
          use_rabbitmq: ${{ inputs.use_rabbitmq }}
          postgres_db: ${{ inputs.postgres_db }}
          postgres_user: ${{ inputs.postgres_user }}
          postgres_password: ${{ inputs.postgres_password }}
          postgres_version: ${{ inputs.postgres_version }}
          mongo_version: ${{ inputs.mongo_version }}
          elasticsearch_version:  ${{ inputs.elasticsearch_version }}
          elasticsearch_port: ${{ inputs.elasticsearch_port }}
          memcached_version:  ${{ inputs.memcached_version }}
          redis_version:  ${{ inputs.redis_version }}
          rabbitmq_version:  ${{ inputs.rabbitmq_version }}

      - name: Start celery worker
        if: inputs.use_celery
        run: |
          celery -A ${{ inputs.celery_app }} worker -E -c 1 -n worker -Q ${{ inputs.celery_queues }} --detach
        working-directory: ${{ inputs.working_directory }}

      - name: Run custom command
        if: inputs.custom_command
        run: |
          COMMAND='${{ inputs.custom_command }}'
          echo "Running command: $COMMAND"
          eval $COMMAND
        working-directory: ${{ inputs.working_directory }}
        shell: bash

      - name: Check migrations
        if: inputs.check_migrations && inputs.django_settings_module
        run: |
          django-admin makemigrations --dry-run >/dev/null
        working-directory: ${{ inputs.working_directory }}
        shell: bash

      - name: Run unittest
        run: |
          CMD="python"
          if [[ '${{ inputs.use_coverage }}' != 'false' ]]; then
              CMD="coverage run --rcfile=${{ inputs.coverage_config_path }}"
          
              if [[ -n '${{ inputs.django_settings_module }}' ]]; then
                CMD="${CMD} -a manage.py test"
                # manage manual tags
                if [[ -n '${{ inputs.tags_for_manual_tests }}' ]]; then
                  for tag in ${{ inputs.tags_for_manual_tests }}; do
                    CMD="${CMD} --exclude-tag=$tag"
                  done
                fi
                # manage slow tags
                if [[ '${{ github.base_ref }}' != 'master' && '${{ github.base_ref }}' != 'main' ]]; then
                  if [[ -n '${{ inputs.tags_for_slow_tests }}' ]]; then
                    for tag in ${{ inputs.tags_for_slow_tests }}; do
                      CMD="${CMD} --exclude-tag=$tag"
                    done
                  fi
                fi
          
              else
                CMD="${CMD} -m unittest discover"
              fi

          else
            CMD="python -m unittest discover"
          fi
          CMD="${CMD} --failfast"
          echo "Running command: ${CMD}"
          $CMD
        working-directory: ${{ inputs.working_directory }}
        env: ${{ secrets }}
        shell: bash


      - name: Create coverage output
        if: inputs.use_coverage && inputs.upload_coverage
        id: coverage-output
        run: |
          echo "Uploading coverage.py report"
          echo "## Coverage.py report" >> $GITHUB_STEP_SUMMARY
          echo "$(coverage report -m --format=markdown)" >> $GITHUB_STEP_SUMMARY
        working-directory: ${{ inputs.working_directory }}

      - name: Generate coverage XML
        if: inputs.use_coverage && inputs.upload_coverage
        run: |
          coverage xml
        working-directory: ${{ inputs.working_directory }}

      - name: Upload coverage report as artifact
        if: inputs.use_coverage && inputs.upload_coverage
        uses: actions/upload-artifact@v4
        with:
          name: coverage-report-py${{ matrix.python_version }}
          path: ${{ inputs.working_directory }}/coverage.xml
          retention-days: 30