benchmark.yml

name: Memory Tracker Benchmark

on:
  workflow_dispatch:
    inputs:
      commit_range:
        description: 'Commit range to benchmark (e.g., HEAD~10..HEAD)'
        required: true
        default: 'HEAD~5..HEAD'
      binary_id:
        description: 'Binary ID to use for benchmarking'
        required: true
        default: 'default'
      environment_id:
        description: 'Environment ID'
        required: true
        default: 'linux-x86_64'
      server_url:
        description: 'Memory tracker server URL'
        required: false
        default: 'https://memory.python.org'
      cpython_repo:
        description: 'CPython repository URL'
        required: false
        default: 'https://github.com/python/cpython.git'
      configure_flags:
        description: 'Configure flags for CPython build'
        required: false
        default: '-C'
      make_flags:
        description: 'Make flags for CPython build'
        required: false
        default: '-j'
      llvm:
        description: 'LLVM version to use'
        required: false
        default: '19'

permissions:
  contents: read

jobs:
  benchmark:
    runs-on: ubuntu-latest
    
    steps:
    - name: Checkout memory tracker
      uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
      with:
        persist-credentials: false
      
    - name: Set up Python
      uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
      with:
        python-version: '3.11'
        
    - name: Clone CPython repository
      run: |
        git clone ${GITHUB_EVENT_INPUTS_CPYTHON_REPO} cpython
        cd cpython
        git fetch --depth=200
      env:
        GITHUB_EVENT_INPUTS_CPYTHON_REPO: ${{ github.event.inputs.cpython_repo }}
        
    - name: Install memory tracker worker
      run: |
        cd worker
        pip install -e .
        
    - name: Install build dependencies
      run: |
        # Install CPython dependencies using their script
        cd cpython
        sudo .github/workflows/posix-deps-apt.sh

        # Install JIT dependencies
        if [ "${INPUTS_BINARY_ID}" = "jit" ]; then
          sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${INPUTS_LLVM}
        fi

        # Install Memray dependencies
        sudo apt-get install -y \
          python3-dev \
          libdebuginfod-dev \
          libunwind-dev \
          liblz4-dev
      env:
        INPUTS_BINARY_ID: ${{ inputs.binary_id }}
        INPUTS_LLVM: ${{ inputs.llvm }}
          
    - name: Run memory benchmarks
      env:
        MEMORY_TRACKER_TOKEN: ${{ secrets.MEMORY_TRACKER_TOKEN }}
        INPUTS_BINARY_ID: ${{ inputs.binary_id }}
        INPUTS_LLVM: ${{ inputs.llvm }}
        GITHUB_EVENT_INPUTS_COMMIT_RANGE: ${{ github.event.inputs.commit_range }}
        GITHUB_EVENT_INPUTS_BINARY_ID: ${{ github.event.inputs.binary_id }}
        GITHUB_EVENT_INPUTS_ENVIRONMENT_ID: ${{ github.event.inputs.environment_id }}
        GITHUB_EVENT_INPUTS_SERVER_URL: ${{ github.event.inputs.server_url }}
        GITHUB_EVENT_INPUTS_CONFIGURE_FLAGS: ${{ github.event.inputs.configure_flags }}
        GITHUB_EVENT_INPUTS_MAKE_FLAGS: ${{ github.event.inputs.make_flags }}
      run: |
        if [ "${INPUTS_BINARY_ID}" = "jit" ]; then
          export PATH="$(llvm-config-${INPUTS_LLVM} --bindir):$PATH"
        fi

        # Build command with conditional flags
        CMD="memory-tracker benchmark '${GITHUB_EVENT_INPUTS_COMMIT_RANGE}'"
        CMD="$CMD --repo-path ./cpython"
        CMD="$CMD --binary-id '${GITHUB_EVENT_INPUTS_BINARY_ID}'"
        CMD="$CMD --environment-id '${GITHUB_EVENT_INPUTS_ENVIRONMENT_ID}'"
        CMD="$CMD --api-base '${GITHUB_EVENT_INPUTS_SERVER_URL}'"
        CMD="$CMD --output-dir ./benchmark_results"
        CMD="$CMD --force"
        CMD="$CMD -vv"
        
        # Add configure flags if provided
        if [ -n "${GITHUB_EVENT_INPUTS_CONFIGURE_FLAGS}" ]; then
          CMD="$CMD --configure-flags='${GITHUB_EVENT_INPUTS_CONFIGURE_FLAGS}'"
        fi
        
        # Add make flags if provided
        if [ -n "${GITHUB_EVENT_INPUTS_MAKE_FLAGS}" ]; then
          CMD="$CMD --make-flags='${GITHUB_EVENT_INPUTS_MAKE_FLAGS}'"
        fi
        
        echo "Running: $CMD"
        eval $CMD
          
    - name: Upload benchmark results (if failed)
      if: failure()
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
      with:
        name: benchmark-logs
        path: |
          *.log
          ./benchmark_results/
        retention-days: 7
        
    - name: Upload benchmark results (on success)
      if: success()
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
      with:
        name: benchmark-results
        path: ./benchmark_results/
        retention-days: 30