From 9d6ef1104dc2cfda6f1811942731ba9dd88ff8f9 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 14 Dec 2022 15:10:20 -0600 Subject: [PATCH 1/4] add GHA workflow for ./pants test --- .github/workflows/test.yaml | 91 +++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 .github/workflows/test.yaml diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000000..beb713ca4b --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,91 @@ +--- +# This Lint workflow uses pants +name: Lint + +on: + push: + branches: + # only on merges to master branch + - master + # and version branches, which only include minor versions (eg: v3.4) + - v[0-9]+.[0-9]+ + tags: + # also version tags, which include bugfix releases (eg: v3.4.0) + - v[0-9]+.[0-9]+.[0-9]+ + pull_request: + type: [opened, reopened, edited] + branches: + # Only for PRs targeting those branches + - master + - v[0-9]+.[0-9]+ + #schedule: + # # run every night at midnight + # - cron: '0 0 * * *' + +jobs: + # Lint checks which don't depend on any service containes, etc. to be running. + lint-checks: + name: 'Tests: (pants runs: pytest)' + runs-on: ubuntu-20.04 + + env: + COLUMNS: '120' + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + # a test uses a submodule, and pants needs access to it to calculate deps. + submodules: 'true' + + #- name: Cache APT Dependencies + # id: cache-apt-deps + # uses: actions/cache@v2 + # with: + # path: | + # ~/apt_cache + # key: ${{ runner.os }}-apt-v7-${{ hashFiles('scripts/github/apt-packages.txt') }} + # restore-keys: | + # ${{ runner.os }}-apt-v7- + - name: Install APT Depedencies + env: + CACHE_HIT: 'false' # cache doesn't work + #CACHE_HIT: ${{steps.cache-apt-deps.outputs.cache-hit}} + run: | + # install dev dependencies for Python YAML and LDAP packages + # https://github.com/StackStorm/st2-auth-ldap + ./scripts/github/install-apt-packages-use-cache.sh + + - name: Initialize Pants and its GHA caches + uses: pantsbuild/actions/init-pants@e63d2d0e3c339bdffbe5e51e7c39550e3bc527bb + # This action adds an env var to make pants use both pants.ci.toml & pants.toml. + # This action also creates 3 GHA caches (1 is optional). + # - `pants-setup` has the bootsrapped pants install + # - `pants-named-caches` has pip/wheel and PEX caches + # - `pants-lmdb-store` has the fine-grained process cache. + # If we ever use a remote cache, then we can drop this. + # Otherwise, we may need an additional workflow or job to delete old caches + # if they are not expiring fast enough, and we hit the GHA 10GB per repo max. + with: + base-branch: master + # To ignore a bad cache, bump the cache* integer. + gha-cache-key: cache0 + # This hash should include all of our lockfiles so that the pip/pex caches + # get invalidated on any transitive dependency update. + named-caches-hash: ${{ hashFiles('requirements.txt') }} + # enable the optional lmdb_store cache since we're not using remote caching. + cache-lmdb-store: 'true' + + - name: Test + # We do not support running pytest everywhere yet. When we do it will be simply: + # ./pants lint :: + # Until then, we need to manually adjust this command line to test what we can. + run: | + ./pants test pylint_plugins/:: pants-plugins/:: + + - name: Upload pants log + uses: actions/upload-artifact@v2 + with: + name: pants-log-py${{ matrix.python-version }} + path: .pants.d/pants.log + if: always() # We want the log even on failures. From 794baf2af9f9ac4641a0aea4001f6704d0e15f42 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 14 Dec 2022 15:12:46 -0600 Subject: [PATCH 2/4] Update GHA action to pantsbuild/actions/init-pants@v2 --- .github/workflows/lint.yaml | 2 +- .github/workflows/pants.yaml | 2 +- .github/workflows/test.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 731857aa23..5e338f26ed 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -57,7 +57,7 @@ jobs: ./scripts/github/install-apt-packages-use-cache.sh - name: Initialize Pants and its GHA caches - uses: pantsbuild/actions/init-pants@e63d2d0e3c339bdffbe5e51e7c39550e3bc527bb + uses: pantsbuild/actions/init-pants@v2 # This action adds an env var to make pants use both pants.ci.toml & pants.toml. # This action also creates 3 GHA caches (1 is optional). # - `pants-setup` has the bootsrapped pants install diff --git a/.github/workflows/pants.yaml b/.github/workflows/pants.yaml index 50d68ba428..64fddfbc77 100644 --- a/.github/workflows/pants.yaml +++ b/.github/workflows/pants.yaml @@ -30,7 +30,7 @@ jobs: submodules: 'true' - name: Initialize Pants and its GHA caches - uses: pantsbuild/actions/init-pants@e63d2d0e3c339bdffbe5e51e7c39550e3bc527bb + uses: pantsbuild/actions/init-pants@v2 # This action adds an env var to make pants use both pants.ci.toml & pants.toml. # This action also creates 3 GHA caches (1 is optional). # - `pants-setup` has the bootsrapped pants install diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index beb713ca4b..7e7031ffc6 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -57,7 +57,7 @@ jobs: ./scripts/github/install-apt-packages-use-cache.sh - name: Initialize Pants and its GHA caches - uses: pantsbuild/actions/init-pants@e63d2d0e3c339bdffbe5e51e7c39550e3bc527bb + uses: pantsbuild/actions/init-pants@v2 # This action adds an env var to make pants use both pants.ci.toml & pants.toml. # This action also creates 3 GHA caches (1 is optional). # - `pants-setup` has the bootsrapped pants install From 2d15004598223b525068b3d5812a5469b17b2ed2 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 14 Dec 2022 15:20:47 -0600 Subject: [PATCH 3/4] add matrix of python versions to GHA test workflow --- .github/workflows/test.yaml | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7e7031ffc6..cc236afdb9 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,6 +1,6 @@ --- -# This Lint workflow uses pants -name: Lint +# This Test workflow uses pants +name: Test on: push: @@ -23,10 +23,21 @@ on: # - cron: '0 0 * * *' jobs: - # Lint checks which don't depend on any service containes, etc. to be running. - lint-checks: - name: 'Tests: (pants runs: pytest)' + test: + name: '${{ matrix.name }} - Python ${{ matrix.python-version-short }}' runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + # NOTE: We need to use full Python version as part of Python deps cache key otherwise + # setup virtualenv step will fail. + include: + - name: 'Test (pants runs: pytest)' + python-version-short: '3.6' + python-version: '3.6.13' + - name: 'Test (pants runs: pytest)' + python-version-short: '3.8' + python-version: '3.8.10' env: COLUMNS: '120' @@ -38,6 +49,12 @@ jobs: # a test uses a submodule, and pants needs access to it to calculate deps. submodules: 'true' + - name: 'Set up Python (${{ matrix.python-version }})' + uses: actions/setup-python@v2 + with: + python-version: '${{ matrix.python-version }}' + + #- name: Cache APT Dependencies # id: cache-apt-deps # uses: actions/cache@v2 @@ -69,7 +86,7 @@ jobs: with: base-branch: master # To ignore a bad cache, bump the cache* integer. - gha-cache-key: cache0 + gha-cache-key: cache0-py${{ matrix.python-version }} # This hash should include all of our lockfiles so that the pip/pex caches # get invalidated on any transitive dependency update. named-caches-hash: ${{ hashFiles('requirements.txt') }} @@ -78,7 +95,7 @@ jobs: - name: Test # We do not support running pytest everywhere yet. When we do it will be simply: - # ./pants lint :: + # ./pants test :: # Until then, we need to manually adjust this command line to test what we can. run: | ./pants test pylint_plugins/:: pants-plugins/:: From 0e6dc462f842b71a58d17a19a76d7286a2b0a8d3 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 14 Dec 2022 15:14:10 -0600 Subject: [PATCH 4/4] update changelog entry --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ea3d93bd04..e14c784128 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,7 +14,7 @@ Added working on StackStorm, improve our security posture, and improve CI reliability thanks in part to pants' use of PEX lockfiles. This is not a user-facing addition. #5778 #5789 #5817 #5795 #5830 #5833 #5834 #5841 #5840 #5838 #5842 #5837 #5849 #5850 - #5846 #5853 + #5846 #5853 #5848 Contributed by @cognifloyd * Added a joint index to solve the problem of slow mongo queries for scheduled executions. #5805