Tell pants about tests that use globs to test with many pack fixtures

CHANGELOG.rst

@@ -14,7 +14,7 @@ Added
   working on StackStorm, improve our security posture, and improve CI reliability thanks in part
   to pants' use of PEX lockfiles. This is not a user-facing addition.
   #5778 #5789 #5817 #5795 #5830 #5833 #5834 #5841 #5840 #5838 #5842 #5837 #5849 #5850
-  #5846 #5853 #5848 #5847 #5858 #5857 #5860 #5868 #5871 #5864
+  #5846 #5853 #5848 #5847 #5858 #5857 #5860 #5868 #5871 #5864 #5874
   Contributed by @cognifloyd
 
 * Added a joint index to solve the problem of slow mongo queries for scheduled executions. #5805

pants-plugins/pack_metadata/register.py

@@ -11,13 +11,24 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from pack_metadata import tailor
-from pack_metadata.target_types import PackMetadata, PackMetadataInGitSubmodule
+from pack_metadata import tailor, target_types_rules
+from pack_metadata.target_types import (
+    PackMetadata,
+    PackMetadataInGitSubmodule,
+    PacksGlob,
+)
 
 
 def rules():
-    return tailor.rules()
+    return [
+        *tailor.rules(),
+        *target_types_rules.rules(),
+    ]
 
 
 def target_types():
-    return [PackMetadata, PackMetadataInGitSubmodule]
+    return [
+        PackMetadata,
+        PackMetadataInGitSubmodule,
+        PacksGlob,
+    ]

pants-plugins/pack_metadata/target_types.py

@@ -17,6 +17,7 @@
 from pants.core.target_types import (
     ResourcesGeneratingSourcesField,
     ResourcesGeneratorTarget,
+    GenericTarget,
 )
 
 
@@ -77,3 +78,18 @@ class PackMetadataInGitSubmodule(PackMetadata):
         "has unmatched globs. It prints instructions on how to checkout git "
         "submodules."
     )
+
+
+class PacksGlobDependencies(Dependencies):
+    pass
+
+
+class PacksGlob(GenericTarget):
+    alias = "packs_glob"
+    core_fields = (*COMMON_TARGET_FIELDS, PacksGlobDependencies)
+    help = (
+        "Packs glob.\n\n"
+        "Avoid using this target. It gets automatic dependencies on all "
+        "subdirectories (packs) except those listed with ! in dependencies. "
+        "This is unfortunately needed by tests that use a glob to load pack fixtures."
+    )

pants-plugins/pack_metadata/target_types_rules.py

@@ -0,0 +1,83 @@
+# Copyright 2023 The StackStorm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+from dataclasses import dataclass
+
+from pants.engine.addresses import Address
+from pants.engine.fs import GlobMatchErrorBehavior, PathGlobs, Paths
+from pants.engine.rules import Get, collect_rules, MultiGet, rule, UnionRule
+from pants.engine.target import (
+    DependenciesRequest,
+    ExplicitlyProvidedDependencies,
+    FieldSet,
+    InferDependenciesRequest,
+    InferredDependencies,
+)
+from pants.util.logging import LogLevel
+
+from pack_metadata.target_types import PacksGlobDependencies
+
+
+@dataclass(frozen=True)
+class PacksGlobInferenceFieldSet(FieldSet):
+    required_fields = (PacksGlobDependencies,)
+
+    dependencies: PacksGlobDependencies
+
+
+class InferPacksGlobDependencies(InferDependenciesRequest):
+    infer_from = PacksGlobInferenceFieldSet
+
+
+@rule(
+    desc="Inferring packs glob dependencies",
+    level=LogLevel.DEBUG,
+)
+async def infer_packs_globs_dependencies(
+    request: InferPacksGlobDependencies,
+) -> InferredDependencies:
+    address = request.field_set.address
+
+    pack_build_paths, explicitly_provided_deps = await MultiGet(
+        Get(
+            Paths,
+            PathGlobs(
+                [os.path.join(address.spec_path, "*", "BUILD")],
+                glob_match_error_behavior=GlobMatchErrorBehavior.error,
+                description_of_origin=f"{address}'s packs glob",
+            ),
+        ),
+        Get(
+            ExplicitlyProvidedDependencies,
+            DependenciesRequest(request.field_set.dependencies),
+        ),
+    )
+
+    implicit_packs_deps = {
+        Address(os.path.dirname(path)) for path in pack_build_paths.files
+    }
+
+    inferred_packs_deps = (
+        implicit_packs_deps
+        - explicitly_provided_deps.ignores  # FrozenOrderedSet[Address]
+        - explicitly_provided_deps.includes  # FrozenOrderedSet[Address]
+    )
+    return InferredDependencies(inferred_packs_deps)
+
+
+def rules():
+    return [
+        *collect_rules(),
+        UnionRule(InferDependenciesRequest, InferPacksGlobDependencies),
+    ]

pants-plugins/pack_metadata/target_types_rules_test.py

@@ -0,0 +1,125 @@
+# Copyright 2023 The StackStorm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+from textwrap import dedent
+
+from pants.backend.python.target_types import (
+    PythonSourceTarget,
+    PythonSourcesGeneratorTarget,
+)
+from pants.backend.python.target_types_rules import rules as python_target_types_rules
+from pants.engine.addresses import Address
+from pants.engine.target import InferredDependencies
+from pants.testutil.rule_runner import QueryRule, RuleRunner
+
+from .target_types_rules import (
+    InferPacksGlobDependencies,
+    PacksGlobInferenceFieldSet,
+    rules as pack_metadata_target_types_rules,
+)
+from .target_types import PacksGlob
+
+
+def test_infer_packs_globs_dependencies() -> None:
+    rule_runner = RuleRunner(
+        rules=[
+            *python_target_types_rules(),
+            *pack_metadata_target_types_rules(),
+            QueryRule(InferredDependencies, (InferPacksGlobDependencies,)),
+        ],
+        target_types=[
+            PythonSourceTarget,
+            PythonSourcesGeneratorTarget,
+            PacksGlob,
+        ],
+    )
+    rule_runner.write_files(
+        {
+            "packs/BUILD": dedent(
+                """\
+                python_sources(
+                    name="git_submodule",
+                    sources=["./git_submodule/*.py"],
+                )
+
+                packs_glob(
+                    name="all_packs_glob",
+                    dependencies=[
+                        "!./configs",  # explicit ignore
+                        "./a",         # explicit include
+                    ],
+                )
+                """
+            ),
+            "packs/a/BUILD": "python_sources()",
+            "packs/a/__init__.py": "",
+            "packs/a/fixture.py": "",
+            "packs/b/BUILD": dedent(
+                """\
+                python_sources(
+                    dependencies=["packs/configs/b.yaml"],
+                )
+                """
+            ),
+            "packs/b/__init__.py": "",
+            "packs/b/fixture.py": "",
+            "packs/c/BUILD": "python_sources()",
+            "packs/c/__init__.py": "",
+            "packs/c/fixture.py": "",
+            "packs/d/BUILD": "python_sources()",
+            "packs/d/__init__.py": "",
+            "packs/d/fixture.py": "",
+            # imitate a pack in a git submodule (should NOT have a BUILD file)
+            "packs/git_submodule/__init__.py": "",
+            "packs/git_submodule/fixture.py": "",
+            "packs/configs/BUILD": dedent(
+                """\
+                resources(
+                    sources=["*.yaml"],
+                )
+                """
+            ),
+            "packs/configs/b.yaml": dedent(
+                """\
+                ---
+                # pack config for pack b
+                """
+            ),
+        }
+    )
+
+    def run_dep_inference(address: Address) -> InferredDependencies:
+        args = [
+            "--source-root-patterns=/packs",
+        ]
+        rule_runner.set_options(args, env_inherit={"PATH", "PYENV_ROOT", "HOME"})
+        target = rule_runner.get_target(address)
+        return rule_runner.request(
+            InferredDependencies,
+            [InferPacksGlobDependencies(PacksGlobInferenceFieldSet.create(target))],
+        )
+
+    assert run_dep_inference(
+        Address("packs", target_name="all_packs_glob")
+    ) == InferredDependencies(
+        [
+            # should not have packs/a (explicit dep does not need to be inferred)
+            # should not have packs/configs (explicitly ignored)
+            # should not have packs/git_submodule (no BUILD file = no targets to add)
+            Address("packs/b"),
+            Address("packs/c"),
+            Address("packs/d"),
+        ],
+    )

st2api/tests/unit/controllers/v1/test_pack_config_schema.py

@@ -17,12 +17,12 @@
 
 from st2tests.api import FunctionalTest
 
-from st2tests.fixturesloader import get_fixtures_packs_base_path
+# import this so that pants can infer dependencies for the glob below
+from st2tests.fixtures.packs.all_packs_glob import PACKS_PATH
 
 __all__ = ["PackConfigSchemasControllerTestCase"]
 
-PACKS_PATH = get_fixtures_packs_base_path()
-CONFIG_SCHEMA_COUNT = len(glob.glob("%s/*/config.schema.yaml" % (PACKS_PATH)))
+CONFIG_SCHEMA_COUNT = len(glob.glob(f"{PACKS_PATH}/*/config.schema.yaml"))
 assert CONFIG_SCHEMA_COUNT > 1
 
 

st2api/tests/unit/controllers/v1/test_pack_configs.py

@@ -19,12 +19,13 @@
 
 from st2tests.api import FunctionalTest
 from st2api.controllers.v1.pack_configs import PackConfigsController
-from st2tests.fixturesloader import get_fixtures_packs_base_path
+
+# import this so that pants can infer dependencies for the glob below
+from st2tests.fixtures.packs.all_packs_glob import PACKS_PATH
 
 __all__ = ["PackConfigsControllerTestCase"]
 
-PACKS_PATH = get_fixtures_packs_base_path()
-CONFIGS_COUNT = len(glob.glob("%s/configs/*.yaml" % (PACKS_PATH)))
+CONFIGS_COUNT = len(glob.glob(f"{PACKS_PATH}/configs/*.yaml"))
 assert CONFIGS_COUNT > 1
 
 

st2common/tests/integration/test_register_content_script.py

@@ -22,9 +22,9 @@
 from st2tests.base import IntegrationTestCase
 from st2common.util.shell import run_command
 from st2tests import config as test_config
-from st2tests.fixturesloader import get_fixtures_packs_base_path
 
 # import this so that pants can infer dependencies for the glob below
+from st2tests.fixtures.packs.all_packs_glob import PACKS_PATH
 from st2tests.fixtures.packs.dummy_pack_1.fixture import PACK_PATH as DUMMY_PACK_1_PATH
 from st2tests.fixtures.packs.dummy_pack_4.fixture import PACK_PATH as DUMMY_PACK_4_PATH
 from st2tests.fixtures.packs.runners.fixture import FIXTURE_PATH as RUNNER_DIRS
@@ -38,8 +38,7 @@
 BASE_CMD_ARGS = [sys.executable, SCRIPT_PATH, "--config-file=conf/st2.tests.conf", "-v"]
 BASE_REGISTER_ACTIONS_CMD_ARGS = BASE_CMD_ARGS + ["--register-actions"]
 
-PACKS_PATH = get_fixtures_packs_base_path()
-PACKS_COUNT = len(glob.glob("%s/*/pack.yaml" % (PACKS_PATH)))
+PACKS_COUNT = len(glob.glob(f"{PACKS_PATH}/*/pack.yaml"))
 assert PACKS_COUNT >= 2
 
 

st2common/tests/unit/test_policies_registrar.py

@@ -27,7 +27,7 @@
 from st2common.persistence.policy import Policy
 from st2common.persistence.policy import PolicyType
 from st2tests.base import CleanDbTestCase
-from st2tests.fixturesloader import get_fixtures_packs_base_path
+from st2tests.fixtures.packs.all_packs_glob import PACKS_PATH
 from st2tests.fixtures.packs.dummy_pack_1.fixture import (
     PACK_NAME as DUMMY_PACK_1,
     PACK_PATH as DUMMY_PACK_1_PATH,
@@ -63,8 +63,7 @@ def test_register_all_policies(self):
         policies_dbs = Policy.get_all()
         self.assertEqual(len(policies_dbs), 0)
 
-        packs_base_path = get_fixtures_packs_base_path()
-        count = policies_registrar.register_policies(packs_base_paths=[packs_base_path])
+        count = policies_registrar.register_policies(packs_base_paths=[PACKS_PATH])
 
         # Verify PolicyDB objects have been created
         policies_dbs = Policy.get_all()

st2common/tests/unit/test_triggers_registrar.py

@@ -19,7 +19,7 @@
 from st2common.persistence.trigger import Trigger
 from st2common.persistence.trigger import TriggerType
 from st2tests.base import CleanDbTestCase
-from st2tests.fixturesloader import get_fixtures_packs_base_path
+from st2tests.fixtures.packs.all_packs_glob import PACKS_PATH
 from st2tests.fixtures.packs.dummy_pack_1.fixture import (
     PACK_NAME as DUMMY_PACK_1,
     PACK_PATH as DUMMY_PACK_1_PATH,
@@ -33,8 +33,7 @@ def test_register_all_triggers(self):
         trigger_type_dbs = TriggerType.get_all()
         self.assertEqual(len(trigger_type_dbs), 0)
 
-        packs_base_path = get_fixtures_packs_base_path()
-        count = triggers_registrar.register_triggers(packs_base_paths=[packs_base_path])
+        count = triggers_registrar.register_triggers(packs_base_paths=[PACKS_PATH])
         self.assertEqual(count, 2)
 
         # Verify TriggerTypeDB and corresponding TriggerDB objects have been created

st2tests/st2tests/fixtures/packs/BUILD

@@ -34,3 +34,19 @@ python_sources(
         "test_content_version/**/*.py",
     ],
 )
+
+packs_glob(
+    name="all_packs",
+    dependencies=[
+        # core is a symlink instead of a dir
+        "./core",
+        # use :test_content_version instead because of the git submodule
+        ":test_content_version",
+        "!./test_content_version_fixture",
+        # these are not packs
+        "!./configs",
+        "!./executions",
+        "!./runners",
+        "!./all_packs_glob",  # the fixture that pulls in this target
+    ],
+)

st2tests/st2tests/fixtures/packs/all_packs_glob/BUILD

@@ -0,0 +1,3 @@
+python_sources(
+    dependencies=["st2tests/st2tests/fixtures/packs:all_packs"],
+)