StackStorm · cognifloyd · Mar 17, 2023 · Feb 10, 2023 · Mar 11, 2023 · Feb 10, 2023
diff --git a/BUILD b/BUILD
@@ -1,16 +1,6 @@
 python_requirements(
     name="reqs",
     source="requirements-pants.txt",
-    # module_mapping can be removed once pants is released with
-    # https://github.com/pantsbuild/pants/pull/17390
-    module_mapping={
-        "python-editor": ["editor"],
-        "python-json-logger": ["pythonjsonlogger"],
-        "python-statsd": ["statsd"],
-        "sseclient-py": ["sseclient"],
-        "oslo.config": ["oslo_config"],
-        "RandomWords": ["random_words"],
-    },
     overrides={
         # flex and stevedore uses pkg_resources w/o declaring the dep
         ("flex", "stevedore"): {

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -17,7 +17,7 @@ Added
   #5778 #5789 #5817 #5795 #5830 #5833 #5834 #5841 #5840 #5838 #5842 #5837 #5849 #5850
   #5846 #5853 #5848 #5847 #5858 #5857 #5860 #5868 #5871 #5864 #5874 #5884 #5893 #5891
   #5890 #5898 #5901 #5906 #5899 #5907 #5909 #5922 #5926 #5927 #5925 #5928 #5929 #5930
-  #5931
+  #5931 #5932
   Contributed by @cognifloyd
 
 * Added a joint index to solve the problem of slow mongo queries for scheduled executions. #5805

diff --git a/contrib/core/BUILD b/contrib/core/BUILD
@@ -5,9 +5,6 @@ pack_metadata(
 python_requirements(
     name="reqs",
     source="requirements-tests.txt",
-    # module_mapping can be removed once pants is released with
-    # https://github.com/pantsbuild/pants/pull/17390
-    module_mapping={"mail-parser": ["mailparser"]},
 )
 
 python_sources(

diff --git a/contrib/packs/tests/BUILD b/contrib/packs/tests/BUILD
@@ -1,11 +1,3 @@
 python_tests(
     skip_pylint=True,
-    overrides={
-        "test_action_download.py": {
-            "dependencies": [
-                # imports tests.fixtures which is ambiguous. Tell pants which one to use.
-                "./fixtures",
-            ],
-        }
-    },
 )
diff --git a/contrib/runners/orquesta_runner/tests/unit/BUILD b/contrib/runners/orquesta_runner/tests/unit/BUILD
@@ -5,10 +5,6 @@ __defaults__(
 
 python_tests(
     name="tests",
-    dependencies=[
-        # most files import tests.unit.base which is ambiguous. Tell pants which one to use.
-        "contrib/runners/orquesta_runner/tests/unit/base.py",
-    ],
 )
 
 python_test_utils(

diff --git a/lockfiles/pants-plugins.lock b/lockfiles/pants-plugins.lock
diff --git a/lockfiles/st2.lock b/lockfiles/st2.lock
diff --git a/pants-plugins/macros.py b/pants-plugins/macros.py
@@ -13,11 +13,6 @@
 # limitations under the License.
 
 
-# this is only here temporarily until we update to pants 2.16+
-def stevedore_namespace(ns):
-    return ns
-
-
 def st2_publish_repos():
     """Return the list of repos twine should publish to.
 
@@ -29,9 +24,7 @@ def st2_publish_repos():
 
     Credentials for pypi should be in ~/.pypirc or in TWINE_* env vars.
     """
-    # TODO: switch from hard-coded to env() once we upgrade to pants 2.16
-    # return [env("ST2_PUBLISH_REPO", "@pypi")]  # noqa: F821
-    return ["@pypi"]
+    return [env("ST2_PUBLISH_REPO", "@pypi")]  # noqa: F821
 
 
 def st2_license(**kwargs):

diff --git a/pants.toml b/pants.toml
@@ -6,13 +6,14 @@ enabled = false
 repo_id = "de0dea7a-9f6a-4c6e-aa20-6ba5ad969b8a"
 
 [GLOBAL]
-pants_version = "2.15.0"
+pants_version = "2.16.0a0"
 pythonpath = ["%(buildroot)s/pants-plugins"]
 build_file_prelude_globs = ["pants-plugins/macros.py"]
 backend_packages = [
   # python
   "pants.backend.python",
   "pants.backend.experimental.python", # activates twine `publish` support
+  "pants.backend.experimental.python.framework.stevedore",
   "pants.backend.python.mixed_interpreter_constraints",
   "pants.backend.python.lint.bandit",
   "pants.backend.python.lint.black",
@@ -130,14 +131,26 @@ pants-plugins = [
 st2 = "lockfiles/st2-constraints.txt"
 
 [python-infer]
+# https://www.pantsbuild.org/docs/reference-python-infer#string_imports
+# https://www.pantsbuild.org/docs/reference-python-infer#string_imports_min_dots
+# Infer a target's dependencies based on strings that look like dynamic deps with >=1 dots.
+# To debug the imports and see if a string is used in dep inference or if it is ignored, use:
+#   ./pants python-dump-source-analysis --analysis-flavor=raw_dependency_inference <path(s) to file(s)> | jq '.[].resolved'
+string_imports = true
+string_imports_min_dots = 1
 # https://www.pantsbuild.org/docs/reference-python-infer#unowned_dependency_behavior
 # The default changed from "ignore" to "warning" in pants 2.14.
-# Many of the new warnings however have been adressed via explicit deps,
-# so the warnings are not helpful. In pants 2.16, a "visibility" feature might help
-# us to disambiguate deps between files without those explicit BUILD dependencies,
-# and without adding "# pants: no-infer-dep" comments all over the codebase.
-# Revisit this in pants 2.16 to see if it is feasible to use the default "warning".
+# The ambiguity_resolution setting/feature (below) added in 2.16 resolves most of
+# our ambiguous dependency inference issues, which allowed us to remove the explicit
+# deps in various BUILD files. But, there is not a good way to tell pants about our
+# custom PYTHONPATH for packs, so actions that import other actions are still showing
+# up as unowned. Maybe we can extend pants-plugins/pack_metadata so we can use "warn".
 unowned_dependency_behavior = "ignore"
+# https://www.pantsbuild.org/v2.16/docs/reference-python-infer#ambiguity_resolution
+# When resolving ambiguous deps prefer one that is in the same source root as the
+# file that uses it. So, without manually disambiguating the dep in the BUILD file,
+# importing tests.unit.base in st2common/tests/unit will get a dep on st2common/tests/unit/base.py
+ambiguity_resolution = "by_source_root"
 
 [setup-py-generation]
 # when building the package (with ./pants package ::), pants will,
@@ -176,6 +189,9 @@ extra_requirements = [
 ]
 config = "lint-configs/python/.flake8"
 
+[generate-lockfiles]
+diff = true
+
 [pylint]
 lockfile = "lockfiles/pylint.lock"
 version = "pylint~=2.8.2"

diff --git a/st2auth/st2auth/backends/__init__.py b/st2auth/st2auth/backends/__init__.py
@@ -26,7 +26,7 @@
 
 LOG = logging.getLogger(__name__)
 
-BACKENDS_NAMESPACE = "st2auth.backends.backend"
+BACKENDS_NAMESPACE = "st2auth.backends.backend"  # pants: no-infer-dep
 
 
 def get_available_backends():

diff --git a/st2auth/st2auth/sso/__init__.py b/st2auth/st2auth/sso/__init__.py
@@ -29,7 +29,7 @@
 
 LOG = logging.getLogger(__name__)
 
-BACKENDS_NAMESPACE = "st2auth.sso.backends"
+BACKENDS_NAMESPACE = "st2auth.sso.backends"  # pants: no-infer-dep
 
 
 def get_available_backends():

diff --git a/st2auth/tests/unit/BUILD b/st2auth/tests/unit/BUILD
@@ -5,8 +5,4 @@ __defaults__(
 
 python_tests(
     name="tests",
-    dependencies=[
-        # most files import tests.base which is ambiguous. Tell pants which one to use.
-        "st2auth/tests/base.py",
-    ],
 )
diff --git a/st2auth/tests/unit/controllers/v1/BUILD b/st2auth/tests/unit/controllers/v1/BUILD
@@ -1,7 +1,3 @@
 python_tests(
     name="tests",
-    dependencies=[
-        # most files import tests.base which is ambiguous. Tell pants which one to use.
-        "st2auth/tests/base.py",
-    ],
 )
diff --git a/st2client/tests/unit/BUILD b/st2client/tests/unit/BUILD
@@ -7,7 +7,5 @@ python_tests(
     name="tests",
     dependencies=[
         "st2client/tests/fixtures:st2client_ini",
-        # most files import tests.base which is ambiguous. Tell pants which one to use.
-        "st2client/tests/base.py",
     ],
 )
diff --git a/st2common/st2common/constants/runners.py b/st2common/st2common/constants/runners.py
@@ -78,4 +78,4 @@
 ]
 
 # Namespaces for dynamically loaded runner modules
-RUNNERS_NAMESPACE = "st2common.runners.runner"
+RUNNERS_NAMESPACE = "st2common.runners.runner"  # pants: no-infer-dep
diff --git a/st2common/st2common/metrics/base.py b/st2common/st2common/metrics/base.py
@@ -43,7 +43,7 @@
 
 LOG = logging.getLogger(__name__)
 
-PLUGIN_NAMESPACE = "st2common.metrics.driver"
+PLUGIN_NAMESPACE = "st2common.metrics.driver"  # pants: no-infer-dep
 
 # Stores reference to the metrics driver class instance.
 # NOTE: This value is populated lazily on the first get_driver() function call

diff --git a/st2common/st2common/rbac/backends/__init__.py b/st2common/st2common/rbac/backends/__init__.py
@@ -26,7 +26,7 @@
 
 LOG = logging.getLogger(__name__)
 
-BACKENDS_NAMESPACE = "st2common.rbac.backend"
+BACKENDS_NAMESPACE = "st2common.rbac.backend"  # pants: no-infer-dep
 
 # Cache which maps backed name -> backend class instance
 # NOTE: We use cache to avoid slow stevedore dynamic filesystem instrospection on every

diff --git a/st2common/st2common/runners/__init__.py b/st2common/st2common/runners/__init__.py
@@ -21,7 +21,7 @@
 
 __all__ = ["BACKENDS_NAMESPACE", "get_available_backends", "get_backend_driver"]
 
-BACKENDS_NAMESPACE = "st2common.runners.runner"
+BACKENDS_NAMESPACE = "st2common.runners.runner"  # pants: no-infer-dep
 
 
 def get_available_backends():

diff --git a/st2common/st2common/util/driver_loader.py b/st2common/st2common/util/driver_loader.py
@@ -25,7 +25,7 @@
 
 LOG = logging.getLogger(__name__)
 
-BACKENDS_NAMESPACE = "st2common.rbac.backend"
+BACKENDS_NAMESPACE = "st2common.rbac.backend"  # pants: no-infer-dep
 
 
 def get_available_backends(namespace, invoke_on_load=False):

diff --git a/st2common/tests/integration/BUILD b/st2common/tests/integration/BUILD
@@ -10,6 +10,12 @@ python_tests(
         "conf/st2.tests.conf:st2_tests_conf",
         "conf/st2.tests1.conf:st2_tests_conf",
     ],
+    stevedore_namespaces=[
+        "orquesta.expressions.functions",
+        "st2common.runners.runner",
+        "st2common.rbac.backend",
+        "st2common.metrics.driver",
+    ],
 )
 
 python_test_utils(

diff --git a/st2common/tests/unit/BUILD b/st2common/tests/unit/BUILD
@@ -5,9 +5,11 @@ __defaults__(
 
 python_tests(
     name="tests",
-    dependencies=[
-        # several files import tests.unit.base which is ambiguous. Tell pants which one to use.
-        "st2common/tests/unit/base.py",
+    stevedore_namespaces=[
+        "orquesta.expressions.functions",
+        "st2common.runners.runner",
+        "st2common.rbac.backend",
+        "st2common.metrics.driver",
     ],
     uses=["mongo", "rabbitmq"],
     overrides={

diff --git a/st2common/tests/unit/migrations/BUILD b/st2common/tests/unit/migrations/BUILD
@@ -1,3 +1,10 @@
 python_tests(
     name="tests",
+    overrides={
+        "test_v35_migrate_db_dict_field_values.py": dict(
+            dependencies=[
+                "st2common/bin/migrations/v3.5/st2_migrate_db_dict_field_values.py",
+            ],
+        ),
+    },
 )
diff --git a/st2common/tests/unit/services/BUILD b/st2common/tests/unit/services/BUILD
@@ -1,4 +1,10 @@
 python_tests(
     name="tests",
+    stevedore_namespaces=[
+        "orquesta.expressions.functions",
+        "st2common.runners.runner",
+        "st2common.rbac.backend",
+        "st2common.metrics.driver",
+    ],
     uses=["mongo", "rabbitmq"],
 )
diff --git a/st2common/tests/unit/test_action_db_utils.py b/st2common/tests/unit/test_action_db_utils.py
@@ -532,7 +532,7 @@ def setup_runner(cls):
                     "default": "runnerdummy",
                 },
             },
-            "runner_module": "tests.test_runner",
+            "runner_module": "tests.test_runner",  # pants: no-infer-dep
         }
         runnertype_api = RunnerTypeAPI(**test_runner)
         ActionDBUtilsTestCase.runnertype_db = RunnerType.add_or_update(

diff --git a/tools/BUILD b/tools/BUILD
@@ -16,20 +16,10 @@ python_sources(
             "dependencies": [
                 # the auth backends get listed in the conf file
                 "//:auth_backends",
-                # the following should match CONFIGS in config_gen.py
+                # We are using string import detection to gather the imports
+                # from CONFIGS in config_gen.py. The following command is
+                # helpful in validating that dependencies include everything:
                 # grep -rl '^def register_opts(ignore_errors=False):' st2*
-                "st2actions/st2actions/scheduler/config.py",
-                "st2actions/st2actions/workflows/config.py",
-                "st2actions/st2actions/notifier/config.py",
-                "st2actions/st2actions/config.py",
-                "st2api/st2api/config.py",
-                "st2auth/st2auth/config.py",
-                "st2common/st2common/config.py",
-                "st2reactor/st2reactor/garbage_collector/config.py",
-                "st2reactor/st2reactor/timer/config.py",
-                "st2reactor/st2reactor/sensor/config.py",
-                "st2reactor/st2reactor/rules/config.py",
-                "st2stream/st2stream/config.py",
             ]
         },
     },

diff --git a/tools/config_gen.py b/tools/config_gen.py
@@ -25,10 +25,9 @@
 
 
 CONFIGS = [
-    # this is duplicated in tools/BUILD
-    # TODO: replace this with a heuristic that searches for config.py
-    #       maybe with an exclude list (eg st2tests.config st2client)
-    #       grep -rl 'def register_opts(ignore_errors=False):' st2*
+    # pants uses these strings to infer dependencies. Compare this list
+    # with the output of this command to make sure everything is present:
+    #   grep -rl 'def register_opts(ignore_errors=False):' st2*
     "st2actions.config",
     "st2actions.scheduler.config",
     "st2actions.notifier.config",