From d86a31cd94e88448222f368354c50adb42ca6c31 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 16 Sep 2024 17:47:13 -0500 Subject: [PATCH 1/3] pants-plugins/uses_services: correct env var usage in mongo_rules --- pants-plugins/uses_services/mongo_rules.py | 16 ++++++++++++---- .../uses_services/scripts/is_mongo_running.py | 6 +++++- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/pants-plugins/uses_services/mongo_rules.py b/pants-plugins/uses_services/mongo_rules.py index ad8333132f..b537ff7aff 100644 --- a/pants-plugins/uses_services/mongo_rules.py +++ b/pants-plugins/uses_services/mongo_rules.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import os from dataclasses import dataclass from textwrap import dedent @@ -20,6 +19,7 @@ PytestPluginSetupRequest, PytestPluginSetup, ) +from pants.backend.python.subsystems.pytest import PyTest from pants.backend.python.util_rules.pex import ( PexRequest, PexRequirements, @@ -64,9 +64,12 @@ class UsesMongoRequest: db_host: str = "127.0.0.1" # localhost in test_db.DbConnectionTestCase db_port: int = 27017 # db_name is "st2" in test_db.DbConnectionTestCase - db_name: str = f"st2-test{os.environ.get('ST2TESTS_PARALLEL_SLOT', '')}" + db_name: str = "st2-test{}" # {} will be replaced by test slot (a format string) + db_connection_timeout: int = 3000 + execution_slot_var: str = "ST2TESTS_PARALLEL_SLOT" + @dataclass(frozen=True) class MongoIsRunning: @@ -87,7 +90,7 @@ def is_applicable(cls, target: Target) -> bool: level=LogLevel.DEBUG, ) async def mongo_is_running_for_pytest( - request: PytestUsesMongoRequest, + request: PytestUsesMongoRequest, pytest: PyTest ) -> PytestPluginSetup: # TODO: delete these comments once the Makefile becomes irrelevant. # the comments explore how the Makefile prepares to run and runs tests @@ -104,7 +107,10 @@ async def mongo_is_running_for_pytest( # nosetests $(NOSE_OPTS) -s -v $(NOSE_COVERAGE_FLAGS) $(NOSE_COVERAGE_PACKAGES) $$component/tests/unit # this will raise an error if mongo is not running - _ = await Get(MongoIsRunning, UsesMongoRequest()) + _ = await Get( + MongoIsRunning, + UsesMongoRequest(execution_slot_var=pytest.execution_slot_var or ""), + ) return PytestPluginSetup() @@ -145,8 +151,10 @@ async def mongo_is_running( str(request.db_port), request.db_name, str(request.db_connection_timeout), + request.execution_slot_var, ), input_digest=script_digest, + execution_slot_variable=request.execution_slot_var, description="Checking to see if Mongo is up and accessible.", # this can change from run to run, so don't cache results. cache_scope=ProcessCacheScope.PER_SESSION, diff --git a/pants-plugins/uses_services/scripts/is_mongo_running.py b/pants-plugins/uses_services/scripts/is_mongo_running.py index 8d5ecfce8a..637fc9ac1c 100644 --- a/pants-plugins/uses_services/scripts/is_mongo_running.py +++ b/pants-plugins/uses_services/scripts/is_mongo_running.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os import sys @@ -48,9 +49,12 @@ def _is_mongo_running( args = dict((k, v) for k, v in enumerate(sys.argv)) db_host = args.get(1, "127.0.0.1") db_port = args.get(2, 27017) - db_name = args.get(3, "st2-test") + db_name = args.get(3, "st2-test{}") connection_timeout_ms = args.get(4, 3000) + slot_var = args.get(5, "ST2TESTS_PARALLEL_SLOT") + db_name = db_name.format(os.environ.get(slot_var) or "") + is_running = _is_mongo_running( db_host, int(db_port), db_name, int(connection_timeout_ms) ) From c980a6f3b73157ac5db2f181e9ac10e788797fe3 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 16 Sep 2024 20:38:38 -0500 Subject: [PATCH 2/3] pants-plugins/uses_services: add detection for system_user --- .../runners/orquesta_runner/tests/unit/BUILD | 12 ++ pants-plugins/uses_services/BUILD | 1 + pants-plugins/uses_services/register.py | 9 +- .../uses_services/scripts/has_system_user.py | 43 +++++ .../uses_services/system_user_rules.py | 158 ++++++++++++++++++ .../uses_services/system_user_rules_test.py | 96 +++++++++++ pants-plugins/uses_services/target_types.py | 2 +- st2actions/tests/unit/BUILD | 9 + st2api/tests/unit/controllers/v1/BUILD | 11 ++ st2common/tests/unit/BUILD | 3 + 10 files changed, 342 insertions(+), 2 deletions(-) create mode 100644 pants-plugins/uses_services/scripts/has_system_user.py create mode 100644 pants-plugins/uses_services/system_user_rules.py create mode 100644 pants-plugins/uses_services/system_user_rules_test.py diff --git a/contrib/runners/orquesta_runner/tests/unit/BUILD b/contrib/runners/orquesta_runner/tests/unit/BUILD index 1daa501cd5..5b3cb7900c 100644 --- a/contrib/runners/orquesta_runner/tests/unit/BUILD +++ b/contrib/runners/orquesta_runner/tests/unit/BUILD @@ -5,6 +5,18 @@ __defaults__( python_tests( name="tests", + overrides={ + ( + "test_basic.py", + "test_cancel.py", + "test_context.py", + "test_error_handling.py", + "test_pause_and_resume.py", + "test_with_items.py", + ): dict( + uses=["system_user"], + ), + }, ) python_test_utils( diff --git a/pants-plugins/uses_services/BUILD b/pants-plugins/uses_services/BUILD index 8c63bf16d4..0bce3fe934 100644 --- a/pants-plugins/uses_services/BUILD +++ b/pants-plugins/uses_services/BUILD @@ -17,5 +17,6 @@ python_tests( # "mongo_rules_test.py": dict(uses=["mongo"]), # "rabbitmq_rules_test.py": dict(uses=["rabbitmq"]), # "redis_rules_test.py": dict(uses=["redis"]), + # "system_user_test.py": dict(uses=["system_user"]), # }, ) diff --git a/pants-plugins/uses_services/register.py b/pants-plugins/uses_services/register.py index 83ab32b3a7..1b5b6e91a2 100644 --- a/pants-plugins/uses_services/register.py +++ b/pants-plugins/uses_services/register.py @@ -16,7 +16,13 @@ PythonTestsGeneratorTarget, ) -from uses_services import mongo_rules, platform_rules, rabbitmq_rules, redis_rules +from uses_services import ( + mongo_rules, + platform_rules, + rabbitmq_rules, + redis_rules, + system_user_rules, +) from uses_services.target_types import UsesServicesField @@ -28,4 +34,5 @@ def rules(): *mongo_rules.rules(), *rabbitmq_rules.rules(), *redis_rules.rules(), + *system_user_rules.rules(), ] diff --git a/pants-plugins/uses_services/scripts/has_system_user.py b/pants-plugins/uses_services/scripts/has_system_user.py new file mode 100644 index 0000000000..29a4cfb6de --- /dev/null +++ b/pants-plugins/uses_services/scripts/has_system_user.py @@ -0,0 +1,43 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import os +import pwd +import sys + + +def _has_system_user(system_user: str) -> bool: + """Make sure the system_user exists. + + This should not import the st2 code as it should be self-contained. + """ + try: + pwd.getpwnam(system_user) + except KeyError: + # put current user (for use in error msg instructions) + print(pwd.getpwuid(os.getuid()).pw_name) + return False + print(system_user) + return True + + +if __name__ == "__main__": + args = dict((k, v) for k, v in enumerate(sys.argv)) + + system_user = args.get(1, "stanley") + + is_running = _has_system_user(system_user) + exit_code = 0 if is_running else 1 + sys.exit(exit_code) diff --git a/pants-plugins/uses_services/system_user_rules.py b/pants-plugins/uses_services/system_user_rules.py new file mode 100644 index 0000000000..bd2c8e86e6 --- /dev/null +++ b/pants-plugins/uses_services/system_user_rules.py @@ -0,0 +1,158 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +from dataclasses import dataclass +from textwrap import dedent + +from pants.backend.python.goals.pytest_runner import ( + PytestPluginSetupRequest, + PytestPluginSetup, +) +from pants.backend.python.target_types import Executable +from pants.backend.python.util_rules.pex import ( + PexRequest, + VenvPex, + VenvPexProcess, + rules as pex_rules, +) +from pants.core.goals.test import TestExtraEnv +from pants.engine.fs import CreateDigest, Digest, FileContent +from pants.engine.rules import collect_rules, Get, rule +from pants.engine.process import FallibleProcessResult, ProcessCacheScope +from pants.engine.target import Target +from pants.engine.unions import UnionRule +from pants.util.logging import LogLevel + +from uses_services.exceptions import ServiceMissingError +from uses_services.platform_rules import Platform +from uses_services.scripts.has_system_user import ( + __file__ as has_system_user_full_path, +) +from uses_services.target_types import UsesServicesField + + +@dataclass(frozen=True) +class UsesSystemUserRequest: + """One or more targets need the system_user (like stanley) using these settings. + + The system_user attributes represent the system_user.user settings from st2.conf. + In st2 code, they come from: + oslo_config.cfg.CONF.system_user.user + """ + + system_user: str = "stanley" + + +@dataclass(frozen=True) +class HasSystemUser: + pass + + +class PytestUsesSystemUserRequest(PytestPluginSetupRequest): + @classmethod + def is_applicable(cls, target: Target) -> bool: + if not target.has_field(UsesServicesField): + return False + uses = target.get(UsesServicesField).value + return uses is not None and "system_user" in uses + + +@rule( + desc="Ensure system_user is present before running tests.", + level=LogLevel.DEBUG, +) +async def has_system_user_for_pytest( + request: PytestUsesSystemUserRequest, + test_extra_env: TestExtraEnv, +) -> PytestPluginSetup: + system_user = test_extra_env.env.get("ST2TESTS_SYSTEM_USER", "stanley") + + # this will raise an error if system_user is not present + _ = await Get(HasSystemUser, UsesSystemUserRequest(system_user=system_user)) + + return PytestPluginSetup() + + +@rule( + desc="Test to see if system_user is present.", + level=LogLevel.DEBUG, +) +async def has_system_user( + request: UsesSystemUserRequest, platform: Platform +) -> HasSystemUser: + script_path = "./has_system_user.py" + + # pants is already watching this directory as it is under a source root. + # So, we don't need to double watch with PathGlobs, just open it. + with open(has_system_user_full_path, "rb") as script_file: + script_contents = script_file.read() + + script_digest = await Get( + Digest, CreateDigest([FileContent(script_path, script_contents)]) + ) + script_pex = await Get( + VenvPex, + PexRequest( + output_filename="script.pex", + internal_only=True, + sources=script_digest, + main=Executable(script_path), + ), + ) + + result = await Get( + FallibleProcessResult, + VenvPexProcess( + script_pex, + argv=(request.system_user,), + description="Checking to see if system_user is present.", + # this can change from run to run, so don't cache results. + cache_scope=ProcessCacheScope.PER_SESSION, + level=LogLevel.DEBUG, + ), + ) + has_user = result.exit_code == 0 + + if has_user: + return HasSystemUser() + + current_user = result.stdout.decode().strip() + + # system_user is not present, so raise an error with instructions. + raise ServiceMissingError( + service="system_user", + platform=platform, + msg=dedent( + f"""\ + The system_user ({request.system_user}) does not seem to be present! + + Please export the ST2TESTS_SYSTEM_USER env var to specify which user + tests should use as the system_user. This user must be present on + your system. + + To use your current user ({current_user}) as the system_user, run: + + export ST2TESTS_SYSTEM_USER=$(id -un) + """ + ), + ) + + +def rules(): + return [ + *collect_rules(), + UnionRule(PytestPluginSetupRequest, PytestUsesSystemUserRequest), + *pex_rules(), + ] diff --git a/pants-plugins/uses_services/system_user_rules_test.py b/pants-plugins/uses_services/system_user_rules_test.py new file mode 100644 index 0000000000..a41371a382 --- /dev/null +++ b/pants-plugins/uses_services/system_user_rules_test.py @@ -0,0 +1,96 @@ +# Copyright 2024 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import os + +import pytest + +from pants.engine.internals.scheduler import ExecutionError +from pants.testutil.rule_runner import QueryRule, RuleRunner + +from .data_fixtures import platform, platform_samples +from .exceptions import ServiceMissingError +from .system_user_rules import ( + HasSystemUser, + UsesSystemUserRequest, + rules as system_user_rules, +) +from .platform_rules import Platform + + +@pytest.fixture +def rule_runner() -> RuleRunner: + return RuleRunner( + rules=[ + *system_user_rules(), + QueryRule(HasSystemUser, (UsesSystemUserRequest, Platform)), + ], + target_types=[], + ) + + +def run_has_system_user( + rule_runner: RuleRunner, + uses_system_user_request: UsesSystemUserRequest, + mock_platform: Platform, + *, + extra_args: list[str] | None = None, +) -> HasSystemUser: + rule_runner.set_options( + [ + "--backend-packages=uses_services", + *(extra_args or ()), + ], + env_inherit={"PATH", "PYENV_ROOT", "HOME", "ST2TESTS_SYSTEM_USER"}, + ) + result = rule_runner.request( + HasSystemUser, + [uses_system_user_request, mock_platform], + ) + return result + + +# Warning this requires that system_user is present +def test_system_user_is_present(rule_runner: RuleRunner) -> None: + request = UsesSystemUserRequest( + system_user=os.environ.get("ST2TESTS_SYSTEM_USER", "stanley") + ) + mock_platform = platform(os="TestMock") + + # we are asserting that this does not raise an exception + has_user = run_has_system_user(rule_runner, request, mock_platform) + assert has_user + + +@pytest.mark.parametrize("mock_platform", platform_samples) +def test_system_user_is_absent( + rule_runner: RuleRunner, mock_platform: Platform +) -> None: + request = UsesSystemUserRequest( + system_user="bogus-stanley", + ) + + with pytest.raises(ExecutionError) as exception_info: + run_has_system_user(rule_runner, request, mock_platform) + + execution_error = exception_info.value + assert len(execution_error.wrapped_exceptions) == 1 + + exc = execution_error.wrapped_exceptions[0] + assert isinstance(exc, ServiceMissingError) + + assert exc.service == "system_user" + assert "The system_user (bogus-stanley) does not seem to be present" in str(exc) + assert not exc.instructions diff --git a/pants-plugins/uses_services/target_types.py b/pants-plugins/uses_services/target_types.py index 5723ceb9ae..0a0f2d89bd 100644 --- a/pants-plugins/uses_services/target_types.py +++ b/pants-plugins/uses_services/target_types.py @@ -14,7 +14,7 @@ from pants.engine.target import StringSequenceField -supported_services = ("mongo", "rabbitmq", "redis") +supported_services = ("mongo", "rabbitmq", "redis", "system_user") class UsesServicesField(StringSequenceField): diff --git a/st2actions/tests/unit/BUILD b/st2actions/tests/unit/BUILD index 9a24dba70a..4e577b340a 100644 --- a/st2actions/tests/unit/BUILD +++ b/st2actions/tests/unit/BUILD @@ -5,4 +5,13 @@ __defaults__( python_tests( name="tests", + overrides={ + ( + "test_execution_cancellation.py", + "test_runner_container.py", + "test_worker.py", + ): dict( + uses=["system_user"], + ), + }, ) diff --git a/st2api/tests/unit/controllers/v1/BUILD b/st2api/tests/unit/controllers/v1/BUILD index 57341b1358..99064bb94c 100644 --- a/st2api/tests/unit/controllers/v1/BUILD +++ b/st2api/tests/unit/controllers/v1/BUILD @@ -1,3 +1,14 @@ python_tests( name="tests", + overrides={ + ( + "test_alias_execution.py", + "test_auth.py", + "test_auth_api_keys.py", + "test_executions.py", + "test_inquiries.py", + ): dict( + uses=["system_user"], + ), + }, ) diff --git a/st2common/tests/unit/BUILD b/st2common/tests/unit/BUILD index a1da37051b..d8c6fe4709 100644 --- a/st2common/tests/unit/BUILD +++ b/st2common/tests/unit/BUILD @@ -19,6 +19,9 @@ python_tests( "st2tests/st2tests/policies/meta", ], ), + "test_param_utils.py": dict( + uses=["system_user"], + ), }, ) From dc5a86b2e7da4f7b22374c9b239016cb4e109ebe Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 16 Sep 2024 20:55:56 -0500 Subject: [PATCH 3/3] update changelog entry --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4b4dd3961a..87db58e182 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -33,7 +33,7 @@ Added * Continue introducing `pants `_ to improve DX (Developer Experience) working on StackStorm, improve our security posture, and improve CI reliability thanks in part to pants' use of PEX lockfiles. This is not a user-facing addition. - #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 #6240 #6241 + #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 #6240 #6241 #6244 Contributed by @cognifloyd * Build of ST2 EL9 packages #6153 Contributed by @amanda11