From b7e53da477adcd4a9c35521eeca5b3cb201477ed Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 13 May 2021 15:04:35 -0500 Subject: [PATCH 01/71] add v1 pants-plugin/uses_services To assert services (mongo) are running before running tests that require a database or similar. --- pants-plugins/uses_services/BUILD | 1 + pants-plugins/uses_services/__init__.py | 0 pants-plugins/uses_services/register.py | 41 +++++++++++++++++++++++++ pants.toml | 1 + st2common/tests/unit/BUILD | 1 + 5 files changed, 44 insertions(+) create mode 100644 pants-plugins/uses_services/BUILD create mode 100644 pants-plugins/uses_services/__init__.py create mode 100644 pants-plugins/uses_services/register.py diff --git a/pants-plugins/uses_services/BUILD b/pants-plugins/uses_services/BUILD new file mode 100644 index 0000000000..db46e8d6c9 --- /dev/null +++ b/pants-plugins/uses_services/BUILD @@ -0,0 +1 @@ +python_sources() diff --git a/pants-plugins/uses_services/__init__.py b/pants-plugins/uses_services/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pants-plugins/uses_services/register.py b/pants-plugins/uses_services/register.py new file mode 100644 index 0000000000..590f66e304 --- /dev/null +++ b/pants-plugins/uses_services/register.py @@ -0,0 +1,41 @@ +from enum import Enum +from typing import Iterable, Optional, Tuple + +# from pants.backend.python.goals.pytest_runner import PytestPluginSetupRequest, PytestPluginSetup +from pants.backend.python.target_types import PythonTests +from pants.engine.addresses import Address +from pants.engine.target import InvalidFieldChoiceException, StringSequenceField, StringField + + +class Service(Enum): + MONGO = "mongo" + RABBITMQ = "rabbitmq" + REDIS = "redis" + + +class UsesServices(StringSequenceField): + alias = "uses" + help = "Define the services that a test target depends on (mongo, rabbitmq, redis)." + valid_choices = Service + + @classmethod + def compute_value( + cls, raw_value: Optional[Iterable[str]], address: Address + ) -> Optional[Tuple[str, ...]]: + services = super().compute_value(raw_value, address) + if not services: + return services + valid_choices = (choice.value for choice in cls.valid_choices) + for service in services: + if service not in valid_choices: + raise InvalidFieldChoiceException( + address, cls.alias, service, valid_choices=valid_choices + ) + + +def rules(): + return [PythonTests.register_plugin_field(UsesServices)] + + +# def target_types(): +# return [CustomTargetType] diff --git a/pants.toml b/pants.toml index 375f9a1843..86b9499a34 100644 --- a/pants.toml +++ b/pants.toml @@ -28,6 +28,7 @@ backend_packages = [ "pack_metadata", "sample_conf", "schemas", + "uses_services", ] # pants ignores files in .gitignore, .*/ directories, /dist/ directory, and __pycache__. pants_ignore.add = [ diff --git a/st2common/tests/unit/BUILD b/st2common/tests/unit/BUILD index fe53c9f265..bc4f9fb0b4 100644 --- a/st2common/tests/unit/BUILD +++ b/st2common/tests/unit/BUILD @@ -9,6 +9,7 @@ python_tests( # several files import tests.unit.base which is ambiguous. Tell pants which one to use. "st2common/tests/unit/base.py", ], + uses=["mongo"], ) python_sources() From fbf9caa705aa5e4eeacb212ffaabc6f8b949f750 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 13 May 2021 16:02:20 -0500 Subject: [PATCH 02/71] stub instructions for mongo service --- pants-plugins/uses_services/exceptions.py | 3 ++ pants-plugins/uses_services/mongo.py | 52 +++++++++++++++++++++++ pants-plugins/uses_services/register.py | 26 +++++++----- 3 files changed, 70 insertions(+), 11 deletions(-) create mode 100644 pants-plugins/uses_services/exceptions.py create mode 100644 pants-plugins/uses_services/mongo.py diff --git a/pants-plugins/uses_services/exceptions.py b/pants-plugins/uses_services/exceptions.py new file mode 100644 index 0000000000..4167392835 --- /dev/null +++ b/pants-plugins/uses_services/exceptions.py @@ -0,0 +1,3 @@ +class ServiceMissingError(Exception): + # TODO add special platform handling to DRY instructions across services + pass diff --git a/pants-plugins/uses_services/mongo.py b/pants-plugins/uses_services/mongo.py new file mode 100644 index 0000000000..22a9f06f19 --- /dev/null +++ b/pants-plugins/uses_services/mongo.py @@ -0,0 +1,52 @@ +# TODO: this is planned / does not exist yet +from pants.backend.python.goals.pytest_runner import ( + PytestPluginSetupRequest, + PytestPluginSetup, +) +from pants.engine.rules import collect_rules, rule +from pants.engine.target import Target + +from .exceptions import ServiceMissingError + + +class UsesMongoRequest(PytestPluginSetupRequest): + @classmethod + def is_applicable(cls, target: Target) -> bool: + return "mongo" in target.get(UsesServicesField).value + + +@rule +def assert_mongo_is_running(request: UsesMongoRequest) -> PytestPluginSetup: + mongo_is_running = True + # TODO: logic to determine if it is running + if not mongo_is_running: + platform = "" # TODO: lookup + + if platform == "CentOS7": + insturctions = """ + helpful instructions for installation / running required service + """ + elif platform == "CentOS8": + insturctions = """ + helpful instructions for installation / running required service + """ + elif platform == "Ubuntu": + insturctions = """ + helpful instructions for installation / running required service + """ + elif platform == "MacOSX": + insturctions = """ + helpful instructions for installation / running required service + """ + else: + insturctions = """ + helpful instructions for installation / running required service + """ + + raise ServiceMissingError(instructions) + + return PytestPluginSetup() + + +def rules(): + return collect_rules() diff --git a/pants-plugins/uses_services/register.py b/pants-plugins/uses_services/register.py index 590f66e304..54ef165e2d 100644 --- a/pants-plugins/uses_services/register.py +++ b/pants-plugins/uses_services/register.py @@ -1,22 +1,24 @@ from enum import Enum from typing import Iterable, Optional, Tuple -# from pants.backend.python.goals.pytest_runner import PytestPluginSetupRequest, PytestPluginSetup from pants.backend.python.target_types import PythonTests from pants.engine.addresses import Address -from pants.engine.target import InvalidFieldChoiceException, StringSequenceField, StringField +from pants.engine.target import ( + InvalidFieldChoiceException, + StringSequenceField, + StringField, +) +# from . import mongo -class Service(Enum): - MONGO = "mongo" - RABBITMQ = "rabbitmq" - REDIS = "redis" +supported_services = ("mongo", "rabbitmq", "redis") -class UsesServices(StringSequenceField): + +class UsesServicesField(StringSequenceField): alias = "uses" help = "Define the services that a test target depends on (mongo, rabbitmq, redis)." - valid_choices = Service + valid_choices = supported_services @classmethod def compute_value( @@ -25,16 +27,18 @@ def compute_value( services = super().compute_value(raw_value, address) if not services: return services - valid_choices = (choice.value for choice in cls.valid_choices) for service in services: - if service not in valid_choices: + if service not in cls.valid_choices: raise InvalidFieldChoiceException( address, cls.alias, service, valid_choices=valid_choices ) def rules(): - return [PythonTests.register_plugin_field(UsesServices)] + return [ + PythonTests.register_plugin_field(UsesServicesField), + # *mongo.rules(), + ] # def target_types(): From 896c87bc16593be2998ebaf07cb25864a52716c3 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 13 May 2021 16:47:46 -0500 Subject: [PATCH 03/71] improve mongo_is_running stub to check only once per pants run --- pants-plugins/uses_services/mongo.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/pants-plugins/uses_services/mongo.py b/pants-plugins/uses_services/mongo.py index 22a9f06f19..2ecdff9367 100644 --- a/pants-plugins/uses_services/mongo.py +++ b/pants-plugins/uses_services/mongo.py @@ -1,9 +1,11 @@ +from dataclasses import dataclass + # TODO: this is planned / does not exist yet from pants.backend.python.goals.pytest_runner import ( PytestPluginSetupRequest, PytestPluginSetup, ) -from pants.engine.rules import collect_rules, rule +from pants.engine.rules import collect_rules, rule, _uncacheable_rule from pants.engine.target import Target from .exceptions import ServiceMissingError @@ -15,11 +17,22 @@ def is_applicable(cls, target: Target) -> bool: return "mongo" in target.get(UsesServicesField).value -@rule -def assert_mongo_is_running(request: UsesMongoRequest) -> PytestPluginSetup: - mongo_is_running = True +@dataclass(frozen=True) +class MongoStatus: + is_running: bool + + +@_uncacheable_rule +async def mongo_is_running() -> MongoStatus: # TODO: logic to determine if it is running - if not mongo_is_running: + # maybe something like https://stackoverflow.com/a/53640204 + # https://github.com/Lucas-C/dotfiles_and_notes/blob/master/languages/python/mongo_ping_client.py + return MongoStatus(True) + + +@rule +def assert_mongo_is_running(request: UsesMongoRequest, mongo_status: MongoStatus) -> PytestPluginSetup: + if not mongo_status.is_running: platform = "" # TODO: lookup if platform == "CentOS7": From de0a29723c82e4a828e40c18f168a6b60a25ee19 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 13 May 2021 16:48:51 -0500 Subject: [PATCH 04/71] rules are async --- pants-plugins/uses_services/mongo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pants-plugins/uses_services/mongo.py b/pants-plugins/uses_services/mongo.py index 2ecdff9367..71aa4c0b75 100644 --- a/pants-plugins/uses_services/mongo.py +++ b/pants-plugins/uses_services/mongo.py @@ -31,7 +31,7 @@ async def mongo_is_running() -> MongoStatus: @rule -def assert_mongo_is_running(request: UsesMongoRequest, mongo_status: MongoStatus) -> PytestPluginSetup: +async def assert_mongo_is_running(request: UsesMongoRequest, mongo_status: MongoStatus) -> PytestPluginSetup: if not mongo_status.is_running: platform = "" # TODO: lookup From bf26223528c8f020011508b7b2a503666a630374 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 13 May 2021 17:03:14 -0500 Subject: [PATCH 05/71] add rule for platform introspection --- pants-plugins/uses_services/mongo.py | 15 ++++++++------- pants-plugins/uses_services/platform.py | 20 ++++++++++++++++++++ pants-plugins/uses_services/register.py | 2 ++ 3 files changed, 30 insertions(+), 7 deletions(-) create mode 100644 pants-plugins/uses_services/platform.py diff --git a/pants-plugins/uses_services/mongo.py b/pants-plugins/uses_services/mongo.py index 71aa4c0b75..d3bc303eb6 100644 --- a/pants-plugins/uses_services/mongo.py +++ b/pants-plugins/uses_services/mongo.py @@ -9,6 +9,7 @@ from pants.engine.target import Target from .exceptions import ServiceMissingError +from .platform import Platform class UsesMongoRequest(PytestPluginSetupRequest): @@ -31,23 +32,23 @@ async def mongo_is_running() -> MongoStatus: @rule -async def assert_mongo_is_running(request: UsesMongoRequest, mongo_status: MongoStatus) -> PytestPluginSetup: +async def assert_mongo_is_running( + request: UsesMongoRequest, mongo_status: MongoStatus, platform: Platform +) -> PytestPluginSetup: if not mongo_status.is_running: - platform = "" # TODO: lookup - - if platform == "CentOS7": + if platform.os == "CentOS7": insturctions = """ helpful instructions for installation / running required service """ - elif platform == "CentOS8": + elif platform.os == "CentOS8": insturctions = """ helpful instructions for installation / running required service """ - elif platform == "Ubuntu": + elif platform.os == "Ubuntu": insturctions = """ helpful instructions for installation / running required service """ - elif platform == "MacOSX": + elif platform.os == "MacOSX": insturctions = """ helpful instructions for installation / running required service """ diff --git a/pants-plugins/uses_services/platform.py b/pants-plugins/uses_services/platform.py new file mode 100644 index 0000000000..010265d9be --- /dev/null +++ b/pants-plugins/uses_services/platform.py @@ -0,0 +1,20 @@ +from dataclasses import dataclass + +from pants.engine.rules import collect_rules, _uncacheable_rule + + +@dataclass(frozen=True) +class Platform: + os: str + arch: str + # etc + + +@_uncacheable_rule +async def get_platform() -> Platform: + # TODO: lookup details (use Process if needed, but prefer python's introspection) + return Platform("", "") + + +def rules(): + return collect_rules() diff --git a/pants-plugins/uses_services/register.py b/pants-plugins/uses_services/register.py index 54ef165e2d..f4b3f2aeea 100644 --- a/pants-plugins/uses_services/register.py +++ b/pants-plugins/uses_services/register.py @@ -10,6 +10,7 @@ ) # from . import mongo +from . import platform supported_services = ("mongo", "rabbitmq", "redis") @@ -37,6 +38,7 @@ def compute_value( def rules(): return [ PythonTests.register_plugin_field(UsesServicesField), + *platform.rules(), # *mongo.rules(), ] From a0cd7894bfa7b18bdcb49748fd140f11363e1553 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 14 May 2021 15:36:00 -0500 Subject: [PATCH 06/71] introspect platform --- pants-plugins/uses_services/platform.py | 28 +++++++++++++++++++++---- pants.toml | 4 ++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/pants-plugins/uses_services/platform.py b/pants-plugins/uses_services/platform.py index 010265d9be..6e9c3f96c7 100644 --- a/pants-plugins/uses_services/platform.py +++ b/pants-plugins/uses_services/platform.py @@ -1,19 +1,39 @@ +import platform from dataclasses import dataclass +import distro + from pants.engine.rules import collect_rules, _uncacheable_rule @dataclass(frozen=True) class Platform: - os: str arch: str - # etc + os: str + distro: str + distro_name: str + distro_codename: str + distro_like: str + distro_major_version: str + distro_version: str + mac_release: str + win_release: str @_uncacheable_rule async def get_platform() -> Platform: - # TODO: lookup details (use Process if needed, but prefer python's introspection) - return Platform("", "") + return Platform( + arch=platform.machine(), # x86_64 + os=platform.system(), # Linux, Darwin + distro=distro.id(), # rhel, ubuntu, centos, gentoo, darwin + distro_name=distro.name(), # Ubuntu, Centos Linux, Gentoo, Darwin + distro_codename=distro.codename(), # xenial, Core, n/a, '' + distro_like=distro.like(), # debian, rhel fedora, '', '' + distro_major_version=distro.major_version(), # 16, 7, 2, 19 + distro_version=distro.version(), # 16.04, 7, 2.7, 19.6.0 + mac_release=platform.mac_ver()[0], # '', 10.15.7 + win_release=platform.win32_ver()[0], # '' + ) def rules(): diff --git a/pants.toml b/pants.toml index 86b9499a34..14c8c5020a 100644 --- a/pants.toml +++ b/pants.toml @@ -30,6 +30,10 @@ backend_packages = [ "schemas", "uses_services", ] +plugins = [ + # dependencies for internal plugins + "distro", +] # pants ignores files in .gitignore, .*/ directories, /dist/ directory, and __pycache__. pants_ignore.add = [ # TODO: remove these once we start building wheels with pants. From 4c4f5b6fb4e0dc9900adeff525732fe497533bbb Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 14 May 2021 16:40:45 -0500 Subject: [PATCH 07/71] platform introspection in a Process() --- .../uses_services/inspect_platform.py | 41 +++++++++++++++ pants-plugins/uses_services/platform.py | 52 +++++++++---------- 2 files changed, 65 insertions(+), 28 deletions(-) create mode 100644 pants-plugins/uses_services/inspect_platform.py diff --git a/pants-plugins/uses_services/inspect_platform.py b/pants-plugins/uses_services/inspect_platform.py new file mode 100644 index 0000000000..5760a13eaa --- /dev/null +++ b/pants-plugins/uses_services/inspect_platform.py @@ -0,0 +1,41 @@ +import json + +from dataclasses import asdict, dataclass + + +@dataclass(frozen=True) +class Platform: + arch: str + os: str + distro: str + distro_name: str + distro_codename: str + distro_like: str + distro_major_version: str + distro_version: str + mac_release: str + win_release: str + + +def _get_platform() -> Platform: + # late import so that Platform can be imported in the pants plugin as well + import platform, distro + + return Platform( + arch=platform.machine(), # x86_64 + os=platform.system(), # Linux, Darwin + distro=distro.id(), # rhel, ubuntu, centos, gentoo, darwin + distro_name=distro.name(), # Ubuntu, Centos Linux, Gentoo, Darwin + distro_codename=distro.codename(), # xenial, Core, n/a, '' + distro_like=distro.like(), # debian, rhel fedora, '', '' + distro_major_version=distro.major_version(), # 16, 7, 2, 19 + distro_version=distro.version(), # 16.04, 7, 2.7, 19.6.0 + mac_release=platform.mac_ver()[0], # '', 10.15.7 + win_release=platform.win32_ver()[0], # '' + ) + + +if __name__ == "__main__": + platform = _get_platform() + platform_dict = asdict(platform) + print(json.dumps(platform_dict)) diff --git a/pants-plugins/uses_services/platform.py b/pants-plugins/uses_services/platform.py index 6e9c3f96c7..9dfe2f1b3f 100644 --- a/pants-plugins/uses_services/platform.py +++ b/pants-plugins/uses_services/platform.py @@ -1,39 +1,35 @@ -import platform -from dataclasses import dataclass - -import distro +import json +from pants.engine.fs import EMPTY_DIGEST, CreateDigest, Digest, FileContent, FileDigest +from pants.engine.process import Process, ProcessCacheScope, ProcessResult from pants.engine.rules import collect_rules, _uncacheable_rule +from .inspect_platform import Platform - -@dataclass(frozen=True) -class Platform: - arch: str - os: str - distro: str - distro_name: str - distro_codename: str - distro_like: str - distro_major_version: str - distro_version: str - mac_release: str - win_release: str +__all__ = ["Platform", "get_platform", "rules"] @_uncacheable_rule async def get_platform() -> Platform: - return Platform( - arch=platform.machine(), # x86_64 - os=platform.system(), # Linux, Darwin - distro=distro.id(), # rhel, ubuntu, centos, gentoo, darwin - distro_name=distro.name(), # Ubuntu, Centos Linux, Gentoo, Darwin - distro_codename=distro.codename(), # xenial, Core, n/a, '' - distro_like=distro.like(), # debian, rhel fedora, '', '' - distro_major_version=distro.major_version(), # 16, 7, 2, 19 - distro_version=distro.version(), # 16.04, 7, 2.7, 19.6.0 - mac_release=platform.mac_ver()[0], # '', 10.15.7 - win_release=platform.win32_ver()[0], # '' + + script_path = "./inspect_platform.py" + script_digest = await Get( + Digest, + # TODO: I need both the script and `distro`, the dependency. Do I need to build a PEX? + CreateDigest([FileContent(script_path, "", is_executable=True)]), + ) + + result = await Get( + ProcessResult, + Process( + description=f"Introspecting platform (arch, os, distro)", + input_digest=script_digest, + argv=[script_path], + # this can change from run to run, so don't cache results. + cache_scope=ProcessCacheScope.PER_RESTART, # NEVER? + ), ) + platform = json.loads(result.stdout) + return Platform(**platform) def rules(): From 90bc769f23898b59271c9dff2ce1b79d8cdd3f18 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 15 May 2021 02:06:58 -0500 Subject: [PATCH 08/71] use pex to provide distro requirement --- pants-plugins/uses_services/platform.py | 26 +++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/pants-plugins/uses_services/platform.py b/pants-plugins/uses_services/platform.py index 9dfe2f1b3f..a6d00d1c61 100644 --- a/pants-plugins/uses_services/platform.py +++ b/pants-plugins/uses_services/platform.py @@ -1,8 +1,15 @@ import json +from pants.backend.python.util_rules.pex import ( + PexRequest, + PexRequirements, + VenvPex, + VenvPexProcess, +) from pants.engine.fs import EMPTY_DIGEST, CreateDigest, Digest, FileContent, FileDigest from pants.engine.process import Process, ProcessCacheScope, ProcessResult from pants.engine.rules import collect_rules, _uncacheable_rule +from pants.util.logging import LogLevel from .inspect_platform import Platform __all__ = ["Platform", "get_platform", "rules"] @@ -11,21 +18,32 @@ @_uncacheable_rule async def get_platform() -> Platform: + distro_pex = await Get( + VenvPex, + PexRequest( + output_filename="distro.pex", + internal_only=True, + requirements=[PexRequirements({"distro"})], + ), + ) + script_path = "./inspect_platform.py" script_digest = await Get( Digest, - # TODO: I need both the script and `distro`, the dependency. Do I need to build a PEX? + # TODO: get script contents CreateDigest([FileContent(script_path, "", is_executable=True)]), ) result = await Get( ProcessResult, - Process( - description=f"Introspecting platform (arch, os, distro)", - input_digest=script_digest, + VenvPexProcess( + distro_pex, argv=[script_path], + input_digest=script_digest, + description=f"Introspecting platform (arch, os, distro)", # this can change from run to run, so don't cache results. cache_scope=ProcessCacheScope.PER_RESTART, # NEVER? + level=LogLevl.DEBUG, ), ) platform = json.loads(result.stdout) From b00fbb22adc2b606e298ba94468c9cb2b0d6dd8b Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 15 May 2021 10:03:01 -0500 Subject: [PATCH 09/71] use open to get script contents --- pants-plugins/uses_services/platform.py | 8 +++++--- pants.toml | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pants-plugins/uses_services/platform.py b/pants-plugins/uses_services/platform.py index a6d00d1c61..582f2dc8b3 100644 --- a/pants-plugins/uses_services/platform.py +++ b/pants-plugins/uses_services/platform.py @@ -10,7 +10,7 @@ from pants.engine.process import Process, ProcessCacheScope, ProcessResult from pants.engine.rules import collect_rules, _uncacheable_rule from pants.util.logging import LogLevel -from .inspect_platform import Platform +from .inspect_platform import Platform, __file__ as inspect_platform_full_path __all__ = ["Platform", "get_platform", "rules"] @@ -28,10 +28,12 @@ async def get_platform() -> Platform: ) script_path = "./inspect_platform.py" + with open(inspect_platform_full_path, "rb") as script_file: + script_contents = script_file.read() + script_digest = await Get( Digest, - # TODO: get script contents - CreateDigest([FileContent(script_path, "", is_executable=True)]), + CreateDigest([FileContent(script_path, script_contents, is_executable=True)]), ) result = await Get( diff --git a/pants.toml b/pants.toml index 14c8c5020a..485f0b5065 100644 --- a/pants.toml +++ b/pants.toml @@ -32,7 +32,7 @@ backend_packages = [ ] plugins = [ # dependencies for internal plugins - "distro", + #"distro", # dep of the pex ] # pants ignores files in .gitignore, .*/ directories, /dist/ directory, and __pycache__. pants_ignore.add = [ From 3690617a3d29491993d629dcd6a3d8076b6254e8 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 15 May 2021 10:33:57 -0500 Subject: [PATCH 10/71] Use PathGlobs to get the inspect_platform script --- pants-plugins/uses_services/platform.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/pants-plugins/uses_services/platform.py b/pants-plugins/uses_services/platform.py index 582f2dc8b3..e8ea0e08de 100644 --- a/pants-plugins/uses_services/platform.py +++ b/pants-plugins/uses_services/platform.py @@ -6,16 +6,16 @@ VenvPex, VenvPexProcess, ) -from pants.engine.fs import EMPTY_DIGEST, CreateDigest, Digest, FileContent, FileDigest +from pants.engine.fs import Digest, PathGlobs from pants.engine.process import Process, ProcessCacheScope, ProcessResult -from pants.engine.rules import collect_rules, _uncacheable_rule +from pants.engine.rules import collect_rules, rule from pants.util.logging import LogLevel -from .inspect_platform import Platform, __file__ as inspect_platform_full_path +from .inspect_platform import Platform __all__ = ["Platform", "get_platform", "rules"] -@_uncacheable_rule +@rule async def get_platform() -> Platform: distro_pex = await Get( @@ -27,14 +27,8 @@ async def get_platform() -> Platform: ), ) - script_path = "./inspect_platform.py" - with open(inspect_platform_full_path, "rb") as script_file: - script_contents = script_file.read() - - script_digest = await Get( - Digest, - CreateDigest([FileContent(script_path, script_contents, is_executable=True)]), - ) + script_path = "pants-plugins/uses_services/inspect_platform.py" + script_digest = await Get(Digest, PathGlobs([script_path])) result = await Get( ProcessResult, From 03feaf17c6e9475bf75490f48b1569507ae8ff3a Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 15 May 2021 10:36:25 -0500 Subject: [PATCH 11/71] Error if script is moved --- pants-plugins/uses_services/platform.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pants-plugins/uses_services/platform.py b/pants-plugins/uses_services/platform.py index e8ea0e08de..a2a0ad7464 100644 --- a/pants-plugins/uses_services/platform.py +++ b/pants-plugins/uses_services/platform.py @@ -9,6 +9,7 @@ from pants.engine.fs import Digest, PathGlobs from pants.engine.process import Process, ProcessCacheScope, ProcessResult from pants.engine.rules import collect_rules, rule +from pants.option.global_options import GlobMatchErrorBehavior from pants.util.logging import LogLevel from .inspect_platform import Platform @@ -28,7 +29,11 @@ async def get_platform() -> Platform: ) script_path = "pants-plugins/uses_services/inspect_platform.py" - script_digest = await Get(Digest, PathGlobs([script_path])) + script_digest = await Get( + Digest, + PathGlobs([script_path]), + glob_match_error_behavior=GlobMatchErrorBehavior.error, + ) result = await Get( ProcessResult, From a9590cc57bd3e7663df25299da17a46bce9a9f7b Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 15 May 2021 10:52:44 -0500 Subject: [PATCH 12/71] use platform object in mongo --- pants-plugins/uses_services/mongo.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/pants-plugins/uses_services/mongo.py b/pants-plugins/uses_services/mongo.py index d3bc303eb6..b1c1bce9b9 100644 --- a/pants-plugins/uses_services/mongo.py +++ b/pants-plugins/uses_services/mongo.py @@ -36,19 +36,35 @@ async def assert_mongo_is_running( request: UsesMongoRequest, mongo_status: MongoStatus, platform: Platform ) -> PytestPluginSetup: if not mongo_status.is_running: - if platform.os == "CentOS7": + if platform.distro == "centos" and platform.major_version == "7": insturctions = """ helpful instructions for installation / running required service """ - elif platform.os == "CentOS8": + elif ( + (platform.distro == "centos" and platform.major_version == "8") + or platform.distro == "rhel" + or "rhel" in platform.like + ): insturctions = """ helpful instructions for installation / running required service """ - elif platform.os == "Ubuntu": + elif platform.distro == "ubuntu" and platform.codename == "xenial": insturctions = """ helpful instructions for installation / running required service """ - elif platform.os == "MacOSX": + elif platform.distro == "ubuntu" and platform.codename == "bionic": + insturctions = """ + helpful instructions for installation / running required service + """ + elif ( + (platform.distro == "ubuntu" and platform.codename == "focal") + or platform.distro == "debian" + or "debian" in distro.like + ): + insturctions = """ + helpful instructions for installation / running required service + """ + elif platform.os == "Darwin": # Mac OS X insturctions = """ helpful instructions for installation / running required service """ From e92c2193afc52a5d0493be22560c6b17ef3f534c Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 15 May 2021 11:36:03 -0500 Subject: [PATCH 13/71] Add mongo install instructions --- pants-plugins/uses_services/exceptions.py | 11 +- pants-plugins/uses_services/mongo.py | 122 +++++++++++++++++----- 2 files changed, 107 insertions(+), 26 deletions(-) diff --git a/pants-plugins/uses_services/exceptions.py b/pants-plugins/uses_services/exceptions.py index 4167392835..b58859fcac 100644 --- a/pants-plugins/uses_services/exceptions.py +++ b/pants-plugins/uses_services/exceptions.py @@ -1,3 +1,12 @@ class ServiceMissingError(Exception): + """Error raised when a test uses a service but that service is missing.""" # TODO add special platform handling to DRY instructions across services - pass + + def __init__(self, service, instructions="", msg=None): + if msg is None: + msg = f"The {service} service does not seem to be running or is not accessible!" + if instructions: + msg += f"\n{instructions}" + super().__init__(msg) + self.service = service + self.instructions = instructions diff --git a/pants-plugins/uses_services/mongo.py b/pants-plugins/uses_services/mongo.py index b1c1bce9b9..f8c1553a43 100644 --- a/pants-plugins/uses_services/mongo.py +++ b/pants-plugins/uses_services/mongo.py @@ -1,5 +1,7 @@ from dataclasses import dataclass +from textwrap import dedent + # TODO: this is planned / does not exist yet from pants.backend.python.goals.pytest_runner import ( PytestPluginSetupRequest, @@ -36,44 +38,114 @@ async def assert_mongo_is_running( request: UsesMongoRequest, mongo_status: MongoStatus, platform: Platform ) -> PytestPluginSetup: if not mongo_status.is_running: - if platform.distro == "centos" and platform.major_version == "7": - insturctions = """ - helpful instructions for installation / running required service + if platform.distro in ["ubuntu", "debian"] or "debian" in distro.like: + insturctions = dedent( + """\ + If mongo is installed, but not running try: + + systemctl start mongod + + If mongo is not installed, this is one way to install it: + + # Add key and repo for the latest stable MongoDB (4.0) + rpm --import https://www.mongodb.org/static/pgp/server-4.0.asc + sh -c "cat < /etc/yum.repos.d/mongodb-org-4.repo + [mongodb-org-4] + name=MongoDB Repository + baseurl=https://repo.mongodb.org/yum/redhat/${OSRELEASE_VERSION}/mongodb-org/4.0/x86_64/ + gpgcheck=1 + enabled=1 + gpgkey=https://www.mongodb.org/static/pgp/server-4.0.asc + EOT" + # install mongo + yum install mongodb-org + # Don't forget to start mongo. + """ + ) + elif platform.distro == "centos" and platform.major_version == "7": + insturctions = dedent( + """\ + If mongo is installed, but not running try: + + service mongo start + + If mongo is not installed, this is one way to install it: + + apt-get install mongodb mongodb-server """ elif ( (platform.distro == "centos" and platform.major_version == "8") or platform.distro == "rhel" or "rhel" in platform.like ): - insturctions = """ - helpful instructions for installation / running required service - """ - elif platform.distro == "ubuntu" and platform.codename == "xenial": - insturctions = """ - helpful instructions for installation / running required service - """ - elif platform.distro == "ubuntu" and platform.codename == "bionic": - insturctions = """ - helpful instructions for installation / running required service + insturctions = dedent( + """\ + If mongo is installed, but not running try: + + systemctl start mongod + + If mongo is not installed, this is one way to install it: + + apt-get install mongodb mongodb-server """ - elif ( - (platform.distro == "ubuntu" and platform.codename == "focal") - or platform.distro == "debian" - or "debian" in distro.like - ): - insturctions = """ - helpful instructions for installation / running required service + ) + elif platform.os == "Linux": + insturctions = dedent( + f"""\ + You are on Linux using {platform.distro_name}, which is not + one of our generally supported distributions. We recommend + you use vagrant for local development with something like: + + vagrant init stackstorm/st2 + vagrant up + vagrant ssh + + Please see: https://docs.stackstorm.com/install/vagrant.html + + For anyone who wants to attempt local development without vagrant, + you are pretty much on your own. At a minimum you need to install + and start mongo with something like: + + systemctl start mongod + + We would be interested to hear about alternative distros people + are using for development. If you are able, please let us know + on slack which distro you are using: + + Arch: {platform.arch} + Distro: {platform.distro} + Distro Codename: {platform.distro_codename} + Distro Version: {platform.distro_version} + + Thanks and Good Luck! """ + ) elif platform.os == "Darwin": # Mac OS X - insturctions = """ - helpful instructions for installation / running required service + insturctions = dedent( + """\ + TODO: Mac OS X instructions for installing/starting mongo """ + ) else: - insturctions = """ - helpful instructions for installation / running required service + insturctions = dedent( + """\ + You are not on Linux. In this case we recommend using vagrant + for local development with something like: + + vagrant init stackstorm/st2 + vagrant up + vagrant ssh + + Please see: https://docs.stackstorm.com/install/vagrant.html + + For anyone who wants to attempt local development without vagrant, + you are pretty much on your own. At a minimum you need to install + and start mongo. Good luck! """ + ) + - raise ServiceMissingError(instructions) + raise ServiceMissingError("mongo", instructions) return PytestPluginSetup() From acb5b30664102cf8059fdc0f62744ff11356006e Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 15 May 2021 11:41:12 -0500 Subject: [PATCH 14/71] add basic instructions for Mac --- pants-plugins/uses_services/mongo.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/pants-plugins/uses_services/mongo.py b/pants-plugins/uses_services/mongo.py index f8c1553a43..d13753c431 100644 --- a/pants-plugins/uses_services/mongo.py +++ b/pants-plugins/uses_services/mongo.py @@ -120,10 +120,22 @@ async def assert_mongo_is_running( Thanks and Good Luck! """ ) - elif platform.os == "Darwin": # Mac OS X + elif platform.os == "Darwin": # MacOS insturctions = dedent( """\ - TODO: Mac OS X instructions for installing/starting mongo + You are on Mac OS. Generally we recommend using vagrant for local + development on Mac OS with something like: + + vagrant init stackstorm/st2 + vagrant up + vagrant ssh + + Please see: https://docs.stackstorm.com/install/vagrant.html + + For anyone who wants to attempt local development without vagrant, + you may run into some speed bumps. Others StackStorm developers have + been known to use Mac OS for development, so feel free to ask for + help in slack. At a minimum you need to install and start mongo. """ ) else: From 172247f363ef68e6a559f8875230955d0a9afe40 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 15 May 2021 11:43:10 -0500 Subject: [PATCH 15/71] enhance exception to include platform info --- pants-plugins/uses_services/exceptions.py | 3 ++- pants-plugins/uses_services/mongo.py | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pants-plugins/uses_services/exceptions.py b/pants-plugins/uses_services/exceptions.py index b58859fcac..823f7a9250 100644 --- a/pants-plugins/uses_services/exceptions.py +++ b/pants-plugins/uses_services/exceptions.py @@ -2,11 +2,12 @@ class ServiceMissingError(Exception): """Error raised when a test uses a service but that service is missing.""" # TODO add special platform handling to DRY instructions across services - def __init__(self, service, instructions="", msg=None): + def __init__(self, service, platform: "Platform", instructions="", msg=None): if msg is None: msg = f"The {service} service does not seem to be running or is not accessible!" if instructions: msg += f"\n{instructions}" super().__init__(msg) self.service = service + self.platform = platform self.instructions = instructions diff --git a/pants-plugins/uses_services/mongo.py b/pants-plugins/uses_services/mongo.py index d13753c431..fdf94f5f7c 100644 --- a/pants-plugins/uses_services/mongo.py +++ b/pants-plugins/uses_services/mongo.py @@ -156,8 +156,7 @@ async def assert_mongo_is_running( """ ) - - raise ServiceMissingError("mongo", instructions) + raise ServiceMissingError("mongo", platform, instructions) return PytestPluginSetup() From cfc932a3c2f8b4af7e5aac06f96b020157978f9b Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 15 May 2021 11:48:42 -0500 Subject: [PATCH 16/71] use right instructions per platform --- pants-plugins/uses_services/mongo.py | 31 +++++++++++----------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/pants-plugins/uses_services/mongo.py b/pants-plugins/uses_services/mongo.py index fdf94f5f7c..93969baacf 100644 --- a/pants-plugins/uses_services/mongo.py +++ b/pants-plugins/uses_services/mongo.py @@ -38,13 +38,21 @@ async def assert_mongo_is_running( request: UsesMongoRequest, mongo_status: MongoStatus, platform: Platform ) -> PytestPluginSetup: if not mongo_status.is_running: - if platform.distro in ["ubuntu", "debian"] or "debian" in distro.like: + elif platform.distro in ["centos", "rhel"] or "rhel" in platform.like: insturctions = dedent( - """\ + f"""\ If mongo is installed, but not running try: - systemctl start mongod + """ + ) + + if platform.major_version == "7" + instructions += "\nservice mongo start\n" + else: + instructions += "\nsystemctl start mongod\n" + instructions += dedent( + """ If mongo is not installed, this is one way to install it: # Add key and repo for the latest stable MongoDB (4.0) @@ -62,22 +70,7 @@ async def assert_mongo_is_running( # Don't forget to start mongo. """ ) - elif platform.distro == "centos" and platform.major_version == "7": - insturctions = dedent( - """\ - If mongo is installed, but not running try: - - service mongo start - - If mongo is not installed, this is one way to install it: - - apt-get install mongodb mongodb-server - """ - elif ( - (platform.distro == "centos" and platform.major_version == "8") - or platform.distro == "rhel" - or "rhel" in platform.like - ): + elif platform.distro in ["ubuntu", "debian"] or "debian" in distro.like: insturctions = dedent( """\ If mongo is installed, but not running try: From 59e72e87c7f37a7ed3f4f75578e16dacdeeb6ffe Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 15 May 2021 11:49:46 -0500 Subject: [PATCH 17/71] add note --- pants-plugins/uses_services/mongo.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pants-plugins/uses_services/mongo.py b/pants-plugins/uses_services/mongo.py index 93969baacf..3e94b140db 100644 --- a/pants-plugins/uses_services/mongo.py +++ b/pants-plugins/uses_services/mongo.py @@ -80,6 +80,7 @@ async def assert_mongo_is_running( If mongo is not installed, this is one way to install it: apt-get install mongodb mongodb-server + # Don't forget to start mongo. """ ) elif platform.os == "Linux": From c9bfed1c96229d4c1eb788ac4648799dc574db21 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 15 May 2021 11:56:26 -0500 Subject: [PATCH 18/71] revert to open() file per feedback in slack --- pants-plugins/uses_services/platform.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/pants-plugins/uses_services/platform.py b/pants-plugins/uses_services/platform.py index a2a0ad7464..2da907f2c5 100644 --- a/pants-plugins/uses_services/platform.py +++ b/pants-plugins/uses_services/platform.py @@ -6,12 +6,12 @@ VenvPex, VenvPexProcess, ) -from pants.engine.fs import Digest, PathGlobs +from pants.engine.fs import CreateDigest, Digest, FileContent from pants.engine.process import Process, ProcessCacheScope, ProcessResult from pants.engine.rules import collect_rules, rule from pants.option.global_options import GlobMatchErrorBehavior from pants.util.logging import LogLevel -from .inspect_platform import Platform +from .inspect_platform import Platform, __file__ as inspect_platform_full_path __all__ = ["Platform", "get_platform", "rules"] @@ -28,11 +28,16 @@ async def get_platform() -> Platform: ), ) - script_path = "pants-plugins/uses_services/inspect_platform.py" + script_path = "./inspect_platform.py" + + # pants is already watching this directory as it is under a source root. + # So, we don't need to double watch with PathGlobs, just open it. + with open(inspect_platform_full_path, "rb") as script_file: + script_contents = script_file.read() + script_digest = await Get( Digest, - PathGlobs([script_path]), - glob_match_error_behavior=GlobMatchErrorBehavior.error, + CreateDigest([FileContent(script_path, script_contents)]), ) result = await Get( From 392f1b9814d593d1f0e141d87cb46428d031f645 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sun, 16 May 2021 01:36:26 -0500 Subject: [PATCH 19/71] notes on db host,port,name used in tests --- pants-plugins/uses_services/mongo.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/pants-plugins/uses_services/mongo.py b/pants-plugins/uses_services/mongo.py index 3e94b140db..8d0bd2f958 100644 --- a/pants-plugins/uses_services/mongo.py +++ b/pants-plugins/uses_services/mongo.py @@ -27,9 +27,34 @@ class MongoStatus: @_uncacheable_rule async def mongo_is_running() -> MongoStatus: + # These config opts are used via oslo_config.cfg.CONF.database.{host,port,db_name} + # These config opts currently hard-coded in: + # for unit tests: st2tests/st2tests/config.py + # for integration tests: conf/st2.tests*.conf st2tests/st2tests/fixtures/conf/st2.tests*.conf + # (changed by setting ST2_CONFIG_PATH env var inside the tests) + # TODO: for unit tests: modify code to pull from an env var and then use per-pantsd-slot db_name + # TODO: for int tests: modify st2.tests*.conf on the fly to set the per-pantsd-slot db_name + _db_host = "127.0.0.1" # localhost in test_db.DbConnectionTestCase + _db_port = 27017 + _db_name = "st2-test" # st2 in test_db.DbConnectionTestCase + + # so st2-test database gets dropped between: + # - each component (st2*/ && various config/ dirs) in Makefile + # - DbTestCase/CleanDbTestCase setUpClass + + # with our version of oslo.config (newer are slower) we can't directly override opts w/ environment variables. + + # Makefile + # .run-unit-tests-with-coverage (<- .combine-unit-tests-coverage <- .coverage.unit <- .unit-tests-coverage-html <- ci-unit <- ci) + # echo "----- Dropping st2-test db -----" + # mongo st2-test --eval "db.dropDatabase();" + # for component in $(COMPONENTS_TEST) + # nosetests $(NOSE_OPTS) -s -v $(NOSE_COVERAGE_FLAGS) $(NOSE_COVERAGE_PACKAGES) $$component/tests/unit + # TODO: logic to determine if it is running # maybe something like https://stackoverflow.com/a/53640204 # https://github.com/Lucas-C/dotfiles_and_notes/blob/master/languages/python/mongo_ping_client.py + # download it? return MongoStatus(True) From acdc8fa82ff05338e33300d804c15ce3ecd049ec Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sun, 16 May 2021 15:42:03 -0500 Subject: [PATCH 20/71] add __all__ to inspect_platform --- pants-plugins/uses_services/inspect_platform.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pants-plugins/uses_services/inspect_platform.py b/pants-plugins/uses_services/inspect_platform.py index 5760a13eaa..519ae8ebee 100644 --- a/pants-plugins/uses_services/inspect_platform.py +++ b/pants-plugins/uses_services/inspect_platform.py @@ -2,6 +2,8 @@ from dataclasses import asdict, dataclass +__all__ = ["__file__", "Platform"] + @dataclass(frozen=True) class Platform: From e07ea59cac5456a4137d20062d26836ddc451e5b Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sun, 16 May 2021 15:42:48 -0500 Subject: [PATCH 21/71] add is_mongo_running check to pants-plugin --- .../uses_services/is_mongo_running.py | 39 ++++++++++++++ pants-plugins/uses_services/mongo.py | 52 +++++++++++++++---- 2 files changed, 81 insertions(+), 10 deletions(-) create mode 100644 pants-plugins/uses_services/is_mongo_running.py diff --git a/pants-plugins/uses_services/is_mongo_running.py b/pants-plugins/uses_services/is_mongo_running.py new file mode 100644 index 0000000000..b2d36fff0e --- /dev/null +++ b/pants-plugins/uses_services/is_mongo_running.py @@ -0,0 +1,39 @@ +import sys + +__all__ = ["__file__"] + + +def _is_mongo_running(db_host: str, db_port: int, db_name: str, connection_timeout_ms: int) -> bool: + # late import so that __file__ can be imported in the pants plugin without these imports + import mongoengine + from pymongo.errors import ConnectionFailure + from pymongo.errors import ServerSelectionTimeoutError + + # cf st2common.models.db.setup() + connection = mongoengine.connection.connect( + db_name, + host=db_host, + port=db_port, + connectTimeoutMS=connection_timeout_ms, + serverSelectionTimeoutMS=connection_timeout_ms, + ) + + # connection.connect() is lazy. Make a command to test connection. + try: + # The ismaster command is cheap and does not require auth + connection.admin.command("ismaster") + except (ConnectionFailure, ServerSelectionTimeoutError): + return False + return True + + +if __name__ == "__main__": + args = dict(enumerate(sys.argv)) + db_host = args.get(1, "127.0.0.1") + db_port = args.get(2, 27017) + db_name = args.get(3, "st2-test") + connection_timeout_ms = args.get(4, 3000) + + is_running = _is_mongo_running(db_host, int(db_port), db_name, int(connection_timeout_ms)) + exit_code = 0 if is_running else 1 + sys.exit(exit_code) diff --git a/pants-plugins/uses_services/mongo.py b/pants-plugins/uses_services/mongo.py index 8d0bd2f958..707a91176f 100644 --- a/pants-plugins/uses_services/mongo.py +++ b/pants-plugins/uses_services/mongo.py @@ -11,6 +11,7 @@ from pants.engine.target import Target from .exceptions import ServiceMissingError +from .is_mongo_running import __file__ as is_mongo_running_full_path from .platform import Platform @@ -25,18 +26,20 @@ class MongoStatus: is_running: bool -@_uncacheable_rule +@rule async def mongo_is_running() -> MongoStatus: - # These config opts are used via oslo_config.cfg.CONF.database.{host,port,db_name} + # These config opts are used via oslo_config.cfg.CONF.database.{host,port,db_name,connection_timeout} # These config opts currently hard-coded in: # for unit tests: st2tests/st2tests/config.py # for integration tests: conf/st2.tests*.conf st2tests/st2tests/fixtures/conf/st2.tests*.conf # (changed by setting ST2_CONFIG_PATH env var inside the tests) # TODO: for unit tests: modify code to pull from an env var and then use per-pantsd-slot db_name # TODO: for int tests: modify st2.tests*.conf on the fly to set the per-pantsd-slot db_name - _db_host = "127.0.0.1" # localhost in test_db.DbConnectionTestCase - _db_port = 27017 - _db_name = "st2-test" # st2 in test_db.DbConnectionTestCase + + db_host = "127.0.0.1" # localhost in test_db.DbConnectionTestCase + db_port = 27017 + db_name = "st2-test" # st2 in test_db.DbConnectionTestCase + connection_timeout = 3000 # so st2-test database gets dropped between: # - each component (st2*/ && various config/ dirs) in Makefile @@ -51,11 +54,40 @@ async def mongo_is_running() -> MongoStatus: # for component in $(COMPONENTS_TEST) # nosetests $(NOSE_OPTS) -s -v $(NOSE_COVERAGE_FLAGS) $(NOSE_COVERAGE_PACKAGES) $$component/tests/unit - # TODO: logic to determine if it is running - # maybe something like https://stackoverflow.com/a/53640204 - # https://github.com/Lucas-C/dotfiles_and_notes/blob/master/languages/python/mongo_ping_client.py - # download it? - return MongoStatus(True) + mongoengine_pex = await Get( + VenvPex, + PexRequest( + output_filename="mongoengine.pex", + internal_only=True, + requirements=[PexRequirements({"mongoengine", "pymongo"})], + ), + ) + + script_path = "./is_mongo_running.py" + + # pants is already watching this directory as it is under a source root. + # So, we don't need to double watch with PathGlobs, just open it. + with open(is_mongo_running_full_path, "rb") as script_file: + script_contents = script_file.read() + + script_digest = await Get( + Digest, + CreateDigest([FileContent(script_path, script_contents)]), + ) + + result = await Get( + FallibleProcessResult, + VenvPexProcess( + mongoengine_pex, + argv=[script_path, db_host, db_port, db_name, connection_timeout], + input_digest=script_digest, + description=f"Checking to see if Mongo is up and accessible.", + # this can change from run to run, so don't cache results. + cache_scope=ProcessCacheScope.PER_RESTART, # NEVER? + level=LogLevl.DEBUG, + ), + ) + return MongoStatus(result.exit_code == 0) @rule From 299f962a94fe975ad05f4305e54c00d84902a175 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 18 May 2021 11:47:04 -0500 Subject: [PATCH 22/71] do not shadow platform module --- pants-plugins/uses_services/mongo.py | 2 +- pants-plugins/uses_services/{platform.py => platform_.py} | 0 pants-plugins/uses_services/register.py | 4 ++-- 3 files changed, 3 insertions(+), 3 deletions(-) rename pants-plugins/uses_services/{platform.py => platform_.py} (100%) diff --git a/pants-plugins/uses_services/mongo.py b/pants-plugins/uses_services/mongo.py index 707a91176f..0c3737b7de 100644 --- a/pants-plugins/uses_services/mongo.py +++ b/pants-plugins/uses_services/mongo.py @@ -12,7 +12,7 @@ from .exceptions import ServiceMissingError from .is_mongo_running import __file__ as is_mongo_running_full_path -from .platform import Platform +from .platform_ import Platform class UsesMongoRequest(PytestPluginSetupRequest): diff --git a/pants-plugins/uses_services/platform.py b/pants-plugins/uses_services/platform_.py similarity index 100% rename from pants-plugins/uses_services/platform.py rename to pants-plugins/uses_services/platform_.py diff --git a/pants-plugins/uses_services/register.py b/pants-plugins/uses_services/register.py index f4b3f2aeea..c6fe7f6ad7 100644 --- a/pants-plugins/uses_services/register.py +++ b/pants-plugins/uses_services/register.py @@ -10,7 +10,7 @@ ) # from . import mongo -from . import platform +from . import platform_ supported_services = ("mongo", "rabbitmq", "redis") @@ -38,7 +38,7 @@ def compute_value( def rules(): return [ PythonTests.register_plugin_field(UsesServicesField), - *platform.rules(), + *platform_.rules(), # *mongo.rules(), ] From 090b4cbdadacf89d009e6cc859247fd9f29ab0cd Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 19 May 2021 11:23:05 -0500 Subject: [PATCH 23/71] pants: clean up uses_services pants plugin --- pants-plugins/uses_services/mongo.py | 30 ++++++++++++------ pants-plugins/uses_services/platform_.py | 7 ++--- pants-plugins/uses_services/register.py | 32 +------------------- pants-plugins/uses_services/uses_services.py | 27 +++++++++++++++++ 4 files changed, 51 insertions(+), 45 deletions(-) create mode 100644 pants-plugins/uses_services/uses_services.py diff --git a/pants-plugins/uses_services/mongo.py b/pants-plugins/uses_services/mongo.py index 0c3737b7de..8a75d35f85 100644 --- a/pants-plugins/uses_services/mongo.py +++ b/pants-plugins/uses_services/mongo.py @@ -7,12 +7,22 @@ PytestPluginSetupRequest, PytestPluginSetup, ) -from pants.engine.rules import collect_rules, rule, _uncacheable_rule +from pants.backend.python.util_rules.pex import ( + PexRequest, + PexRequirements, + VenvPex, + VenvPexProcess, +) +from pants.engine.fs import CreateDigest, Digest, FileContent +from pants.engine.rules import collect_rules, Get, rule +from pants.engine.process import FallibleProcessResult, ProcessCacheScope from pants.engine.target import Target +from pants.util.logging import LogLevel from .exceptions import ServiceMissingError from .is_mongo_running import __file__ as is_mongo_running_full_path from .platform_ import Platform +from .uses_services import UsesServicesField class UsesMongoRequest(PytestPluginSetupRequest): @@ -84,7 +94,7 @@ async def mongo_is_running() -> MongoStatus: description=f"Checking to see if Mongo is up and accessible.", # this can change from run to run, so don't cache results. cache_scope=ProcessCacheScope.PER_RESTART, # NEVER? - level=LogLevl.DEBUG, + level=LogLevel.DEBUG, ), ) return MongoStatus(result.exit_code == 0) @@ -95,15 +105,15 @@ async def assert_mongo_is_running( request: UsesMongoRequest, mongo_status: MongoStatus, platform: Platform ) -> PytestPluginSetup: if not mongo_status.is_running: - elif platform.distro in ["centos", "rhel"] or "rhel" in platform.like: - insturctions = dedent( + if platform.distro in ["centos", "rhel"] or "rhel" in platform.distro_like: + instructions = dedent( f"""\ If mongo is installed, but not running try: """ ) - if platform.major_version == "7" + if platform.distro_major_version == "7": instructions += "\nservice mongo start\n" else: instructions += "\nsystemctl start mongod\n" @@ -127,8 +137,8 @@ async def assert_mongo_is_running( # Don't forget to start mongo. """ ) - elif platform.distro in ["ubuntu", "debian"] or "debian" in distro.like: - insturctions = dedent( + elif platform.distro in ["ubuntu", "debian"] or "debian" in platform.distro_like: + instructions = dedent( """\ If mongo is installed, but not running try: @@ -141,7 +151,7 @@ async def assert_mongo_is_running( """ ) elif platform.os == "Linux": - insturctions = dedent( + instructions = dedent( f"""\ You are on Linux using {platform.distro_name}, which is not one of our generally supported distributions. We recommend @@ -172,7 +182,7 @@ async def assert_mongo_is_running( """ ) elif platform.os == "Darwin": # MacOS - insturctions = dedent( + instructions = dedent( """\ You are on Mac OS. Generally we recommend using vagrant for local development on Mac OS with something like: @@ -190,7 +200,7 @@ async def assert_mongo_is_running( """ ) else: - insturctions = dedent( + instructions = dedent( """\ You are not on Linux. In this case we recommend using vagrant for local development with something like: diff --git a/pants-plugins/uses_services/platform_.py b/pants-plugins/uses_services/platform_.py index 2da907f2c5..3e74ed23c0 100644 --- a/pants-plugins/uses_services/platform_.py +++ b/pants-plugins/uses_services/platform_.py @@ -7,9 +7,8 @@ VenvPexProcess, ) from pants.engine.fs import CreateDigest, Digest, FileContent -from pants.engine.process import Process, ProcessCacheScope, ProcessResult -from pants.engine.rules import collect_rules, rule -from pants.option.global_options import GlobMatchErrorBehavior +from pants.engine.process import ProcessCacheScope, ProcessResult +from pants.engine.rules import collect_rules, Get, rule from pants.util.logging import LogLevel from .inspect_platform import Platform, __file__ as inspect_platform_full_path @@ -49,7 +48,7 @@ async def get_platform() -> Platform: description=f"Introspecting platform (arch, os, distro)", # this can change from run to run, so don't cache results. cache_scope=ProcessCacheScope.PER_RESTART, # NEVER? - level=LogLevl.DEBUG, + level=LogLevel.DEBUG, ), ) platform = json.loads(result.stdout) diff --git a/pants-plugins/uses_services/register.py b/pants-plugins/uses_services/register.py index c6fe7f6ad7..4b816960f2 100644 --- a/pants-plugins/uses_services/register.py +++ b/pants-plugins/uses_services/register.py @@ -1,38 +1,8 @@ -from enum import Enum -from typing import Iterable, Optional, Tuple - from pants.backend.python.target_types import PythonTests -from pants.engine.addresses import Address -from pants.engine.target import ( - InvalidFieldChoiceException, - StringSequenceField, - StringField, -) # from . import mongo from . import platform_ - - -supported_services = ("mongo", "rabbitmq", "redis") - - -class UsesServicesField(StringSequenceField): - alias = "uses" - help = "Define the services that a test target depends on (mongo, rabbitmq, redis)." - valid_choices = supported_services - - @classmethod - def compute_value( - cls, raw_value: Optional[Iterable[str]], address: Address - ) -> Optional[Tuple[str, ...]]: - services = super().compute_value(raw_value, address) - if not services: - return services - for service in services: - if service not in cls.valid_choices: - raise InvalidFieldChoiceException( - address, cls.alias, service, valid_choices=valid_choices - ) +from .uses_services import UsesServicesField def rules(): diff --git a/pants-plugins/uses_services/uses_services.py b/pants-plugins/uses_services/uses_services.py new file mode 100644 index 0000000000..be4e74031f --- /dev/null +++ b/pants-plugins/uses_services/uses_services.py @@ -0,0 +1,27 @@ +# coding: utf-8 +from typing import Iterable, Optional, Tuple + +from pants.build_graph.address import Address +from pants.engine.target import InvalidFieldChoiceException, StringSequenceField + + +supported_services = ("mongo", "rabbitmq", "redis") + + +class UsesServicesField(StringSequenceField): + alias = "uses" + help = "Define the services that a test target depends on (mongo, rabbitmq, redis)." + valid_choices = supported_services + + @classmethod + def compute_value( + cls, raw_value: Optional[Iterable[str]], address: Address + ) -> Optional[Tuple[str, ...]]: + services = super().compute_value(raw_value, address) + if not services: + return services + for service in services: + if service not in cls.valid_choices: + raise InvalidFieldChoiceException( + address, cls.alias, service, valid_choices=cls.valid_choices + ) \ No newline at end of file From 6e05dca578fa5c169abb4511220bfedcb2c42dd9 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 19 May 2021 12:21:27 -0500 Subject: [PATCH 24/71] pants: shortform Get has to have only 1 comma plus a few other fixes in uses_services plugin --- pants-plugins/uses_services/inspect_platform.py | 2 +- pants-plugins/uses_services/mongo.py | 6 +++--- pants-plugins/uses_services/register.py | 8 ++------ pants-plugins/uses_services/uses_services.py | 2 +- 4 files changed, 7 insertions(+), 11 deletions(-) diff --git a/pants-plugins/uses_services/inspect_platform.py b/pants-plugins/uses_services/inspect_platform.py index 519ae8ebee..dfa095acfa 100644 --- a/pants-plugins/uses_services/inspect_platform.py +++ b/pants-plugins/uses_services/inspect_platform.py @@ -2,7 +2,7 @@ from dataclasses import asdict, dataclass -__all__ = ["__file__", "Platform"] +__all__ = ["Platform"] @dataclass(frozen=True) diff --git a/pants-plugins/uses_services/mongo.py b/pants-plugins/uses_services/mongo.py index 8a75d35f85..26bb4dd6f8 100644 --- a/pants-plugins/uses_services/mongo.py +++ b/pants-plugins/uses_services/mongo.py @@ -70,7 +70,7 @@ async def mongo_is_running() -> MongoStatus: output_filename="mongoengine.pex", internal_only=True, requirements=[PexRequirements({"mongoengine", "pymongo"})], - ), + ) ) script_path = "./is_mongo_running.py" @@ -82,7 +82,7 @@ async def mongo_is_running() -> MongoStatus: script_digest = await Get( Digest, - CreateDigest([FileContent(script_path, script_contents)]), + CreateDigest([FileContent(script_path, script_contents)]) ) result = await Get( @@ -95,7 +95,7 @@ async def mongo_is_running() -> MongoStatus: # this can change from run to run, so don't cache results. cache_scope=ProcessCacheScope.PER_RESTART, # NEVER? level=LogLevel.DEBUG, - ), + ) ) return MongoStatus(result.exit_code == 0) diff --git a/pants-plugins/uses_services/register.py b/pants-plugins/uses_services/register.py index 4b816960f2..8c9c1d9b8b 100644 --- a/pants-plugins/uses_services/register.py +++ b/pants-plugins/uses_services/register.py @@ -1,6 +1,6 @@ from pants.backend.python.target_types import PythonTests -# from . import mongo +from . import mongo from . import platform_ from .uses_services import UsesServicesField @@ -9,9 +9,5 @@ def rules(): return [ PythonTests.register_plugin_field(UsesServicesField), *platform_.rules(), - # *mongo.rules(), + *mongo.rules(), ] - - -# def target_types(): -# return [CustomTargetType] diff --git a/pants-plugins/uses_services/uses_services.py b/pants-plugins/uses_services/uses_services.py index be4e74031f..51fcc1fde9 100644 --- a/pants-plugins/uses_services/uses_services.py +++ b/pants-plugins/uses_services/uses_services.py @@ -24,4 +24,4 @@ def compute_value( if service not in cls.valid_choices: raise InvalidFieldChoiceException( address, cls.alias, service, valid_choices=cls.valid_choices - ) \ No newline at end of file + ) From 6bb5a2e206b0231b793444634ff00a5a3cd46db3 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 19 May 2021 23:55:11 -0500 Subject: [PATCH 25/71] pants: cleanup use_services plugin --- pants-plugins/uses_services/is_mongo_running.py | 6 +++--- pants-plugins/uses_services/mongo.py | 12 ++++++------ pants-plugins/uses_services/register.py | 8 ++------ pants-plugins/uses_services/uses_services.py | 5 +++++ 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/pants-plugins/uses_services/is_mongo_running.py b/pants-plugins/uses_services/is_mongo_running.py index b2d36fff0e..8a0e9c6263 100644 --- a/pants-plugins/uses_services/is_mongo_running.py +++ b/pants-plugins/uses_services/is_mongo_running.py @@ -1,6 +1,5 @@ import sys - -__all__ = ["__file__"] +from typing import cast, Iterable, Tuple def _is_mongo_running(db_host: str, db_port: int, db_name: str, connection_timeout_ms: int) -> bool: @@ -28,7 +27,8 @@ def _is_mongo_running(db_host: str, db_port: int, db_name: str, connection_timeo if __name__ == "__main__": - args = dict(enumerate(sys.argv)) + args = cast(Iterable[Tuple[int, str]], enumerate(sys.argv)) + args = dict(args) db_host = args.get(1, "127.0.0.1") db_port = args.get(2, 27017) db_name = args.get(3, "st2-test") diff --git a/pants-plugins/uses_services/mongo.py b/pants-plugins/uses_services/mongo.py index 26bb4dd6f8..b5373a3a2e 100644 --- a/pants-plugins/uses_services/mongo.py +++ b/pants-plugins/uses_services/mongo.py @@ -19,10 +19,10 @@ from pants.engine.target import Target from pants.util.logging import LogLevel -from .exceptions import ServiceMissingError -from .is_mongo_running import __file__ as is_mongo_running_full_path -from .platform_ import Platform -from .uses_services import UsesServicesField +from uses_services.exceptions import ServiceMissingError +from uses_services.is_mongo_running import __file__ as is_mongo_running_full_path +from uses_services.platform_ import Platform +from uses_services.uses_services import UsesServicesField class UsesMongoRequest(PytestPluginSetupRequest): @@ -36,7 +36,7 @@ class MongoStatus: is_running: bool -@rule +@rule(desc="Test to see if mongodb is running and accessible for tests.") async def mongo_is_running() -> MongoStatus: # These config opts are used via oslo_config.cfg.CONF.database.{host,port,db_name,connection_timeout} # These config opts currently hard-coded in: @@ -100,7 +100,7 @@ async def mongo_is_running() -> MongoStatus: return MongoStatus(result.exit_code == 0) -@rule +@rule("Report that mongodb is required for test runs.") async def assert_mongo_is_running( request: UsesMongoRequest, mongo_status: MongoStatus, platform: Platform ) -> PytestPluginSetup: diff --git a/pants-plugins/uses_services/register.py b/pants-plugins/uses_services/register.py index 8c9c1d9b8b..85c3a566bf 100644 --- a/pants-plugins/uses_services/register.py +++ b/pants-plugins/uses_services/register.py @@ -1,13 +1,9 @@ -from pants.backend.python.target_types import PythonTests - -from . import mongo -from . import platform_ -from .uses_services import UsesServicesField +from uses_services import mongo, platform_, uses_services def rules(): return [ - PythonTests.register_plugin_field(UsesServicesField), + *uses_services.rules(), *platform_.rules(), *mongo.rules(), ] diff --git a/pants-plugins/uses_services/uses_services.py b/pants-plugins/uses_services/uses_services.py index 51fcc1fde9..7838501518 100644 --- a/pants-plugins/uses_services/uses_services.py +++ b/pants-plugins/uses_services/uses_services.py @@ -1,6 +1,7 @@ # coding: utf-8 from typing import Iterable, Optional, Tuple +from pants.backend.python.target_types import PythonTests from pants.build_graph.address import Address from pants.engine.target import InvalidFieldChoiceException, StringSequenceField @@ -25,3 +26,7 @@ def compute_value( raise InvalidFieldChoiceException( address, cls.alias, service, valid_choices=cls.valid_choices ) + + +def rules(): + return [PythonTests.register_plugin_field(UsesServicesField)] From c4f3e24d0c21d402182d3751c95776661154ec8b Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 19 May 2021 23:56:47 -0500 Subject: [PATCH 26/71] pants: fix uses_services plugin registration --- pants-plugins/uses_services/mongo.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pants-plugins/uses_services/mongo.py b/pants-plugins/uses_services/mongo.py index b5373a3a2e..1649dabb42 100644 --- a/pants-plugins/uses_services/mongo.py +++ b/pants-plugins/uses_services/mongo.py @@ -17,6 +17,7 @@ from pants.engine.rules import collect_rules, Get, rule from pants.engine.process import FallibleProcessResult, ProcessCacheScope from pants.engine.target import Target +from pants.engine.unions import UnionRule from pants.util.logging import LogLevel from uses_services.exceptions import ServiceMissingError @@ -223,4 +224,7 @@ async def assert_mongo_is_running( def rules(): - return collect_rules() + return [ + *collect_rules(), + UnionRule(PytestPluginSetupRequest, UsesMongoRequest), + ] From 1def4e5b13136d441623155baad5ff5d679dd35c Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 20 May 2021 00:43:32 -0500 Subject: [PATCH 27/71] pants: fix uses_services so it works --- pants-plugins/uses_services/is_mongo_running.py | 4 +--- pants-plugins/uses_services/mongo.py | 9 ++++++--- pants-plugins/uses_services/platform_.py | 11 ++++++----- pants-plugins/uses_services/uses_services.py | 1 + 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/pants-plugins/uses_services/is_mongo_running.py b/pants-plugins/uses_services/is_mongo_running.py index 8a0e9c6263..871cea2793 100644 --- a/pants-plugins/uses_services/is_mongo_running.py +++ b/pants-plugins/uses_services/is_mongo_running.py @@ -1,5 +1,4 @@ import sys -from typing import cast, Iterable, Tuple def _is_mongo_running(db_host: str, db_port: int, db_name: str, connection_timeout_ms: int) -> bool: @@ -27,8 +26,7 @@ def _is_mongo_running(db_host: str, db_port: int, db_name: str, connection_timeo if __name__ == "__main__": - args = cast(Iterable[Tuple[int, str]], enumerate(sys.argv)) - args = dict(args) + args = dict((k, v) for k, v in enumerate(sys.argv)) db_host = args.get(1, "127.0.0.1") db_port = args.get(2, 27017) db_name = args.get(3, "st2-test") diff --git a/pants-plugins/uses_services/mongo.py b/pants-plugins/uses_services/mongo.py index 1649dabb42..d9532ed619 100644 --- a/pants-plugins/uses_services/mongo.py +++ b/pants-plugins/uses_services/mongo.py @@ -29,7 +29,10 @@ class UsesMongoRequest(PytestPluginSetupRequest): @classmethod def is_applicable(cls, target: Target) -> bool: - return "mongo" in target.get(UsesServicesField).value + if not target.has_field(UsesServicesField): + return False + uses = target.get(UsesServicesField).value + return uses is not None and "mongo" in uses @dataclass(frozen=True) @@ -70,7 +73,7 @@ async def mongo_is_running() -> MongoStatus: PexRequest( output_filename="mongoengine.pex", internal_only=True, - requirements=[PexRequirements({"mongoengine", "pymongo"})], + requirements=PexRequirements({"mongoengine", "pymongo"}), ) ) @@ -90,7 +93,7 @@ async def mongo_is_running() -> MongoStatus: FallibleProcessResult, VenvPexProcess( mongoengine_pex, - argv=[script_path, db_host, db_port, db_name, connection_timeout], + argv=(script_path, db_host, str(db_port), db_name, str(connection_timeout)), input_digest=script_digest, description=f"Checking to see if Mongo is up and accessible.", # this can change from run to run, so don't cache results. diff --git a/pants-plugins/uses_services/platform_.py b/pants-plugins/uses_services/platform_.py index 3e74ed23c0..698c36ee9d 100644 --- a/pants-plugins/uses_services/platform_.py +++ b/pants-plugins/uses_services/platform_.py @@ -10,12 +10,13 @@ from pants.engine.process import ProcessCacheScope, ProcessResult from pants.engine.rules import collect_rules, Get, rule from pants.util.logging import LogLevel +# noinspection PyProtectedMember from .inspect_platform import Platform, __file__ as inspect_platform_full_path __all__ = ["Platform", "get_platform", "rules"] -@rule +@rule(desc="Get details (os, distro, etc) about platform running tests.") async def get_platform() -> Platform: distro_pex = await Get( @@ -23,8 +24,8 @@ async def get_platform() -> Platform: PexRequest( output_filename="distro.pex", internal_only=True, - requirements=[PexRequirements({"distro"})], - ), + requirements=PexRequirements({"distro"}), + ) ) script_path = "./inspect_platform.py" @@ -43,13 +44,13 @@ async def get_platform() -> Platform: ProcessResult, VenvPexProcess( distro_pex, - argv=[script_path], + argv=(script_path,), input_digest=script_digest, description=f"Introspecting platform (arch, os, distro)", # this can change from run to run, so don't cache results. cache_scope=ProcessCacheScope.PER_RESTART, # NEVER? level=LogLevel.DEBUG, - ), + ) ) platform = json.loads(result.stdout) return Platform(**platform) diff --git a/pants-plugins/uses_services/uses_services.py b/pants-plugins/uses_services/uses_services.py index 7838501518..9f4879ce55 100644 --- a/pants-plugins/uses_services/uses_services.py +++ b/pants-plugins/uses_services/uses_services.py @@ -26,6 +26,7 @@ def compute_value( raise InvalidFieldChoiceException( address, cls.alias, service, valid_choices=cls.valid_choices ) + return tuple(services) def rules(): From e7c1e60db7e173f6b3ca7ff480b46ef4cc0b34da Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 20 May 2021 00:49:19 -0500 Subject: [PATCH 28/71] pants: do not cache check for mongodb service If you start it without stopping pantsd, it will keep reporting that it failed. PER_RESTART_IF_SUCCESSFUL would be ideal, but it doesn't exist. So, we go with ProcessCacheScope.NEVER --- pants-plugins/uses_services/mongo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pants-plugins/uses_services/mongo.py b/pants-plugins/uses_services/mongo.py index d9532ed619..9e12ba964e 100644 --- a/pants-plugins/uses_services/mongo.py +++ b/pants-plugins/uses_services/mongo.py @@ -97,7 +97,7 @@ async def mongo_is_running() -> MongoStatus: input_digest=script_digest, description=f"Checking to see if Mongo is up and accessible.", # this can change from run to run, so don't cache results. - cache_scope=ProcessCacheScope.PER_RESTART, # NEVER? + cache_scope=ProcessCacheScope.NEVER, # PER_RESTART isn't enough level=LogLevel.DEBUG, ) ) From a42676faff9652fec2d73657e95c253603059fc3 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 21 May 2021 12:48:05 -0500 Subject: [PATCH 29/71] pants: clean up imports in uses_services plugin Move the scripts that run remotely to a separate dir so that their purpose is more obvious. --- pants-plugins/uses_services/exceptions.py | 5 +++- pants-plugins/uses_services/mongo.py | 4 +-- pants-plugins/uses_services/platform_.py | 30 +++++++++---------- pants-plugins/uses_services/register.py | 4 +-- .../uses_services/scripts/__init__.py | 1 + .../{ => scripts}/inspect_platform.py | 0 .../{ => scripts}/is_mongo_running.py | 0 .../{uses_services.py => target_types.py} | 0 8 files changed, 24 insertions(+), 20 deletions(-) create mode 100644 pants-plugins/uses_services/scripts/__init__.py rename pants-plugins/uses_services/{ => scripts}/inspect_platform.py (100%) rename pants-plugins/uses_services/{ => scripts}/is_mongo_running.py (100%) rename pants-plugins/uses_services/{uses_services.py => target_types.py} (100%) diff --git a/pants-plugins/uses_services/exceptions.py b/pants-plugins/uses_services/exceptions.py index 823f7a9250..ad4b22006e 100644 --- a/pants-plugins/uses_services/exceptions.py +++ b/pants-plugins/uses_services/exceptions.py @@ -1,8 +1,11 @@ +from uses_services.platform_ import Platform + + class ServiceMissingError(Exception): """Error raised when a test uses a service but that service is missing.""" # TODO add special platform handling to DRY instructions across services - def __init__(self, service, platform: "Platform", instructions="", msg=None): + def __init__(self, service, platform: Platform, instructions="", msg=None): if msg is None: msg = f"The {service} service does not seem to be running or is not accessible!" if instructions: diff --git a/pants-plugins/uses_services/mongo.py b/pants-plugins/uses_services/mongo.py index 9e12ba964e..35be238d65 100644 --- a/pants-plugins/uses_services/mongo.py +++ b/pants-plugins/uses_services/mongo.py @@ -21,9 +21,9 @@ from pants.util.logging import LogLevel from uses_services.exceptions import ServiceMissingError -from uses_services.is_mongo_running import __file__ as is_mongo_running_full_path from uses_services.platform_ import Platform -from uses_services.uses_services import UsesServicesField +from uses_services.scripts.is_mongo_running import __file__ as is_mongo_running_full_path +from uses_services.target_types import UsesServicesField class UsesMongoRequest(PytestPluginSetupRequest): diff --git a/pants-plugins/uses_services/platform_.py b/pants-plugins/uses_services/platform_.py index 698c36ee9d..c380705b8a 100644 --- a/pants-plugins/uses_services/platform_.py +++ b/pants-plugins/uses_services/platform_.py @@ -8,26 +8,16 @@ ) from pants.engine.fs import CreateDigest, Digest, FileContent from pants.engine.process import ProcessCacheScope, ProcessResult -from pants.engine.rules import collect_rules, Get, rule +from pants.engine.rules import collect_rules, Get, MultiGet, rule from pants.util.logging import LogLevel # noinspection PyProtectedMember -from .inspect_platform import Platform, __file__ as inspect_platform_full_path +from uses_services.scripts.inspect_platform import Platform, __file__ as inspect_platform_full_path __all__ = ["Platform", "get_platform", "rules"] @rule(desc="Get details (os, distro, etc) about platform running tests.") async def get_platform() -> Platform: - - distro_pex = await Get( - VenvPex, - PexRequest( - output_filename="distro.pex", - internal_only=True, - requirements=PexRequirements({"distro"}), - ) - ) - script_path = "./inspect_platform.py" # pants is already watching this directory as it is under a source root. @@ -35,9 +25,19 @@ async def get_platform() -> Platform: with open(inspect_platform_full_path, "rb") as script_file: script_contents = script_file.read() - script_digest = await Get( - Digest, - CreateDigest([FileContent(script_path, script_contents)]), + script_digest, distro_pex = await MultiGet( + Get( + Digest, + CreateDigest([FileContent(script_path, script_contents)]), + ), + Get( + VenvPex, + PexRequest( + output_filename="distro.pex", + internal_only=True, + requirements=PexRequirements({"distro"}), + ) + ) ) result = await Get( diff --git a/pants-plugins/uses_services/register.py b/pants-plugins/uses_services/register.py index 85c3a566bf..93ca1324cc 100644 --- a/pants-plugins/uses_services/register.py +++ b/pants-plugins/uses_services/register.py @@ -1,9 +1,9 @@ -from uses_services import mongo, platform_, uses_services +from uses_services import mongo, platform_, target_types def rules(): return [ - *uses_services.rules(), + *target_types.rules(), *platform_.rules(), *mongo.rules(), ] diff --git a/pants-plugins/uses_services/scripts/__init__.py b/pants-plugins/uses_services/scripts/__init__.py new file mode 100644 index 0000000000..57d631c3f0 --- /dev/null +++ b/pants-plugins/uses_services/scripts/__init__.py @@ -0,0 +1 @@ +# coding: utf-8 diff --git a/pants-plugins/uses_services/inspect_platform.py b/pants-plugins/uses_services/scripts/inspect_platform.py similarity index 100% rename from pants-plugins/uses_services/inspect_platform.py rename to pants-plugins/uses_services/scripts/inspect_platform.py diff --git a/pants-plugins/uses_services/is_mongo_running.py b/pants-plugins/uses_services/scripts/is_mongo_running.py similarity index 100% rename from pants-plugins/uses_services/is_mongo_running.py rename to pants-plugins/uses_services/scripts/is_mongo_running.py diff --git a/pants-plugins/uses_services/uses_services.py b/pants-plugins/uses_services/target_types.py similarity index 100% rename from pants-plugins/uses_services/uses_services.py rename to pants-plugins/uses_services/target_types.py From 33aef472777192e8dbb765454af9069e09382d0a Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 21 May 2021 12:52:20 -0500 Subject: [PATCH 30/71] pants: use MultiGet in pants plugin --- pants-plugins/uses_services/mongo.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/pants-plugins/uses_services/mongo.py b/pants-plugins/uses_services/mongo.py index 35be238d65..c984f723f5 100644 --- a/pants-plugins/uses_services/mongo.py +++ b/pants-plugins/uses_services/mongo.py @@ -2,7 +2,6 @@ from textwrap import dedent -# TODO: this is planned / does not exist yet from pants.backend.python.goals.pytest_runner import ( PytestPluginSetupRequest, PytestPluginSetup, @@ -14,7 +13,7 @@ VenvPexProcess, ) from pants.engine.fs import CreateDigest, Digest, FileContent -from pants.engine.rules import collect_rules, Get, rule +from pants.engine.rules import collect_rules, Get, MultiGet, rule from pants.engine.process import FallibleProcessResult, ProcessCacheScope from pants.engine.target import Target from pants.engine.unions import UnionRule @@ -68,15 +67,6 @@ async def mongo_is_running() -> MongoStatus: # for component in $(COMPONENTS_TEST) # nosetests $(NOSE_OPTS) -s -v $(NOSE_COVERAGE_FLAGS) $(NOSE_COVERAGE_PACKAGES) $$component/tests/unit - mongoengine_pex = await Get( - VenvPex, - PexRequest( - output_filename="mongoengine.pex", - internal_only=True, - requirements=PexRequirements({"mongoengine", "pymongo"}), - ) - ) - script_path = "./is_mongo_running.py" # pants is already watching this directory as it is under a source root. @@ -84,9 +74,19 @@ async def mongo_is_running() -> MongoStatus: with open(is_mongo_running_full_path, "rb") as script_file: script_contents = script_file.read() - script_digest = await Get( - Digest, - CreateDigest([FileContent(script_path, script_contents)]) + script_digest, mongoengine_pex = await MultiGet( + Get( + Digest, + CreateDigest([FileContent(script_path, script_contents)]) + ), + Get( + VenvPex, + PexRequest( + output_filename="mongoengine.pex", + internal_only=True, + requirements=PexRequirements({"mongoengine", "pymongo"}), + ) + ) ) result = await Get( From a445005f09ac13b789da5f646eb2b3f1b2887ede Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 21 May 2021 12:57:43 -0500 Subject: [PATCH 31/71] pants: combine mongo_is_running rules --- pants-plugins/uses_services/mongo.py | 213 +++++++++++++-------------- 1 file changed, 101 insertions(+), 112 deletions(-) diff --git a/pants-plugins/uses_services/mongo.py b/pants-plugins/uses_services/mongo.py index c984f723f5..3dba917ec3 100644 --- a/pants-plugins/uses_services/mongo.py +++ b/pants-plugins/uses_services/mongo.py @@ -1,5 +1,3 @@ -from dataclasses import dataclass - from textwrap import dedent from pants.backend.python.goals.pytest_runner import ( @@ -34,13 +32,9 @@ def is_applicable(cls, target: Target) -> bool: return uses is not None and "mongo" in uses -@dataclass(frozen=True) -class MongoStatus: - is_running: bool - - @rule(desc="Test to see if mongodb is running and accessible for tests.") -async def mongo_is_running() -> MongoStatus: +async def mongo_is_running(request: UsesMongoRequest, platform: Platform) -> PytestPluginSetup: + # These config opts are used via oslo_config.cfg.CONF.database.{host,port,db_name,connection_timeout} # These config opts currently hard-coded in: # for unit tests: st2tests/st2tests/config.py @@ -101,129 +95,124 @@ async def mongo_is_running() -> MongoStatus: level=LogLevel.DEBUG, ) ) - return MongoStatus(result.exit_code == 0) + is_running = result.exit_code == 0 + if is_running: + return PytestPluginSetup() -@rule("Report that mongodb is required for test runs.") -async def assert_mongo_is_running( - request: UsesMongoRequest, mongo_status: MongoStatus, platform: Platform -) -> PytestPluginSetup: - if not mongo_status.is_running: - if platform.distro in ["centos", "rhel"] or "rhel" in platform.distro_like: - instructions = dedent( - f"""\ - If mongo is installed, but not running try: - - """ - ) + if platform.distro in ["centos", "rhel"] or "rhel" in platform.distro_like: + instructions = dedent( + f"""\ + If mongo is installed, but not running try: - if platform.distro_major_version == "7": - instructions += "\nservice mongo start\n" - else: - instructions += "\nsystemctl start mongod\n" + """ + ) - instructions += dedent( - """ - If mongo is not installed, this is one way to install it: - - # Add key and repo for the latest stable MongoDB (4.0) - rpm --import https://www.mongodb.org/static/pgp/server-4.0.asc - sh -c "cat < /etc/yum.repos.d/mongodb-org-4.repo - [mongodb-org-4] - name=MongoDB Repository - baseurl=https://repo.mongodb.org/yum/redhat/${OSRELEASE_VERSION}/mongodb-org/4.0/x86_64/ - gpgcheck=1 - enabled=1 - gpgkey=https://www.mongodb.org/static/pgp/server-4.0.asc - EOT" - # install mongo - yum install mongodb-org - # Don't forget to start mongo. - """ - ) - elif platform.distro in ["ubuntu", "debian"] or "debian" in platform.distro_like: - instructions = dedent( - """\ - If mongo is installed, but not running try: + if platform.distro_major_version == "7": + instructions += "\nservice mongo start\n" + else: + instructions += "\nsystemctl start mongod\n" + + instructions += dedent( + """ + If mongo is not installed, this is one way to install it: + + # Add key and repo for the latest stable MongoDB (4.0) + rpm --import https://www.mongodb.org/static/pgp/server-4.0.asc + sh -c "cat < /etc/yum.repos.d/mongodb-org-4.repo + [mongodb-org-4] + name=MongoDB Repository + baseurl=https://repo.mongodb.org/yum/redhat/${OSRELEASE_VERSION}/mongodb-org/4.0/x86_64/ + gpgcheck=1 + enabled=1 + gpgkey=https://www.mongodb.org/static/pgp/server-4.0.asc + EOT" + # install mongo + yum install mongodb-org + # Don't forget to start mongo. + """ + ) + elif platform.distro in ["ubuntu", "debian"] or "debian" in platform.distro_like: + instructions = dedent( + """\ + If mongo is installed, but not running try: - systemctl start mongod + systemctl start mongod - If mongo is not installed, this is one way to install it: + If mongo is not installed, this is one way to install it: - apt-get install mongodb mongodb-server - # Don't forget to start mongo. - """ - ) - elif platform.os == "Linux": - instructions = dedent( - f"""\ - You are on Linux using {platform.distro_name}, which is not - one of our generally supported distributions. We recommend - you use vagrant for local development with something like: + apt-get install mongodb mongodb-server + # Don't forget to start mongo. + """ + ) + elif platform.os == "Linux": + instructions = dedent( + f"""\ + You are on Linux using {platform.distro_name}, which is not + one of our generally supported distributions. We recommend + you use vagrant for local development with something like: - vagrant init stackstorm/st2 - vagrant up - vagrant ssh + vagrant init stackstorm/st2 + vagrant up + vagrant ssh - Please see: https://docs.stackstorm.com/install/vagrant.html + Please see: https://docs.stackstorm.com/install/vagrant.html - For anyone who wants to attempt local development without vagrant, - you are pretty much on your own. At a minimum you need to install - and start mongo with something like: + For anyone who wants to attempt local development without vagrant, + you are pretty much on your own. At a minimum you need to install + and start mongo with something like: - systemctl start mongod + systemctl start mongod - We would be interested to hear about alternative distros people - are using for development. If you are able, please let us know - on slack which distro you are using: + We would be interested to hear about alternative distros people + are using for development. If you are able, please let us know + on slack which distro you are using: - Arch: {platform.arch} - Distro: {platform.distro} - Distro Codename: {platform.distro_codename} - Distro Version: {platform.distro_version} + Arch: {platform.arch} + Distro: {platform.distro} + Distro Codename: {platform.distro_codename} + Distro Version: {platform.distro_version} - Thanks and Good Luck! - """ - ) - elif platform.os == "Darwin": # MacOS - instructions = dedent( - """\ - You are on Mac OS. Generally we recommend using vagrant for local - development on Mac OS with something like: - - vagrant init stackstorm/st2 - vagrant up - vagrant ssh - - Please see: https://docs.stackstorm.com/install/vagrant.html - - For anyone who wants to attempt local development without vagrant, - you may run into some speed bumps. Others StackStorm developers have - been known to use Mac OS for development, so feel free to ask for - help in slack. At a minimum you need to install and start mongo. - """ - ) - else: - instructions = dedent( - """\ - You are not on Linux. In this case we recommend using vagrant - for local development with something like: + Thanks and Good Luck! + """ + ) + elif platform.os == "Darwin": # MacOS + instructions = dedent( + """\ + You are on Mac OS. Generally we recommend using vagrant for local + development on Mac OS with something like: + + vagrant init stackstorm/st2 + vagrant up + vagrant ssh + + Please see: https://docs.stackstorm.com/install/vagrant.html + + For anyone who wants to attempt local development without vagrant, + you may run into some speed bumps. Others StackStorm developers have + been known to use Mac OS for development, so feel free to ask for + help in slack. At a minimum you need to install and start mongo. + """ + ) + else: + instructions = dedent( + """\ + You are not on Linux. In this case we recommend using vagrant + for local development with something like: - vagrant init stackstorm/st2 - vagrant up - vagrant ssh + vagrant init stackstorm/st2 + vagrant up + vagrant ssh - Please see: https://docs.stackstorm.com/install/vagrant.html + Please see: https://docs.stackstorm.com/install/vagrant.html - For anyone who wants to attempt local development without vagrant, - you are pretty much on your own. At a minimum you need to install - and start mongo. Good luck! + For anyone who wants to attempt local development without vagrant, + you are pretty much on your own. At a minimum you need to install + and start mongo. Good luck! """ - ) - - raise ServiceMissingError("mongo", platform, instructions) + ) - return PytestPluginSetup() + raise ServiceMissingError("mongo", platform, instructions) def rules(): From 6346ffa8a810ad9b7fc5e0f657daff535e97e7ad Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 21 May 2021 13:10:34 -0500 Subject: [PATCH 32/71] pants: add LogLevel to rules so they show up in UI apparently they are TRACE level by default. --- pants-plugins/uses_services/mongo.py | 2 +- pants-plugins/uses_services/platform_.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pants-plugins/uses_services/mongo.py b/pants-plugins/uses_services/mongo.py index 3dba917ec3..221eec1c0c 100644 --- a/pants-plugins/uses_services/mongo.py +++ b/pants-plugins/uses_services/mongo.py @@ -32,7 +32,7 @@ def is_applicable(cls, target: Target) -> bool: return uses is not None and "mongo" in uses -@rule(desc="Test to see if mongodb is running and accessible for tests.") +@rule(desc="Test to see if mongodb is running and accessible for tests.", level=LogLevel.DEBUG) async def mongo_is_running(request: UsesMongoRequest, platform: Platform) -> PytestPluginSetup: # These config opts are used via oslo_config.cfg.CONF.database.{host,port,db_name,connection_timeout} diff --git a/pants-plugins/uses_services/platform_.py b/pants-plugins/uses_services/platform_.py index c380705b8a..a5326d0db0 100644 --- a/pants-plugins/uses_services/platform_.py +++ b/pants-plugins/uses_services/platform_.py @@ -16,7 +16,7 @@ __all__ = ["Platform", "get_platform", "rules"] -@rule(desc="Get details (os, distro, etc) about platform running tests.") +@rule(desc="Get details (os, distro, etc) about platform running tests.", level=LogLevel.DEBUG) async def get_platform() -> Platform: script_path = "./inspect_platform.py" From b221cb0de01ad69412a2c8b4cd0a3652153c00a5 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 21 May 2021 22:47:57 -0500 Subject: [PATCH 33/71] pants: tailor --- pants-plugins/uses_services/scripts/BUILD | 1 + 1 file changed, 1 insertion(+) create mode 100644 pants-plugins/uses_services/scripts/BUILD diff --git a/pants-plugins/uses_services/scripts/BUILD b/pants-plugins/uses_services/scripts/BUILD new file mode 100644 index 0000000000..db46e8d6c9 --- /dev/null +++ b/pants-plugins/uses_services/scripts/BUILD @@ -0,0 +1 @@ +python_sources() From 73315a50cd1ef944623e7ac1cde803c61be3e294 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 21 May 2021 22:53:46 -0500 Subject: [PATCH 34/71] pants: fix error importing uses_services plugin I'm not sure why having a rules() in target_types caused this issue, but moving it back to register seemed to work. 22:39:25.92 [ERROR] Entrypoint target_types in uses_services.register must be a zero-arg callable: TypeError("'module' object is not callable") --- pants-plugins/uses_services/register.py | 7 +++++-- pants-plugins/uses_services/target_types.py | 5 ----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/pants-plugins/uses_services/register.py b/pants-plugins/uses_services/register.py index 93ca1324cc..a2176595e2 100644 --- a/pants-plugins/uses_services/register.py +++ b/pants-plugins/uses_services/register.py @@ -1,9 +1,12 @@ -from uses_services import mongo, platform_, target_types +from pants.backend.python.target_types import PythonTests + +from uses_services import mongo, platform_ +from uses_services.target_types import UsesServicesField def rules(): return [ - *target_types.rules(), + PythonTests.register_plugin_field(UsesServicesField), *platform_.rules(), *mongo.rules(), ] diff --git a/pants-plugins/uses_services/target_types.py b/pants-plugins/uses_services/target_types.py index 9f4879ce55..b6d94f417a 100644 --- a/pants-plugins/uses_services/target_types.py +++ b/pants-plugins/uses_services/target_types.py @@ -1,7 +1,6 @@ # coding: utf-8 from typing import Iterable, Optional, Tuple -from pants.backend.python.target_types import PythonTests from pants.build_graph.address import Address from pants.engine.target import InvalidFieldChoiceException, StringSequenceField @@ -27,7 +26,3 @@ def compute_value( address, cls.alias, service, valid_choices=cls.valid_choices ) return tuple(services) - - -def rules(): - return [PythonTests.register_plugin_field(UsesServicesField)] From 036e1ac70d84f8f73346ba4d502d85b788ec5e81 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sun, 30 May 2021 19:22:18 -0500 Subject: [PATCH 35/71] pants: reformat plugins with black But drop the final , in the Get() entries because it confuses the pants AST parsing. --- pants-plugins/uses_services/exceptions.py | 1 + pants-plugins/uses_services/mongo.py | 20 +++++++++++-------- pants-plugins/uses_services/platform_.py | 13 +++++++++--- .../uses_services/scripts/is_mongo_running.py | 8 ++++++-- 4 files changed, 29 insertions(+), 13 deletions(-) diff --git a/pants-plugins/uses_services/exceptions.py b/pants-plugins/uses_services/exceptions.py index ad4b22006e..eb3fb47b90 100644 --- a/pants-plugins/uses_services/exceptions.py +++ b/pants-plugins/uses_services/exceptions.py @@ -3,6 +3,7 @@ class ServiceMissingError(Exception): """Error raised when a test uses a service but that service is missing.""" + # TODO add special platform handling to DRY instructions across services def __init__(self, service, platform: Platform, instructions="", msg=None): diff --git a/pants-plugins/uses_services/mongo.py b/pants-plugins/uses_services/mongo.py index 221eec1c0c..9640c0fa2d 100644 --- a/pants-plugins/uses_services/mongo.py +++ b/pants-plugins/uses_services/mongo.py @@ -19,7 +19,9 @@ from uses_services.exceptions import ServiceMissingError from uses_services.platform_ import Platform -from uses_services.scripts.is_mongo_running import __file__ as is_mongo_running_full_path +from uses_services.scripts.is_mongo_running import ( + __file__ as is_mongo_running_full_path, +) from uses_services.target_types import UsesServicesField @@ -32,8 +34,13 @@ def is_applicable(cls, target: Target) -> bool: return uses is not None and "mongo" in uses -@rule(desc="Test to see if mongodb is running and accessible for tests.", level=LogLevel.DEBUG) -async def mongo_is_running(request: UsesMongoRequest, platform: Platform) -> PytestPluginSetup: +@rule( + desc="Test to see if mongodb is running and accessible for tests.", + level=LogLevel.DEBUG, +) +async def mongo_is_running( + request: UsesMongoRequest, platform: Platform +) -> PytestPluginSetup: # These config opts are used via oslo_config.cfg.CONF.database.{host,port,db_name,connection_timeout} # These config opts currently hard-coded in: @@ -69,10 +76,7 @@ async def mongo_is_running(request: UsesMongoRequest, platform: Platform) -> Pyt script_contents = script_file.read() script_digest, mongoengine_pex = await MultiGet( - Get( - Digest, - CreateDigest([FileContent(script_path, script_contents)]) - ), + Get(Digest, CreateDigest([FileContent(script_path, script_contents)])), Get( VenvPex, PexRequest( @@ -80,7 +84,7 @@ async def mongo_is_running(request: UsesMongoRequest, platform: Platform) -> Pyt internal_only=True, requirements=PexRequirements({"mongoengine", "pymongo"}), ) - ) + ), ) result = await Get( diff --git a/pants-plugins/uses_services/platform_.py b/pants-plugins/uses_services/platform_.py index a5326d0db0..db84390982 100644 --- a/pants-plugins/uses_services/platform_.py +++ b/pants-plugins/uses_services/platform_.py @@ -10,13 +10,20 @@ from pants.engine.process import ProcessCacheScope, ProcessResult from pants.engine.rules import collect_rules, Get, MultiGet, rule from pants.util.logging import LogLevel + # noinspection PyProtectedMember -from uses_services.scripts.inspect_platform import Platform, __file__ as inspect_platform_full_path +from uses_services.scripts.inspect_platform import ( + Platform, + __file__ as inspect_platform_full_path, +) __all__ = ["Platform", "get_platform", "rules"] -@rule(desc="Get details (os, distro, etc) about platform running tests.", level=LogLevel.DEBUG) +@rule( + desc="Get details (os, distro, etc) about platform running tests.", + level=LogLevel.DEBUG, +) async def get_platform() -> Platform: script_path = "./inspect_platform.py" @@ -37,7 +44,7 @@ async def get_platform() -> Platform: internal_only=True, requirements=PexRequirements({"distro"}), ) - ) + ), ) result = await Get( diff --git a/pants-plugins/uses_services/scripts/is_mongo_running.py b/pants-plugins/uses_services/scripts/is_mongo_running.py index 871cea2793..254e125e6f 100644 --- a/pants-plugins/uses_services/scripts/is_mongo_running.py +++ b/pants-plugins/uses_services/scripts/is_mongo_running.py @@ -1,7 +1,9 @@ import sys -def _is_mongo_running(db_host: str, db_port: int, db_name: str, connection_timeout_ms: int) -> bool: +def _is_mongo_running( + db_host: str, db_port: int, db_name: str, connection_timeout_ms: int +) -> bool: # late import so that __file__ can be imported in the pants plugin without these imports import mongoengine from pymongo.errors import ConnectionFailure @@ -32,6 +34,8 @@ def _is_mongo_running(db_host: str, db_port: int, db_name: str, connection_timeo db_name = args.get(3, "st2-test") connection_timeout_ms = args.get(4, 3000) - is_running = _is_mongo_running(db_host, int(db_port), db_name, int(connection_timeout_ms)) + is_running = _is_mongo_running( + db_host, int(db_port), db_name, int(connection_timeout_ms) + ) exit_code = 0 if is_running else 1 sys.exit(exit_code) From 3f4d70ac9c3dd2b8951533508ebb635c374bebd2 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 8 Jun 2021 11:30:44 -0500 Subject: [PATCH 36/71] pants: update ProcessCacheScope in uses_services plugin ProcessCacheScope.* names were adjusted (in pants 2.6.0.dev1) based on my feedback. This makes the scope clearer in our pants plugins. --- pants-plugins/uses_services/mongo.py | 2 +- pants-plugins/uses_services/platform_.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pants-plugins/uses_services/mongo.py b/pants-plugins/uses_services/mongo.py index 9640c0fa2d..47050d2e37 100644 --- a/pants-plugins/uses_services/mongo.py +++ b/pants-plugins/uses_services/mongo.py @@ -95,7 +95,7 @@ async def mongo_is_running( input_digest=script_digest, description=f"Checking to see if Mongo is up and accessible.", # this can change from run to run, so don't cache results. - cache_scope=ProcessCacheScope.NEVER, # PER_RESTART isn't enough + cache_scope=ProcessCacheScope.PER_SESSION, level=LogLevel.DEBUG, ) ) diff --git a/pants-plugins/uses_services/platform_.py b/pants-plugins/uses_services/platform_.py index db84390982..03087cfb4c 100644 --- a/pants-plugins/uses_services/platform_.py +++ b/pants-plugins/uses_services/platform_.py @@ -55,7 +55,7 @@ async def get_platform() -> Platform: input_digest=script_digest, description=f"Introspecting platform (arch, os, distro)", # this can change from run to run, so don't cache results. - cache_scope=ProcessCacheScope.PER_RESTART, # NEVER? + cache_scope=ProcessCacheScope.PER_RESTART_SUCCESSFULL, level=LogLevel.DEBUG, ) ) From fcd9376e3daac97d49d2856fd62c3040b666475a Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 10 Dec 2022 22:45:13 -0600 Subject: [PATCH 37/71] reformat with black --- pants-plugins/uses_services/mongo.py | 4 ++-- pants-plugins/uses_services/platform_.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pants-plugins/uses_services/mongo.py b/pants-plugins/uses_services/mongo.py index 47050d2e37..91a1e4c1d6 100644 --- a/pants-plugins/uses_services/mongo.py +++ b/pants-plugins/uses_services/mongo.py @@ -83,7 +83,7 @@ async def mongo_is_running( output_filename="mongoengine.pex", internal_only=True, requirements=PexRequirements({"mongoengine", "pymongo"}), - ) + ), ), ) @@ -97,7 +97,7 @@ async def mongo_is_running( # this can change from run to run, so don't cache results. cache_scope=ProcessCacheScope.PER_SESSION, level=LogLevel.DEBUG, - ) + ), ) is_running = result.exit_code == 0 diff --git a/pants-plugins/uses_services/platform_.py b/pants-plugins/uses_services/platform_.py index 03087cfb4c..56cdd03b71 100644 --- a/pants-plugins/uses_services/platform_.py +++ b/pants-plugins/uses_services/platform_.py @@ -43,7 +43,7 @@ async def get_platform() -> Platform: output_filename="distro.pex", internal_only=True, requirements=PexRequirements({"distro"}), - ) + ), ), ) @@ -57,7 +57,7 @@ async def get_platform() -> Platform: # this can change from run to run, so don't cache results. cache_scope=ProcessCacheScope.PER_RESTART_SUCCESSFULL, level=LogLevel.DEBUG, - ) + ), ) platform = json.loads(result.stdout) return Platform(**platform) From 5e2e60954b2082db60ba323d50b5b086c132804b Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 14 Jun 2021 12:48:29 -0500 Subject: [PATCH 38/71] add license header --- pants-plugins/uses_services/exceptions.py | 13 +++++++++++++ pants-plugins/uses_services/mongo.py | 13 +++++++++++++ pants-plugins/uses_services/platform_.py | 13 +++++++++++++ pants-plugins/uses_services/register.py | 13 +++++++++++++ pants-plugins/uses_services/scripts/__init__.py | 1 - .../uses_services/scripts/inspect_platform.py | 13 +++++++++++++ .../uses_services/scripts/is_mongo_running.py | 13 +++++++++++++ pants-plugins/uses_services/target_types.py | 14 +++++++++++++- 8 files changed, 91 insertions(+), 2 deletions(-) diff --git a/pants-plugins/uses_services/exceptions.py b/pants-plugins/uses_services/exceptions.py index eb3fb47b90..a3d9f6265b 100644 --- a/pants-plugins/uses_services/exceptions.py +++ b/pants-plugins/uses_services/exceptions.py @@ -1,3 +1,16 @@ +# Copyright 2021 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from uses_services.platform_ import Platform diff --git a/pants-plugins/uses_services/mongo.py b/pants-plugins/uses_services/mongo.py index 91a1e4c1d6..a578c77a5f 100644 --- a/pants-plugins/uses_services/mongo.py +++ b/pants-plugins/uses_services/mongo.py @@ -1,3 +1,16 @@ +# Copyright 2021 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from textwrap import dedent from pants.backend.python.goals.pytest_runner import ( diff --git a/pants-plugins/uses_services/platform_.py b/pants-plugins/uses_services/platform_.py index 56cdd03b71..7645cca4cb 100644 --- a/pants-plugins/uses_services/platform_.py +++ b/pants-plugins/uses_services/platform_.py @@ -1,3 +1,16 @@ +# Copyright 2021 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import json from pants.backend.python.util_rules.pex import ( diff --git a/pants-plugins/uses_services/register.py b/pants-plugins/uses_services/register.py index a2176595e2..bd964cbb4d 100644 --- a/pants-plugins/uses_services/register.py +++ b/pants-plugins/uses_services/register.py @@ -1,3 +1,16 @@ +# Copyright 2021 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from pants.backend.python.target_types import PythonTests from uses_services import mongo, platform_ diff --git a/pants-plugins/uses_services/scripts/__init__.py b/pants-plugins/uses_services/scripts/__init__.py index 57d631c3f0..e69de29bb2 100644 --- a/pants-plugins/uses_services/scripts/__init__.py +++ b/pants-plugins/uses_services/scripts/__init__.py @@ -1 +0,0 @@ -# coding: utf-8 diff --git a/pants-plugins/uses_services/scripts/inspect_platform.py b/pants-plugins/uses_services/scripts/inspect_platform.py index dfa095acfa..7953bc7d5e 100644 --- a/pants-plugins/uses_services/scripts/inspect_platform.py +++ b/pants-plugins/uses_services/scripts/inspect_platform.py @@ -1,3 +1,16 @@ +# Copyright 2021 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import json from dataclasses import asdict, dataclass diff --git a/pants-plugins/uses_services/scripts/is_mongo_running.py b/pants-plugins/uses_services/scripts/is_mongo_running.py index 254e125e6f..2746d2b997 100644 --- a/pants-plugins/uses_services/scripts/is_mongo_running.py +++ b/pants-plugins/uses_services/scripts/is_mongo_running.py @@ -1,3 +1,16 @@ +# Copyright 2021 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import sys diff --git a/pants-plugins/uses_services/target_types.py b/pants-plugins/uses_services/target_types.py index b6d94f417a..c51c03c92f 100644 --- a/pants-plugins/uses_services/target_types.py +++ b/pants-plugins/uses_services/target_types.py @@ -1,4 +1,16 @@ -# coding: utf-8 +# Copyright 2021 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from typing import Iterable, Optional, Tuple from pants.build_graph.address import Address From e741af50d04b5a9dde889122df5b5ea8515b5e15 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 14 Jun 2021 13:02:42 -0500 Subject: [PATCH 39/71] fix flake8 identified issues --- pants-plugins/uses_services/mongo.py | 4 ++-- pants-plugins/uses_services/platform_.py | 2 +- pants-plugins/uses_services/scripts/inspect_platform.py | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pants-plugins/uses_services/mongo.py b/pants-plugins/uses_services/mongo.py index a578c77a5f..4bbee378c1 100644 --- a/pants-plugins/uses_services/mongo.py +++ b/pants-plugins/uses_services/mongo.py @@ -106,7 +106,7 @@ async def mongo_is_running( mongoengine_pex, argv=(script_path, db_host, str(db_port), db_name, str(connection_timeout)), input_digest=script_digest, - description=f"Checking to see if Mongo is up and accessible.", + description="Checking to see if Mongo is up and accessible.", # this can change from run to run, so don't cache results. cache_scope=ProcessCacheScope.PER_SESSION, level=LogLevel.DEBUG, @@ -119,7 +119,7 @@ async def mongo_is_running( if platform.distro in ["centos", "rhel"] or "rhel" in platform.distro_like: instructions = dedent( - f"""\ + """\ If mongo is installed, but not running try: """ diff --git a/pants-plugins/uses_services/platform_.py b/pants-plugins/uses_services/platform_.py index 7645cca4cb..91b92bc93e 100644 --- a/pants-plugins/uses_services/platform_.py +++ b/pants-plugins/uses_services/platform_.py @@ -66,7 +66,7 @@ async def get_platform() -> Platform: distro_pex, argv=(script_path,), input_digest=script_digest, - description=f"Introspecting platform (arch, os, distro)", + description="Introspecting platform (arch, os, distro)", # this can change from run to run, so don't cache results. cache_scope=ProcessCacheScope.PER_RESTART_SUCCESSFULL, level=LogLevel.DEBUG, diff --git a/pants-plugins/uses_services/scripts/inspect_platform.py b/pants-plugins/uses_services/scripts/inspect_platform.py index 7953bc7d5e..b3154172da 100644 --- a/pants-plugins/uses_services/scripts/inspect_platform.py +++ b/pants-plugins/uses_services/scripts/inspect_platform.py @@ -34,7 +34,8 @@ class Platform: def _get_platform() -> Platform: # late import so that Platform can be imported in the pants plugin as well - import platform, distro + import distro + import platform return Platform( arch=platform.machine(), # x86_64 From f02f6de86e5147b19309ad41ff0d37380a649f00 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 26 May 2022 11:17:52 -0500 Subject: [PATCH 40/71] update pants-plugin apis from pants 2.7 to 2.8 --- pants-plugins/uses_services/register.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pants-plugins/uses_services/register.py b/pants-plugins/uses_services/register.py index bd964cbb4d..f81d54c105 100644 --- a/pants-plugins/uses_services/register.py +++ b/pants-plugins/uses_services/register.py @@ -11,7 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from pants.backend.python.target_types import PythonTests +from pants.backend.python.target_types import ( + PythonTestTarget, + PythonTestsGeneratorTarget, +) from uses_services import mongo, platform_ from uses_services.target_types import UsesServicesField @@ -19,7 +22,8 @@ def rules(): return [ - PythonTests.register_plugin_field(UsesServicesField), + PythonTestsGeneratorTarget.register_plugin_field(UsesServicesField), + PythonTestTarget.register_plugin_field(UsesServicesField), *platform_.rules(), *mongo.rules(), ] From 0d73cd76b50cb0f3fa33c1e51cb87d406bf5214f Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 1 Jul 2022 18:01:47 -0500 Subject: [PATCH 41/71] update pants-plugins plugin apis from 2.12 to 2.13.0a1 --- pants-plugins/uses_services/target_types.py | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/pants-plugins/uses_services/target_types.py b/pants-plugins/uses_services/target_types.py index c51c03c92f..c0d5242db0 100644 --- a/pants-plugins/uses_services/target_types.py +++ b/pants-plugins/uses_services/target_types.py @@ -11,10 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Iterable, Optional, Tuple - -from pants.build_graph.address import Address -from pants.engine.target import InvalidFieldChoiceException, StringSequenceField +from pants.engine.target import StringSequenceField supported_services = ("mongo", "rabbitmq", "redis") @@ -24,17 +21,3 @@ class UsesServicesField(StringSequenceField): alias = "uses" help = "Define the services that a test target depends on (mongo, rabbitmq, redis)." valid_choices = supported_services - - @classmethod - def compute_value( - cls, raw_value: Optional[Iterable[str]], address: Address - ) -> Optional[Tuple[str, ...]]: - services = super().compute_value(raw_value, address) - if not services: - return services - for service in services: - if service not in cls.valid_choices: - raise InvalidFieldChoiceException( - address, cls.alias, service, valid_choices=cls.valid_choices - ) - return tuple(services) From 8c0d6b87af09b81b1711cd685bcab246009cd8fb Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 10 Dec 2022 23:02:19 -0600 Subject: [PATCH 42/71] drop now unused [GLOBAL].plugins in pants.toml --- pants.toml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pants.toml b/pants.toml index 485f0b5065..86b9499a34 100644 --- a/pants.toml +++ b/pants.toml @@ -30,10 +30,6 @@ backend_packages = [ "schemas", "uses_services", ] -plugins = [ - # dependencies for internal plugins - #"distro", # dep of the pex -] # pants ignores files in .gitignore, .*/ directories, /dist/ directory, and __pycache__. pants_ignore.add = [ # TODO: remove these once we start building wheels with pants. From e517c7a45ad355050845bf551e9b7604495e48b7 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 19 May 2021 18:49:17 -0500 Subject: [PATCH 43/71] pants: Mark more tests with uses=["mongo"] --- st2common/tests/unit/services/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/st2common/tests/unit/services/BUILD b/st2common/tests/unit/services/BUILD index 57341b1358..44c0254066 100644 --- a/st2common/tests/unit/services/BUILD +++ b/st2common/tests/unit/services/BUILD @@ -1,3 +1,4 @@ python_tests( name="tests", + uses=["mongo"], ) From 8c9941afa0d055986128b6fc502f32785493cd77 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 10 Dec 2022 23:31:57 -0600 Subject: [PATCH 44/71] update db notes in pants-plugins/uses_services --- pants-plugins/uses_services/mongo.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pants-plugins/uses_services/mongo.py b/pants-plugins/uses_services/mongo.py index 4bbee378c1..4d6c9e3057 100644 --- a/pants-plugins/uses_services/mongo.py +++ b/pants-plugins/uses_services/mongo.py @@ -60,22 +60,24 @@ async def mongo_is_running( # for unit tests: st2tests/st2tests/config.py # for integration tests: conf/st2.tests*.conf st2tests/st2tests/fixtures/conf/st2.tests*.conf # (changed by setting ST2_CONFIG_PATH env var inside the tests) - # TODO: for unit tests: modify code to pull from an env var and then use per-pantsd-slot db_name + # TODO: for unit tests: modify code to pull db connect settings from env vars # TODO: for int tests: modify st2.tests*.conf on the fly to set the per-pantsd-slot db_name + # and either add env vars for db connect settings or modify conf files as well db_host = "127.0.0.1" # localhost in test_db.DbConnectionTestCase db_port = 27017 - db_name = "st2-test" # st2 in test_db.DbConnectionTestCase + db_name = f"st2-test{os.environ.get('ST2TESTS_PARALLEL_SLOT', '')}" + # db_name = "st2-test" # st2 in test_db.DbConnectionTestCase connection_timeout = 3000 - # so st2-test database gets dropped between: + # The st2-test database gets dropped between (in Makefile based testing): # - each component (st2*/ && various config/ dirs) in Makefile # - DbTestCase/CleanDbTestCase setUpClass # with our version of oslo.config (newer are slower) we can't directly override opts w/ environment variables. # Makefile - # .run-unit-tests-with-coverage (<- .combine-unit-tests-coverage <- .coverage.unit <- .unit-tests-coverage-html <- ci-unit <- ci) + # .run-unit-tests-coverage (<- .combine-unit-tests-coverage <- .coverage.unit <- .unit-tests-coverage-html <- ci-unit <- ci) # echo "----- Dropping st2-test db -----" # mongo st2-test --eval "db.dropDatabase();" # for component in $(COMPONENTS_TEST) @@ -226,7 +228,7 @@ async def mongo_is_running( For anyone who wants to attempt local development without vagrant, you are pretty much on your own. At a minimum you need to install and start mongo. Good luck! - """ + """ ) raise ServiceMissingError("mongo", platform, instructions) From 0352ccfa703abaa579b718a290a54809ac02a2a3 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 5 Jan 2023 11:28:19 -0600 Subject: [PATCH 45/71] update pants-plugins/uses_services copyright to 2023 --- pants-plugins/uses_services/exceptions.py | 2 +- pants-plugins/uses_services/mongo.py | 2 +- pants-plugins/uses_services/platform_.py | 2 +- pants-plugins/uses_services/register.py | 2 +- pants-plugins/uses_services/scripts/inspect_platform.py | 2 +- pants-plugins/uses_services/scripts/is_mongo_running.py | 2 +- pants-plugins/uses_services/target_types.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pants-plugins/uses_services/exceptions.py b/pants-plugins/uses_services/exceptions.py index a3d9f6265b..4fbd330e65 100644 --- a/pants-plugins/uses_services/exceptions.py +++ b/pants-plugins/uses_services/exceptions.py @@ -1,4 +1,4 @@ -# Copyright 2021 The StackStorm Authors. +# Copyright 2023 The StackStorm Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/pants-plugins/uses_services/mongo.py b/pants-plugins/uses_services/mongo.py index 4d6c9e3057..e638c157ad 100644 --- a/pants-plugins/uses_services/mongo.py +++ b/pants-plugins/uses_services/mongo.py @@ -1,4 +1,4 @@ -# Copyright 2021 The StackStorm Authors. +# Copyright 2023 The StackStorm Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/pants-plugins/uses_services/platform_.py b/pants-plugins/uses_services/platform_.py index 91b92bc93e..0d743b4a73 100644 --- a/pants-plugins/uses_services/platform_.py +++ b/pants-plugins/uses_services/platform_.py @@ -1,4 +1,4 @@ -# Copyright 2021 The StackStorm Authors. +# Copyright 2023 The StackStorm Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/pants-plugins/uses_services/register.py b/pants-plugins/uses_services/register.py index f81d54c105..55647048a9 100644 --- a/pants-plugins/uses_services/register.py +++ b/pants-plugins/uses_services/register.py @@ -1,4 +1,4 @@ -# Copyright 2021 The StackStorm Authors. +# Copyright 2023 The StackStorm Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/pants-plugins/uses_services/scripts/inspect_platform.py b/pants-plugins/uses_services/scripts/inspect_platform.py index b3154172da..383535c9b1 100644 --- a/pants-plugins/uses_services/scripts/inspect_platform.py +++ b/pants-plugins/uses_services/scripts/inspect_platform.py @@ -1,4 +1,4 @@ -# Copyright 2021 The StackStorm Authors. +# Copyright 2023 The StackStorm Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/pants-plugins/uses_services/scripts/is_mongo_running.py b/pants-plugins/uses_services/scripts/is_mongo_running.py index 2746d2b997..787c7fefc4 100644 --- a/pants-plugins/uses_services/scripts/is_mongo_running.py +++ b/pants-plugins/uses_services/scripts/is_mongo_running.py @@ -1,4 +1,4 @@ -# Copyright 2021 The StackStorm Authors. +# Copyright 2023 The StackStorm Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/pants-plugins/uses_services/target_types.py b/pants-plugins/uses_services/target_types.py index c0d5242db0..5723ceb9ae 100644 --- a/pants-plugins/uses_services/target_types.py +++ b/pants-plugins/uses_services/target_types.py @@ -1,4 +1,4 @@ -# Copyright 2021 The StackStorm Authors. +# Copyright 2023 The StackStorm Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 048d03536b96f9f89292b45058fca5a5193c35c5 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 6 Jan 2023 21:21:26 -0600 Subject: [PATCH 46/71] include "rules" in uses_services rule file names --- pants-plugins/uses_services/exceptions.py | 2 +- pants-plugins/uses_services/{mongo.py => mongo_rules.py} | 2 +- .../uses_services/{platform_.py => platform_rules.py} | 0 pants-plugins/uses_services/register.py | 6 +++--- 4 files changed, 5 insertions(+), 5 deletions(-) rename pants-plugins/uses_services/{mongo.py => mongo_rules.py} (99%) rename pants-plugins/uses_services/{platform_.py => platform_rules.py} (100%) diff --git a/pants-plugins/uses_services/exceptions.py b/pants-plugins/uses_services/exceptions.py index 4fbd330e65..c020e33a63 100644 --- a/pants-plugins/uses_services/exceptions.py +++ b/pants-plugins/uses_services/exceptions.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from uses_services.platform_ import Platform +from uses_services.platform_rules import Platform class ServiceMissingError(Exception): diff --git a/pants-plugins/uses_services/mongo.py b/pants-plugins/uses_services/mongo_rules.py similarity index 99% rename from pants-plugins/uses_services/mongo.py rename to pants-plugins/uses_services/mongo_rules.py index e638c157ad..d3701098ac 100644 --- a/pants-plugins/uses_services/mongo.py +++ b/pants-plugins/uses_services/mongo_rules.py @@ -31,7 +31,7 @@ from pants.util.logging import LogLevel from uses_services.exceptions import ServiceMissingError -from uses_services.platform_ import Platform +from uses_services.platform_rules import Platform from uses_services.scripts.is_mongo_running import ( __file__ as is_mongo_running_full_path, ) diff --git a/pants-plugins/uses_services/platform_.py b/pants-plugins/uses_services/platform_rules.py similarity index 100% rename from pants-plugins/uses_services/platform_.py rename to pants-plugins/uses_services/platform_rules.py diff --git a/pants-plugins/uses_services/register.py b/pants-plugins/uses_services/register.py index 55647048a9..5ca4363ab3 100644 --- a/pants-plugins/uses_services/register.py +++ b/pants-plugins/uses_services/register.py @@ -16,7 +16,7 @@ PythonTestsGeneratorTarget, ) -from uses_services import mongo, platform_ +from uses_services import mongo_rules, platform_rules from uses_services.target_types import UsesServicesField @@ -24,6 +24,6 @@ def rules(): return [ PythonTestsGeneratorTarget.register_plugin_field(UsesServicesField), PythonTestTarget.register_plugin_field(UsesServicesField), - *platform_.rules(), - *mongo.rules(), + *platform_rules.rules(), + *mongo_rules.rules(), ] From ee4a4cc95fc2fb9142a3a0fee7d1a4d70d884e88 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 6 Jan 2023 21:22:31 -0600 Subject: [PATCH 47/71] add dependent rules to uses_services plugin --- pants-plugins/uses_services/mongo_rules.py | 2 ++ pants-plugins/uses_services/platform_rules.py | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pants-plugins/uses_services/mongo_rules.py b/pants-plugins/uses_services/mongo_rules.py index d3701098ac..c13e1979ec 100644 --- a/pants-plugins/uses_services/mongo_rules.py +++ b/pants-plugins/uses_services/mongo_rules.py @@ -22,6 +22,7 @@ PexRequirements, VenvPex, VenvPexProcess, + rules as pex_rules, ) from pants.engine.fs import CreateDigest, Digest, FileContent from pants.engine.rules import collect_rules, Get, MultiGet, rule @@ -238,4 +239,5 @@ def rules(): return [ *collect_rules(), UnionRule(PytestPluginSetupRequest, UsesMongoRequest), + *pex_rules(), ] diff --git a/pants-plugins/uses_services/platform_rules.py b/pants-plugins/uses_services/platform_rules.py index 0d743b4a73..9db8bdfc00 100644 --- a/pants-plugins/uses_services/platform_rules.py +++ b/pants-plugins/uses_services/platform_rules.py @@ -18,6 +18,7 @@ PexRequirements, VenvPex, VenvPexProcess, + rules as pex_rules, ) from pants.engine.fs import CreateDigest, Digest, FileContent from pants.engine.process import ProcessCacheScope, ProcessResult @@ -77,4 +78,7 @@ async def get_platform() -> Platform: def rules(): - return collect_rules() + return [ + *collect_rules(), + *pex_rules(), + ] From 4605244229b707caf7c3305840c1d8e1b25107a4 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 6 Jan 2023 22:23:32 -0600 Subject: [PATCH 48/71] fix typos in uses_services plugin --- pants-plugins/uses_services/mongo_rules.py | 2 ++ pants-plugins/uses_services/platform_rules.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pants-plugins/uses_services/mongo_rules.py b/pants-plugins/uses_services/mongo_rules.py index c13e1979ec..495f3f443d 100644 --- a/pants-plugins/uses_services/mongo_rules.py +++ b/pants-plugins/uses_services/mongo_rules.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os + from textwrap import dedent from pants.backend.python.goals.pytest_runner import ( diff --git a/pants-plugins/uses_services/platform_rules.py b/pants-plugins/uses_services/platform_rules.py index 9db8bdfc00..92408d8ca8 100644 --- a/pants-plugins/uses_services/platform_rules.py +++ b/pants-plugins/uses_services/platform_rules.py @@ -69,7 +69,7 @@ async def get_platform() -> Platform: input_digest=script_digest, description="Introspecting platform (arch, os, distro)", # this can change from run to run, so don't cache results. - cache_scope=ProcessCacheScope.PER_RESTART_SUCCESSFULL, + cache_scope=ProcessCacheScope.PER_RESTART_SUCCESSFUL, level=LogLevel.DEBUG, ), ) From d83ab195601cb5ad89d7d10026f62928a7f5d327 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 6 Jan 2023 22:25:13 -0600 Subject: [PATCH 49/71] add test for uses_services get_platform rule --- pants-plugins/uses_services/BUILD | 4 ++ .../uses_services/platform_rules_test.py | 49 +++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 pants-plugins/uses_services/platform_rules_test.py diff --git a/pants-plugins/uses_services/BUILD b/pants-plugins/uses_services/BUILD index db46e8d6c9..0eea8b1cf1 100644 --- a/pants-plugins/uses_services/BUILD +++ b/pants-plugins/uses_services/BUILD @@ -1 +1,5 @@ python_sources() + +python_tests( + name="tests", +) diff --git a/pants-plugins/uses_services/platform_rules_test.py b/pants-plugins/uses_services/platform_rules_test.py new file mode 100644 index 0000000000..f1c07ac087 --- /dev/null +++ b/pants-plugins/uses_services/platform_rules_test.py @@ -0,0 +1,49 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import dataclasses + +import pytest + +from pants.testutil.rule_runner import QueryRule, RuleRunner + +from .platform_rules import Platform, rules as platform_rules + + +@pytest.fixture +def rule_runner() -> RuleRunner: + return RuleRunner( + rules=[ + *platform_rules(), + QueryRule(Platform, ()), + ], + target_types=[], + ) + + +def test_get_platform(rule_runner: RuleRunner) -> None: + rule_runner.set_options( + ["--backend-packages=uses_services"], + env_inherit={"PATH", "PYENV_ROOT", "HOME"}, + ) + + platform = rule_runner.request(Platform, ()) + + assert isinstance(platform, Platform) + assert dataclasses.is_dataclass(platform) + # there isn't a good way to inject mocks into the script that + # the rule_runner runs in a venv. So, there isn't a nice way + # to test the values of the Platform fields as people could + # run tests on any platform. From c1ea512af3b36e1bb9f00cb59602ad5572d07e3a Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 6 Jan 2023 23:11:43 -0600 Subject: [PATCH 50/71] separate pytest-specific rule from generic mongo_is_running rule --- pants-plugins/uses_services/mongo_rules.py | 37 ++++++++++++++++++---- 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/pants-plugins/uses_services/mongo_rules.py b/pants-plugins/uses_services/mongo_rules.py index 495f3f443d..f4ce3b1f56 100644 --- a/pants-plugins/uses_services/mongo_rules.py +++ b/pants-plugins/uses_services/mongo_rules.py @@ -13,6 +13,7 @@ # limitations under the License. import os +from dataclasses import dataclass from textwrap import dedent from pants.backend.python.goals.pytest_runner import ( @@ -41,7 +42,17 @@ from uses_services.target_types import UsesServicesField -class UsesMongoRequest(PytestPluginSetupRequest): +@dataclass(frozen=True) +class UsesMongoRequest: + pass + + +@dataclass(frozen=True) +class MongoIsRunning: + pass + + +class PytestUsesMongoRequest(PytestPluginSetupRequest): @classmethod def is_applicable(cls, target: Target) -> bool: if not target.has_field(UsesServicesField): @@ -51,13 +62,25 @@ def is_applicable(cls, target: Target) -> bool: @rule( - desc="Test to see if mongodb is running and accessible for tests.", + desc="Ensure mongodb is running and accessible before running tests.", level=LogLevel.DEBUG, ) -async def mongo_is_running( - request: UsesMongoRequest, platform: Platform +async def mongo_is_running_for_pytest( + request: PytestUsesMongoRequest ) -> PytestPluginSetup: + # this will raise an error if mongo is not running + _ = await Get(MongoIsRunning, UsesMongoRequest()) + + return PytestPluginSetup() + +@rule( + desc="Test to see if mongodb is running and accessible.", + level=LogLevel.DEBUG, +) +async def mongo_is_running( + request: UsesMongoRequest, platform: Platform +) -> MongoIsRunning: # These config opts are used via oslo_config.cfg.CONF.database.{host,port,db_name,connection_timeout} # These config opts currently hard-coded in: # for unit tests: st2tests/st2tests/config.py @@ -120,7 +143,9 @@ async def mongo_is_running( is_running = result.exit_code == 0 if is_running: - return PytestPluginSetup() + return MongoIsRunning() + + # mongo is not running, so raise an error with instructions. if platform.distro in ["centos", "rhel"] or "rhel" in platform.distro_like: instructions = dedent( @@ -240,6 +265,6 @@ async def mongo_is_running( def rules(): return [ *collect_rules(), - UnionRule(PytestPluginSetupRequest, UsesMongoRequest), + UnionRule(PytestPluginSetupRequest, PytestUsesMongoRequest), *pex_rules(), ] From 9ef1f3806188e375d64af6a546a15fcb2f0d28bb Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 6 Jan 2023 23:33:11 -0600 Subject: [PATCH 51/71] refactor uses_services plugin + comments Now the db connection settings come from the generic request object (which for now just defaults to what is hard-coded in the st2 tests). And the comments make a bit more sense in where they are. --- pants-plugins/uses_services/mongo_rules.py | 72 +++++++++++++--------- 1 file changed, 42 insertions(+), 30 deletions(-) diff --git a/pants-plugins/uses_services/mongo_rules.py b/pants-plugins/uses_services/mongo_rules.py index f4ce3b1f56..00d3389532 100644 --- a/pants-plugins/uses_services/mongo_rules.py +++ b/pants-plugins/uses_services/mongo_rules.py @@ -44,7 +44,27 @@ @dataclass(frozen=True) class UsesMongoRequest: - pass + """One or more targets need a running mongo service using these settings. + + The db_* attributes represent the db connection settings from st2.conf. + In st2 code, they come from: + oslo_config.cfg.CONF.database.{host,port,db_name,connection_timeout} + """ + # These config opts currently hard-coded in: + # for unit tests: st2tests/st2tests/config.py + # for integration tests: conf/st2.tests*.conf st2tests/st2tests/fixtures/conf/st2.tests*.conf + # (changed by setting ST2_CONFIG_PATH env var inside the tests) + # TODO: for unit tests: modify code to pull db connect settings from env vars + # TODO: for int tests: modify st2.tests*.conf on the fly to set the per-pantsd-slot db_name + # and either add env vars for db connect settings or modify conf files as well + + # with our version of oslo.config (newer are slower) we can't directly override opts w/ environment variables. + + db_host: str = "127.0.0.1" # localhost in test_db.DbConnectionTestCase + db_port: int = 27017 + # db_name is "st2" in test_db.DbConnectionTestCase + db_name: str = f"st2-test{os.environ.get('ST2TESTS_PARALLEL_SLOT', '')}" + db_connection_timeout: int = 3000 @dataclass(frozen=True) @@ -68,6 +88,20 @@ def is_applicable(cls, target: Target) -> bool: async def mongo_is_running_for_pytest( request: PytestUsesMongoRequest ) -> PytestPluginSetup: + # TODO: delete these comments once the Makefile becomes irrelevant. + # the comments explore how the Makefile prepares to run and runs tests + + # The st2-test database gets dropped between (in Makefile based testing): + # - each component (st2*/ && various config/ dirs) in Makefile + # - DbTestCase/CleanDbTestCase setUpClass + + # Makefile + # .run-unit-tests-coverage (<- .combine-unit-tests-coverage <- .coverage.unit <- .unit-tests-coverage-html <- ci-unit <- ci) + # echo "----- Dropping st2-test db -----" + # mongo st2-test --eval "db.dropDatabase();" + # for component in $(COMPONENTS_TEST) + # nosetests $(NOSE_OPTS) -s -v $(NOSE_COVERAGE_FLAGS) $(NOSE_COVERAGE_PACKAGES) $$component/tests/unit + # this will raise an error if mongo is not running _ = await Get(MongoIsRunning, UsesMongoRequest()) @@ -81,34 +115,6 @@ async def mongo_is_running_for_pytest( async def mongo_is_running( request: UsesMongoRequest, platform: Platform ) -> MongoIsRunning: - # These config opts are used via oslo_config.cfg.CONF.database.{host,port,db_name,connection_timeout} - # These config opts currently hard-coded in: - # for unit tests: st2tests/st2tests/config.py - # for integration tests: conf/st2.tests*.conf st2tests/st2tests/fixtures/conf/st2.tests*.conf - # (changed by setting ST2_CONFIG_PATH env var inside the tests) - # TODO: for unit tests: modify code to pull db connect settings from env vars - # TODO: for int tests: modify st2.tests*.conf on the fly to set the per-pantsd-slot db_name - # and either add env vars for db connect settings or modify conf files as well - - db_host = "127.0.0.1" # localhost in test_db.DbConnectionTestCase - db_port = 27017 - db_name = f"st2-test{os.environ.get('ST2TESTS_PARALLEL_SLOT', '')}" - # db_name = "st2-test" # st2 in test_db.DbConnectionTestCase - connection_timeout = 3000 - - # The st2-test database gets dropped between (in Makefile based testing): - # - each component (st2*/ && various config/ dirs) in Makefile - # - DbTestCase/CleanDbTestCase setUpClass - - # with our version of oslo.config (newer are slower) we can't directly override opts w/ environment variables. - - # Makefile - # .run-unit-tests-coverage (<- .combine-unit-tests-coverage <- .coverage.unit <- .unit-tests-coverage-html <- ci-unit <- ci) - # echo "----- Dropping st2-test db -----" - # mongo st2-test --eval "db.dropDatabase();" - # for component in $(COMPONENTS_TEST) - # nosetests $(NOSE_OPTS) -s -v $(NOSE_COVERAGE_FLAGS) $(NOSE_COVERAGE_PACKAGES) $$component/tests/unit - script_path = "./is_mongo_running.py" # pants is already watching this directory as it is under a source root. @@ -132,7 +138,13 @@ async def mongo_is_running( FallibleProcessResult, VenvPexProcess( mongoengine_pex, - argv=(script_path, db_host, str(db_port), db_name, str(connection_timeout)), + argv=( + script_path, + request.db_host, + str(request.db_port), + request.db_name, + str(request.db_connection_timeout), + ), input_digest=script_digest, description="Checking to see if Mongo is up and accessible.", # this can change from run to run, so don't cache results. From 7d9320d63cb24960285238965b7c0244d4264074 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 7 Jan 2023 00:44:26 -0600 Subject: [PATCH 52/71] add test for uses_services plugin mongo rule --- .../uses_services/mongo_rules_test.py | 193 ++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 pants-plugins/uses_services/mongo_rules_test.py diff --git a/pants-plugins/uses_services/mongo_rules_test.py b/pants-plugins/uses_services/mongo_rules_test.py new file mode 100644 index 0000000000..363f0c29f9 --- /dev/null +++ b/pants-plugins/uses_services/mongo_rules_test.py @@ -0,0 +1,193 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import os + +import pytest + +from pants.backend.python import target_types_rules +from pants.backend.python.target_types import PythonSourcesGeneratorTarget + +from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest +from pants.engine.addresses import Address +from pants.engine.internals.scheduler import ExecutionError +from pants.engine.fs import CreateDigest, Digest, FileContent, Snapshot +from pants.engine.target import Target +from pants.core.goals.fmt import FmtResult +from pants.testutil.rule_runner import QueryRule, RuleRunner + +from .exceptions import ServiceMissingError +from .mongo_rules import ( + MongoIsRunning, + PytestUsesMongoRequest, + UsesMongoRequest, + rules as mongo_rules +) +from .platform_rules import Platform +from .target_types import UsesServicesField + + +class MockTarget(Target): + alias = "mock_target" + core_fields = (UsesServicesField,) + + +@pytest.fixture +def rule_runner() -> RuleRunner: + return RuleRunner( + rules=[ + *mongo_rules(), + # *target_types_rules.rules(), + QueryRule(MongoIsRunning, (UsesMongoRequest, Platform)), + # QueryRule(PytestPluginSetup, (PytestUsesMongoRequest)), + ], + target_types=[ + # MockTarget, + # PythonTestsGeneratorTarget, + # PythonTestTarget, + ], + ) + + +def run_mongo_is_running( + rule_runner: RuleRunner, + uses_mongo_request: UsesMongoRequest, + mock_platform: Platform, + *, + extra_args: list[str] | None = None, +) -> MongoIsRunning: + rule_runner.set_options( + [ + "--backend-packages=uses_services", + *(extra_args or ()), + ], + env_inherit={"PATH", "PYENV_ROOT", "HOME"}, + ) + result = rule_runner.request( + MongoIsRunning, + [uses_mongo_request, mock_platform], + ) + return result + + +def platform( + arch="", + os="", + distro="", + distro_name="", + distro_codename="", + distro_like="", + distro_major_version="", + distro_version="", + mac_release="", + win_release="", +) -> Platform: + """Create a Platform with all values defaulted to the empty string.""" + return Platform( + arch=arch, + os=os, + distro=distro, + distro_name=distro_name, + distro_codename=distro_codename, + distro_like=distro_like, + distro_major_version=distro_major_version, + distro_version=distro_version, + mac_release=mac_release, + win_release=win_release, + ) + + +# TODO: this requires that mongo be running... +#def test_is_running(rule_runner: RuleRunner) -> None: +# request = UsesMongoRequest() +# mock_platform = platform() + + # we are asserting that this does not raise an exception +# is_running = run_mongo_is_running(rule_runner, request, mock_platform) +# assert is_running + + +@pytest.mark.parametrize( + "mock_platform", + ( + platform(), # empty + # platform( + # arch="x86_64", + # os="Linux", + # distro="", + # distro_name="", + # distro_codename="", + # distro_like="", + # distro_major_version="", + # distro_version="", + # ), + platform( + arch="x86_64", + os="Linux", + distro="centos", + distro_name="Centos Linux", + distro_codename="Core", + distro_like="rhel fedora", + distro_major_version="7", + distro_version="7", + ), + platform( + arch="x86_64", + os="Linux", + distro="ubuntu", + distro_name="Ubuntu", + distro_codename="xenial", + distro_like="debian", + distro_major_version="16", + distro_version="16.04", + ), + platform( + arch="x86_64", + os="Linux", + distro="gentoo", + distro_name="Gentoo", + distro_codename="n/a", + distro_major_version="2", + distro_version="2.7", + ), + platform( + arch="x86_64", + os="Darwin", + distro="darwin", + distro_name="Darwin", + distro_major_version="19", + distro_version="19.6.0", + mac_release="10.15.7", + ), + ), +) +def test_not_running(rule_runner: RuleRunner, mock_platform: Platform) -> None: + request = UsesMongoRequest( + db_host="127.100.20.7", + db_port=10, # unassigned port, unlikely to be used + ) + + with pytest.raises(ExecutionError) as exception_info: + run_mongo_is_running(rule_runner, request, mock_platform) + + execution_error = exception_info.value + assert len(execution_error.wrapped_exceptions) == 1 + + exc = execution_error.wrapped_exceptions[0] + assert isinstance(exc, ServiceMissingError) + + assert exc.service == "mongo" + assert "The mongo service does not seem to be running" in str(exc) + assert exc.instructions != "" From 571c341cfb89d8b81108d39ac35feea116cbc500 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sun, 8 Jan 2023 02:10:03 -0600 Subject: [PATCH 53/71] add description of uses_services plugin to pants-plugins/README.md --- pants-plugins/README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pants-plugins/README.md b/pants-plugins/README.md index 8f99aa3d5e..b995e2b6d5 100644 --- a/pants-plugins/README.md +++ b/pants-plugins/README.md @@ -8,6 +8,9 @@ The plugins here add custom goals or other logic into pants. To see available goals, do "./pants help goals" and "./pants help $goal". +These plugins might be useful outside of the StackStorm project: +- `uses_services` + These StackStorm-specific plugins might be useful in other StackStorm-related repos. - `pack_metadata` @@ -66,3 +69,13 @@ the `fmt` goal (eg `./pants fmt contrib/schemas::`), the schemas will be regenerated if any of the files used to generate them have changed. Also, running the `lint` goal will fail if the schemas need to be regenerated. + +### `uses_seevices` plugin + +This plugin validates that services are running if required. For example, some tests +need mongo, so this plugin can ensure mongo is running. If it is not running, then +an error with instructions on how to run it are given to the user. + +`uses_services` has some StackStorm-specific assumptions in it, but it might be +generalizable. There are several other StackStorm-specific plugins, but some of +them are only useful in the st2 repo. From 29fbefaa298a97e4e94e35608f339b49ab1ec480 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 9 Jan 2023 10:04:06 -0600 Subject: [PATCH 54/71] reformat with black --- pants-plugins/uses_services/mongo_rules.py | 3 ++- pants-plugins/uses_services/mongo_rules_test.py | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/pants-plugins/uses_services/mongo_rules.py b/pants-plugins/uses_services/mongo_rules.py index 00d3389532..eaddc4a42d 100644 --- a/pants-plugins/uses_services/mongo_rules.py +++ b/pants-plugins/uses_services/mongo_rules.py @@ -50,6 +50,7 @@ class UsesMongoRequest: In st2 code, they come from: oslo_config.cfg.CONF.database.{host,port,db_name,connection_timeout} """ + # These config opts currently hard-coded in: # for unit tests: st2tests/st2tests/config.py # for integration tests: conf/st2.tests*.conf st2tests/st2tests/fixtures/conf/st2.tests*.conf @@ -86,7 +87,7 @@ def is_applicable(cls, target: Target) -> bool: level=LogLevel.DEBUG, ) async def mongo_is_running_for_pytest( - request: PytestUsesMongoRequest + request: PytestUsesMongoRequest, ) -> PytestPluginSetup: # TODO: delete these comments once the Makefile becomes irrelevant. # the comments explore how the Makefile prepares to run and runs tests diff --git a/pants-plugins/uses_services/mongo_rules_test.py b/pants-plugins/uses_services/mongo_rules_test.py index 363f0c29f9..869641220b 100644 --- a/pants-plugins/uses_services/mongo_rules_test.py +++ b/pants-plugins/uses_services/mongo_rules_test.py @@ -33,7 +33,7 @@ MongoIsRunning, PytestUsesMongoRequest, UsesMongoRequest, - rules as mongo_rules + rules as mongo_rules, ) from .platform_rules import Platform from .target_types import UsesServicesField @@ -110,11 +110,11 @@ def platform( # TODO: this requires that mongo be running... -#def test_is_running(rule_runner: RuleRunner) -> None: +# def test_is_running(rule_runner: RuleRunner) -> None: # request = UsesMongoRequest() # mock_platform = platform() - # we are asserting that this does not raise an exception +# we are asserting that this does not raise an exception # is_running = run_mongo_is_running(rule_runner, request, mock_platform) # assert is_running @@ -176,7 +176,7 @@ def platform( def test_not_running(rule_runner: RuleRunner, mock_platform: Platform) -> None: request = UsesMongoRequest( db_host="127.100.20.7", - db_port=10, # unassigned port, unlikely to be used + db_port=10, # unassigned port, unlikely to be used ) with pytest.raises(ExecutionError) as exception_info: @@ -187,7 +187,7 @@ def test_not_running(rule_runner: RuleRunner, mock_platform: Platform) -> None: exc = execution_error.wrapped_exceptions[0] assert isinstance(exc, ServiceMissingError) - + assert exc.service == "mongo" assert "The mongo service does not seem to be running" in str(exc) assert exc.instructions != "" From 37816fc6ef9795664d42a84fc2c4de66276bf539 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 9 Jan 2023 10:10:25 -0600 Subject: [PATCH 55/71] enable pants-plugins/uses_services mongo test_is_running --- .github/workflows/test.yaml | 6 ++++++ pants-plugins/uses_services/BUILD | 4 ++++ pants-plugins/uses_services/mongo_rules_test.py | 16 ++++++++-------- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index cc236afdb9..8391949d30 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -39,6 +39,12 @@ jobs: python-version-short: '3.8' python-version: '3.8.10' + services: + mongo: + image: mongo:4.4 + ports: + - 27017:27017 + env: COLUMNS: '120' diff --git a/pants-plugins/uses_services/BUILD b/pants-plugins/uses_services/BUILD index 0eea8b1cf1..33bccad5cb 100644 --- a/pants-plugins/uses_services/BUILD +++ b/pants-plugins/uses_services/BUILD @@ -2,4 +2,8 @@ python_sources() python_tests( name="tests", + overrides={ + # We use this plugin to validate we can run the tests for it. + "mongo_rules_test.py": {"uses": ["mongo"]}, + } ) diff --git a/pants-plugins/uses_services/mongo_rules_test.py b/pants-plugins/uses_services/mongo_rules_test.py index 869641220b..a3565f743f 100644 --- a/pants-plugins/uses_services/mongo_rules_test.py +++ b/pants-plugins/uses_services/mongo_rules_test.py @@ -109,14 +109,14 @@ def platform( ) -# TODO: this requires that mongo be running... -# def test_is_running(rule_runner: RuleRunner) -> None: -# request = UsesMongoRequest() -# mock_platform = platform() - -# we are asserting that this does not raise an exception -# is_running = run_mongo_is_running(rule_runner, request, mock_platform) -# assert is_running +# Warning this requires that mongo be running +def test_is_running(rule_runner: RuleRunner) -> None: + request = UsesMongoRequest() + mock_platform = platform() + + # we are asserting that this does not raise an exception + is_running = run_mongo_is_running(rule_runner, request, mock_platform) + assert is_running @pytest.mark.parametrize( From a46328c97aa2471bad30564d3da78bce2cf06b44 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 9 Jan 2023 10:53:50 -0600 Subject: [PATCH 56/71] satisfy flake8 --- .../uses_services/mongo_rules_test.py | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/pants-plugins/uses_services/mongo_rules_test.py b/pants-plugins/uses_services/mongo_rules_test.py index a3565f743f..f1d11a0720 100644 --- a/pants-plugins/uses_services/mongo_rules_test.py +++ b/pants-plugins/uses_services/mongo_rules_test.py @@ -13,25 +13,15 @@ # limitations under the License. from __future__ import annotations -import os - import pytest -from pants.backend.python import target_types_rules -from pants.backend.python.target_types import PythonSourcesGeneratorTarget - -from pants.core.util_rules.source_files import SourceFiles, SourceFilesRequest -from pants.engine.addresses import Address from pants.engine.internals.scheduler import ExecutionError -from pants.engine.fs import CreateDigest, Digest, FileContent, Snapshot from pants.engine.target import Target -from pants.core.goals.fmt import FmtResult from pants.testutil.rule_runner import QueryRule, RuleRunner from .exceptions import ServiceMissingError from .mongo_rules import ( MongoIsRunning, - PytestUsesMongoRequest, UsesMongoRequest, rules as mongo_rules, ) @@ -49,15 +39,9 @@ def rule_runner() -> RuleRunner: return RuleRunner( rules=[ *mongo_rules(), - # *target_types_rules.rules(), QueryRule(MongoIsRunning, (UsesMongoRequest, Platform)), - # QueryRule(PytestPluginSetup, (PytestUsesMongoRequest)), - ], - target_types=[ - # MockTarget, - # PythonTestsGeneratorTarget, - # PythonTestTarget, ], + target_types=[], ) From 64b149d0aafb203d08bdad9b9b1430957858f6fe Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 9 Jan 2023 12:01:43 -0600 Subject: [PATCH 57/71] reformat with black --- pants-plugins/uses_services/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pants-plugins/uses_services/BUILD b/pants-plugins/uses_services/BUILD index 33bccad5cb..3f90050ff8 100644 --- a/pants-plugins/uses_services/BUILD +++ b/pants-plugins/uses_services/BUILD @@ -5,5 +5,5 @@ python_tests( overrides={ # We use this plugin to validate we can run the tests for it. "mongo_rules_test.py": {"uses": ["mongo"]}, - } + }, ) From f82a4f34776418209ab37b9f3dc2ad3ccc62d931 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 9 Jan 2023 12:17:34 -0600 Subject: [PATCH 58/71] cleanup uses_services plugin test names/comments --- pants-plugins/uses_services/mongo_rules_test.py | 4 ++-- pants-plugins/uses_services/scripts/is_mongo_running.py | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pants-plugins/uses_services/mongo_rules_test.py b/pants-plugins/uses_services/mongo_rules_test.py index f1d11a0720..667450afd2 100644 --- a/pants-plugins/uses_services/mongo_rules_test.py +++ b/pants-plugins/uses_services/mongo_rules_test.py @@ -94,7 +94,7 @@ def platform( # Warning this requires that mongo be running -def test_is_running(rule_runner: RuleRunner) -> None: +def test_mongo_is_running(rule_runner: RuleRunner) -> None: request = UsesMongoRequest() mock_platform = platform() @@ -157,7 +157,7 @@ def test_is_running(rule_runner: RuleRunner) -> None: ), ), ) -def test_not_running(rule_runner: RuleRunner, mock_platform: Platform) -> None: +def test_mongo_not_running(rule_runner: RuleRunner, mock_platform: Platform) -> None: request = UsesMongoRequest( db_host="127.100.20.7", db_port=10, # unassigned port, unlikely to be used diff --git a/pants-plugins/uses_services/scripts/is_mongo_running.py b/pants-plugins/uses_services/scripts/is_mongo_running.py index 787c7fefc4..8d5ecfce8a 100644 --- a/pants-plugins/uses_services/scripts/is_mongo_running.py +++ b/pants-plugins/uses_services/scripts/is_mongo_running.py @@ -17,12 +17,16 @@ def _is_mongo_running( db_host: str, db_port: int, db_name: str, connection_timeout_ms: int ) -> bool: + """Connect to mongo with connection logic that mirrors the st2 code. + + In particular, this is based on st2common.models.db.db_setup(). + This should not import the st2 code as it should be self-contained. + """ # late import so that __file__ can be imported in the pants plugin without these imports import mongoengine from pymongo.errors import ConnectionFailure from pymongo.errors import ServerSelectionTimeoutError - # cf st2common.models.db.setup() connection = mongoengine.connection.connect( db_name, host=db_host, @@ -31,7 +35,7 @@ def _is_mongo_running( serverSelectionTimeoutMS=connection_timeout_ms, ) - # connection.connect() is lazy. Make a command to test connection. + # connection.connect() is lazy. Make a command to test the connection. try: # The ismaster command is cheap and does not require auth connection.admin.command("ismaster") From 84dd957ccdf69d5eb6e905bee706ea8085017389 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 9 Jan 2023 18:11:57 -0600 Subject: [PATCH 59/71] more consistent & drop unused test bits --- pants-plugins/uses_services/mongo_rules.py | 2 +- pants-plugins/uses_services/mongo_rules_test.py | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/pants-plugins/uses_services/mongo_rules.py b/pants-plugins/uses_services/mongo_rules.py index eaddc4a42d..c143690480 100644 --- a/pants-plugins/uses_services/mongo_rules.py +++ b/pants-plugins/uses_services/mongo_rules.py @@ -187,7 +187,7 @@ async def mongo_is_running( enabled=1 gpgkey=https://www.mongodb.org/static/pgp/server-4.0.asc EOT" - # install mongo + # Install mongo yum install mongodb-org # Don't forget to start mongo. """ diff --git a/pants-plugins/uses_services/mongo_rules_test.py b/pants-plugins/uses_services/mongo_rules_test.py index 667450afd2..9162973034 100644 --- a/pants-plugins/uses_services/mongo_rules_test.py +++ b/pants-plugins/uses_services/mongo_rules_test.py @@ -26,12 +26,6 @@ rules as mongo_rules, ) from .platform_rules import Platform -from .target_types import UsesServicesField - - -class MockTarget(Target): - alias = "mock_target" - core_fields = (UsesServicesField,) @pytest.fixture From cabfdfe608150e0bfe2cb7aac1b4ebf212da926d Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 10 Jan 2023 15:32:00 -0600 Subject: [PATCH 60/71] flake8 --- pants-plugins/uses_services/mongo_rules_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pants-plugins/uses_services/mongo_rules_test.py b/pants-plugins/uses_services/mongo_rules_test.py index 9162973034..ca11e6df03 100644 --- a/pants-plugins/uses_services/mongo_rules_test.py +++ b/pants-plugins/uses_services/mongo_rules_test.py @@ -16,7 +16,6 @@ import pytest from pants.engine.internals.scheduler import ExecutionError -from pants.engine.target import Target from pants.testutil.rule_runner import QueryRule, RuleRunner from .exceptions import ServiceMissingError From d8b423fcc7cb6b87420602cd8c0f33c539b7c82e Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 23 Jan 2023 09:47:46 -0600 Subject: [PATCH 61/71] more samples --- pants-plugins/uses_services/mongo_rules_test.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pants-plugins/uses_services/mongo_rules_test.py b/pants-plugins/uses_services/mongo_rules_test.py index ca11e6df03..edf80e2f50 100644 --- a/pants-plugins/uses_services/mongo_rules_test.py +++ b/pants-plugins/uses_services/mongo_rules_test.py @@ -148,6 +148,16 @@ def test_mongo_is_running(rule_runner: RuleRunner) -> None: distro_version="19.6.0", mac_release="10.15.7", ), + platform( + arch="AMD64", + os="Windows", + win_release="", + ), + platform( + arch="aarch64", + os="Linux", + # no distro in termux on android + ), ), ) def test_mongo_not_running(rule_runner: RuleRunner, mock_platform: Platform) -> None: From b76147d198b77795856ff4688ab883fbe65330b5 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 25 Jan 2023 11:29:36 -0600 Subject: [PATCH 62/71] move platform_sample test cases to separate file for reusability --- pants-plugins/uses_services/BUILD | 10 +- pants-plugins/uses_services/data_fixtures.py | 150 ++++++++++++++++++ .../uses_services/mongo_rules_test.py | 93 +---------- 3 files changed, 161 insertions(+), 92 deletions(-) create mode 100644 pants-plugins/uses_services/data_fixtures.py diff --git a/pants-plugins/uses_services/BUILD b/pants-plugins/uses_services/BUILD index 3f90050ff8..dd9d873fc4 100644 --- a/pants-plugins/uses_services/BUILD +++ b/pants-plugins/uses_services/BUILD @@ -1,4 +1,12 @@ -python_sources() +python_sources( + sources=["*.py", "!*_test.py", "!data_fixtures.py"], +) + +python_test_utils( + sources=[ + "data_fixtures.py", + ], +) python_tests( name="tests", diff --git a/pants-plugins/uses_services/data_fixtures.py b/pants-plugins/uses_services/data_fixtures.py new file mode 100644 index 0000000000..df03c89809 --- /dev/null +++ b/pants-plugins/uses_services/data_fixtures.py @@ -0,0 +1,150 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +from .platform_rules import Platform + + +def platform( + arch="", + os="", + distro="", + distro_name="", + distro_codename="", + distro_like="", + distro_major_version="", + distro_version="", + mac_release="", + win_release="", +) -> Platform: + """Create a Platform with all values defaulted to the empty string.""" + return Platform( + arch=arch, + os=os, + distro=distro, + distro_name=distro_name, + distro_codename=distro_codename, + distro_like=distro_like, + distro_major_version=distro_major_version, + distro_version=distro_version, + mac_release=mac_release, + win_release=win_release, + ) + + +platform_samples = ( + platform(), # empty + # EL distros ################## + platform( + arch="x86_64", + os="Linux", + distro="centos", + distro_name="Centos Linux", + distro_codename="Core", + distro_like="rhel fedora", + distro_major_version="7", + distro_version="7", + ), + platform( + arch="x86_64", + os="Linux", + distro="rocky", + distro_name="Rocky Linux", + distro_codename="Green Obsidian", + distro_like="rhel centos fedora", + distro_major_version="8", + distro_version="8.7", + ), + # debian distros ############## + platform( + arch="x86_64", + os="Linux", + distro="ubuntu", + distro_name="Ubuntu", + distro_codename="xenial", + distro_like="debian", + distro_major_version="16", + distro_version="16.04", + ), + platform( + arch="x86_64", + os="Linux", + distro="ubuntu", + distro_name="Ubuntu", + distro_codename="bionic", + distro_like="debian", + distro_major_version="18", + distro_version="18.04", + ), + platform( + arch="x86_64", + os="Linux", + distro="ubuntu", + distro_name="Ubuntu", + distro_codename="focal", + distro_like="debian", + distro_major_version="20", + distro_version="20.04", + ), + # other Linux distros ######### + platform( + arch="x86_64", + os="Linux", + distro="gentoo", + distro_name="Gentoo", + distro_codename="n/a", + distro_major_version="2", + distro_version="2.7", + ), + platform( + arch="aarch64", + os="Linux", + # no distro in termux on android + ), + # platform( + # arch="x86_64", + # os="Linux", + # distro="", + # distro_name="", + # distro_codename="", + # distro_like="", + # distro_major_version="", + # distro_version="", + # ), + # Mac OS X #################### + platform( + arch="x86_64", + os="Darwin", + distro="darwin", + distro_name="Darwin", + distro_major_version="19", + distro_version="19.6.0", + mac_release="10.15.7", + ), + platform( + arch="x86_64", + os="Darwin", + distro="darwin", + distro_name="Darwin", + distro_major_version="21", + distro_version="21.6.0", + mac_release="12.6.2", + ), + # Windows ##################### + platform( + arch="AMD64", + os="Windows", + win_release="", + ), +) diff --git a/pants-plugins/uses_services/mongo_rules_test.py b/pants-plugins/uses_services/mongo_rules_test.py index edf80e2f50..ca4b15a2f0 100644 --- a/pants-plugins/uses_services/mongo_rules_test.py +++ b/pants-plugins/uses_services/mongo_rules_test.py @@ -18,6 +18,7 @@ from pants.engine.internals.scheduler import ExecutionError from pants.testutil.rule_runner import QueryRule, RuleRunner +from .data_fixtures import platform, platform_samples from .exceptions import ServiceMissingError from .mongo_rules import ( MongoIsRunning, @@ -59,33 +60,6 @@ def run_mongo_is_running( return result -def platform( - arch="", - os="", - distro="", - distro_name="", - distro_codename="", - distro_like="", - distro_major_version="", - distro_version="", - mac_release="", - win_release="", -) -> Platform: - """Create a Platform with all values defaulted to the empty string.""" - return Platform( - arch=arch, - os=os, - distro=distro, - distro_name=distro_name, - distro_codename=distro_codename, - distro_like=distro_like, - distro_major_version=distro_major_version, - distro_version=distro_version, - mac_release=mac_release, - win_release=win_release, - ) - - # Warning this requires that mongo be running def test_mongo_is_running(rule_runner: RuleRunner) -> None: request = UsesMongoRequest() @@ -96,70 +70,7 @@ def test_mongo_is_running(rule_runner: RuleRunner) -> None: assert is_running -@pytest.mark.parametrize( - "mock_platform", - ( - platform(), # empty - # platform( - # arch="x86_64", - # os="Linux", - # distro="", - # distro_name="", - # distro_codename="", - # distro_like="", - # distro_major_version="", - # distro_version="", - # ), - platform( - arch="x86_64", - os="Linux", - distro="centos", - distro_name="Centos Linux", - distro_codename="Core", - distro_like="rhel fedora", - distro_major_version="7", - distro_version="7", - ), - platform( - arch="x86_64", - os="Linux", - distro="ubuntu", - distro_name="Ubuntu", - distro_codename="xenial", - distro_like="debian", - distro_major_version="16", - distro_version="16.04", - ), - platform( - arch="x86_64", - os="Linux", - distro="gentoo", - distro_name="Gentoo", - distro_codename="n/a", - distro_major_version="2", - distro_version="2.7", - ), - platform( - arch="x86_64", - os="Darwin", - distro="darwin", - distro_name="Darwin", - distro_major_version="19", - distro_version="19.6.0", - mac_release="10.15.7", - ), - platform( - arch="AMD64", - os="Windows", - win_release="", - ), - platform( - arch="aarch64", - os="Linux", - # no distro in termux on android - ), - ), -) +@pytest.mark.parametrize("mock_platform", platform_samples) def test_mongo_not_running(rule_runner: RuleRunner, mock_platform: Platform) -> None: request = UsesMongoRequest( db_host="127.100.20.7", From a44d3ad0c88029c37adaa773be2253c5bd46d499 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 25 Jan 2023 12:57:48 -0600 Subject: [PATCH 63/71] drop windows platform fixture I guessed on Reintroduce windows platofrm tests if someone submits a sample. --- pants-plugins/uses_services/data_fixtures.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pants-plugins/uses_services/data_fixtures.py b/pants-plugins/uses_services/data_fixtures.py index df03c89809..39304cb2fb 100644 --- a/pants-plugins/uses_services/data_fixtures.py +++ b/pants-plugins/uses_services/data_fixtures.py @@ -141,10 +141,4 @@ def platform( distro_version="21.6.0", mac_release="12.6.2", ), - # Windows ##################### - platform( - arch="AMD64", - os="Windows", - win_release="", - ), ) From fc69b6684377fc3fd28cd53d5208e2d33df04409 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 25 Jan 2023 13:10:10 -0600 Subject: [PATCH 64/71] expand instructions --- pants-plugins/uses_services/mongo_rules.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pants-plugins/uses_services/mongo_rules.py b/pants-plugins/uses_services/mongo_rules.py index c143690480..43930d7f04 100644 --- a/pants-plugins/uses_services/mongo_rules.py +++ b/pants-plugins/uses_services/mongo_rules.py @@ -230,7 +230,10 @@ async def mongo_is_running( Arch: {platform.arch} Distro: {platform.distro} + Distro Name: {platform.distro_name} Distro Codename: {platform.distro_codename} + Distro Family: {platform.distro_like} + Distro Major Version: {platform.distro_major_version} Distro Version: {platform.distro_version} Thanks and Good Luck! From 2df21cba595b3a264db0ff0fefcaea058291396d Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 25 Jan 2023 17:45:54 -0600 Subject: [PATCH 65/71] standardize ServiceMissingError messages --- pants-plugins/uses_services/exceptions.py | 131 +++++++++++++++++- pants-plugins/uses_services/mongo_rules.py | 154 +++++---------------- 2 files changed, 165 insertions(+), 120 deletions(-) diff --git a/pants-plugins/uses_services/exceptions.py b/pants-plugins/uses_services/exceptions.py index c020e33a63..0db3e1ac6a 100644 --- a/pants-plugins/uses_services/exceptions.py +++ b/pants-plugins/uses_services/exceptions.py @@ -11,14 +11,30 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from dataclasses import dataclass + from uses_services.platform_rules import Platform +@dataclass(frozen=True) +class ServiceSpecificMessages: + service: str + + service_start_cmd_el_7: str + service_start_cmd_el: str + not_installed_clause_el: str + install_instructions_el: str + + service_start_cmd_deb: str + not_installed_clause_deb: str + install_instructions_deb: str + + service_start_cmd_generic: str + + class ServiceMissingError(Exception): """Error raised when a test uses a service but that service is missing.""" - # TODO add special platform handling to DRY instructions across services - def __init__(self, service, platform: Platform, instructions="", msg=None): if msg is None: msg = f"The {service} service does not seem to be running or is not accessible!" @@ -28,3 +44,114 @@ def __init__(self, service, platform: Platform, instructions="", msg=None): self.service = service self.platform = platform self.instructions = instructions + + @classmethod + def generate_instructions( + cls, platform: Platform, messages: ServiceSpecificMessages + ): + service = messages.service + + supported = False + if platform.distro in ["centos", "rhel"] or "rhel" in platform.distro_like: + supported = True + if platform.distro_major_version == "7": + service_start_cmd = messages.service_start_cmd_el_7 + else: + service_start_cmd = messages.service_start_cmd_el + not_installed_clause = messages.not_installed_clause_el + install_instructions = messages.install_instructions_el + + elif ( + platform.distro in ["ubuntu", "debian"] or "debian" in platform.distro_like + ): + supported = True + service_start_cmd = messages.service_start_cmd_deb + not_installed_clause = messages.not_installed_clause_deb + install_instructions = messages.install_instructions_deb + + if supported: + instructions = dedent( + f"""\ + If {service} is installed, but not running try: + + {service_start_cmd} + + If {service} is not installed, {not_installed_clause}: + + """ + ).format( + service=service, + service_start_cmd=service_start_cmd, + not_installed_clause=not_installed_clause, + ) + instructions += install_instructions + elif platform.os == "Linux": + instructions = dedent( + f"""\ + You are on Linux using {platform.distro_name}, which is not + one of our generally supported distributions. We recommend + you use vagrant for local development with something like: + + vagrant init stackstorm/st2 + vagrant up + vagrant ssh + + Please see: https://docs.stackstorm.com/install/vagrant.html + + For anyone who wants to attempt local development without vagrant, + you are pretty much on your own. At a minimum you need to install + and start {service} with something like: + + {messages.service_start_cmd_generic} + + We would be interested to hear about alternative distros people + are using for development. If you are able, please let us know + on slack which distro you are using: + + Arch: {platform.arch} + Distro: {platform.distro} + Distro Name: {platform.distro_name} + Distro Codename: {platform.distro_codename} + Distro Family: {platform.distro_like} + Distro Major Version: {platform.distro_major_version} + Distro Version: {platform.distro_version} + + Thanks and Good Luck! + """ + ) + elif platform.os == "Darwin": # MacOS + instructions = dedent( + f"""\ + You are on Mac OS. Generally we recommend using vagrant for local + development on Mac OS with something like: + + vagrant init stackstorm/st2 + vagrant up + vagrant ssh + + Please see: https://docs.stackstorm.com/install/vagrant.html + + For anyone who wants to attempt local development without vagrant, + you may run into some speed bumps. Others StackStorm developers have + been known to use Mac OS for development, so feel free to ask for + help in slack. At a minimum you need to install and start {service}. + """ + ) + else: + instructions = dedent( + f"""\ + You are not on Linux. In this case we recommend using vagrant + for local development with something like: + + vagrant init stackstorm/st2 + vagrant up + vagrant ssh + + Please see: https://docs.stackstorm.com/install/vagrant.html + + For anyone who wants to attempt local development without vagrant, + you are pretty much on your own. At a minimum you need to install + and start {service}. Good luck! + """ + ) + return instructions diff --git a/pants-plugins/uses_services/mongo_rules.py b/pants-plugins/uses_services/mongo_rules.py index 43930d7f04..7d93335d0a 100644 --- a/pants-plugins/uses_services/mongo_rules.py +++ b/pants-plugins/uses_services/mongo_rules.py @@ -34,7 +34,7 @@ from pants.engine.unions import UnionRule from pants.util.logging import LogLevel -from uses_services.exceptions import ServiceMissingError +from uses_services.exceptions import ServiceMissingError, ServiceSpecificMessages from uses_services.platform_rules import Platform from uses_services.scripts.is_mongo_running import ( __file__ as is_mongo_running_full_path, @@ -159,123 +159,41 @@ async def mongo_is_running( return MongoIsRunning() # mongo is not running, so raise an error with instructions. - - if platform.distro in ["centos", "rhel"] or "rhel" in platform.distro_like: - instructions = dedent( - """\ - If mongo is installed, but not running try: - - """ - ) - - if platform.distro_major_version == "7": - instructions += "\nservice mongo start\n" - else: - instructions += "\nsystemctl start mongod\n" - - instructions += dedent( - """ - If mongo is not installed, this is one way to install it: - - # Add key and repo for the latest stable MongoDB (4.0) - rpm --import https://www.mongodb.org/static/pgp/server-4.0.asc - sh -c "cat < /etc/yum.repos.d/mongodb-org-4.repo - [mongodb-org-4] - name=MongoDB Repository - baseurl=https://repo.mongodb.org/yum/redhat/${OSRELEASE_VERSION}/mongodb-org/4.0/x86_64/ - gpgcheck=1 - enabled=1 - gpgkey=https://www.mongodb.org/static/pgp/server-4.0.asc - EOT" - # Install mongo - yum install mongodb-org - # Don't forget to start mongo. - """ - ) - elif platform.distro in ["ubuntu", "debian"] or "debian" in platform.distro_like: - instructions = dedent( - """\ - If mongo is installed, but not running try: - - systemctl start mongod - - If mongo is not installed, this is one way to install it: - - apt-get install mongodb mongodb-server - # Don't forget to start mongo. - """ - ) - elif platform.os == "Linux": - instructions = dedent( - f"""\ - You are on Linux using {platform.distro_name}, which is not - one of our generally supported distributions. We recommend - you use vagrant for local development with something like: - - vagrant init stackstorm/st2 - vagrant up - vagrant ssh - - Please see: https://docs.stackstorm.com/install/vagrant.html - - For anyone who wants to attempt local development without vagrant, - you are pretty much on your own. At a minimum you need to install - and start mongo with something like: - - systemctl start mongod - - We would be interested to hear about alternative distros people - are using for development. If you are able, please let us know - on slack which distro you are using: - - Arch: {platform.arch} - Distro: {platform.distro} - Distro Name: {platform.distro_name} - Distro Codename: {platform.distro_codename} - Distro Family: {platform.distro_like} - Distro Major Version: {platform.distro_major_version} - Distro Version: {platform.distro_version} - - Thanks and Good Luck! - """ - ) - elif platform.os == "Darwin": # MacOS - instructions = dedent( - """\ - You are on Mac OS. Generally we recommend using vagrant for local - development on Mac OS with something like: - - vagrant init stackstorm/st2 - vagrant up - vagrant ssh - - Please see: https://docs.stackstorm.com/install/vagrant.html - - For anyone who wants to attempt local development without vagrant, - you may run into some speed bumps. Others StackStorm developers have - been known to use Mac OS for development, so feel free to ask for - help in slack. At a minimum you need to install and start mongo. - """ - ) - else: - instructions = dedent( - """\ - You are not on Linux. In this case we recommend using vagrant - for local development with something like: - - vagrant init stackstorm/st2 - vagrant up - vagrant ssh - - Please see: https://docs.stackstorm.com/install/vagrant.html - - For anyone who wants to attempt local development without vagrant, - you are pretty much on your own. At a minimum you need to install - and start mongo. Good luck! - """ - ) - - raise ServiceMissingError("mongo", platform, instructions) + raise ServiceMissingError( + platform, + ServiceSpecificMessages( + service="mongo", + service_start_cmd_el_7="service mongo start", + service_start_cmd_el="systemctl start mongod", + not_installed_clause_el="this is one way to install it:", + install_instructions_el=dedent( + """\ + # Add key and repo for the latest stable MongoDB (4.0) + rpm --import https://www.mongodb.org/static/pgp/server-4.0.asc + sh -c "cat < /etc/yum.repos.d/mongodb-org-4.repo + [mongodb-org-4] + name=MongoDB Repository + baseurl=https://repo.mongodb.org/yum/redhat/${OSRELEASE_VERSION}/mongodb-org/4.0/x86_64/ + gpgcheck=1 + enabled=1 + gpgkey=https://www.mongodb.org/static/pgp/server-4.0.asc + EOT" + # Install mongo + yum install mongodb-org + # Don't forget to start mongo. + """ + ), + service_start_cmd_deb="systemctl start mongod", + not_installed_clause_deb="this is one way to install it:", + install_instructions_deb=dedent( + """\ + apt-get install mongodb mongodb-server + # Don't forget to start mongo. + """ + ), + service_start_cmd_generic="systemctl start mongod", + ), + ) def rules(): From 6f41b21510eb74ce2644921e0508382a614c3ee1 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 25 Jan 2023 18:12:29 -0600 Subject: [PATCH 66/71] include sudo in instructions --- pants-plugins/uses_services/mongo_rules.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pants-plugins/uses_services/mongo_rules.py b/pants-plugins/uses_services/mongo_rules.py index 7d93335d0a..f8c83a9f86 100644 --- a/pants-plugins/uses_services/mongo_rules.py +++ b/pants-plugins/uses_services/mongo_rules.py @@ -169,8 +169,8 @@ async def mongo_is_running( install_instructions_el=dedent( """\ # Add key and repo for the latest stable MongoDB (4.0) - rpm --import https://www.mongodb.org/static/pgp/server-4.0.asc - sh -c "cat < /etc/yum.repos.d/mongodb-org-4.repo + sudo rpm --import https://www.mongodb.org/static/pgp/server-4.0.asc + sudo sh -c "cat < /etc/yum.repos.d/mongodb-org-4.repo [mongodb-org-4] name=MongoDB Repository baseurl=https://repo.mongodb.org/yum/redhat/${OSRELEASE_VERSION}/mongodb-org/4.0/x86_64/ @@ -179,7 +179,7 @@ async def mongo_is_running( gpgkey=https://www.mongodb.org/static/pgp/server-4.0.asc EOT" # Install mongo - yum install mongodb-org + sudo yum -y install mongodb-org # Don't forget to start mongo. """ ), @@ -187,7 +187,7 @@ async def mongo_is_running( not_installed_clause_deb="this is one way to install it:", install_instructions_deb=dedent( """\ - apt-get install mongodb mongodb-server + sudo apt-get install -y mongodb-org # Don't forget to start mongo. """ ), From c800d3b0d31521f86e4d14fc8f46799832e76529 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 26 Jan 2023 10:54:53 -0600 Subject: [PATCH 67/71] cleanup BUILD file --- pants-plugins/uses_services/BUILD | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pants-plugins/uses_services/BUILD b/pants-plugins/uses_services/BUILD index dd9d873fc4..2a462c1e58 100644 --- a/pants-plugins/uses_services/BUILD +++ b/pants-plugins/uses_services/BUILD @@ -3,9 +3,8 @@ python_sources( ) python_test_utils( - sources=[ - "data_fixtures.py", - ], + name="test_utils", + sources=["data_fixtures.py"], ) python_tests( From f9b607b8ddaeef746179e4e8c77740a881612731 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Thu, 26 Jan 2023 11:13:29 -0600 Subject: [PATCH 68/71] finish updating interface of ServiceSpecificMessages --- pants-plugins/uses_services/exceptions.py | 18 ++++++++++++++---- pants-plugins/uses_services/mongo_rules.py | 6 +++--- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/pants-plugins/uses_services/exceptions.py b/pants-plugins/uses_services/exceptions.py index 0db3e1ac6a..c32423f735 100644 --- a/pants-plugins/uses_services/exceptions.py +++ b/pants-plugins/uses_services/exceptions.py @@ -11,7 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations + from dataclasses import dataclass +from textwrap import dedent from uses_services.platform_rules import Platform @@ -35,7 +38,9 @@ class ServiceSpecificMessages: class ServiceMissingError(Exception): """Error raised when a test uses a service but that service is missing.""" - def __init__(self, service, platform: Platform, instructions="", msg=None): + def __init__( + self, service: str, platform: Platform, instructions: str = "", msg=None + ): if msg is None: msg = f"The {service} service does not seem to be running or is not accessible!" if instructions: @@ -46,9 +51,9 @@ def __init__(self, service, platform: Platform, instructions="", msg=None): self.instructions = instructions @classmethod - def generate_instructions( + def generate( cls, platform: Platform, messages: ServiceSpecificMessages - ): + ) -> ServiceMissingError: service = messages.service supported = False @@ -154,4 +159,9 @@ def generate_instructions( and start {service}. Good luck! """ ) - return instructions + + return cls( + service=service, + platform=platform, + instructions=instructions, + ) diff --git a/pants-plugins/uses_services/mongo_rules.py b/pants-plugins/uses_services/mongo_rules.py index f8c83a9f86..ad8333132f 100644 --- a/pants-plugins/uses_services/mongo_rules.py +++ b/pants-plugins/uses_services/mongo_rules.py @@ -159,9 +159,9 @@ async def mongo_is_running( return MongoIsRunning() # mongo is not running, so raise an error with instructions. - raise ServiceMissingError( - platform, - ServiceSpecificMessages( + raise ServiceMissingError.generate( + platform=platform, + messages=ServiceSpecificMessages( service="mongo", service_start_cmd_el_7="service mongo start", service_start_cmd_el="systemctl start mongod", From 30e62637678f0bd6746e8cea8ed34b2add11c76c Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Fri, 27 Jan 2023 15:26:36 -0600 Subject: [PATCH 69/71] avoid cirucular use of `pants-plugins/uses_services` before the tests that test it --- pants-plugins/uses_services/BUILD | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pants-plugins/uses_services/BUILD b/pants-plugins/uses_services/BUILD index 2a462c1e58..19808c0b61 100644 --- a/pants-plugins/uses_services/BUILD +++ b/pants-plugins/uses_services/BUILD @@ -9,8 +9,11 @@ python_test_utils( python_tests( name="tests", - overrides={ - # We use this plugin to validate we can run the tests for it. - "mongo_rules_test.py": {"uses": ["mongo"]}, - }, + # Do not enable `uses` for these tests. + # These tests still need the running services, but if the plugin + # is somehow broken, then any failures would prevent these tests + # from running to tell us what is wrong. + # overrides={ + # "mongo_rules_test.py": {"uses": ["mongo"]}, + # }, ) From ffe161fe2f5373d3efe4830e45e0c8dada24f381 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 9 Jan 2023 10:53:37 -0600 Subject: [PATCH 70/71] update changelog entry --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3d37b236c2..7274fdbbdd 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,7 +14,7 @@ Added working on StackStorm, improve our security posture, and improve CI reliability thanks in part to pants' use of PEX lockfiles. This is not a user-facing addition. #5778 #5789 #5817 #5795 #5830 #5833 #5834 #5841 #5840 #5838 #5842 #5837 #5849 #5850 - #5846 #5853 #5848 #5847 #5858 #5857 #5860 #5868 #5871 + #5846 #5853 #5848 #5847 #5858 #5857 #5860 #5868 #5871 #5864 Contributed by @cognifloyd * Added a joint index to solve the problem of slow mongo queries for scheduled executions. #5805 From 5e18f3ea397472cc4406e724153cc5c851a49139 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 28 Jan 2023 10:50:52 -0600 Subject: [PATCH 71/71] grammar/typo fix --- pants-plugins/uses_services/exceptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pants-plugins/uses_services/exceptions.py b/pants-plugins/uses_services/exceptions.py index c32423f735..6da31cf46b 100644 --- a/pants-plugins/uses_services/exceptions.py +++ b/pants-plugins/uses_services/exceptions.py @@ -137,7 +137,7 @@ def generate( Please see: https://docs.stackstorm.com/install/vagrant.html For anyone who wants to attempt local development without vagrant, - you may run into some speed bumps. Others StackStorm developers have + you may run into some speed bumps. Other StackStorm developers have been known to use Mac OS for development, so feel free to ask for help in slack. At a minimum you need to install and start {service}. """