Replace Tito releases with rhpkg and automatic Koji tagging

.github/workflows/main.yml

@@ -34,7 +34,7 @@ jobs:
           sudo curl --create-dirs -o /usr/local/bin/rpmdev-bumpspec https://codeberg.org/rpm-community/rpmdevtools/raw/commit/30eca1b7a06a2df5bbfa3bfd6a4c89c19c6ebd60/rpmdev-bumpspec
           sudo curl --create-dirs -o /usr/local/bin/rpmdev-packager https://codeberg.org/rpm-community/rpmdevtools/raw/branch/main/rpmdev-packager
           sudo chmod +x /usr/local/bin/spectool /usr/local/bin/rpmdev-bumpspec /usr/local/bin/rpmdev-packager
-          sudo apt-get update && sudo apt-get install -y git-annex rpm
+          sudo apt-get update && sudo apt-get install -y git-annex rpm cpio
       - name: Run tests
         run: tox
 

obal/data/module_utils/koji_wrapper.py

@@ -1,7 +1,7 @@
 """
 A koji wrapper
 """
-from subprocess import check_output, CalledProcessError
+from subprocess import check_output, CalledProcessError, STDOUT
 
 class KojiCommandError(Exception):
     """Raised when Koji command fails"""
@@ -18,6 +18,18 @@ def koji(command, executable=None):
         executable = 'koji'
 
     try:
-        return check_output([executable] + command, universal_newlines=True)
+        return check_output([executable] + command, universal_newlines=True, stderr=STDOUT)
     except CalledProcessError as error:
         raise KojiCommandError(error.output, error.cmd)
+
+
+def package_whitelisted(executable, tag, package):
+    """Check effective tag registration, excluding blocked packages."""
+    try:
+        output = koji(['list-pkgs', '--tag', tag, '--package', package, '--quiet'], executable)
+    except KojiCommandError as error:
+        if error.message.strip() == '(no matching packages)':
+            return False
+        raise
+    return any(line.split() and line.split()[0] == package and '[BLOCKED]' not in line
+               for line in output.splitlines())

obal/data/module_utils/obal.py

@@ -5,10 +5,10 @@
 import os
 
 try:
-    from ansible.module_utils.koji_wrapper import koji, KojiCommandError # pylint:disable=import-error,no-name-in-module
+    from ansible.module_utils.koji_wrapper import package_whitelisted, KojiCommandError # pylint:disable=import-error,no-name-in-module
 except ImportError:
     # when trying to import this file outside the ansible context, we cannot rely on the magic ansible import path
-    from .koji_wrapper import koji, KojiCommandError # pylint:disable=import-error,no-name-in-module
+    from .koji_wrapper import package_whitelisted, KojiCommandError # pylint:disable=import-error,no-name-in-module
 
 
 def macro_lookup(command, scl=None, dist=None, macros=None):
@@ -128,18 +128,8 @@ def get_whitelist_status(build_command, tag, package):
 
     Return `True` if the package is whitelisted, `False` otherwise.
     """
-    cmd = [
-        'list-pkgs',
-        '--tag',
-        tag,
-        '--package',
-        package,
-        '--quiet'
-    ]
-
     try:
-        koji(cmd, build_command)
-        return True
+        return package_whitelisted(build_command, tag, package)
     except KojiCommandError:
         return False
 

obal/data/modules/check_koji_build.py

@@ -5,6 +5,7 @@
 
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.koji_wrapper import koji, KojiCommandError # pylint:disable=import-error,no-name-in-module
+from ansible.module_utils.rhpkg import build_info, ReleaseError # pylint:disable=import-error,no-name-in-module
 
 def main():
     """
@@ -24,22 +25,15 @@ def main():
     package = module.params['package']
     koji_executable = module.params['koji_executable']
 
-    command = ['buildinfo', nvr]
     try:
-        output = koji(command, koji_executable)
-        exists = "BUILD: %s" % (nvr) in output and "State: COMPLETE" in output
-    except KojiCommandError:
-        output = None
-        exists = False
+        info = build_info(koji_executable or 'koji', nvr)
+    except ReleaseError as error:
+        module.fail_json(msg=str(error), changed=False)
+    exists = info['state'] == 'COMPLETE'
+    result = dict(changed=False, exists=exists, state=info['state'], task=info['task'])
 
     if tag:
-        exists_for_tag = False
-
-        if output is not None:
-            for line in output.split("\n"):
-                if line.startswith("Tags:"):
-                    tags = line.split()[1:]
-                    exists_for_tag = tag in tags
+        exists_for_tag = exists and tag in info['tags']
 
         if not exists_for_tag:
             command = ['latest-build', '--quiet', tag, package]
@@ -50,11 +44,11 @@ def main():
                 module.fail_json(changed=False, msg=error.message, command=error.command)
 
             build = build.split(' ')[0]
-            module.exit_json(changed=False, exists=exists, tagged_version=build, exists_for_tag=False)
+            module.exit_json(tagged_version=build, exists_for_tag=False, **result)
         else:
-            module.exit_json(changed=False, exists=exists, tagged_version=nvr, exists_for_tag=True)
+            module.exit_json(tagged_version=nvr, exists_for_tag=True, **result)
     else:
-        module.exit_json(changed=False, exists=exists)
+        module.exit_json(**result)
 
 if __name__ == '__main__':
     main()

obal/data/modules/koji_find_tags.py

@@ -4,6 +4,7 @@
 """
 
 from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.rhpkg import inventory_tags, ReleaseError  # pylint:disable=import-error,no-name-in-module
 
 def main():
     """
@@ -19,9 +20,11 @@ def main():
 
     tags = set()
 
-    for attributes in packages.values():
-        if 'koji_tags' in attributes:
-            tags.update(tag['name'] for tag in attributes['koji_tags'])
+    try:
+        for package, attributes in packages.items():
+            tags.update(tag['name'] for tag in inventory_tags(attributes, package))
+    except ReleaseError as error:
+        module.fail_json(msg=str(error))
 
     module.exit_json(changed=False, tags=sorted(tags))
 

obal/data/modules/koji_verify_tag.py

@@ -7,8 +7,9 @@
 import subprocess
 
 from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils.obal import get_specfile_name # pylint:disable=import-error,no-name-in-module
+from ansible.module_utils.obal import specfile_macro_lookup # pylint:disable=import-error,no-name-in-module
 from ansible.module_utils.koji_wrapper import koji # pylint:disable=import-error,no-name-in-module
+from ansible.module_utils.rhpkg import inventory_tags, uses_rhpkg, ReleaseError  # pylint:disable=import-error,no-name-in-module
 
 def main():
     """
@@ -19,31 +20,48 @@ def main():
             packages=dict(type='dict', required=True),
             tag=dict(type='str', required=True),
             directory=dict(type='str', required=True),
-            koji_executable=dict(type='str', require=False, default='koji')
+            koji_executable=dict(type='str', required=False)
         )
     )
 
     packages_for_tag = set()
+    executables = set()
 
     for (package, attributes) in module.params['packages'].items():
         tag = None
 
-        if 'koji_tags' in attributes:
-            for koji_tag in attributes['koji_tags']:
+        try:
+            tags = inventory_tags(attributes, package)
+        except ReleaseError as error:
+            module.fail_json(msg=str(error))
+
+        if tags:
+            for koji_tag in tags:
                 if module.params['tag'] == koji_tag['name']:
                     tag = koji_tag
 
-            if tag and 'package_base_dir' in attributes:
-                specfile = os.path.join(attributes['package_base_dir'], package, "{}.spec".format(package))
-                scl = tag.get('scl')
-                name = get_specfile_name(os.path.join(module.params['directory'], specfile), scl)
+            if tag and ('package_base_dir' in attributes or uses_rhpkg(attributes)):
+                specfile = os.path.join(attributes.get('package_base_dir', 'packages'), package,
+                                        "{}.spec".format(package))
+                directory = attributes.get('inventory_dir', module.params['directory'])
+                name, _ = specfile_macro_lookup(os.path.join(directory, specfile), '%{name}',
+                                                scl=tag.get('scl'), dist=tag.get('dist'), macros=tag.get('macros'))
                 packages_for_tag.add(name)
+                default_command = 'brew' if uses_rhpkg(attributes) else 'koji'
+                executables.add(attributes.get('build_package_koji_command',
+                                                attributes.get('koji_executable', default_command)))
+
+    executable = module.params['koji_executable']
+    if not executable:
+        if len(executables) > 1:
+            module.fail_json(msg='Packages for this tag select different Koji executables')
+        executable = next(iter(executables), 'koji')
 
     try:
         command = ['list-pkgs', '--quiet', '--tag', module.params['tag']]
         koji_output = koji(
             command,
-            executable=module.params['koji_executable']
+            executable=executable
         )
 
         packages_in_koji = {item.split(' ', 1)[0] for item in koji_output.split("\n") if item}

obal/data/modules/package_whitelist_check.py

@@ -1,70 +1,62 @@
 #!/usr/bin/python
-# pylint: disable=C0111,C0301,R1710
+"""Check Koji package registration for explicit tags or legacy releaser configuration."""
 
-try:
-    import configparser
-except ImportError:
-    import ConfigParser as configparser
+import configparser
 
-import os
-
-from ansible.module_utils.basic import AnsibleModule  # pylint: disable=C0413
-from ansible.module_utils.obal import get_specfile_name, get_whitelist_status  # pylint:disable=import-error,no-name-in-module
-
-ANSIBLE_METADATA = {
-    'metadata_version': '1.2',
-    'status': ['preview'],
-    'supported_by': 'community'
-}
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.obal import specfile_macro_lookup  # pylint:disable=import-error,no-name-in-module
+from ansible.module_utils.koji_wrapper import package_whitelisted, KojiCommandError  # pylint:disable=import-error,no-name-in-module
 
 
 def run_module():
+    """Fail before tagging if the package is absent or blocked in any destination."""
     module = AnsibleModule(
         argument_spec=dict(
-            releasers_conf=dict(type='str', required=True),
-            spec_file_path=dict(type='str', required=True),
-            releasers=dict(type='list', required=True),
+            releasers_conf=dict(type='path'),
+            spec_file_path=dict(type='path'),
+            releasers=dict(type='list', elements='str', default=[]),
             build_command=dict(type='str', required=True),
-        )
-    )
-
-    result = dict(
-        changed=False,
-        branches=[],
-        autobuild_tags=[],
-        whitelist_status={},
+            package=dict(type='str'),
+            tags=dict(type='list', elements='str'),
+            scl=dict(type='str'),
+            dist=dict(type='str'),
+            macros=dict(type='dict'),
+        ),
+        required_one_of=[['package', 'spec_file_path'], ['tags', 'releasers_conf']],
+        mutually_exclusive=[['tags', 'releasers_conf']],
+        supports_check_mode=True,
     )
-
-    releasers_config = module.params['releasers_conf']
-
-    # fail if we can't find a releasers.conf
-    if not os.path.isfile(releasers_config):
-        module.fail_json(msg="Config file [ {} ] could not be found.".format(releasers_config), **result)
-
-    config = configparser.ConfigParser(allow_no_value=True)
-    config.read(releasers_config)
-
-    package_name = get_specfile_name(module.params['spec_file_path'])
-
-    # get branches and tags
-    for releaser in module.params['releasers']:
-        if config.has_option(releaser, 'branches'):
-            result['branches'] = config.get(releaser, 'branches').split(' ')
-        if config.has_option(releaser, 'autobuild_tags'):
-            result['autobuild_tags'] = config.get(releaser, 'autobuild_tags').split(' ')
-
-    # fail if we don't find any branches or autobuild_tags
-    if not result['branches'] and not result['autobuild_tags']:
-        module.fail_json(msg="No branches or autobuild_tags were found mapped to {}.".format(module.params['releasers']), **result)
-
-    # check whitelist status
-    for tag in result['branches'] + result['autobuild_tags']:
-        status = get_whitelist_status(module.params['build_command'], tag, package_name)
-        result['whitelist_status'][tag] = status
-
-    if not all(result['whitelist_status'].values()):
-        module.fail_json(msg="Package has not been whitelisted for given branches and autobuild_tags", **result)
-
+    result = dict(changed=False, branches=[], autobuild_tags=[], whitelist_status={})
+    tags = module.params['tags'] or []
+    if module.params['releasers_conf']:
+        config = configparser.ConfigParser(allow_no_value=True)
+        try:
+            with open(module.params['releasers_conf']) as stream:
+                config.read_file(stream)
+            for releaser in module.params['releasers']:
+                for field in ('branches', 'autobuild_tags'):
+                    if config.has_option(releaser, field):
+                        result[field].extend((config.get(releaser, field) or '').split())
+        except (OSError, configparser.Error) as error:
+            module.fail_json(msg=str(error), **result)
+        tags = result['branches'] + result['autobuild_tags']
+    if not tags:
+        module.fail_json(msg='No destination tags configured for package registration checks', **result)
+
+    package = module.params['package']
+    if not package:
+        package, _ = specfile_macro_lookup(module.params['spec_file_path'], '%{name}',
+                                           scl=module.params['scl'], dist=module.params['dist'],
+                                           macros=module.params['macros'])
+    try:
+        for tag in dict.fromkeys(tags):
+            result['whitelist_status'][tag] = package_whitelisted(module.params['build_command'], tag, package)
+    except KojiCommandError as error:
+        module.fail_json(msg=error.message, command=error.command, **result)
+    missing = [tag for tag, registered in result['whitelist_status'].items() if not registered]
+    if missing:
+        module.fail_json(msg='Package {} is not registered or is blocked in tags: {}'.format(
+            package, ', '.join(missing)), **result)
     module.exit_json(**result)
 
 

obal/data/modules/rhpkg_release.py

@@ -0,0 +1,45 @@
+#!/usr/bin/python
+"""Release packages through rhpkg, reusing and tagging existing Koji builds."""
+
+from ansible.module_utils.basic import AnsibleModule
+from ansible.module_utils.rhpkg import RhpkgRelease, ReleaseError, inventory_tags  # pylint:disable=import-error,no-name-in-module
+
+
+def main():
+    """Submit builds and reconcile output tags without invoking Tito."""
+    module = AnsibleModule(
+        argument_spec=dict(
+            directory=dict(type='path', required=True),
+            spec_file=dict(type='path', required=True),
+            targets=dict(type='list', elements='dict', required=True),
+            koji_tags=dict(type='list', elements='dict', default=[]),
+            koji_executable=dict(type='str', default='brew'),
+            rhpkg_executable=dict(type='str', default='rhpkg'),
+            receipt=dict(type='path', required=True),
+            output_directory=dict(type='path', required=True),
+            scratch=dict(type='bool', default=False),
+            wait=dict(type='bool', default=True),
+            waitrepo=dict(type='bool', default=False),
+            tag_check=dict(type='bool', default=True),
+            download_logs=dict(type='bool', default=False),
+            download_rpms=dict(type='bool', default=False),
+        ),
+        supports_check_mode=True,
+    )
+    release = None
+    try:
+        package = module.params['directory'].rstrip('/').split('/')[-1]
+        inventory_tags(dict(build_package_releaser='rhpkg', rhpkg_targets=module.params['targets'],
+                            koji_tags=module.params['koji_tags']), package)
+        release = RhpkgRelease(dict(module.params, check_mode=module.check_mode))
+        result = release.execute()
+    except (ReleaseError, OSError, ValueError) as error:
+        module.fail_json(msg=str(error), changed=release.changed if release else False,
+                         results=release.results if release else [])
+    if result['errors']:
+        module.fail_json(msg='; '.join(result['errors']), **result)
+    module.exit_json(**result)
+
+
+if __name__ == '__main__':
+    main()