diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6cdb7088..737bffc1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -34,7 +34,7 @@ jobs: sudo curl --create-dirs -o /usr/local/bin/rpmdev-bumpspec https://codeberg.org/rpm-community/rpmdevtools/raw/commit/30eca1b7a06a2df5bbfa3bfd6a4c89c19c6ebd60/rpmdev-bumpspec sudo curl --create-dirs -o /usr/local/bin/rpmdev-packager https://codeberg.org/rpm-community/rpmdevtools/raw/branch/main/rpmdev-packager sudo chmod +x /usr/local/bin/spectool /usr/local/bin/rpmdev-bumpspec /usr/local/bin/rpmdev-packager - sudo apt-get update && sudo apt-get install -y git-annex rpm + sudo apt-get update && sudo apt-get install -y git-annex rpm cpio - name: Run tests run: tox diff --git a/obal/data/module_utils/koji_wrapper.py b/obal/data/module_utils/koji_wrapper.py index 407ef8bd..f59f6c72 100644 --- a/obal/data/module_utils/koji_wrapper.py +++ b/obal/data/module_utils/koji_wrapper.py @@ -1,7 +1,7 @@ """ A koji wrapper """ -from subprocess import check_output, CalledProcessError +from subprocess import check_output, CalledProcessError, STDOUT class KojiCommandError(Exception): """Raised when Koji command fails""" @@ -18,6 +18,18 @@ def koji(command, executable=None): executable = 'koji' try: - return check_output([executable] + command, universal_newlines=True) + return check_output([executable] + command, universal_newlines=True, stderr=STDOUT) except CalledProcessError as error: raise KojiCommandError(error.output, error.cmd) + + +def package_whitelisted(executable, tag, package): + """Check effective tag registration, excluding blocked packages.""" + try: + output = koji(['list-pkgs', '--tag', tag, '--package', package, '--quiet'], executable) + except KojiCommandError as error: + if error.message.strip() == '(no matching packages)': + return False + raise + return any(line.split() and line.split()[0] == package and '[BLOCKED]' not in line + for line in output.splitlines()) diff --git a/obal/data/module_utils/obal.py b/obal/data/module_utils/obal.py index 07ee1652..f9fef465 100644 --- a/obal/data/module_utils/obal.py +++ b/obal/data/module_utils/obal.py @@ -5,10 +5,10 @@ import os try: - from ansible.module_utils.koji_wrapper import koji, KojiCommandError # pylint:disable=import-error,no-name-in-module + from ansible.module_utils.koji_wrapper import package_whitelisted, KojiCommandError # pylint:disable=import-error,no-name-in-module except ImportError: # when trying to import this file outside the ansible context, we cannot rely on the magic ansible import path - from .koji_wrapper import koji, KojiCommandError # pylint:disable=import-error,no-name-in-module + from .koji_wrapper import package_whitelisted, KojiCommandError # pylint:disable=import-error,no-name-in-module def macro_lookup(command, scl=None, dist=None, macros=None): @@ -128,18 +128,8 @@ def get_whitelist_status(build_command, tag, package): Return `True` if the package is whitelisted, `False` otherwise. """ - cmd = [ - 'list-pkgs', - '--tag', - tag, - '--package', - package, - '--quiet' - ] - try: - koji(cmd, build_command) - return True + return package_whitelisted(build_command, tag, package) except KojiCommandError: return False diff --git a/obal/data/module_utils/rhpkg.py b/obal/data/module_utils/rhpkg.py new file mode 100644 index 00000000..c5283126 --- /dev/null +++ b/obal/data/module_utils/rhpkg.py @@ -0,0 +1,445 @@ +"""Release dist-git packages through rhpkg and reconcile Koji build tags.""" + +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +from urllib.parse import urlsplit + +from ansible.module_utils.parsing.convert_bool import boolean + +try: + from ansible.module_utils.obal import specfile_macro_lookup + from ansible.module_utils.koji_wrapper import package_whitelisted, KojiCommandError +except ImportError: + from .obal import specfile_macro_lookup + from .koji_wrapper import package_whitelisted, KojiCommandError + + +class ReleaseError(Exception): + """A release could not be completed.""" + + +def run(command, directory=None, check=True): + """Run an argument vector, preserving command diagnostics.""" + result = subprocess.run(command, cwd=directory, text=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, env=dict(os.environ, LC_ALL='C'), check=False) + if check and result.returncode: + raise ReleaseError('{} failed ({}): {}'.format(command, result.returncode, result.stdout)) + return result + + +def normalize_targets(targets, package): + """Validate explicit build destinations independently of output tags.""" + if not isinstance(targets, list) or not targets: + raise ReleaseError('rhpkg_targets must contain at least one build target') + normalized = [] + names = set() + for value in targets: + if not isinstance(value, dict): + raise ReleaseError('Each rhpkg target must be a dictionary') + target = dict(value) + target.setdefault('distgit_package', package) + target.setdefault('macros', {}) + for key in ('name', 'distgit_package', 'distgit_branch', 'build_target', 'dist'): + if not isinstance(target.get(key), str) or not target[key] or target[key].startswith('-'): + raise ReleaseError('Each rhpkg target requires a nonempty {}'.format(key)) + if not re.fullmatch(r'[A-Za-z0-9_.-]+', target['name']) or target['name'] in names: + raise ReleaseError('rhpkg target names must be unique path-safe identifiers') + names.add(target['name']) + if not isinstance(target['macros'], dict): + raise ReleaseError('rhpkg target macros must be a dictionary') + if set(target['macros']) & {'dist', 'scl'}: + raise ReleaseError('Use the target dist and scl fields instead of redefining them in macros') + tags = target.get('tags') + if not isinstance(tags, list) or not tags or any( + not isinstance(tag, str) or not tag or tag.startswith('-') for tag in tags): + raise ReleaseError('Each rhpkg target requires a nonempty list of destination tag names') + target['tags'] = list(dict.fromkeys(tags)) + normalized.append(target) + return normalized + + +def uses_rhpkg(attributes): + """Recognize configured dist-git targets outside build role defaults.""" + direct = boolean(attributes.get('build_package_use_koji_build', False)) + default = 'rhpkg' if attributes.get('rhpkg_targets') and not direct else 'koji' + return attributes.get('build_package_releaser', default) == 'rhpkg' + + +def inventory_tags(attributes, package): + """Expose the same tag configuration to release and inventory verification.""" + if not uses_rhpkg(attributes): + return attributes.get('koji_tags', []) + tags = [] + for target in normalize_targets(attributes.get('rhpkg_targets'), package): + for name in target['tags']: + tag = {key: target[key] for key in ('dist', 'scl', 'macros') if key in target} + tags.append(dict(tag, name=name)) + configured = attributes.get('koji_tags', []) + def comparable(values): + return {json.dumps({key: value.get(key, {} if key == 'macros' else None) + for key in ('name', 'dist', 'scl', 'macros')}, sort_keys=True) for value in values} + if configured and comparable(configured) != comparable(tags): + raise ReleaseError('koji_tags conflicts with rhpkg_targets; configure the destination tags in rhpkg_targets') + return tags + + +def build_info(executable, nvr): + """Distinguish an absent build from a failed query or an incomplete build.""" + result = run([executable, 'buildinfo', nvr], check=False) + if result.returncode: + if result.stdout.strip() == 'No such build: {}'.format(nvr): + return {'state': 'MISSING', 'tags': [], 'task': None, 'source': None} + raise ReleaseError('Unable to query {}: {}'.format(nvr, result.stdout)) + fields = dict(re.findall(r'^(BUILD|State|Tags|Task|Source):[ \t]*(.*)$', result.stdout, re.MULTILINE)) + if fields.get('BUILD', '').split(' ', 1)[0] != nvr or not fields.get('State'): + raise ReleaseError('Unrecognized buildinfo response for {}: {}'.format(nvr, result.stdout)) + task = re.match(r'\d+', fields.get('Task', '')) + return {'state': fields['State'], 'tags': fields.get('Tags', '').split(), + 'task': task.group() if task else None, 'source': fields.get('Source')} + + +def rpm_identity(spec, target): + """Query name and NVR with one consistent macro context.""" + try: + value, _ = specfile_macro_lookup(str(spec), '%{name}\n%{nvr}', scl=target.get('scl'), + dist=target['dist'], macros=target['macros']) + except subprocess.CalledProcessError as error: + raise ReleaseError('Cannot query {}: {}'.format(spec, error.stderr or error.stdout)) from error + name, nvr = value.strip().splitlines() + return name, nvr + + +def macro_arguments(target): + """Build the macro options shared by source expansion and scratch SRPMs.""" + macros = dict(target['macros'], dist=target['dist']) + if target.get('scl'): + macros['scl'] = target['scl'] + return [part for key, value in macros.items() for part in ('--define', '{} {}'.format(key, value))] + + +def spec_inputs(spec, target): + """List source and patch filenames using the destination macro context.""" + output = run(['spectool', '--list-files', '--all'] + macro_arguments(target) + [str(spec)]).stdout + for kind, value in re.findall(r'^(Source\d*|Patch\d*):\s*(.+)$', output, re.MULTILINE): + filename = os.path.basename(urlsplit(value).path) + if not filename or filename in ('.', '..'): + raise ReleaseError('Invalid spec input {}'.format(value)) + yield kind, filename + + +class RhpkgRelease: # pylint: disable=too-many-instance-attributes + """Submit independent build targets before waiting and tagging their results.""" + + # Preserve dist-git service configuration when replacing packaging inputs. + protected = {'Makefile', 'branch', 'sources', 'package.cfg', 'gating.yaml', 'tests', 'plans'} + + def __init__(self, options): + self.options = options + self.directory = Path(options['directory']).resolve() + self.spec = Path(options['spec_file']).resolve() + self.brew = options['koji_executable'] + self.rhpkg = options['rhpkg_executable'] + self.targets = normalize_targets(options['targets'], self.directory.name) + self.receipt = Path(options['receipt']) + self.previous = {} + if self.receipt.exists(): + self.previous = json.loads(self.receipt.read_text()).get('targets', {}) + self.results = [] + self.errors = [] + self.changed = False + + def save(self): + """Persist submitted tasks so a later invocation can finish tagging.""" + if self.options.get('check_mode'): + return + self.receipt.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(mode='w', dir=self.receipt.parent, delete=False) as stream: + targets = dict(self.previous) + for item in self.results: + previous = targets.get(item['target']['name'], {}) + # A failed query or resume validation must not discard a pending task. + if item.get('processed') and (item['tasks'] or not previous.get('tasks') or + previous.get('state') == 'COMPLETE' or item['state'] == 'COMPLETE'): + targets[item['target']['name']] = item + json.dump({'targets': targets}, stream, indent=2) + temporary = stream.name + os.replace(temporary, self.receipt) + + def execute(self): + """Return all successes and failures, including recoverable pending tags.""" + identities = set() + for target in self.targets: + name, nvr = rpm_identity(self.spec, target) + if nvr in identities: + raise ReleaseError('Multiple build targets produce {}; combine their output tags'.format(nvr)) + identities.add(nvr) + self.results.append({'target': target, 'package': name, 'nvr': nvr, 'tasks': [], + 'task_urls': [], 'source_sha': None, + 'pending_tags': [] if self.options['scratch'] else target['tags'], + 'state': 'PENDING', 'changed': False}) + for item in self.results: + try: + self.submit(item) + except (ReleaseError, OSError, ValueError) as error: + item['state'] = 'FAILED' + self.errors.append('{}: {}'.format(item['target']['name'], error)) + item['processed'] = True + self.save() + for item in self.results: + if item['state'] == 'FAILED': + continue + try: + self.finish(item) + except (ReleaseError, OSError, ValueError) as error: + item['state'] = 'FAILED' + self.errors.append('{}: {}'.format(item['target']['name'], error)) + self.save() + return {'changed': self.changed, 'results': self.results, 'errors': self.errors, + 'pending': any(item['state'] == 'PENDING' for item in self.results)} + + def submit(self, item): + """Reuse complete/in-flight builds, otherwise prepare and submit dist-git.""" + target = item['target'] + if not self.options['scratch']: + info = build_info(self.brew, item['nvr']) + resumed = self.resume(item) + if info['state'] == 'COMPLETE': + item['state'] = 'COMPLETE' + return + if info['state'] == 'BUILDING' and info['task']: + item['tasks'] = [info['task']] + return + if info['state'] != 'MISSING': + raise ReleaseError('{} is {}; refusing another submission'.format(item['nvr'], info['state'])) + if resumed: + return + elif self.resume(item): + return + if self.options.get('check_mode'): + item['changed'] = self.changed = True + return + output = run([self.brew, 'list-targets', '--quiet', '--name', target['build_target']]).stdout + if not any(line.split() and line.split()[0] == target['build_target'] for line in output.splitlines()): + raise ReleaseError('Unknown build target {}'.format(target['build_target'])) + self.check_registration(item, target['tags']) + item['input_digest'] = self.input_digest(target) + if self.options['scratch']: + self.scratch_build(item) + else: + self.prepare_and_build(item) + + def check_registration(self, item, tags): + """Validate destination eligibility before a release submission or tag operation.""" + if not self.options['tag_check'] or self.options['scratch'] or self.options.get('check_mode'): + return + for tag in tags: + try: + registered = package_whitelisted(self.brew, tag, item['package']) + except KojiCommandError as error: + raise ReleaseError('Unable to check package registration: {}'.format(error.message)) from error + if not registered: + raise ReleaseError('{} is not registered or is blocked in {}'.format(item['package'], tag)) + + def resume(self, item): + """Recover tasks that may not have reserved a build record yet.""" + previous = self.previous.get(item['target']['name'], {}) + if not previous.get('tasks') or previous.get('state') == 'COMPLETE': + return False + if previous['nvr'] != item['nvr'] or previous['target'] != item['target']: + # Scratch tasks do not reserve NVRs, so buildinfo cannot establish + # whether an earlier scratch task has finished. + if not self.options['scratch']: + old = build_info(self.brew, previous['nvr']) + if old['state'] in ('COMPLETE', 'FAILED', 'CANCELED', 'DELETED'): + return False + raise ReleaseError('An earlier task is pending for a different release; finish it first') + if not self.options.get('check_mode') and previous.get('input_digest') != self.input_digest(item['target']): + raise ReleaseError('Package sources changed while a release task was pending; finish it first') + item.update({key: previous.get(key) for key in ('tasks', 'task_urls', 'source_sha', 'input_digest')}) + return True + + def input_digest(self, target): + """Bind pending tasks to the exact spec and source bytes requested.""" + digest = hashlib.sha256(self.spec.read_bytes()) + for filename, (source, _) in sorted(self.source_files(target).items()): + digest.update(filename.encode()) + with source.open('rb') as stream: + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + def source_files(self, target): + """Resolve spec inputs without copying annex links into dist-git.""" + files = {} + for kind, filename in spec_inputs(self.spec, target): + if filename.startswith('.') or filename in self.protected: + raise ReleaseError('Spec input conflicts with dist-git configuration: {}'.format(filename)) + source = self.directory / filename + if not source.is_file(): + run(['git', 'annex', 'get', '--', filename], self.directory) + if not source.is_file(): + raise ReleaseError('Missing source {}'.format(source)) + tracked = kind.startswith('Patch') or filename.endswith(('.patch', '.changes', '.rpmlintrc')) + files[filename] = (source, tracked) + return files + + def sync(self, checkout, target): + """Replace packaging inputs while preserving dist-git service files.""" + files = self.source_files(target) + files[self.spec.name] = (self.spec, True) + previous_inputs = set() + for spec in checkout.glob('*.spec'): + previous_inputs.add(spec.name) + previous_inputs.update(filename for _, filename in spec_inputs(spec, target)) + tracked = run(['git', 'ls-files', '-z'], checkout).stdout.split('\0') + for filename in tracked: + if filename in previous_inputs and not filename.startswith('.') and filename not in self.protected: + if filename not in files: + run(['git', 'rm', '--', filename], checkout) + for filename, (source, _) in files.items(): + destination = checkout / filename + if destination.is_symlink(): + destination.unlink() + shutil.copyfile(source, destination) + return files + + def prepare_and_build(self, item): + """Submit a normal release from dist-git through rhpkg.""" + target = item['target'] + with tempfile.TemporaryDirectory(prefix='obal-rhpkg-') as temporary: + checkout = Path(temporary) / 'distgit' + run([self.rhpkg, 'clone', '--branch', target['distgit_branch'], target['distgit_package'], str(checkout)]) + files = self.sync(checkout, target) + command = [self.rhpkg, '--release', target['distgit_branch'], 'build', '--nowait', + '--target', target['build_target']] + self.commit(checkout, files, item) + item['source_sha'] = run(['git', 'rev-parse', 'HEAD'], checkout).stdout.strip() + command.append('--skip-nvr-check') + result = run(command, checkout, check=False) + if result.returncode: + # A concurrent release can win the NVR race after our initial query. + if 'already been built' in result.stdout: + info = build_info(self.brew, item['nvr']) + if info['state'] == 'COMPLETE': + item['source_sha'] = None + item['state'] = 'COMPLETE' + return + raise ReleaseError('rhpkg build failed: {}'.format(result.stdout)) + self.record_submission(item, result.stdout) + + def scratch_build(self, item): + """Build an SRPM from local/annex sources and submit it directly to Koji.""" + target = item['target'] + with tempfile.TemporaryDirectory(prefix='obal-scratch-') as temporary: + directory = Path(temporary) + sources = directory / 'SOURCES' + sources.mkdir() + for filename, (source, _) in self.source_files(target).items(): + shutil.copyfile(source, sources / filename) + spec = directory / self.spec.name + shutil.copyfile(self.spec, spec) + output = run(['rpmbuild', '-bs', str(spec), '--define', '_topdir ' + temporary, + '--define', '_sourcedir ' + str(sources), '--define', '_srcrpmdir ' + temporary] + + macro_arguments(target), directory).stdout + matches = re.findall(r'^Wrote:\s*(.*\.src\.rpm)\s*$', output, re.MULTILINE) + if not matches: + raise ReleaseError('rpmbuild returned no source RPM: {}'.format(output)) + srpm = str((directory / matches[-1]).resolve()) + result = run([self.brew, 'build', '--scratch', '--nowait', target['build_target'], srpm]) + self.record_submission(item, result.stdout) + + def record_submission(self, item, output): + """Capture Koji task identifiers from either submission client.""" + item['tasks'] = re.findall(r'^Created task:\s*(\d+)', output, re.MULTILINE) + item['task_urls'] = re.findall(r'^Task info:\s*(.+)', output, re.MULTILINE) + if not item['tasks']: + raise ReleaseError('Build submission returned no task ID: {}'.format(output)) + item['changed'] = self.changed = True + + def commit(self, checkout, files, item): + """Upload sources and push an actual packaging change before building.""" + uploads = sorted(filename for filename, (_, tracked) in files.items() if not tracked) + if uploads: + run([self.rhpkg, 'new-sources'] + uploads, checkout) + tracked_sources = set(run(['git', 'ls-files', '-z'], checkout).stdout.split('\0')) + for filename in uploads: + if filename in tracked_sources: + run(['git', 'rm', '--cached', '--', filename], checkout) + elif (checkout / 'sources').exists(): + (checkout / 'sources').write_text('') + tracked = sorted(filename for filename, (_, keep) in files.items() if keep) + for filename in ('sources', '.gitignore'): + if (checkout / filename).is_file(): + tracked.append(filename) + run(['git', 'add', '--'] + tracked, checkout) + difference = run(['git', 'diff', '--cached', '--quiet'], checkout, check=False) + if difference.returncode not in (0, 1): + raise ReleaseError(difference.stdout) + if difference.returncode: + for field in ('user.name', 'user.email'): + value = run(['git', 'config', field], self.directory).stdout.strip() + run(['git', 'config', field, value], checkout) + run([self.rhpkg, 'commit', '-m', 'Update {}'.format(item['nvr'])], checkout) + item['changed'] = self.changed = True + run([self.rhpkg, 'push'], checkout) + + def finish(self, item): + """Wait for builds, then wait for and verify every requested output tag.""" + if self.options.get('check_mode'): + info = build_info(self.brew, item['nvr']) + missing = set(item['target']['tags']) - set(info['tags']) + item['changed'] = info['state'] != 'COMPLETE' or bool(missing) + self.changed |= item['changed'] + return + if item['tasks'] and item['state'] != 'COMPLETE': + if not self.options['wait']: + return + for task in item['tasks']: + watched = run([self.brew, 'watch-task', task], check=False) + if self.options['download_logs']: + run([self.brew, 'download-logs', '-r', task], self.options['output_directory']) + if watched.returncode: + raise ReleaseError('Build task {} failed: {}'.format(task, watched.stdout)) + if not self.options['scratch']: + self.reconcile(item) + else: + item['pending_tags'] = [] + if self.options['download_rpms']: + self.download(item) + item['state'] = 'COMPLETE' + + def download(self, item): + """Download completed results, including builds reused by tagging only.""" + destination = Path(self.options['output_directory']) / 'downloaded_rpms' / item['target']['name'] + destination.mkdir(parents=True, exist_ok=True) + if self.options['scratch']: + for task in item['tasks']: + run([self.brew, 'download-task', '--arch=noarch', '--arch=x86_64', task], destination) + else: + run([self.brew, 'download-build', item['nvr']], destination) + + def reconcile(self, item): + """A complete build must have the expected identity before it can be tagged.""" + info = build_info(self.brew, item['nvr']) + if info['state'] != 'COMPLETE': + raise ReleaseError('Expected complete build {}; found {}'.format(item['nvr'], info['state'])) + if item['source_sha'] and (not info['source'] or not info['source'].endswith('#' + item['source_sha'])): + raise ReleaseError('Build source does not match the submitted dist-git commit for {}'.format(item['nvr'])) + self.check_registration(item, [tag for tag in item['target']['tags'] if tag not in info['tags']]) + for tag in item['target']['tags']: + if tag not in info['tags']: + result = run([self.brew, 'tag-build', '--wait', tag, item['nvr']], check=False) + info = build_info(self.brew, item['nvr']) + if tag not in info['tags']: + raise ReleaseError('Unable to tag {} into {}: {}'.format(item['nvr'], tag, result.stdout)) + item['changed'] = self.changed = True + item['pending_tags'] = [name for name in item['target']['tags'] if name not in info['tags']] + self.save() + if self.options['waitrepo']: + run([self.brew, 'wait-repo', '--build=' + item['nvr'], '--target', item['target']['build_target']]) diff --git a/obal/data/module_utils/tito_wrapper.py b/obal/data/module_utils/tito_wrapper.py deleted file mode 100644 index 51fdbc17..00000000 --- a/obal/data/module_utils/tito_wrapper.py +++ /dev/null @@ -1,11 +0,0 @@ -""" -A tito wrapper -""" -from subprocess import STDOUT, check_output - - -def tito(command, directory): - """ - Run a tito command - """ - return check_output(command, stderr=STDOUT, universal_newlines=True, cwd=directory) diff --git a/obal/data/modules/check_koji_build.py b/obal/data/modules/check_koji_build.py index 7e5a28c5..02bdf711 100644 --- a/obal/data/modules/check_koji_build.py +++ b/obal/data/modules/check_koji_build.py @@ -5,6 +5,7 @@ from ansible.module_utils.basic import AnsibleModule from ansible.module_utils.koji_wrapper import koji, KojiCommandError # pylint:disable=import-error,no-name-in-module +from ansible.module_utils.rhpkg import build_info, ReleaseError # pylint:disable=import-error,no-name-in-module def main(): """ @@ -24,22 +25,15 @@ def main(): package = module.params['package'] koji_executable = module.params['koji_executable'] - command = ['buildinfo', nvr] try: - output = koji(command, koji_executable) - exists = "BUILD: %s" % (nvr) in output and "State: COMPLETE" in output - except KojiCommandError: - output = None - exists = False + info = build_info(koji_executable or 'koji', nvr) + except ReleaseError as error: + module.fail_json(msg=str(error), changed=False) + exists = info['state'] == 'COMPLETE' + result = dict(changed=False, exists=exists, state=info['state'], task=info['task']) if tag: - exists_for_tag = False - - if output is not None: - for line in output.split("\n"): - if line.startswith("Tags:"): - tags = line.split()[1:] - exists_for_tag = tag in tags + exists_for_tag = exists and tag in info['tags'] if not exists_for_tag: command = ['latest-build', '--quiet', tag, package] @@ -50,11 +44,11 @@ def main(): module.fail_json(changed=False, msg=error.message, command=error.command) build = build.split(' ')[0] - module.exit_json(changed=False, exists=exists, tagged_version=build, exists_for_tag=False) + module.exit_json(tagged_version=build, exists_for_tag=False, **result) else: - module.exit_json(changed=False, exists=exists, tagged_version=nvr, exists_for_tag=True) + module.exit_json(tagged_version=nvr, exists_for_tag=True, **result) else: - module.exit_json(changed=False, exists=exists) + module.exit_json(**result) if __name__ == '__main__': main() diff --git a/obal/data/modules/koji_find_tags.py b/obal/data/modules/koji_find_tags.py index 075f9c94..e12766ec 100644 --- a/obal/data/modules/koji_find_tags.py +++ b/obal/data/modules/koji_find_tags.py @@ -4,6 +4,7 @@ """ from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils.rhpkg import inventory_tags, ReleaseError # pylint:disable=import-error,no-name-in-module def main(): """ @@ -19,9 +20,11 @@ def main(): tags = set() - for attributes in packages.values(): - if 'koji_tags' in attributes: - tags.update(tag['name'] for tag in attributes['koji_tags']) + try: + for package, attributes in packages.items(): + tags.update(tag['name'] for tag in inventory_tags(attributes, package)) + except ReleaseError as error: + module.fail_json(msg=str(error)) module.exit_json(changed=False, tags=sorted(tags)) diff --git a/obal/data/modules/koji_verify_tag.py b/obal/data/modules/koji_verify_tag.py index b5ba5812..25f44364 100644 --- a/obal/data/modules/koji_verify_tag.py +++ b/obal/data/modules/koji_verify_tag.py @@ -7,8 +7,9 @@ import subprocess from ansible.module_utils.basic import AnsibleModule -from ansible.module_utils.obal import get_specfile_name # pylint:disable=import-error,no-name-in-module +from ansible.module_utils.obal import specfile_macro_lookup # pylint:disable=import-error,no-name-in-module from ansible.module_utils.koji_wrapper import koji # pylint:disable=import-error,no-name-in-module +from ansible.module_utils.rhpkg import inventory_tags, uses_rhpkg, ReleaseError # pylint:disable=import-error,no-name-in-module def main(): """ @@ -19,31 +20,48 @@ def main(): packages=dict(type='dict', required=True), tag=dict(type='str', required=True), directory=dict(type='str', required=True), - koji_executable=dict(type='str', require=False, default='koji') + koji_executable=dict(type='str', required=False) ) ) packages_for_tag = set() + executables = set() for (package, attributes) in module.params['packages'].items(): tag = None - if 'koji_tags' in attributes: - for koji_tag in attributes['koji_tags']: + try: + tags = inventory_tags(attributes, package) + except ReleaseError as error: + module.fail_json(msg=str(error)) + + if tags: + for koji_tag in tags: if module.params['tag'] == koji_tag['name']: tag = koji_tag - if tag and 'package_base_dir' in attributes: - specfile = os.path.join(attributes['package_base_dir'], package, "{}.spec".format(package)) - scl = tag.get('scl') - name = get_specfile_name(os.path.join(module.params['directory'], specfile), scl) + if tag and ('package_base_dir' in attributes or uses_rhpkg(attributes)): + specfile = os.path.join(attributes.get('package_base_dir', 'packages'), package, + "{}.spec".format(package)) + directory = attributes.get('inventory_dir', module.params['directory']) + name, _ = specfile_macro_lookup(os.path.join(directory, specfile), '%{name}', + scl=tag.get('scl'), dist=tag.get('dist'), macros=tag.get('macros')) packages_for_tag.add(name) + default_command = 'brew' if uses_rhpkg(attributes) else 'koji' + executables.add(attributes.get('build_package_koji_command', + attributes.get('koji_executable', default_command))) + + executable = module.params['koji_executable'] + if not executable: + if len(executables) > 1: + module.fail_json(msg='Packages for this tag select different Koji executables') + executable = next(iter(executables), 'koji') try: command = ['list-pkgs', '--quiet', '--tag', module.params['tag']] koji_output = koji( command, - executable=module.params['koji_executable'] + executable=executable ) packages_in_koji = {item.split(' ', 1)[0] for item in koji_output.split("\n") if item} diff --git a/obal/data/modules/package_whitelist_check.py b/obal/data/modules/package_whitelist_check.py index 675a7e54..62dd5a35 100644 --- a/obal/data/modules/package_whitelist_check.py +++ b/obal/data/modules/package_whitelist_check.py @@ -1,70 +1,62 @@ #!/usr/bin/python -# pylint: disable=C0111,C0301,R1710 +"""Check Koji package registration for explicit tags or legacy releaser configuration.""" -try: - import configparser -except ImportError: - import ConfigParser as configparser +import configparser -import os - -from ansible.module_utils.basic import AnsibleModule # pylint: disable=C0413 -from ansible.module_utils.obal import get_specfile_name, get_whitelist_status # pylint:disable=import-error,no-name-in-module - -ANSIBLE_METADATA = { - 'metadata_version': '1.2', - 'status': ['preview'], - 'supported_by': 'community' -} +from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils.obal import specfile_macro_lookup # pylint:disable=import-error,no-name-in-module +from ansible.module_utils.koji_wrapper import package_whitelisted, KojiCommandError # pylint:disable=import-error,no-name-in-module def run_module(): + """Fail before tagging if the package is absent or blocked in any destination.""" module = AnsibleModule( argument_spec=dict( - releasers_conf=dict(type='str', required=True), - spec_file_path=dict(type='str', required=True), - releasers=dict(type='list', required=True), + releasers_conf=dict(type='path'), + spec_file_path=dict(type='path'), + releasers=dict(type='list', elements='str', default=[]), build_command=dict(type='str', required=True), - ) - ) - - result = dict( - changed=False, - branches=[], - autobuild_tags=[], - whitelist_status={}, + package=dict(type='str'), + tags=dict(type='list', elements='str'), + scl=dict(type='str'), + dist=dict(type='str'), + macros=dict(type='dict'), + ), + required_one_of=[['package', 'spec_file_path'], ['tags', 'releasers_conf']], + mutually_exclusive=[['tags', 'releasers_conf']], + supports_check_mode=True, ) - - releasers_config = module.params['releasers_conf'] - - # fail if we can't find a releasers.conf - if not os.path.isfile(releasers_config): - module.fail_json(msg="Config file [ {} ] could not be found.".format(releasers_config), **result) - - config = configparser.ConfigParser(allow_no_value=True) - config.read(releasers_config) - - package_name = get_specfile_name(module.params['spec_file_path']) - - # get branches and tags - for releaser in module.params['releasers']: - if config.has_option(releaser, 'branches'): - result['branches'] = config.get(releaser, 'branches').split(' ') - if config.has_option(releaser, 'autobuild_tags'): - result['autobuild_tags'] = config.get(releaser, 'autobuild_tags').split(' ') - - # fail if we don't find any branches or autobuild_tags - if not result['branches'] and not result['autobuild_tags']: - module.fail_json(msg="No branches or autobuild_tags were found mapped to {}.".format(module.params['releasers']), **result) - - # check whitelist status - for tag in result['branches'] + result['autobuild_tags']: - status = get_whitelist_status(module.params['build_command'], tag, package_name) - result['whitelist_status'][tag] = status - - if not all(result['whitelist_status'].values()): - module.fail_json(msg="Package has not been whitelisted for given branches and autobuild_tags", **result) - + result = dict(changed=False, branches=[], autobuild_tags=[], whitelist_status={}) + tags = module.params['tags'] or [] + if module.params['releasers_conf']: + config = configparser.ConfigParser(allow_no_value=True) + try: + with open(module.params['releasers_conf']) as stream: + config.read_file(stream) + for releaser in module.params['releasers']: + for field in ('branches', 'autobuild_tags'): + if config.has_option(releaser, field): + result[field].extend((config.get(releaser, field) or '').split()) + except (OSError, configparser.Error) as error: + module.fail_json(msg=str(error), **result) + tags = result['branches'] + result['autobuild_tags'] + if not tags: + module.fail_json(msg='No destination tags configured for package registration checks', **result) + + package = module.params['package'] + if not package: + package, _ = specfile_macro_lookup(module.params['spec_file_path'], '%{name}', + scl=module.params['scl'], dist=module.params['dist'], + macros=module.params['macros']) + try: + for tag in dict.fromkeys(tags): + result['whitelist_status'][tag] = package_whitelisted(module.params['build_command'], tag, package) + except KojiCommandError as error: + module.fail_json(msg=error.message, command=error.command, **result) + missing = [tag for tag, registered in result['whitelist_status'].items() if not registered] + if missing: + module.fail_json(msg='Package {} is not registered or is blocked in tags: {}'.format( + package, ', '.join(missing)), **result) module.exit_json(**result) diff --git a/obal/data/modules/rhpkg_release.py b/obal/data/modules/rhpkg_release.py new file mode 100644 index 00000000..8802efd1 --- /dev/null +++ b/obal/data/modules/rhpkg_release.py @@ -0,0 +1,45 @@ +#!/usr/bin/python +"""Release packages through rhpkg, reusing and tagging existing Koji builds.""" + +from ansible.module_utils.basic import AnsibleModule +from ansible.module_utils.rhpkg import RhpkgRelease, ReleaseError, inventory_tags # pylint:disable=import-error,no-name-in-module + + +def main(): + """Submit builds and reconcile output tags without invoking Tito.""" + module = AnsibleModule( + argument_spec=dict( + directory=dict(type='path', required=True), + spec_file=dict(type='path', required=True), + targets=dict(type='list', elements='dict', required=True), + koji_tags=dict(type='list', elements='dict', default=[]), + koji_executable=dict(type='str', default='brew'), + rhpkg_executable=dict(type='str', default='rhpkg'), + receipt=dict(type='path', required=True), + output_directory=dict(type='path', required=True), + scratch=dict(type='bool', default=False), + wait=dict(type='bool', default=True), + waitrepo=dict(type='bool', default=False), + tag_check=dict(type='bool', default=True), + download_logs=dict(type='bool', default=False), + download_rpms=dict(type='bool', default=False), + ), + supports_check_mode=True, + ) + release = None + try: + package = module.params['directory'].rstrip('/').split('/')[-1] + inventory_tags(dict(build_package_releaser='rhpkg', rhpkg_targets=module.params['targets'], + koji_tags=module.params['koji_tags']), package) + release = RhpkgRelease(dict(module.params, check_mode=module.check_mode)) + result = release.execute() + except (ReleaseError, OSError, ValueError) as error: + module.fail_json(msg=str(error), changed=release.changed if release else False, + results=release.results if release else []) + if result['errors']: + module.fail_json(msg='; '.join(result['errors']), **result) + module.exit_json(**result) + + +if __name__ == '__main__': + main() diff --git a/obal/data/modules/srpm.py b/obal/data/modules/srpm.py index 81657e0e..54f55f35 100644 --- a/obal/data/modules/srpm.py +++ b/obal/data/modules/srpm.py @@ -9,6 +9,7 @@ from zipfile import ZipFile from contextlib import contextmanager from tempfile import mkdtemp, TemporaryFile +from urllib.parse import urlsplit from ansible.module_utils.six.moves.urllib.request import urlopen # pylint:disable=import-error,no-name-in-module from ansible.module_utils.six.moves.urllib.error import HTTPError # pylint:disable=import-error,no-name-in-module @@ -38,7 +39,14 @@ def copy_sources(spec_file, package_dir, sources_dir): sources = get_specfile_sources(spec_file) for source in sources: - if not source.startswith('http'): + if source.startswith('http'): + # Nightly builds can supply URL-named sources locally without annexing them. + local_source = os.path.join(package_dir, os.path.basename(urlsplit(source).path)) + if os.path.isfile(local_source): + # Keep staging writable if this is a locked annex source: it will + # be copied again below after unlocking. + shutil.copyfile(local_source, os.path.join(sources_dir, os.path.basename(local_source))) + else: shutil.copy(os.path.join(package_dir, source), sources_dir) with chdir(package_dir): diff --git a/obal/data/modules/tito_build.py b/obal/data/modules/tito_build.py deleted file mode 100644 index d34a15f1..00000000 --- a/obal/data/modules/tito_build.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/python -""" -Release a package using tito -""" - -import re -from subprocess import CalledProcessError - -from ansible.module_utils.basic import AnsibleModule -from ansible.module_utils.tito_wrapper import tito # pylint:disable=import-error,no-name-in-module - - -def main(): - """ - Build a package using tito - """ - module = AnsibleModule( - argument_spec=dict( - directory=dict(type='path', required=True), - arguments=dict(type='list', required=False), - build_arguments=dict(type='list', required=False), - srpm=dict(type='bool', required=False, default=False), - offline=dict(type='bool', required=False, default=False), - dist=dict(type='str', required=False), - scl=dict(rtype='str', equired=False), - output=dict(type='path', required=False), - builder=dict(type='str', required=False), - ) - ) - - command = ['tito', 'build'] - - for param in ('srpm', 'offline'): - if module.params[param]: - command.append('--' + param) - - for param in ('dist', 'scl', 'output', 'builder'): - value = module.params[param] - if value: - command += ['--' + param, value] - - if module.params['build_arguments']: - for argument in module.params['build_arguments']: - command += ['--arg', argument] - - if module.params['arguments']: - command += module.params['arguments'] - - directory = module.params['directory'] - - try: - output = tito(command, directory) - except CalledProcessError as error: - module.fail_json(msg='Failed to tito build', command=error.cmd, output=error.output, - directory=directory, code=error.returncode) - - match = re.search(r'^Wrote: (?P.+)', output, re.MULTILINE) - path = match.group('path') if match else None - - module.exit_json(changed=True, path=path, output=output) - - -if __name__ == '__main__': - main() diff --git a/obal/data/modules/tito_release.py b/obal/data/modules/tito_release.py deleted file mode 100644 index 7a2c229f..00000000 --- a/obal/data/modules/tito_release.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/python -""" -Release a package using tito -""" - -import re -from subprocess import CalledProcessError - -from ansible.module_utils.basic import AnsibleModule -from ansible.module_utils.tito_wrapper import tito # pylint:disable=import-error,no-name-in-module - - -def main(): - """ - Release a package using tito - """ - module = AnsibleModule( - argument_spec=dict( - directory=dict(type='path', required=True), - arguments=dict(type='list', required=False), - scratch=dict(type='bool', required=False, default=False), - test=dict(type='bool', required=False, default=False), - releasers=dict(type='list', required=True), - releaser_arguments=dict(type='list', required=False), - ) - ) - - command = ['tito', 'release', '--yes'] - - for param in ('scratch', 'test'): - if module.params[param]: - command.append('--' + param) - - command += module.params['releasers'] - - if module.params['releaser_arguments']: - for argument in module.params['releaser_arguments']: - command += ['--arg', argument] - - if module.params['arguments']: - command += module.params['arguments'] - - directory = module.params['directory'] - - try: - output = tito(command, directory) - except CalledProcessError as error: - module.fail_json(msg='Failed to tito release', command=error.cmd, directory=directory, - output=error.output, code=error.returncode) - - tasks = re.findall(r'^Created task:\s(\d+)', output, re.MULTILINE) - task_urls = re.findall(r'^Task info:\s(.+)', output, re.MULTILINE) - builds = re.findall(r'^Created builds:\s(\d+)', output, re.MULTILINE) - - module.exit_json(changed=True, output=output, tasks=tasks, task_urls=task_urls, builds=builds) - - -if __name__ == '__main__': - main() diff --git a/obal/data/playbooks/nightly/nightly.yaml b/obal/data/playbooks/nightly/nightly.yaml index 3b855555..f5f918e4 100644 --- a/obal/data/playbooks/nightly/nightly.yaml +++ b/obal/data/playbooks/nightly/nightly.yaml @@ -4,19 +4,12 @@ - packages serial: 1 gather_facts: false - vars: - build_package_tito_releaser_args: "{{ nightly_package_tito_releaser_args | default([]) }}" roles: - package_variables tasks: - name: 'Legacy nightly building' when: nightly_sourcefiles is not defined and nightly_githash is not defined block: - - name: 'Set nightly_releaser' - set_fact: - releasers: - - "{{ nightly_releaser }}" - - name: Build legacy nightly package import_role: name: build_package diff --git a/obal/data/playbooks/release/release.yaml b/obal/data/playbooks/release/release.yaml index 2d4960f2..933ece36 100644 --- a/obal/data/playbooks/release/release.yaml +++ b/obal/data/playbooks/release/release.yaml @@ -6,6 +6,10 @@ gather_facts: false roles: - role: diff_package - when: not build_package_use_koji_build | bool + when: + - not build_package_use_koji_build | bool + - build_package_build_system != 'koji' - role: build_package - when: (build_package_use_koji_build | bool or diff_package_changed is not defined or diff_package_changed | bool) + when: >- + build_package_build_system == 'koji' or + build_package_use_koji_build | bool or diff_package_changed is not defined or diff_package_changed | bool diff --git a/obal/data/playbooks/setup/setup.yaml b/obal/data/playbooks/setup/setup.yaml index b3677842..fb0375e4 100644 --- a/obal/data/playbooks/setup/setup.yaml +++ b/obal/data/playbooks/setup/setup.yaml @@ -11,8 +11,8 @@ name: - rpm-build - koji + - git - "{{ 'git-annex' if ansible_distribution == 'Fedora' or ansible_distribution_major_version == '7' else 'git-annex-standalone' }}" - - tito - scl-utils - scl-utils-build - rpmlint @@ -20,3 +20,22 @@ - which - dnf - dnf-plugins-core + + - name: Identify configured dist-git releases + set_fact: + setup_rhpkg_required: true + loop: "{{ groups['packages'] | default([]) }}" + vars: + setup_package: "{{ hostvars[item] }}" + when: + - setup_package.build_package_build_system | default('koji') == 'koji' + - >- + setup_package.build_package_releaser | + default('koji' if setup_package.build_package_use_koji_build | default(false) | bool else 'rhpkg') == 'rhpkg' + + - name: Install rhpkg for dist-git releases + become: true + package: + state: present + name: rhpkg + when: setup_rhpkg_required | default(false) | bool diff --git a/obal/data/roles/build_package/defaults/main.yaml b/obal/data/roles/build_package/defaults/main.yaml index 03878801..06f7cacf 100644 --- a/obal/data/roles/build_package/defaults/main.yaml +++ b/obal/data/roles/build_package/defaults/main.yaml @@ -1,16 +1,17 @@ --- build_package_build_system: koji -build_package_koji_command: koji -build_package_tito_args: [] +build_package_koji_command: >- + {{ 'brew' if build_package_build_system == 'koji' and build_package_releaser == 'rhpkg' else 'koji' }} +build_package_releaser: "{{ 'koji' if build_package_use_koji_build | bool else 'rhpkg' }}" +build_package_rhpkg_command: rhpkg +rhpkg_targets: [] build_package_scratch: false build_package_test: false -build_package_tito_releaser_args: [] build_package_wait: true build_package_download_logs: false build_package_download_rpms: false build_package_waitrepo: false -build_package_koji_whitelist_check: false -build_package_tito_builder: +build_package_koji_whitelist_check: "{{ build_package_releaser == 'rhpkg' and not build_package_scratch | bool }}" build_package_use_koji_build: false build_package_copr_rebuild: false build_package_skip_failed_build: false diff --git a/obal/data/roles/build_package/tasks/koji.yml b/obal/data/roles/build_package/tasks/koji.yml index ac87ab63..bdd49b1a 100644 --- a/obal/data/roles/build_package/tasks/koji.yml +++ b/obal/data/roles/build_package/tasks/koji.yml @@ -1,22 +1,18 @@ --- -- name: 'Use tito to build package' - when: not build_package_use_koji_build | bool - block: - - name: Scratch build with koji_build - when: - - build_package_scratch - - koji_tags is truthy - include_tasks: koji_build.yml - loop: "{{ koji_tags }}" - loop_control: - loop_var: tag +- name: Validate Koji releaser selection + assert: + that: + - build_package_releaser in ['koji', 'rhpkg'] + - not (build_package_releaser == 'rhpkg' and build_package_use_koji_build | bool) + - build_package_releaser != 'koji' or koji_tags | length > 0 + fail_msg: Tito releases are no longer supported. Configure rhpkg_targets for dist-git releases or select koji for direct SRPM builds - - name: Release with tito - when: (build_package_scratch and koji_tags is not truthy) or (not build_package_scratch) - include_tasks: tito_release.yml +- name: Release with rhpkg + include_tasks: rhpkg_release.yml + when: build_package_releaser == 'rhpkg' - name: Build package with Koji - when: build_package_use_koji_build | bool + when: build_package_releaser == 'koji' block: - name: Build package for Koji tag include_tasks: koji_build.yml diff --git a/obal/data/roles/build_package/tasks/koji_build.yml b/obal/data/roles/build_package/tasks/koji_build.yml index b8db00ab..72f17e7a 100644 --- a/obal/data/roles/build_package/tasks/koji_build.yml +++ b/obal/data/roles/build_package/tasks/koji_build.yml @@ -20,11 +20,30 @@ debug: msg: "{{ build_exists }}" +- name: Check that a Koji build can be reused or submitted + assert: + that: + - build_exists.state in ['MISSING', 'COMPLETE'] + fail_msg: "Build {{ package_nvr.nvr }} is {{ build_exists.state }}; refusing another submission" + when: not build_package_scratch | bool + +- name: Check registration before tagging an existing build + package_whitelist_check: + package: "{{ package_nvr.name }}" + tags: ["{{ tag.name }}"] + build_command: "{{ build_package_koji_command }}" + when: + - build_package_koji_whitelist_check | bool + - build_exists is not skipped + - build_exists.exists + - not build_exists.exists_for_tag + - not build_package_scratch | bool + - name: Tag package build into Koji tag koji_tag_package: tag: "{{ tag.name }}" nvr: "{{ package_nvr.nvr }}" - koji_executable: "{{ koji_executable | default('koji') }}" + koji_executable: "{{ build_package_koji_command }}" when: - build_exists is not skipped - not build_exists.exists_for_tag diff --git a/obal/data/roles/build_package/tasks/rhpkg_release.yml b/obal/data/roles/build_package/tasks/rhpkg_release.yml new file mode 100644 index 00000000..c80cb3de --- /dev/null +++ b/obal/data/roles/build_package/tasks/rhpkg_release.yml @@ -0,0 +1,30 @@ +--- +- name: Validate rhpkg source configuration + assert: + that: + - not build_package_test | bool + - setup_sources_git is not defined + - source_location is not defined + fail_msg: The rhpkg releaser requires package spec sources; Tito test and custom source builders are unsupported + +- name: Release and tag packages through rhpkg + rhpkg_release: + directory: "{{ inventory_dir }}/{{ package_base_dir }}/{{ inventory_hostname }}" + spec_file: "{{ spec_file_path }}" + targets: "{{ rhpkg_targets }}" + koji_tags: "{{ koji_tags }}" + koji_executable: "{{ build_package_koji_command }}" + rhpkg_executable: "{{ build_package_rhpkg_command }}" + receipt: "{{ obal_tmp_dir }}/rhpkg/{{ inventory_hostname }}{{ '-scratch' if build_package_scratch else '' }}.json" + output_directory: "{{ inventory_dir }}" + scratch: "{{ build_package_scratch }}" + wait: "{{ build_package_wait }}" + waitrepo: "{{ build_package_waitrepo }}" + tag_check: "{{ build_package_koji_whitelist_check }}" + download_logs: "{{ build_package_download_logs }}" + download_rpms: "{{ build_package_download_rpms }}" + register: build_package_rhpkg_release + +- name: Report rhpkg release results + debug: + var: build_package_rhpkg_release diff --git a/obal/data/roles/build_package/tasks/tito_release.yml b/obal/data/roles/build_package/tasks/tito_release.yml deleted file mode 100644 index f53780ea..00000000 --- a/obal/data/roles/build_package/tasks/tito_release.yml +++ /dev/null @@ -1,78 +0,0 @@ ---- -- name: "Confirm package is whitelisted" - package_whitelist_check: - releasers_conf: "{{ inventory_dir }}/rel-eng/releasers.conf" - spec_file_path: "{{ spec_file_path }}" - releasers: "{{ releasers }}" - build_command: "{{ build_package_koji_command }}" - when: build_package_koji_whitelist_check | bool - -- name: 'Set tito_releasers for brew scratch' - set_fact: - build_package_tito_releasers: "{{ releasers | map('replace', 'dist-git', 'scratch') | list }}" - when: build_package_koji_command == 'brew' and build_package_scratch - -- name: 'Set tito_releasers' - set_fact: - build_package_tito_releasers: "{{ releasers }}" - when: build_package_koji_command != 'brew' or not build_package_scratch - -- name: 'Release to {{ build_package_koji_command }}' - tito_release: - directory: "{{ inventory_dir }}/{{ package_base_dir }}/{{ inventory_hostname }}" - arguments: "{{ build_package_tito_args }}" - scratch: "{{ build_package_scratch and build_package_koji_command != 'brew' }}" - test: "{{ build_package_test }}" - releasers: "{{ build_package_tito_releasers }}" - releaser_arguments: "{{ build_package_tito_releaser_args }}" - register: build_package_tito_release - -- name: 'Created tasks' - debug: - var: build_package_tito_release.task_urls - -- name: 'Wait for tasks to finish' - when: build_package_wait|bool - block: - - name: Define koji_tasks - set_fact: - koji_tasks: "{{ build_package_tito_release.tasks }}" - - - name: "Watch {{ build_package_koji_command }} task(s)" - command: "{{ build_package_koji_command }} watch-task {{ koji_tasks | join(' ') }}" - ignore_errors: true - register: build_package_koji_status - changed_when: false - - - name: 'Download task logs' - include_tasks: download_logs.yml - when: build_package_download_logs|bool - - - name: 'Failed build' - fail: - msg: "The build in {{ build_package_koji_command }} has failed" - when: (build_package_koji_status is failed) and ('Build already exists' not in build_package_koji_status.stdout) - - - name: "Get {{ build_package_koji_command }} build detals" - command: "{{ build_package_koji_command }} taskinfo -v {{ item }}" - with_items: "{{ koji_tasks }}" - changed_when: false - register: build_package_koji_task_info - - - name: "Filter {{ build_package_build_system }} output" - set_fact: - build_package_koji_task_data: - "{{ build_package_koji_task_info.results | map(attribute='stdout') | map('from_yaml') | list }}" - when: - - build_package_waitrepo|bool or build_package_download_rpms|bool - - - name: 'Wait for builds to apear in the repo' - include_tasks: waitrepo.yml - when: - - build_package_waitrepo|bool - - not build_package_scratch - - build_package_koji_status is succeeded - -- name: 'Download task results' - include_tasks: download_rpms.yml - when: build_package_download_rpms|bool diff --git a/obal/data/roles/build_srpm/defaults/main.yaml b/obal/data/roles/build_srpm/defaults/main.yaml index 111c34be..d94868f6 100644 --- a/obal/data/roles/build_srpm/defaults/main.yaml +++ b/obal/data/roles/build_srpm/defaults/main.yaml @@ -1,7 +1,4 @@ --- build_srpm_dist: build_srpm_scl: -build_srpm_tito_build_args: "{{ build_package_tito_releaser_args | default([]) }}" build_srpm_output_dir: "{{ inventory_dir }}/SRPMs" -build_srpm_tito_args: "{{ build_package_tito_args | default([]) }}" -build_srpm_tito_builder: "{{ build_package_tito_builder | default() }}" diff --git a/obal/data/roles/diff_package/defaults/main.yaml b/obal/data/roles/diff_package/defaults/main.yaml index 5b417640..fdb40b05 100644 --- a/obal/data/roles/diff_package/defaults/main.yaml +++ b/obal/data/roles/diff_package/defaults/main.yaml @@ -1,5 +1,8 @@ --- diff_package_type: koji -diff_package_koji_command: koji +diff_package_koji_command: >- + {{ 'brew' if build_package_build_system | default('koji') == 'koji' + and build_package_releaser == 'rhpkg' else 'koji' }} diff_package_skip: false +build_package_releaser: "{{ 'koji' if build_package_use_koji_build | default(false) | bool else 'rhpkg' }}" scl: false diff --git a/obal/data/roles/diff_package/tasks/koji.yml b/obal/data/roles/diff_package/tasks/koji.yml index d70a703e..ec66c233 100644 --- a/obal/data/roles/diff_package/tasks/koji.yml +++ b/obal/data/roles/diff_package/tasks/koji.yml @@ -1,11 +1,21 @@ --- +- name: Compare rhpkg release destinations + include_tasks: koji_rhpkg.yml + when: + - build_package_build_system | default('koji') == 'koji' + - build_package_releaser == 'rhpkg' + - name: Compare Koji tags include_tasks: koji_tag.yml loop: "{{ koji_tags }}" loop_control: loop_var: tag - when: diff_package_tags is undefined + when: + - build_package_build_system | default('koji') != 'koji' or build_package_releaser == 'koji' + - diff_package_tags is undefined - name: Compare legacy Koji tags include_tasks: koji_old.yml - when: diff_package_tags is defined + when: + - build_package_build_system | default('koji') != 'koji' or build_package_releaser == 'koji' + - diff_package_tags is defined diff --git a/obal/data/roles/diff_package/tasks/koji_old.yml b/obal/data/roles/diff_package/tasks/koji_old.yml index 8961d7aa..f99e22c4 100644 --- a/obal/data/roles/diff_package/tasks/koji_old.yml +++ b/obal/data/roles/diff_package/tasks/koji_old.yml @@ -1,9 +1,4 @@ --- -- name: Set koji_tags if diff_package_tags is defined - set_fact: - koji_tags: "{{ diff_package_tags }}" - when: diff_package_tags is defined - - name: Gather local package version include_tasks: git_package_info.yml @@ -15,7 +10,7 @@ shell: "{{ diff_package_koji_command }} list-tagged --quiet --latest {{ item }} {{ scl + '-' if scl else '' }}{{ inventory_hostname }} \ | cut -d' ' -f1" register: koji_package_versions - with_items: "{{ koji_tags }}" + with_items: "{{ diff_package_tags }}" changed_when: false - name: "Set koji_package_versions" diff --git a/obal/data/roles/diff_package/tasks/koji_rhpkg.yml b/obal/data/roles/diff_package/tasks/koji_rhpkg.yml new file mode 100644 index 00000000..c8a3e034 --- /dev/null +++ b/obal/data/roles/diff_package/tasks/koji_rhpkg.yml @@ -0,0 +1,20 @@ +--- +- name: Inspect rhpkg release destinations + rhpkg_release: + directory: "{{ inventory_dir }}/{{ package_base_dir }}/{{ inventory_hostname }}" + spec_file: "{{ spec_file_path }}" + targets: "{{ rhpkg_targets }}" + koji_tags: "{{ koji_tags }}" + koji_executable: "{{ build_package_koji_command | default(diff_package_koji_command) }}" + receipt: "{{ inventory_dir }}/.tmp/rhpkg/{{ inventory_hostname }}.json" + output_directory: "{{ inventory_dir }}" + check_mode: true + register: diff_package_rhpkg + +- name: Record missing rhpkg builds or tags + set_fact: + diff_package_changed: "{{ diff_package_rhpkg.changed }}" + +- name: Report rhpkg release differences + debug: + var: diff_package_rhpkg diff --git a/obal/data/roles/setup_sources/tasks/annex.yml b/obal/data/roles/setup_sources/tasks/annex.yml index 98642d70..07eaa7e7 100644 --- a/obal/data/roles/setup_sources/tasks/annex.yml +++ b/obal/data/roles/setup_sources/tasks/annex.yml @@ -15,7 +15,7 @@ path: "{{ setup_sources_sourcepath }}" register: setup_sources_source_exists -- name: 'Search annex for web link' +- name: 'Search annex for source' command: argv: - git @@ -23,7 +23,7 @@ - find - --branch=HEAD - "--include={{ setup_sources_source_relative_path }}" - - --in=web + - --anything args: chdir: "{{ inventory_dir }}" changed_when: false diff --git a/obal/data/roles/setup_sources/tasks/git.yml b/obal/data/roles/setup_sources/tasks/git.yml deleted file mode 100644 index c14b7e58..00000000 --- a/obal/data/roles/setup_sources/tasks/git.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -- name: 'Set source checkout location' - set_fact: - setup_sources_git_checkout_dir: "/tmp/ansible_git_{{ inventory_hostname }}" - -- name: 'Clone latest git' - git: - repo: "{{ setup_sources_git }}" - dest: "{{ setup_sources_git_checkout_dir }}" - version: "{{ git_branch | default(omit) }}" - force: true - depth: 1 - -- name: 'Register source dir tito argument' - set_fact: - build_package_tito_releaser_args: - - 'source_dir={{ setup_sources_git_checkout_dir }}' diff --git a/obal/data/roles/setup_sources/tasks/main.yml b/obal/data/roles/setup_sources/tasks/main.yml index 3baff89d..a059dbe9 100644 --- a/obal/data/roles/setup_sources/tasks/main.yml +++ b/obal/data/roles/setup_sources/tasks/main.yml @@ -1,12 +1,13 @@ --- +- name: Validate source preparation + assert: + that: + - setup_sources_git is not defined + fail_msg: setup_sources_git required the removed Tito builder. Prepare sources declared by the spec file instead + - name: 'Set package_dir' set_fact: package_dir: "{{ inventory_dir }}/{{ package_base_dir }}/{{ inventory_hostname }}" - name: 'Setup sources from specfile' include_tasks: specfile.yml - when: setup_sources_git is not defined - -- name: 'Clone git repository' - include_tasks: git.yml - when: setup_sources_git is defined diff --git a/obal/data/roles/verify_koji_tag/tasks/verify.yaml b/obal/data/roles/verify_koji_tag/tasks/verify.yaml index bce1f257..5d3a1d26 100644 --- a/obal/data/roles/verify_koji_tag/tasks/verify.yaml +++ b/obal/data/roles/verify_koji_tag/tasks/verify.yaml @@ -4,7 +4,7 @@ tag: "{{ tag_name }}" packages: "{{ hostvars }}" directory: "{{ lookup('env', 'PWD') }}" - koji_executable: "{{ koji_executable | default('koji') }}" + koji_executable: "{{ koji_executable | default(omit) }}" register: missing - name: "Packages in Koji but not in git for {{ tag_name }}" diff --git a/obal/tito_extensions/__init__.py b/obal/tito_extensions/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/obal/tito_extensions/git_annex_spec_builder.py b/obal/tito_extensions/git_annex_spec_builder.py deleted file mode 100644 index 5ab40398..00000000 --- a/obal/tito_extensions/git_annex_spec_builder.py +++ /dev/null @@ -1,173 +0,0 @@ -""" -A tito plugin to build using git-annex -""" - -# pylint: disable=E0401,R0902,R0903,R0913 - -import os -import shutil -from distutils.version import LooseVersion as loose_version # pylint: disable=E0611,deprecated-module -from importlib import metadata - -from tito.compat import getstatusoutput -from tito.config_object import ConfigObject -from tito.builder import GitAnnexBuilder -from tito.builder.main import BuilderBase -from tito.common import error_out, debug, run_command, get_spec_version_and_release, \ - find_spec_like_file, get_relative_project_dir_cwd, warn_out, get_build_commit - - -class GitAnnexSpecBuilder(GitAnnexBuilder): - """ - A tito builder that uses git annex - """ - - # pylint: disable=too-many-positional-arguments - def __init__(self, name=None, tag=None, build_dir=None, - config=None, user_config=None, - args=None, **kwargs): - - """ - name - Package name that is being built. - - version - Version and release being built. - - tag - The git tag being built. - - build_dir - Temporary build directory where we can safely work. - - config - Merged configuration. (global plus package specific) - - user_config - User configuration from ~/.titorc. - - args - Optional arguments specific to each builder. Can be passed - in explicitly by user on the CLI, or via a release target config - entry. Only for things which vary on invocations of the builder, - avoid using these if possible. *Given in the format of a dictionary - of lists.* - """ - ConfigObject.__init__(self, config=config) - BuilderBase.__init__(self, name=name, build_dir=build_dir, config=config, - user_config=user_config, args=args, **kwargs) - self.build_tag = tag - - self.build_version = self._get_build_version() - self.git_commit_id = get_build_commit(tag=self.build_tag, test=True) - - if kwargs and 'options' in kwargs: - warn_out("'options' no longer a supported builder constructor argument.") - - if self.config.has_option("requirements", "tito"): - if loose_version(self.config.get("requirements", "tito")) > \ - loose_version(metadata.version('tito')): - error_out([ - "tito version %s or later is needed to build this project." % - self.config.get("requirements", "tito"), - "Your version: %s" % metadata.version('tito') - ]) - - self.display_version = self._get_display_version() - - self.relative_project_dir = get_relative_project_dir_cwd(self.git_root) - - tgz_base = self._get_tgz_name_and_ver() - self.tgz_filename = tgz_base + ".tar.gz" - self.tgz_dir = tgz_base - self.artifacts = [] - - self.rpmbuild_gitcopy = os.path.join(self.rpmbuild_sourcedir, - self.tgz_dir) - - # Used to make sure we only modify the spec file for a test build - # once. The srpm method may be called multiple times during koji - # releases to create the proper disttags, but we only want to modify - # the spec file once. - self.ran_setup_test_specfile = False - - # NOTE: These are defined later when/if we actually dump a copy of the - # project source at the tag we're building. Only then can we search for - # a spec file. - self.spec_file_name = None - self.spec_file = None - - # Set to path to srpm once we build one. - self.srpm_location = None - - def _get_build_version(self): - """ - Figure out the git tag and version-release we're building. - """ - # Determine which package version we should build: - build_version = None - if self.build_tag: - build_version = self.build_tag[len(self.project_name + "-"):] - else: - build_version = get_spec_version_and_release(self.start_dir, - find_spec_like_file(self.start_dir)) - self.build_tag = self._get_tag_for_version(build_version) - - self.spec_version = build_version.split('-')[0] - self.spec_release = build_version.split('-')[-1] - return build_version - - def _setup_sources(self): - """ - Create a copy of the git source for the project at the point in time - our build tag was created. - - Created in the temporary rpmbuild SOURCES directory. - """ - self._create_build_dirs() - working_path = os.path.join(os.getcwd(), self.relative_project_dir) - - debug('SETUP SOURCES') - if self.relative_project_dir in os.path.join(os.getcwd(), ''): - working_path = os.getcwd() - debug("working_path: %s" % working_path) - - for directory, _, filenames in os.walk(working_path): - debug('WALK') - dir_artifacts_with_path = [os.path.join(directory, f) for f in filenames] - - debug(dir_artifacts_with_path) - for artifact in dir_artifacts_with_path: - debug(" Copying source file %s" % artifact) - if os.path.isfile(artifact): - if not os.path.exists("/".join([self.rpmbuild_gitcopy, os.path.basename(artifact)])): - shutil.copy(artifact, self.rpmbuild_gitcopy) - if not os.path.exists("/".join([self.rpmbuild_sourcedir, os.path.basename(artifact)])): - shutil.copy(artifact, self.rpmbuild_sourcedir) - - - # NOTE: The spec file we actually use is the one exported by git - # archive into the temp build directory. This is done so we can - # modify the version/release on the fly when building test rpms - # that use a git SHA1 for their version. - self.spec_file_name = os.path.basename(find_spec_like_file(self.rpmbuild_sourcedir)) - self.spec_file = os.path.join( - self.rpmbuild_sourcedir, self.spec_file_name) - - self.old_cwd = os.getcwd() # pylint: disable=W0201 - if self.relative_project_dir not in os.path.join(os.getcwd(), ''): - os.chdir(os.path.join(self.old_cwd, self.relative_project_dir)) - - # NOTE: 'which' may not be installed... (docker containers) - status = getstatusoutput("which git-annex")[0] - if status != 0: - msg = "Please run '%s' as root." % self.package_manager.install(["git-annex"]) - error_out('%s' % msg) - - run_command("git-annex lock") - annexed_files = run_command("git-annex find --include='*'").splitlines() - run_command("git-annex get") - run_command("git-annex unlock") - debug(" Annex files: %s" % annexed_files) - - for annex in annexed_files: - debug("Copying unlocked file %s" % annex) - if os.path.isfile(os.path.join(self.rpmbuild_gitcopy, annex)): - os.remove(os.path.join(self.rpmbuild_gitcopy, annex)) - shutil.copy(annex, self.rpmbuild_gitcopy) - - self._lock() - os.chdir(self.old_cwd) diff --git a/tests/fixtures/mockbin/mockbin b/tests/fixtures/mockbin/mockbin index 50ada49d..03d07053 100755 --- a/tests/fixtures/mockbin/mockbin +++ b/tests/fixtures/mockbin/mockbin @@ -44,39 +44,7 @@ if mockbin_log: mockbin_log_contents = [line.strip() for line in logfile.readlines()] if prog == 'tito': - parser = argparse.ArgumentParser() - subparsers = parser.add_subparsers() - - # tito release arguments and options - parser_release = subparsers.add_parser('release') - parser_release.add_argument('dest') - parser_release.add_argument('--scratch', action='store_true') - parser_release.add_argument('--test', action='store_true') - parser_release.add_argument('-y', '--yes', dest='yes', action='store_true', default=False) - parser_release.add_argument('--arg') - - # tito build arguments and options - parser_build = subparsers.add_parser('build') - parser_build.add_argument('--srpm', action='store_true', default=False) - parser_build.add_argument('--output', default=None) - parser_build.add_argument('--scl') - parser_build.add_argument('--offline', action='store_true') - - args = parser.parse_args() - - pkg = os.path.basename(os.getcwd()) - - if 'yes' in args and args.yes: - print('Created task: 1234') - if 'output' in args and args.output: - if not os.path.exists(args.output): - os.makedirs(args.output) - srpm_path = os.path.join(args.output, _FAKE_SRPMS[pkg]) - open(srpm_path, 'w').close() - else: - srpm_path = _FAKE_SRPMS[pkg] - if 'srpm'in args and args.srpm: - print("Wrote: {}".format(srpm_path)) + sys.exit('Tito must not be invoked') elif prog in ['koji', 'brew']: parser = argparse.ArgumentParser() @@ -138,6 +106,9 @@ elif prog in ['koji', 'brew']: if args.subcommand in ['build', 'tag-build']: print('Created task: 1234') + if args.subcommand == 'list-pkgs' and args.package: + print('{} {} test-owner'.format(args.package, args.tag)) + if 'package-with-existing-build' in sys.argv and 'latest-build' in sys.argv: print('package-with-existing-build-1.0-1.el7') @@ -152,6 +123,14 @@ elif prog in ['koji', 'brew']: if 'koji build obaltest-nightly-el8 /tmp/SRPMs/package-with-two-targets-1.0-1.src.rpm' in mockbin_log_contents: print('BUILD: {}\nState: COMPLETE'.format(args.pkg)) + if args.subcommand == 'buildinfo': + known = args.pkg == 'package-with-existing-build-1.0-1.el7' or ( + args.pkg == 'package-with-two-targets-1.0-1.el8' and + 'koji build obaltest-nightly-el8 /tmp/SRPMs/package-with-two-targets-1.0-1.src.rpm' in mockbin_log_contents) + if not known: + print('No such build: {}'.format(args.pkg), file=sys.stderr) + sys.exit(1) + elif prog in ['copr-cli']: parser = argparse.ArgumentParser() subparsers = parser.add_subparsers(dest='subcommand') diff --git a/tests/fixtures/rhpkg/mock_client.py b/tests/fixtures/rhpkg/mock_client.py new file mode 100644 index 00000000..86c841e4 --- /dev/null +++ b/tests/fixtures/rhpkg/mock_client.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""Stateful Brew/rhpkg test clients; all dist-git operations use local Git.""" + +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys + + +state_path = Path(os.environ['OBAL_BREW_STATE']) +state = json.loads(state_path.read_text()) +program = Path(sys.argv[0]).name +arguments = sys.argv[1:] +state['commands'].append([program] + arguments) +state_path.write_text(json.dumps(state)) + + +def execute(command, cwd=None): + return subprocess.check_output(command, cwd=cwd, text=True, stderr=subprocess.STDOUT).strip() + + +def fail(message): + print(message, file=sys.stderr) + sys.exit(1) + + +def build_info(nvr): + if state.get('query_error'): + fail('Authentication failed') + build = state['builds'].get(nvr) + if not build: + fail('No such build: ' + nvr) + print('BUILD: {} [1]'.format(nvr)) + print('State: ' + build['state']) + print('Task: ' + str(build.get('task', 'none'))) + print('Source: ' + build.get('source', 'git://example.test/rpms/package#original')) + print('Tags: ' + ' '.join(build.get('tags', []))) + + +def brew(): + command = arguments[0] + if command == 'buildinfo': + build_info(arguments[1]) + elif command == 'list-targets': + target = arguments[-1] + if target in state['targets']: + print('{} {}-build {}'.format(target, target, target)) + elif command == 'list-pkgs': + if state.get('registration_error'): + fail('Authentication failed while querying package registration') + tag = arguments[arguments.index('--tag') + 1] + packages = ([arguments[arguments.index('--package') + 1]] if '--package' in arguments + else sorted(state['repositories'])) + if state.get('unregistered') or tag in state.get('unregistered_tags', []): + fail('(no matching packages)') + if tag not in state.get('blocked_tags', []): + for package in packages: + print(package + ' ' + tag + ' test-owner') + elif command == 'build': + assert arguments[1:3] == ['--scratch', '--nowait'] + target, srpm = arguments[3:] + if target in state.get('submission_errors', []): + fail('Submission rejected') + assert Path(srpm).is_file() + nvr = execute(['rpmquery', '--queryformat', '%{nvr}', '--package', srpm]) + assert nvr.endswith(state['targets'][target]['dist']) + archive = subprocess.check_output(['rpm2cpio', srpm]) + payload = subprocess.check_output(['cpio', '-i', '--to-stdout', 'payload.txt'], + input=archive, stderr=subprocess.PIPE) + task_id = str(1000 + len(state['tasks'])) + state['tasks'][task_id] = dict(nvr=nvr, source=srpm, target=target, scratch=True, state='open', + source_sha256=hashlib.sha256(payload).hexdigest()) + print('Created task: ' + task_id) + print('Task info: https://brew.example.test/taskinfo?taskID=' + task_id) + elif command == 'watch-task': + task = state['tasks'][arguments[1]] + if task['target'] in state.get('failed_targets', []): + fail('Build failed') + task['state'] = 'closed' + if not task['scratch']: + state['builds'][task['nvr']] = dict(state='COMPLETE', task=arguments[1], + source=task['source'], tags=[task['target']]) + elif command == 'tag-build': + assert arguments[1] == '--wait' + tag, nvr = arguments[-2:] + if tag in state.get('failed_tags', []): + fail('Tag permission denied') + state['builds'][nvr]['tags'].append(tag) + print('Created task 9000') + elif command in ('wait-repo', 'download-logs', 'download-task', 'download-build'): + pass + else: + fail('Unexpected Brew command: ' + command) + + +def rhpkg(): + args = arguments[:] + if args[0] == '--release': + args = args[2:] + command = args[0] + if command == 'clone': + assert args[1] == '--branch' + branch, package, destination = args[2:] + execute(['git', 'clone', '--branch', branch, state['repositories'][package], destination]) + elif command == 'new-sources': + lines = [] + for filename in args[1:]: + digest = hashlib.sha512(Path(filename).read_bytes()).hexdigest() + lines.append('SHA512 ({}) = {}'.format(filename, digest)) + Path('sources').write_text('\n'.join(lines) + '\n') + Path('.gitignore').write_text('\n'.join(args[1:]) + '\n') + elif command == 'commit': + execute(['git', 'commit'] + args[1:]) + elif command == 'push': + if state.get('push_error'): + fail('Push rejected') + execute(['git', 'push', 'origin', 'HEAD']) + elif command == 'build': + assert '--scratch' not in args + assert '--srpm' not in args + target = args[args.index('--target') + 1] + if target in state.get('submission_errors', []): + fail('Submission rejected') + context = state['targets'][target] + spec = next(Path('.').glob('*.spec')) + query = ['rpmspec', '--query', '--srpm', '--queryformat', '%{nvr}', str(spec)] + for name, value in context.items(): + query += ['--define', '{} {}'.format(name, value)] + nvr = execute(query) + assert execute(['git', 'rev-parse', 'HEAD']) == execute(['git', 'rev-parse', '@{upstream}']) + task_id = str(1000 + len(state['tasks'])) + source = 'git://example.test/rpms/package#' + execute(['git', 'rev-parse', 'HEAD']) + task = dict(nvr=nvr, source=source, target=target, scratch=False, state='open') + state['tasks'][task_id] = task + if not state.get('delay_build_record'): + state['builds'][nvr] = dict(state='BUILDING', task=task_id, source=source, tags=[]) + print('Created task: ' + task_id) + print('Task info: https://brew.example.test/taskinfo?taskID=' + task_id) + else: + fail('Unexpected rhpkg command: ' + command) + + +if program in ('brew', 'koji'): + brew() +elif program == 'rhpkg': + rhpkg() +else: + fail('Unexpected client: ' + program) +state_path.write_text(json.dumps(state)) diff --git a/tests/fixtures/setup_actions/package.py b/tests/fixtures/setup_actions/package.py new file mode 100644 index 00000000..9bf65cd7 --- /dev/null +++ b/tests/fixtures/setup_actions/package.py @@ -0,0 +1,14 @@ +"""Record setup dependencies without installing packages on the test host.""" + +import json +import os + +from ansible.plugins.action import ActionBase + + +class ActionModule(ActionBase): + def run(self, tmp=None, task_vars=None): + names = self._task.args['name'] + with open(os.environ['OBAL_SETUP_LOG'], 'a') as stream: + stream.write(json.dumps(names if isinstance(names, list) else [names]) + '\n') + return {'changed': False} diff --git a/tests/fixtures/testrepo/upstream/package_manifest.yaml b/tests/fixtures/testrepo/upstream/package_manifest.yaml index a3e07827..9e07a2b1 100644 --- a/tests/fixtures/testrepo/upstream/package_manifest.yaml +++ b/tests/fixtures/testrepo/upstream/package_manifest.yaml @@ -1,9 +1,7 @@ --- packages: vars: - releasers: - - dist-git - nightly_releaser: dist-git-jenkins + build_package_use_koji_build: true diff_package_skip: false diff_package_type: 'koji' diff_package_koji_command: 'koji' @@ -17,8 +15,6 @@ packages: - el7-base hosts: hello: - nightly_package_tito_releaser_args: - - "jenkins_job=hello-master-release" repoclosure_target_repos: rhel7: - el7-katello diff --git a/tests/fixtures/testrepo/upstream_with_epoch/package_manifest.yaml b/tests/fixtures/testrepo/upstream_with_epoch/package_manifest.yaml index 29e88e2f..02460581 100644 --- a/tests/fixtures/testrepo/upstream_with_epoch/package_manifest.yaml +++ b/tests/fixtures/testrepo/upstream_with_epoch/package_manifest.yaml @@ -1,9 +1,7 @@ --- packages: vars: - releasers: - - dist-git - nightly_releaser: dist-git-jenkins + build_package_use_koji_build: true diff_package_skip: false diff_package_type: 'koji' diff_package_koji_command: 'koji' @@ -11,9 +9,7 @@ packages: - name: obaltest-nightly-rhel7 dist: '.el7' hosts: - hello: - nightly_package_tito_releaser_args: - - "jenkins_job=hello-master-release" + hello: {} repoclosures: hosts: diff --git a/tests/test_functional.py b/tests/test_functional.py index 43600122..9699b94e 100644 --- a/tests/test_functional.py +++ b/tests/test_functional.py @@ -135,8 +135,8 @@ def test_obal_check_upstream_hello(): @obal_cli_test(repotype='upstream') -def test_obal_scratch_with_tito_upstream_hello(): - assert_obal_success(['scratch', 'hello']) +def test_obal_scratch_with_koji_selector_upstream_hello(): + assert_obal_success(['scratch', 'hello', '-e', 'build_package_releaser=koji']) assert os.path.exists('packages/hello/hello-2.10.tar.gz') @@ -150,8 +150,8 @@ def test_obal_scratch_with_tito_upstream_hello(): @obal_cli_test(repotype='upstream') -def test_obal_scratch_with_tito_upstream_hello_nowait(): - assert_obal_success(['scratch', 'hello', '-e', 'build_package_wait=False']) +def test_obal_scratch_with_koji_selector_upstream_hello_nowait(): + assert_obal_success(['scratch', 'hello', '-e', 'build_package_releaser=koji', '-e', 'build_package_wait=False']) assert os.path.exists('packages/hello/hello-2.10.tar.gz') @@ -280,6 +280,7 @@ def test_obal_release_with_koji_upstream_whitelist_check(): expected_log = [ "koji buildinfo package-with-existing-build-1.0-1.el7", "koji latest-build --quiet obaltest-nightly-rhel7 package-with-existing-build", + "koji list-pkgs --tag obaltest-nightly-rhel7 --package package-with-existing-build --quiet", "koji tag-build obaltest-nightly-rhel7 package-with-existing-build-1.0-1.el7", "koji buildinfo package-with-existing-build-1.0-1.el8", "koji latest-build --quiet obaltest-nightly-el8 package-with-existing-build", @@ -298,13 +299,16 @@ def test_obal_release_upstream_hello(): assert os.path.exists('packages/hello/hello-2.10.tar.gz') expected_log = [ - "koji buildinfo hello-2.10-2.el7", - "koji latest-build --quiet obaltest-nightly-rhel7 hello", - "koji buildinfo hello-2.10-2.el8", - "koji latest-build --quiet obaltest-nightly-el8 hello", - "tito release --yes dist-git", - "koji watch-task 1234", - "koji taskinfo -v 1234", + 'koji buildinfo hello-2.10-2.el7', + 'koji latest-build --quiet obaltest-nightly-rhel7 hello', + 'koji latest-build --quiet obaltest-nightly-rhel7 hello', + 'koji build obaltest-nightly-rhel7 /tmp/SRPMs/hello-2.10-2.src.rpm', + 'koji watch-task 1234', + 'koji buildinfo hello-2.10-2.el8', + 'koji latest-build --quiet obaltest-nightly-el8 hello', + 'koji latest-build --quiet obaltest-nightly-el8 hello', + 'koji build obaltest-nightly-el8 /tmp/SRPMs/hello-2.10-2.src.rpm', + 'koji watch-task 1234', ] assert_mockbin_log(expected_log) @@ -332,196 +336,37 @@ def test_obal_release_upstream_hello_nowait(): assert os.path.exists('packages/hello/hello-2.10.tar.gz') expected_log = [ - "koji buildinfo hello-2.10-2.el7", - "koji latest-build --quiet obaltest-nightly-rhel7 hello", - "koji buildinfo hello-2.10-2.el8", - "koji latest-build --quiet obaltest-nightly-el8 hello", - "tito release --yes dist-git", + 'koji buildinfo hello-2.10-2.el7', + 'koji latest-build --quiet obaltest-nightly-rhel7 hello', + 'koji latest-build --quiet obaltest-nightly-rhel7 hello', + 'koji build obaltest-nightly-rhel7 /tmp/SRPMs/hello-2.10-2.src.rpm', + 'koji buildinfo hello-2.10-2.el8', + 'koji latest-build --quiet obaltest-nightly-el8 hello', + 'koji latest-build --quiet obaltest-nightly-el8 hello', + 'koji build obaltest-nightly-el8 /tmp/SRPMs/hello-2.10-2.src.rpm', ] assert_mockbin_log(expected_log) @obal_cli_test(repotype='upstream') -def test_obal_release_upstream_hello_waitrepo(): - assert_obal_success(['release', 'hello', '-e', 'build_package_waitrepo=True']) - - assert os.path.exists('packages/hello/hello-2.10.tar.gz') +def test_obal_nightly_upstream_hello(): + assert_obal_success(['nightly', 'hello', '--source', os.path.join(MOCK_SOURCES_DIR, 'hello-2.10.tar.gz'), '--githash', '0123456789abcdef']) expected_log = [ "koji buildinfo hello-2.10-2.el7", "koji latest-build --quiet obaltest-nightly-rhel7 hello", + "koji latest-build --quiet obaltest-nightly-rhel7 hello", + "koji build obaltest-nightly-rhel7 /tmp/SRPMs/hello-2.10-2.src.rpm", + "koji watch-task 1234", "koji buildinfo hello-2.10-2.el8", "koji latest-build --quiet obaltest-nightly-el8 hello", - "tito release --yes dist-git", - "koji watch-task 1234", - "koji taskinfo -v 1234", - "koji wait-repo --build=hello-2.10-1.el7 --target obaltest-nightly-rhel7" - ] - assert_mockbin_log(expected_log) - - -@obal_cli_test(repotype='upstream') -def test_obal_nightly_upstream_hello(): - assert_obal_success(['nightly', 'hello', '--source', os.path.join(MOCK_SOURCES_DIR, 'hello-2.10.tar.gz'), '--githash', '0123456789abcdef']) - - expected_log = [ - "tito release --yes dist-git --arg jenkins_job=hello-master-release", + "koji latest-build --quiet obaltest-nightly-el8 hello", + "koji build obaltest-nightly-el8 /tmp/SRPMs/hello-2.10-2.src.rpm", "koji watch-task 1234", - "koji taskinfo -v 1234", - ] - assert_mockbin_log(expected_log) - - -@obal_cli_test(repotype='downstream') -def test_obal_release_downstream_hello_whitelist_check(): - assert_obal_success(['release', 'hello', '-e', 'build_package_koji_whitelist_check=true']) - - expected_log_entry = "brew list-pkgs --tag obaltest-dist-git-rhel-7 --package hello --quiet" - assert_in_mockbin_log(expected_log_entry) - - -@obal_cli_test(repotype='downstream') -def test_obal_release_downstream_hello_no_whitelist_check(): - assert_obal_success(['release', 'hello', '-e', 'build_package_koji_whitelist_check=false']) - - unexpected_log_entry = "brew list-pkgs --tag obaltest-dist-git-rhel-7 --package hello --quiet" - assert_not_in_mockbin_log(unexpected_log_entry) - - -@obal_cli_test(repotype='downstream') -def test_obal_scratch_downstream_hello_nowait(): - assert_obal_success(['scratch', 'hello', '-e', 'build_package_wait=False']) - - assert os.path.exists('packages/hello/hello-2.9.tar.gz') - - expected_log = [ - "tito release --yes obaltest-scratch-rhel-7" - ] - assert_mockbin_log(expected_log) - - -@obal_cli_test(repotype='downstream') -def test_obal_release_downstream_hello_nowait(): - assert_obal_success(['release', 'hello', '-e', 'build_package_wait=False']) - - assert os.path.exists('packages/hello/hello-2.9.tar.gz') - - expected_log = [ - "brew list-tagged --quiet --latest obaltest-6.3.0-rhel-7-candidate tfm-hello", # noqa: E501 - "tito release --yes obaltest-dist-git-rhel-7", - ] - assert_mockbin_log(expected_log) - - -@obal_cli_test(repotype='downstream') -def test_obal_scratch_downstream_hello(): - assert_obal_success(['scratch', 'hello']) - - assert os.path.exists('packages/hello/hello-2.9.tar.gz') - - expected_log = [ - "tito release --yes obaltest-scratch-rhel-7", - "brew watch-task 1234", - "brew taskinfo -v 1234", - ] - assert_mockbin_log(expected_log) - - -@obal_cli_test(repotype='downstream') -def test_obal_scratch_downstream_hello_wait_download_logs(): - assert_obal_success(['scratch', 'hello', '-e', 'build_package_download_logs=True']) - - assert os.path.exists('packages/hello/hello-2.9.tar.gz') - - expected_log = [ - "tito release --yes obaltest-scratch-rhel-7", - "brew watch-task 1234", - "brew download-logs -r 1234", - "brew taskinfo -v 1234", - ] - assert_mockbin_log(expected_log) - - -@obal_cli_test(repotype='downstream') -def test_obal_scratch_downstream_hello_wait_download_rpms(): - assert_obal_success(['scratch', 'hello', '-e', 'build_package_download_rpms=True']) - - assert os.path.exists('packages/hello/hello-2.9.tar.gz') - - expected_log = [ - "tito release --yes obaltest-scratch-rhel-7", - "brew watch-task 1234", - "brew taskinfo -v 1234", - "brew download-task --arch=noarch --arch=x86_64 1234", - "createrepo {pwd}/downloaded_rpms/rhel7" - ] - assert_mockbin_log(expected_log) - - -@obal_cli_test(repotype='downstream') -def test_obal_release_downstream_hello(): - assert_obal_success(['release', 'hello']) - - assert os.path.exists('packages/hello/hello-2.9.tar.gz') - - expected_log = [ - "brew list-tagged --quiet --latest obaltest-6.3.0-rhel-7-candidate tfm-hello", # noqa: E501 - "tito release --yes obaltest-dist-git-rhel-7", - "brew watch-task 1234", - "brew taskinfo -v 1234", - ] - assert_mockbin_log(expected_log) - - -@obal_cli_test(repotype='downstream') -def test_obal_release_downstream_hello_wait_download_logs(): - assert_obal_success(['release', 'hello', '-e', 'build_package_download_logs=True']) - - assert os.path.exists('packages/hello/hello-2.9.tar.gz') - - expected_log = [ - "brew list-tagged --quiet --latest obaltest-6.3.0-rhel-7-candidate tfm-hello", # noqa: E501 - "tito release --yes obaltest-dist-git-rhel-7", - "brew watch-task 1234", - "brew download-logs -r 1234", - "brew taskinfo -v 1234", ] assert_mockbin_log(expected_log) -@obal_cli_test(repotype='downstream') -def test_obal_release_downstream_hello_wait_download_rpms(): - assert_obal_success(['release', 'hello', '-e', 'build_package_download_rpms=True']) - - assert os.path.exists('packages/hello/hello-2.9.tar.gz') - - expected_log = [ - "brew list-tagged --quiet --latest obaltest-6.3.0-rhel-7-candidate tfm-hello", # noqa: E501 - "tito release --yes obaltest-dist-git-rhel-7", - "brew watch-task 1234", - "brew taskinfo -v 1234", - "brew download-task --arch=noarch --arch=x86_64 1234", - "createrepo {pwd}/downloaded_rpms/rhel7", - ] - assert_mockbin_log(expected_log) - -@obal_cli_test(repotype='downstream') -def test_obal_release_downstream_hello_waitrepo(): - assert_obal_success(['release', 'hello', '-e', 'build_package_waitrepo=True']) - - assert os.path.exists('packages/hello/hello-2.9.tar.gz') - - expected_log = [ - "brew list-tagged --quiet --latest obaltest-6.3.0-rhel-7-candidate tfm-hello", # noqa: E501 - "tito release --yes obaltest-dist-git-rhel-7", - "brew watch-task 1234", - "brew taskinfo -v 1234", - # the build and target in the next command are "wrong" because the - # output from our mocked brew is not dynamic - "brew wait-repo --build=hello-2.10-1.el7 --target obaltest-nightly-rhel7" - ] - assert_mockbin_log(expected_log) - @obal_cli_test(repotype='upstream') def test_obal_update_upstream_hello(): assert_obal_success(['update', 'hello', '-e', 'version=2.8']) diff --git a/tests/test_rhpkg_release.py b/tests/test_rhpkg_release.py new file mode 100644 index 00000000..7d3ce398 --- /dev/null +++ b/tests/test_rhpkg_release.py @@ -0,0 +1,591 @@ +"""Promote public Foreman package versions as if Foreman used Brew. + +Version provenance: theforeman/foreman-packaging, rpm/develop at +3aaa19d27f769214391072a2e1e595cf596d9f09 and rpm/5.0 at +9cca987afe60291a4165dff084c6bf2d4a728529. Specs are minimal local fixtures; +their Version/Release fields and foremandist contexts model those snapshots. +""" + +import copy +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest +import yaml + +from obal.data.module_utils.rhpkg import build_info, inventory_tags, normalize_targets, ReleaseError + + +DEVELOP = 'foreman-develop-rhel-9-candidate' +DESTINATION = 'foreman-5.0-rhel-9-candidate' +SECONDARY = 'foreman-client-5.0-rhel-9-candidate' +PACKAGES = { + 'foreman_scap_client_bash': ('0.2.2', '1%{?dist}'), + 'nodejs-graphql': ('15.10.3', '1%{?dist}'), + 'yggdrasil-worker-forwarder': ('0.1.0', '2%{?dist}'), + 'rubygem-foreman_templates': ('11.0.4', '1%{?foremandist}%{?dist}'), +} + + +def git(directory, *arguments): + return subprocess.check_output(['git', '-C', str(directory)] + list(arguments), text=True, + stderr=subprocess.STDOUT).strip() + + +class BrewFixture: + def __init__(self, root, monkeypatch): + self.root = root + self.repository = root / 'packaging' + self.repository.mkdir() + self.bin = root / 'bin' + self.bin.mkdir() + client = Path(__file__).parent / 'fixtures' / 'rhpkg' / 'mock_client.py' + for name in ('brew', 'koji', 'rhpkg', 'tito', 'copr-cli'): + executable = self.bin / name + executable.write_text(client.read_text()) + executable.chmod(0o755) + self.state_path = root / 'brew.json' + self.write(dict(commands=[], builds={}, tasks={}, repositories={}, + targets={DESTINATION: {'dist': '.el9', 'foremandist': '.fm5_0'}})) + monkeypatch.setenv('OBAL_BREW_STATE', str(self.state_path)) + monkeypatch.setenv('PATH', str(self.bin) + os.pathsep + os.environ['PATH']) + monkeypatch.setenv('PYTHONPATH', str(Path(__file__).resolve().parent.parent)) + git(self.repository, 'init', '--initial-branch=rpm/5.0') + git(self.repository, 'config', 'user.name', 'Obal Test') + git(self.repository, 'config', 'user.email', 'obal@example.test') + git(self.repository, 'annex', 'init') + self.target = dict(name='rhel9', distgit_branch='foreman-5.0-rhel-9', + build_target=DESTINATION, dist='.el9', macros={'foremandist': '.fm5_0'}, + tags=[DESTINATION]) + + def read(self): + return json.loads(self.state_path.read_text()) + + def write(self, state): + self.state_path.write_text(json.dumps(state)) + + def add(self, package, old=None): + version, release = PACKAGES[package] + directory = self.repository / 'packages' / package + directory.mkdir(parents=True) + spec = ('Name: {}\nVersion: {}\nRelease: {}\nSummary: Local release fixture\n' + 'License: MIT\nSource0: payload.txt\nPatch0: fix.patch\n' + '%description\nLocal fixture.\n%files\n').format(package, version, release) + (directory / (package + '.spec')).write_text(spec) + (directory / 'payload.txt').write_text('new source content\n') + (directory / 'fix.patch').write_text('fixture patch content\n') + remote = self.root / (package + '.git') + seed = self.root / (package + '-seed') + seed.mkdir() + git(seed, 'init', '--initial-branch=foreman-5.0-rhel-9') + git(seed, 'config', 'user.name', 'Obal Test') + git(seed, 'config', 'user.email', 'obal@example.test') + if old: + spec = spec.replace('Version: ' + version, 'Version: ' + old[0]) + spec = spec.replace('Release: ' + release, 'Release: ' + old[1]) + (seed / (package + '.spec')).write_text(spec.replace('Patch0: fix.patch', 'Patch0: old.patch')) + (seed / 'old.patch').write_text('obsolete\n') + (seed / 'gating.yaml').write_text('preserved\n') + (seed / 'ci.fmf').write_text('preserved service configuration\n') + git(seed, 'add', '.') + git(seed, 'commit', '-m', '5.0 packaging') + git(seed, 'clone', '--bare', str(seed), str(remote)) + state = self.read() + state['repositories'][package] = str(remote) + self.write(state) + return directory + + def manifest(self, packages, targets=None, koji_executable=None): + variables = dict(build_package_build_system='koji', + rhpkg_targets=targets or [self.target], + diff_package_tags=[DEVELOP], diff_package_skip=False, + ansible_python_interpreter=sys.executable) + if koji_executable is not None: + variables['build_package_koji_command'] = koji_executable + data = {'packages': {'vars': variables, 'hosts': {name: {} for name in packages}}} + (self.repository / 'package_manifest.yaml').write_text(yaml.safe_dump(data)) + git(self.repository, '-c', 'annex.largefiles=nothing', 'add', '.') + git(self.repository, 'commit', '--allow-empty', '-m', 'Promote package versions into rpm/5.0') + + def release(self, package, *options, action='release', success=True): + result = subprocess.run([sys.executable, '-c', 'import obal; obal.main()', action, package] + list(options), + cwd=self.repository, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + check=False) + assert result.returncode == (0 if success else 2), result.stdout + return result.stdout + + def existing(self, nvr, tags=None, state='COMPLETE'): + data = self.read() + data['builds'][nvr] = dict(state=state, tags=[DEVELOP] if tags is None else tags) + self.write(data) + + def whitelist(self, parameters, success=True): + root = Path(__file__).resolve().parent.parent / 'obal/data' + playbook = self.root / 'whitelist.yaml' + playbook.write_text(yaml.safe_dump([dict(hosts='localhost', gather_facts=False, + vars={'ansible_python_interpreter': sys.executable}, + tasks=[{'package_whitelist_check': dict(parameters, build_command='brew')}])])) + env = dict(os.environ, ANSIBLE_LIBRARY=str(root / 'modules'), ANSIBLE_MODULE_UTILS=str(root / 'module_utils')) + result = subprocess.run(['ansible-playbook', '-i', 'localhost,', '-c', 'local', str(playbook)], + env=env, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False) + assert result.returncode == (0 if success else 2), result.stdout + return result.stdout + + +@pytest.fixture +def brew(tmp_path, monkeypatch): + return BrewFixture(tmp_path, monkeypatch) + + +@pytest.mark.parametrize('action', ['release', 'scratch', 'check', 'verify-koji-tag']) +@pytest.mark.parametrize('executable', [None, 'koji']) +def test_rhpkg_workflows_default_to_brew_and_preserve_command_override(brew, action, executable): + package = 'nodejs-graphql' + brew.add(package) + brew.manifest([package], koji_executable=executable) + brew.release('all' if action == 'verify-koji-tag' else package, action=action) + commands = brew.read()['commands'] + assert commands + assert any(command[0] == (executable or 'brew') for command in commands) + assert all(command[0] in (executable or 'brew', 'rhpkg') for command in commands) + + +@pytest.mark.parametrize('package,nvr', [ + ('foreman_scap_client_bash', 'foreman_scap_client_bash-0.2.2-1.el9'), + ('nodejs-graphql', 'nodejs-graphql-15.10.3-1.el9'), + ('yggdrasil-worker-forwarder', 'yggdrasil-worker-forwarder-0.1.0-2.el9'), +]) +def test_promote_existing_develop_build_without_rebuilding(brew, package, nvr): + brew.add(package) + brew.manifest([package]) + brew.existing(nvr) + brew.release(package) + state = brew.read() + assert set(state['builds'][nvr]['tags']) == {DEVELOP, DESTINATION} + assert not any(command[0] == 'rhpkg' for command in state['commands']) + assert not any('tito' in command or 'copr-cli' in command for command in state['commands']) + brew.release(package) + tags = [command for command in brew.read()['commands'] if command[1] == 'tag-build'] + assert tags == [['brew', 'tag-build', '--wait', DESTINATION, nvr]] + + +def test_missing_build_pushes_sources_then_builds_once_and_tags_all_destinations(brew): + package = 'nodejs-graphql' + brew.add(package, old=('15.10.2', '2%{?dist}')) + brew.target['tags'].append(SECONDARY) + brew.manifest([package]) + brew.release(package) + state = brew.read() + assert set(state['builds']['nodejs-graphql-15.10.3-1.el9']['tags']) == {DESTINATION, SECONDARY} + assert len(state['tasks']) == 1 + remote = state['repositories'][package] + spec = git(remote, 'show', 'HEAD:nodejs-graphql.spec') + assert 'Version: 15.10.3' in spec and 'Release: 1%{?dist}' in spec + assert git(remote, 'show', 'HEAD:fix.patch') == 'fixture patch content' + assert git(remote, 'show', 'HEAD:gating.yaml') == 'preserved' + assert git(remote, 'show', 'HEAD:ci.fmf') == 'preserved service configuration' + assert 'old.patch' not in git(remote, 'ls-tree', '--name-only', 'HEAD') + assert 'payload.txt' in git(remote, 'show', 'HEAD:sources') + assert not any('--srpm' in command for command in state['commands']) + assert state['builds']['nodejs-graphql-15.10.3-1.el9']['source'].endswith(git(remote, 'rev-parse', 'HEAD')) + + +def test_branch_macro_difference_requires_destination_nvr(brew): + package = 'rubygem-foreman_templates' + brew.add(package) + brew.manifest([package]) + brew.existing(package + '-11.0.4-1.fm5_1.el9') + brew.release(package) + state = brew.read() + assert state['builds'][package + '-11.0.4-1.fm5_1.el9']['tags'] == [DEVELOP] + assert state['builds'][package + '-11.0.4-1.fm5_0.el9']['tags'] == [DESTINATION] + assert len(state['tasks']) == 1 + + +def test_nowait_receipt_resumes_before_build_record_exists(brew): + package = 'nodejs-graphql' + brew.add(package) + brew.manifest([package]) + state = brew.read() + state['delay_build_record'] = True + brew.write(state) + brew.release(package, '--nowait') + receipt = json.loads((brew.repository / '.tmp/rhpkg/nodejs-graphql.json').read_text()) + assert receipt['targets']['rhel9']['pending_tags'] == [DESTINATION] + assert not brew.read()['builds'] + brew.release(package) + assert len(brew.read()['tasks']) == 1 + assert brew.read()['builds']['nodejs-graphql-15.10.3-1.el9']['tags'] == [DESTINATION] + + +def test_partial_tag_failure_can_be_retried_without_rebuilding(brew): + package = 'foreman_scap_client_bash' + nvr = package + '-0.2.2-1.el9' + brew.add(package) + brew.target['tags'].append(SECONDARY) + brew.manifest([package]) + brew.existing(nvr) + state = brew.read() + state['failed_tags'] = [SECONDARY] + brew.write(state) + brew.release(package, success=False) + state = brew.read() + assert state['builds'][nvr]['tags'] == [DEVELOP, DESTINATION] + state['failed_tags'] = [] + brew.write(state) + brew.release(package) + assert set(brew.read()['builds'][nvr]['tags']) == {DEVELOP, DESTINATION, SECONDARY} + assert not brew.read()['tasks'] + + +@pytest.mark.parametrize('failure', ['query_error', 'push_error']) +def test_errors_do_not_submit_a_build(brew, failure): + package = 'nodejs-graphql' + brew.add(package) + brew.manifest([package]) + state = brew.read() + state[failure] = True + brew.write(state) + brew.release(package, success=False) + assert not brew.read()['tasks'] + + +def test_failed_build_is_not_resubmitted(brew): + package = 'nodejs-graphql' + brew.add(package) + brew.manifest([package]) + brew.existing(package + '-15.10.3-1.el9', state='FAILED') + brew.release(package, success=False) + assert not brew.read()['tasks'] + + +def test_scratch_submits_all_targets_before_waiting_and_never_tags(brew): + package = 'nodejs-graphql' + brew.add(package) + second = copy.deepcopy(brew.target) + second.update(name='rhel10', build_target='foreman-5.0-rhel-10-candidate', dist='.el10', + tags=['foreman-5.0-rhel-10-candidate']) + brew.target['tags'].append(SECONDARY) + brew.manifest([package], [brew.target, second]) + state = brew.read() + state['targets'][second['build_target']] = dict(dist='.el10', foremandist='.fm5_0') + state['failed_targets'] = [DESTINATION] + brew.write(state) + original = git(state['repositories'][package], 'rev-parse', 'HEAD') + brew.release(package, action='scratch', success=False) + state = brew.read() + assert len(state['tasks']) == 2 + submissions = [i for i, command in enumerate(state['commands']) if command[:2] == ['brew', 'build']] + waits = [i for i, command in enumerate(state['commands']) if command[1] == 'watch-task'] + assert max(submissions) < min(waits) + assert not any(command[1] in ['new-sources', 'commit', 'push', 'tag-build'] for command in state['commands']) + assert not any(command[0] == 'rhpkg' for command in state['commands']) + assert not state['builds'] + assert git(state['repositories'][package], 'rev-parse', 'HEAD') == original + + +def test_normalization_preserves_build_and_tag_distinction(): + target = dict(name='rhel9', distgit_branch='foreman-5.0-rhel-9', build_target=DESTINATION, + dist='.el9', tags=[DESTINATION, SECONDARY]) + attributes = dict(build_package_releaser='rhpkg', rhpkg_targets=[target]) + assert len(normalize_targets([target], 'package')) == 1 + assert [tag['name'] for tag in inventory_tags(attributes, 'package')] == [DESTINATION, SECONDARY] + equivalent = [dict(name=name, dist='.el9') for name in (SECONDARY, DESTINATION)] + assert inventory_tags(dict(attributes, koji_tags=equivalent), 'package') == inventory_tags(attributes, 'package') + with pytest.raises(ReleaseError, match='conflicts'): + inventory_tags(dict(attributes, koji_tags=[{'name': 'wrong'}]), 'package') + + +def test_check_compares_destination_even_when_develop_has_the_build(brew): + package = 'foreman_scap_client_bash' + nvr = package + '-0.2.2-1.el9' + brew.add(package) + brew.manifest([package]) + brew.existing(nvr) + output = brew.release(package, action='check') + assert 'changed: true' in output + assert brew.read()['builds'][nvr]['tags'] == [DEVELOP] + assert not any(command[0] == 'rhpkg' or command[1] == 'tag-build' for command in brew.read()['commands']) + assert not (brew.repository / '.tmp/rhpkg').exists() + + +def test_release_only_change_builds_the_new_release(brew): + package = 'yggdrasil-worker-forwarder' + brew.add(package, old=('0.1.0', '1%{?dist}')) + brew.manifest([package]) + brew.existing(package + '-0.1.0-1.el9', tags=[DESTINATION]) + brew.release(package) + assert len(brew.read()['tasks']) == 1 + assert brew.read()['builds'][package + '-0.1.0-2.el9']['tags'] == [DESTINATION] + + +def test_resume_rejects_changed_source_bytes(brew): + package = 'nodejs-graphql' + directory = brew.add(package) + brew.manifest([package]) + state = brew.read() + state['delay_build_record'] = True + brew.write(state) + brew.release(package, '--nowait') + (directory / 'payload.txt').write_text('changed after submission\n') + output = brew.release(package, success=False) + assert 'sources changed' in output + assert len(brew.read()['tasks']) == 1 + (directory / 'payload.txt').write_text('new source content\n') + brew.release(package) + assert len(brew.read()['tasks']) == 1 + assert brew.read()['builds'][package + '-15.10.3-1.el9']['tags'] == [DESTINATION] + + +def test_reused_build_supports_waitrepo_and_downloads(brew): + package = 'foreman_scap_client_bash' + nvr = package + '-0.2.2-1.el9' + brew.add(package) + brew.manifest([package]) + brew.existing(nvr) + brew.release(package, '-e', 'build_package_waitrepo=true', '-e', 'build_package_download_rpms=true') + commands = brew.read()['commands'] + assert ['brew', 'wait-repo', '--build=' + nvr, '--target', DESTINATION] in commands + assert ['brew', 'download-build', nvr] in commands + assert not brew.read()['tasks'] + + +def test_conflicting_releasers_fail_before_submission(brew): + package = 'nodejs-graphql' + brew.add(package) + brew.manifest([package]) + brew.release(package, '-e', 'build_package_releaser=rhpkg', + '-e', 'build_package_use_koji_build=true', success=False) + assert not brew.read()['commands'] + + +def test_verify_tags_uses_rhpkg_destinations_and_brew(brew): + package = 'foreman_scap_client_bash' + brew.add(package) + brew.target['tags'].append(SECONDARY) + brew.manifest([package]) + brew.release('all', action='verify-koji-tag') + commands = brew.read()['commands'] + assert ['brew', 'list-pkgs', '--quiet', '--tag', DESTINATION] in commands + assert ['brew', 'list-pkgs', '--quiet', '--tag', SECONDARY] in commands + + +def test_submission_error_still_attempts_other_targets(brew): + package = 'nodejs-graphql' + brew.add(package) + second = copy.deepcopy(brew.target) + second.update(name='rhel10', build_target='foreman-5.0-rhel-10-candidate', dist='.el10', + tags=['foreman-5.0-rhel-10-candidate']) + brew.manifest([package], [brew.target, second]) + state = brew.read() + state['targets'][second['build_target']] = dict(dist='.el10', foremandist='.fm5_0') + state['submission_errors'] = [DESTINATION] + brew.write(state) + brew.release(package, success=False) + assert len(brew.read()['tasks']) == 1 + assert brew.read()['builds'][package + '-15.10.3-1.el10']['tags'] == [second['build_target']] + + +def test_query_failure_is_not_a_missing_build(brew): + state = brew.read() + state['query_error'] = True + brew.write(state) + with pytest.raises(ReleaseError, match='Authentication failed'): + build_info('brew', 'package-1-1.el9') + + +@pytest.mark.parametrize('action', ['release', 'scratch']) +def test_build_downloads_logs_and_rpms_without_tito(brew, action): + package = 'nodejs-graphql' + brew.add(package) + brew.manifest([package]) + brew.release(package, '-e', 'build_package_download_logs=true', + '-e', 'build_package_download_rpms=true', action=action) + state = brew.read() + task = next(iter(state['tasks'])) + assert ['brew', 'download-logs', '-r', task] in state['commands'] + expected = (['brew', 'download-task', '--arch=noarch', '--arch=x86_64', task] if action == 'scratch' + else ['brew', 'download-build', package + '-15.10.3-1.el9']) + assert expected in state['commands'] + assert not any(command[0] == 'tito' for command in state['commands']) + + +def test_scratch_nowait_resumes_pending_task_without_release_mutations(brew): + package = 'nodejs-graphql' + brew.add(package) + brew.manifest([package]) + brew.release(package, '-e', 'build_package_wait=false', action='scratch') + state = brew.read() + assert len(state['tasks']) == 1 + assert not state['builds'] + assert not any(command[1] in ('watch-task', 'tag-build', 'new-sources', 'commit', 'push') + for command in state['commands']) + receipt_path = brew.repository / '.tmp/rhpkg/nodejs-graphql-scratch.json' + receipt = json.loads(receipt_path.read_text()) + assert receipt['targets']['rhel9']['state'] == 'PENDING' + assert receipt['targets']['rhel9']['pending_tags'] == [] + task_id = receipt['targets']['rhel9']['tasks'][0] + + brew.release(package, '-e', 'build_package_wait=false', action='scratch') + assert len(brew.read()['tasks']) == 1 + assert json.loads(receipt_path.read_text())['targets']['rhel9']['tasks'] == [task_id] + + brew.release(package, '-e', 'build_package_download_logs=true', + '-e', 'build_package_download_rpms=true', action='scratch') + state = brew.read() + assert len(state['tasks']) == 1 + assert state['tasks'][task_id]['state'] == 'closed' + assert ['brew', 'watch-task', task_id] in state['commands'] + assert ['brew', 'download-logs', '-r', task_id] in state['commands'] + assert ['brew', 'download-task', '--arch=noarch', '--arch=x86_64', task_id] in state['commands'] + assert json.loads(receipt_path.read_text())['targets']['rhel9']['state'] == 'COMPLETE' + assert not state['builds'] + assert not any(command[0] == 'rhpkg' or command[1] in ('buildinfo', 'list-pkgs', 'tag-build') + for command in state['commands']) + + # A completed scratch receipt must allow a genuinely new scratch build. + brew.release(package, action='scratch') + assert len(brew.read()['tasks']) == 2 + + +@pytest.mark.parametrize('change', ['source', 'spec', 'nvr', 'target']) +def test_scratch_resume_rejects_changed_inputs_and_preserves_receipt(brew, change): + package = 'nodejs-graphql' + directory = brew.add(package) + brew.manifest([package]) + brew.release(package, '-e', 'build_package_wait=false', action='scratch') + receipt_path = brew.repository / '.tmp/rhpkg/nodejs-graphql-scratch.json' + receipt = receipt_path.read_bytes() + if change == 'source': + changed_file = directory / 'payload.txt' + original = changed_file.read_text() + changed_file.write_text('changed after submission\n') + elif change in ('spec', 'nvr'): + changed_file = directory / (package + '.spec') + original = changed_file.read_text() + changed_file.write_text(original + '# Changed spec\n' if change == 'spec' + else original.replace('Version: 15.10.3', 'Version: 15.10.4')) + else: + changed_file = brew.repository / 'package_manifest.yaml' + original = changed_file.read_text() + manifest = yaml.safe_load(original) + manifest['packages']['vars']['rhpkg_targets'][0]['build_target'] = SECONDARY + changed_file.write_text(yaml.safe_dump(manifest)) + output = brew.release(package, action='scratch', success=False) + assert 'pending' in output + assert len(brew.read()['tasks']) == 1 + assert receipt_path.read_bytes() == receipt + assert not any(command[1] in ('buildinfo', 'tag-build', 'watch-task') for command in brew.read()['commands']) + changed_file.write_text(original) + brew.release(package, action='scratch') + assert len(brew.read()['tasks']) == 1 + assert json.loads(receipt_path.read_text())['targets']['rhel9']['state'] == 'COMPLETE' + + +@pytest.mark.parametrize('registered', [True, False]) +def test_tag_registration_checked_before_submission(brew, registered): + package = 'nodejs-graphql' + brew.add(package) + brew.manifest([package]) + state = brew.read() + state['unregistered'] = not registered + brew.write(state) + brew.release(package, '-e', 'build_package_koji_whitelist_check=true', success=registered) + state = brew.read() + assert ['brew', 'list-pkgs', '--tag', DESTINATION, '--package', package, '--quiet'] in state['commands'] + assert len(state['tasks']) == int(registered) + + +@pytest.mark.parametrize('action', ['release', 'scratch']) +def test_tito_selector_is_rejected(brew, action): + package = 'nodejs-graphql' + brew.add(package) + brew.manifest([package]) + output = brew.release(package, '-e', 'build_package_releaser=tito', action=action, success=False) + assert 'Tito releases are no longer supported' in output + assert not brew.read()['commands'] + + +def test_legacy_brew_inventory_requires_explicit_distgit_targets(brew): + package = 'nodejs-graphql' + brew.add(package) + brew.manifest([package]) + manifest = brew.repository / 'package_manifest.yaml' + data = yaml.safe_load(manifest.read_text()) + del data['packages']['vars']['rhpkg_targets'] + data['packages']['vars']['releasers'] = ['foreman-5.0-dist-git'] + manifest.write_text(yaml.safe_dump(data)) + output = brew.release(package, success=False) + assert 'rhpkg_targets must contain at least one build target' in output + assert not brew.read()['commands'] + + +@pytest.mark.parametrize('failure', ['unregistered_tags', 'blocked_tags', 'registration_error']) +def test_existing_build_is_not_tagged_without_destination_registration(brew, failure): + package = 'nodejs-graphql' + nvr = package + '-15.10.3-1.el9' + brew.add(package) + brew.target['tags'].append(SECONDARY) + brew.manifest([package]) + brew.existing(nvr) + state = brew.read() + state[failure] = [SECONDARY] if failure.endswith('_tags') else True + brew.write(state) + output = brew.release(package, success=False) + assert 'registration' in output or 'not registered' in output + state = brew.read() + assert state['builds'][nvr]['tags'] == [DEVELOP] + assert not any(command[0] == 'rhpkg' or command[1] == 'tag-build' for command in state['commands']) + + +@pytest.mark.parametrize('legacy', [True, False]) +@pytest.mark.parametrize('registered', [True, False]) +def test_whitelist_module_supports_explicit_tags_and_legacy_config(brew, legacy, registered): + package = 'nodejs-graphql' + directory = brew.add(package) + state = brew.read() + state['unregistered_tags'] = [] if registered else [SECONDARY] + brew.write(state) + if legacy: + config = brew.root / 'releasers.conf' + config.write_text('[release]\nbranches={}\n[client]\nautobuild_tags={}\n'.format(DESTINATION, SECONDARY)) + parameters = dict(releasers_conf=str(config), releasers=['release', 'client'], + spec_file_path=str(directory / (package + '.spec'))) + else: + parameters = dict(package=package, tags=[DESTINATION, SECONDARY]) + brew.whitelist(parameters, success=registered) + commands = brew.read()['commands'] + assert ['brew', 'list-pkgs', '--tag', DESTINATION, '--package', package, '--quiet'] in commands + assert ['brew', 'list-pkgs', '--tag', SECONDARY, '--package', package, '--quiet'] in commands + assert all(command[:2] == ['brew', 'list-pkgs'] for command in commands) + + +def test_koji_scratch_fetches_annex_bytes_without_registration_or_rhpkg(brew): + package = 'nodejs-graphql' + directory = brew.add(package) + source = directory / 'payload.txt' + payload = source.read_bytes() + relative = str(source.relative_to(brew.repository)) + remote = brew.root / 'annex-source-remote' + remote.mkdir() + git(brew.repository, 'annex', 'initremote', 'sources', 'type=directory', + 'directory=' + str(remote), 'encryption=none') + git(brew.repository, 'annex', 'add', '--force-large', relative) + brew.manifest([package]) + git(brew.repository, 'annex', 'copy', '--to=sources', '--', relative) + git(brew.repository, 'annex', 'drop', '--', relative) + assert source.is_symlink() and not source.exists() + state = brew.read() + state['unregistered'] = True + brew.write(state) + brew.release(package, '-e', 'build_package_koji_whitelist_check=true', action='scratch') + state = brew.read() + task = next(iter(state['tasks'].values())) + assert task['source_sha256'] == hashlib.sha256(payload).hexdigest() + assert source.is_symlink() and source.read_bytes() == payload + assert any(command[:4] == ['brew', 'build', '--scratch', '--nowait'] for command in state['commands']) + assert not any(command[0] == 'rhpkg' or command[1] in ('list-pkgs', 'tag-build') for command in state['commands']) diff --git a/tests/test_setup.py b/tests/test_setup.py new file mode 100644 index 00000000..dc939c72 --- /dev/null +++ b/tests/test_setup.py @@ -0,0 +1,36 @@ +"""Verify setup follows package inventory rather than localhost defaults.""" + +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest +import yaml + + +@pytest.mark.parametrize('variables,rhpkg_required', [ + (None, False), + ({'build_package_build_system': 'copr'}, False), + ({'build_package_build_system': 'koji', 'build_package_use_koji_build': True}, False), + ({'build_package_build_system': 'koji', 'build_package_releaser': 'koji'}, False), + ({'build_package_build_system': 'koji', 'build_package_koji_command': 'brew'}, True), +]) +def test_setup_installs_only_the_required_releaser(tmp_path, variables, rhpkg_required): + root = Path(__file__).resolve().parent.parent + if variables is not None: + manifest = {'packages': {'vars': variables, 'hosts': {'package': {}}}} + (tmp_path / 'package_manifest.yaml').write_text(yaml.safe_dump(manifest)) + log = tmp_path / 'packages.jsonl' + env = dict(os.environ, PYTHONPATH=str(root), OBAL_SETUP_LOG=str(log), + ANSIBLE_ACTION_PLUGINS=str(root / 'tests/fixtures/setup_actions')) + result = subprocess.run([sys.executable, '-c', 'import obal; obal.main()', 'setup'], + cwd=tmp_path, env=env, text=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, check=False) + assert result.returncode == 0, result.stdout + packages = {name for line in log.read_text().splitlines() for name in json.loads(line)} + assert 'git' in packages + assert packages & {'git-annex', 'git-annex-standalone'} + assert 'tito' not in packages + assert ('rhpkg' in packages) == rhpkg_required