Update to upstream tag 1.26.3

.gitlab-image-tags.yml

@@ -0,0 +1,18 @@
+variables:
+  ###
+  # IMPORTANT
+  # These are the version tags for the docker images the CI runs against.
+  # If you are hacking on them or need a them to rebuild, its enough
+  # to change any part of the string of the image you want.
+  ###
+  FEDORA_TAG:  '2025-03-13.0'
+
+  CHECKS_FEDORA_TAG: '2025-03-13.0'
+
+  ANDROID_FEDORA_TAG: '2025-06-21.0-1.26'
+
+  UBUNTU_TAG: '2025-03-13.0'
+
+  TESTS_UBUNTU_TAG: '2025-03-13.0'
+
+  WINDOWS_IMAGE: 'registry.freedesktop.org/gstreamer/gstreamer/amd64/windows:2025-03-12.0-1.26'

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.1.8
+    rev: v0.5.6
     hooks:
       # Run the linter.
       - id: ruff

Makefile

@@ -20,13 +20,5 @@ check-format:
 	ruff check .
 
 check:
-	PYTHONPATH=$(PYTHONPATH):./test:./cerbero; trial test
+	CERBERO_UNINSTALLED=1 PYTHONPATH=$(PYTHONPATH):./test:./cerbero python -m unittest discover -s test -f
 	make check-format
-
-coverage:
-	rm -rf _trial_temp
-	PYTHONPATH=$(PYTHONPATH):./test:./cerbero; trial --coverage test
-	make show-coverage
-
-show-coverage:
-	python tools/show-coverage.py _trial_temp/coverage/cerbero.*

README.md

@@ -34,6 +34,9 @@ On macOS you will need to have install the following software:
 
 Cerbero will build all other required packages during [bootstrap](#Bootstrap).
 
+Note that introspection and python support will only be enabled when building
+on ARM64 (Apple Silicon).
+
 ### Windows Setup
 
 The initial setup on Windows is automated with the PowerShell script
@@ -45,7 +48,7 @@ installs the necessary tools with [Chocolatey](https://chocolatey.org/):
 * MSYS2
 * Git
 * Python 3
-* WiX
+* WiX 5.0.1 or higher
 
 Start an admin PowerShell and run:
 

cerbero-uninstalled

@@ -24,7 +24,8 @@
 ARGS=$@
 SCRIPTDIR="`dirname $0`"
 
-if [ "$OSTYPE" = "msys" ]; then
+# https://www.msys2.org/news/#2025-02-14-moving-msys2-closer-to-cygwin
+if [ "$OSTYPE" = "msys" ] || [ "$OSTYPE" = "cygwin" ]; then
    if [ -f "/mingw/bin/mingw-get.exe" ]; then
        MSYS_VERSION=1
    elif  [ -f "/usr/bin/pacman" ]; then

cerbero/bootstrap/__init__.py

@@ -22,9 +22,10 @@
 
 
 class BootstrapTarball(BaseTarball, Source):
-    def __init__(self, config, offline, url, checksum, download_dir, tarball_name=None):
+    def __init__(self, config, offline, name, url, checksum, download_dir, tarball_name=None):
         self.config = config
         self.offline = offline
+        self.name = name
         self.url = url
         self.download_dir = download_dir
         self.tarball_name = tarball_name
@@ -47,9 +48,10 @@ class BootstrapperBase(object):
     # List of extract steps to be performed
     extract_steps = None
 
-    def __init__(self, config, offline):
+    def __init__(self, config, offline, name):
         self.config = config
         self.offline = offline
+        self.name = name
         self.fetch_urls = []
         self.extract_steps = []
         self.sources = {}
@@ -58,9 +60,15 @@ async def start(self):
         raise NotImplementedError("'start' must be implemented by subclasses")
 
     async def fetch_urls_impl(self, urls):
-        for url, name, checksum in urls:
+        for url, tarball_name, checksum in urls:
             source = BootstrapTarball(
-                self.config, self.offline, url, checksum, self.config.local_sources, tarball_name=name
+                self.config,
+                self.offline,
+                self.name,
+                url,
+                checksum,
+                self.config.local_sources,
+                tarball_name=tarball_name,
             )
             self.sources[url] = source
             await source.fetch()

cerbero/bootstrap/android.py

@@ -35,7 +35,7 @@
 
 class AndroidBootstrapper(BootstrapperBase):
     def __init__(self, config, offline, assume_yes):
-        super().__init__(config, offline)
+        super().__init__(config, offline, 'android')
         self.prefix = self.config.toolchain_prefix
         url = NDK_BASE_URL % (NDK_VERSION, self.config.platform)
         self.fetch_urls.append((url, None, NDK_CHECKSUMS[os.path.basename(url)]))

cerbero/bootstrap/build_tools.py

@@ -17,6 +17,7 @@
 # Boston, MA 02111-1307, USA.
 
 import os
+from pathlib import Path
 import venv
 import glob
 import shutil
@@ -26,7 +27,7 @@
 from cerbero.build.cookbook import CookBook
 from cerbero.commands.fetch import Fetch
 from cerbero.utils import shell
-from cerbero.enums import Platform, Distro
+from cerbero.enums import Platform, Distro, Architecture
 
 
 class BuildTools (BootstrapperBase, Fetch):
@@ -39,7 +40,7 @@ class BuildTools (BootstrapperBase, Fetch):
     }
 
     def __init__(self, config, offline):
-        BootstrapperBase.__init__(self, config, offline)
+        BootstrapperBase.__init__(self, config, offline, None)
 
         if self.config.variants.rust:
             self.BUILD_TOOLS.append('cargo-c')
@@ -116,14 +117,21 @@ def setup_venv(self):
             scriptsdir = os.path.join(self.config.build_tools_prefix, 'Scripts')
             bindir = os.path.join(self.config.build_tools_prefix, 'bin')
             os.makedirs(bindir, exist_ok=True)
-            for f in glob.glob('*', root_dir=scriptsdir):
-                tof = os.path.join(bindir, f)
+            for f in Path(scriptsdir).glob('*'):
+                tof = os.path.join(bindir, f.name)
                 if os.path.isfile(tof):
                     os.remove(tof)
-                shutil.move(os.path.join(scriptsdir, f), tof)
+                shutil.move(f, tof)
             os.rmdir(scriptsdir)
         python = os.path.join(self.config.build_tools_prefix, 'bin', 'python')
-        shell.new_call([python, '-m', 'pip', 'install', 'setuptools'])
+        shell.new_call([python, '-m', 'pip', 'install', '-U', 'setuptools', 'packaging'])
+        if self.config.platform == Platform.DARWIN and self.config.arch == Architecture.ARM64:
+            # Create an x86_64 python for introspection in universal builds
+            python_wrapper = os.path.join(self.config.build_tools_prefix, 'bin', 'python3-x86_64')
+            with open(python_wrapper, 'w') as f:
+                f.write('#!/bin/sh\n')
+                f.write('arch -x86_64 python3 "$@"\n')
+            os.chmod(python_wrapper, 0o755)
 
     async def start(self, jobs=0):
         self.setup_venv()

cerbero/bootstrap/ios.py

@@ -23,18 +23,22 @@
 # the Free Software Foundation; either version 3, or (at your option)
 # any later version.
 
+import os
+from cerbero.utils import shell
+from cerbero.utils import messages as m
 from cerbero.bootstrap import BootstrapperBase
 from cerbero.bootstrap.bootstrapper import register_toolchain_bootstrapper
-from cerbero.config import Distro
+from cerbero.enums import Distro, Architecture
 
 
 class IOSBootstrapper(BootstrapperBase):
     def __init__(self, config, offline, assume_yes):
-        super().__init__(config, offline)
+        super().__init__(config, offline, 'ios')
 
     async def start(self, jobs=0):
-        # FIXME: enable it when buildbots are properly configured
-        return
+        if self.config.arch == Architecture.ARM64 and not os.path.exists('/Library/Apple/usr/lib/libRosettaAot.dylib'):
+            m.message('Installing rosetta needed for some package installation scripts')
+            shell.new_call(['/usr/sbin/softwareupdate', '--install-rosetta', '--agree-to-license'])
 
 
 def register_all():

cerbero/bootstrap/linux.py

@@ -16,11 +16,13 @@
 # Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 # Boston, MA 02111-1307, USA.
 
+import sys
+
 from cerbero.bootstrap import BootstrapperBase
 from cerbero.bootstrap.bootstrapper import register_system_bootstrapper
 from cerbero.enums import Platform, Architecture, Distro, DistroVersion
-from cerbero.errors import ConfigurationError, CommandError
-from cerbero.utils import user_is_root, shell
+from cerbero.errors import ConfigurationError, CommandError, FatalError
+from cerbero.utils import user_is_root, shell, split_version
 from cerbero.utils import messages as m
 
 
@@ -32,7 +34,7 @@ class UnixBootstrapper(BootstrapperBase):
     packages = []
 
     def __init__(self, config, offline, assume_yes):
-        BootstrapperBase.__init__(self, config, offline)
+        BootstrapperBase.__init__(self, config, offline, 'linux')
         self.assume_yes = assume_yes
         if user_is_root() and 'sudo' in self.tool:  # no need for sudo as root user
             self.tool.remove('sudo')
@@ -43,9 +45,20 @@ async def start(self, jobs=0):
 
         if self.config.distro_packages_install:
             extra_packages = self.config.extra_bootstrap_packages.get(self.config.platform, None)
+            override_packages = self.config.override_bootstrap_packages.get(self.config.platform, None)
+            if extra_packages and override_packages:
+                raise ConfigurationError(
+                    'You are setting "extra_bootstrap_packages" and "override_bootstrap_packages" '
+                    'on the same CBC. This might cause conflicts.'
+                )
             if extra_packages:
                 self.packages += extra_packages.get(self.config.distro, [])
                 self.packages += extra_packages.get(self.config.distro_version, [])
+            if override_packages:
+                self.packages = []
+                self.packages += override_packages.get(self.config.distro, [])
+                self.packages += override_packages.get(self.config.distro_version, [])
+
             tool = self.tool
             if self.assume_yes:
                 tool += self.yes_arg
@@ -111,7 +124,7 @@ def __init__(self, config, offline, assume_yes):
                 self.packages.append('libc6:i386')
                 self.checks.append(self.create_debian_arch_check('i386'))
             if self.config.arch in [Architecture.X86_64, Architecture.X86]:
-                self.packages.append('wine')
+                self.packages += ['wine', 'xvfb']
 
     def create_debian_arch_check(self, arch):
         def check_arch():
@@ -154,7 +167,6 @@ class RedHatBootstrapper(UnixBootstrapper):
         'curl',
         'rpm-build',
         'redhat-rpm-config',
-        'python3-devel',
         'libXrender-devel',
         'pulseaudio-libs-devel',
         'libXv-devel',
@@ -164,38 +176,82 @@ class RedHatBootstrapper(UnixBootstrapper):
         'libXi-devel',
         'perl-XML-Simple',
         'gperf',
-        'wget',
         'libXrandr-devel',
         'libXtst-devel',
         'git',
         'xorg-x11-util-macros',
         'mesa-libEGL-devel',
-        'ccache',
         'openssl-devel',
         'alsa-lib-devel',
-        'perl-FindBin',
+        'perl-IPC-Cmd',
+        'libatomic',
     ]
 
     def __init__(self, config, offline, assume_yes):
         UnixBootstrapper.__init__(self, config, offline, assume_yes)
 
-        if self.config.distro_version < DistroVersion.FEDORA_23:
-            self.tool = ['yum']
-        elif self.config.distro_version in [DistroVersion.REDHAT_6, DistroVersion.REDHAT_7]:
+        dn, dv = self.config.distro_version.split('_', 1)
+        dv = split_version(dv)
+
+        if (dn == 'fedora' and dv < (23,)) or (dn == 'redhat' and dv < (8, 3)):
             self.tool = ['yum']
-        elif self.config.distro_version == DistroVersion.REDHAT_8:
-            self.tool = ['yum', '--enablerepo=PowerTools']
+
+        if dn == 'redhat' and dv >= (8,):
+            if dv < (8, 3):
+                self.tool += ['--enablerepo=PowerTools']
+            elif dv < (9,):
+                self.tool += ['--enablerepo=powertools']
+            else:
+                self.tool += ['--enablerepo=crb']
+
+        def pkg_available(pkg):
+            return shell.new_call(self.tool + ['list', pkg, '-q'], fail=False) == 0
+
+        def add_pkg(names, required=True):
+            for pkg in names:
+                if pkg_available(pkg):
+                    self.packages.append(pkg)
+                    return True
+            if required:
+                raise FatalError('Required package not found, tried: ' + ', '.join(names))
+            return False
+
+        if not user_is_root():
+            self.tool = ['sudo'] + self.tool
+
+        # In RHEL based distros ccache is only available in the EPEL repo
+        if not add_pkg(['ccache'], required=False):
+            if pkg_available('epel-release'):
+                cmd = self.tool + ['install', 'epel-release'] + (['-y'] if assume_yes else [])
+                self.checks.append(
+                    lambda: shell.new_call(cmd, interactive=True, fail=False) == 0 and self.packages.append('ccache')
+                )
+            else:
+                m.warning('Compilation will not use ccache since it is not available')
+                self.config.use_ccache = False
+
+        # Before RHEL 9 perl-FindBin wasn't a standalone package
+        if dn != 'redhat' or dv >= (9,):
+            add_pkg(['perl-FindBin'])
+        elif dv >= (8,):
+            add_pkg(['perl-interpreter'])
+        else:
+            add_pkg(['perl'])
+
+        # Use curl if wget isn't found, because wget2 is not production-ready yet
+        add_pkg(['wget', 'curl'])
+
+        # Try to get a better matching python3 library version
+        ver = sys.version_info[1]
+        add_pkg([f'python3{v}-devel' for v in [ver, f'.{ver}', '']])
 
         if self.config.target_platform == Platform.WINDOWS:
             if self.config.arch == Architecture.X86_64:
                 self.packages.append('glibc.i686')
-            if self.config.distro_version in [DistroVersion.FEDORA_24, DistroVersion.FEDORA_25]:
+            if self.config.distro_version in ['fedora_24', 'fedora_25']:
                 self.packages.append('libncurses-compat-libs.i686')
             if self.config.arch in [Architecture.X86_64, Architecture.X86]:
-                self.packages.append('wine')
-        if user_is_root():
-            return
-        self.tool = ['sudo'] + self.tool
+                self.packages += ['wine', 'which', 'xorg-x11-server-Xvfb']
 
 
 class OpenSuseBootstrapper(UnixBootstrapper):
@@ -238,7 +294,7 @@ def __init__(self, config, offline, assume_yes):
         UnixBootstrapper.__init__(self, config, offline, assume_yes)
         if self.config.target_platform == Platform.WINDOWS:
             if self.config.arch in [Architecture.X86_64, Architecture.X86]:
-                self.packages.append('wine')
+                self.packages += ['wine', 'xvfb-run']
 
 
 class ArchBootstrapper(UnixBootstrapper):
@@ -267,6 +323,8 @@ class ArchBootstrapper(UnixBootstrapper):
         'ccache',
         'openssl',
         'alsa-lib',
+        'which',
+        'libpulse',
     ]
 
     def __init__(self, config, offline, assume_yes):
@@ -284,7 +342,7 @@ def __init__(self, config, offline, assume_yes):
             self.packages.append('gcc')
         if self.config.target_platform == Platform.WINDOWS:
             if self.config.arch in [Architecture.X86_64, Architecture.X86]:
-                self.packages.append('wine')
+                self.packages += ['wine', 'xorg-server-xvfb']
 
 
 class GentooBootstrapper(UnixBootstrapper):