Merge branch 'facebook:main' into update-release-yml

Author: thecatcore

CMakeLists.txt

@@ -12,7 +12,17 @@ set(CMAKE_MODULE_PATH
   "${CMAKE_CURRENT_SOURCE_DIR}/build/fbcode_builder/CMake"
   ${CMAKE_MODULE_PATH})
 
-set(CMAKE_CXX_STANDARD 17)
+if(NOT CMAKE_CXX_STANDARD)
+  set(CMAKE_CXX_STANDARD 17)
+  set(CMAKE_CXX_STANDARD_REQUIRED ON)
+  message(STATUS "setting C++ standard to C++${CMAKE_CXX_STANDARD}")
+endif()
+
+# Explicitly enable coroutine support, since GCC does not enable it
+# by default when targeting C++17.
+if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+  add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-fcoroutines>)
+endif()
 
 if (WIN32)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DWIN32_LEAN_AND_MEAN -DNOMINMAX -DSTRICT")

build/deps/github_hashes/facebook/fbthrift-rev.txt

@@ -1 +1 @@
-Subproject commit 97518207214e89d6db73da63837f4e1dc2b57acf
+Subproject commit 4662bb3dd9ff5cc4324e794e29509b8acb1c8a4b

build/deps/github_hashes/facebook/folly-rev.txt

@@ -1 +1 @@
-Subproject commit ab576d641d9ae77662e6e54a5db7fbe6d215fa6d
+Subproject commit 819b4c4d46a7d6447584e46a7eb0731297622acf

build/deps/github_hashes/facebook/wangle-rev.txt

@@ -1 +1 @@
-Subproject commit dd5f918c13d1f4c89519cc76edec50e39c0fdc2b
+Subproject commit 7beae5da9cc5293dd785dd424b5ad5f7d819c64b

build/fbcode_builder/getdeps.py

@@ -333,6 +333,12 @@ def run_project_cmd(self, args, loader, manifest):
 
         cache = cache_module.create_cache()
         for m in projects:
+            fetcher = loader.create_fetcher(m)
+            if isinstance(fetcher, SystemPackageFetcher):
+                # We are guaranteed that if the fetcher is set to
+                # SystemPackageFetcher then this item is completely
+                # satisfied by the appropriate system packages
+                continue
             cached_project = CachedProject(cache, loader, m)
             if cached_project.download():
                 continue
@@ -348,7 +354,6 @@ def run_project_cmd(self, args, loader, manifest):
                     continue
 
             # We need to fetch the sources
-            fetcher = loader.create_fetcher(m)
             fetcher.update()
 
 
@@ -923,6 +928,27 @@ def run_project_cmd(self, args, loader, manifest):
         self.create_builder(loader, manifest).debug(reconfigure=False)
 
 
+@cmd(
+    "env",
+    "print the environment in a shell sourceable format",
+)
+class EnvCmd(ProjectCmdBase):
+    def setup_project_cmd_parser(self, parser):
+        parser.add_argument(
+            "--os-type",
+            help="Filter to just this OS type to run",
+            choices=["linux", "darwin", "windows"],
+            action="store",
+            dest="ostype",
+            default=None,
+        )
+
+    def run_project_cmd(self, args, loader, manifest):
+        if args.ostype:
+            loader.build_opts.host_type.ostype = args.ostype
+        self.create_builder(loader, manifest).printenv(reconfigure=False)
+
+
 @cmd("generate-github-actions", "generate a GitHub actions configuration")
 class GenerateGitHubActionsCmd(ProjectCmdBase):
     RUN_ON_ALL = """ [push, pull_request]"""
@@ -995,6 +1021,8 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
         if build_opts.is_linux():
             artifacts = "linux"
             runs_on = f"ubuntu-{args.ubuntu_version}"
+            if args.cpu_cores:
+                runs_on = f"{args.cpu_cores}-core-ubuntu-{args.ubuntu_version}"
         elif build_opts.is_windows():
             artifacts = "windows"
             runs_on = "windows-2019"
@@ -1246,6 +1274,10 @@ def setup_project_cmd_parser(self, parser):
         parser.add_argument(
             "--ubuntu-version", default="22.04", help="Version of Ubuntu to use"
         )
+        parser.add_argument(
+            "--cpu-cores",
+            help="Number of CPU cores to use (applicable for Linux OS)",
+        )
         parser.add_argument(
             "--cron",
             help="Specify that the job runs on a cron schedule instead of on pushes",

build/fbcode_builder/getdeps/builder.py

@@ -15,8 +15,10 @@
 import subprocess
 import sys
 import typing
+from shlex import quote as shellquote
 from typing import Optional
 
+from .copytree import simple_copytree
 from .dyndeps import create_dyn_dep_munger
 from .envfuncs import add_path_entry, Env, path_search
 from .fetcher import copy_if_different
@@ -157,6 +159,29 @@ def debug(self, reconfigure: bool) -> None:
         shell = ["powershell.exe"] if sys.platform == "win32" else ["/bin/sh", "-i"]
         self._run_cmd(shell, cwd=self.build_dir, env=env)
 
+    def printenv(self, reconfigure: bool) -> None:
+        """print the environment in a shell sourcable format"""
+        reconfigure = self._reconfigure(reconfigure)
+        self._apply_patchfile()
+        self._prepare(reconfigure=reconfigure)
+        env = self._compute_env(env=Env(src={}))
+        prefix = "export "
+        sep = ":"
+        expand = "$"
+        expandpost = ""
+        if self.build_opts.is_windows():
+            prefix = "SET "
+            sep = ";"
+            expand = "%"
+            expandpost = "%"
+        for k, v in sorted(env.items()):
+            existing = os.environ.get(k, None)
+            if k.endswith("PATH") and existing:
+                v = shellquote(v) + sep + f"{expand}{k}{expandpost}"
+            else:
+                v = shellquote(v)
+            print("%s%s=%s" % (prefix, k, v))
+
     def build(self, reconfigure: bool) -> None:
         print("Building %s..." % self.manifest.name)
         reconfigure = self._reconfigure(reconfigure)
@@ -225,14 +250,16 @@ def _build(self, reconfigure) -> None:
         system needs to regenerate its rules."""
         pass
 
-    def _compute_env(self):
+    def _compute_env(self, env=None) -> Env:
+        if env is None:
+            env = self.env
         # CMAKE_PREFIX_PATH is only respected when passed through the
         # environment, so we construct an appropriate path to pass down
         return self.build_opts.compute_env_for_install_dirs(
             self.loader,
             self.dep_manifests,
             self.ctx,
-            env=self.env,
+            env=env,
             manifest=self.manifest,
         )
 
@@ -461,6 +488,61 @@ def _build(self, reconfigure) -> None:
         self._run_cmd(install_cmd, env=env)
 
 
+class SystemdBuilder(BuilderBase):
+    # SystemdBuilder assumes that meson build tool has already been installed on
+    # the machine.
+    def __init__(
+        self,
+        loader,
+        dep_manifests,
+        build_opts,
+        ctx,
+        manifest,
+        src_dir,
+        build_dir,
+        inst_dir,
+    ) -> None:
+        super(SystemdBuilder, self).__init__(
+            loader,
+            dep_manifests,
+            build_opts,
+            ctx,
+            manifest,
+            src_dir,
+            build_dir,
+            inst_dir,
+        )
+
+    def _build(self, reconfigure) -> None:
+        env = self._compute_env()
+        meson = path_search(env, "meson")
+        if meson is None:
+            raise Exception("Failed to find Meson")
+
+        # Meson builds typically require setup, compile, and install steps.
+        # During this setup step we ensure that the static library is built and
+        # the prefix is empty.
+        self._run_cmd(
+            [
+                meson,
+                "setup",
+                "-Dstatic-libsystemd=true",
+                "-Dprefix=/",
+                self.build_dir,
+                self.src_dir,
+            ]
+        )
+
+        # Compile step needs to satisfy the build directory that was previously
+        # prepared during setup.
+        self._run_cmd([meson, "compile", "-C", self.build_dir])
+
+        # Install step
+        self._run_cmd(
+            [meson, "install", "-C", self.build_dir, "--destdir", self.inst_dir]
+        )
+
+
 class CMakeBuilder(BuilderBase):
     MANUAL_BUILD_SCRIPT = """\
 #!{sys.executable}
@@ -1084,9 +1166,14 @@ def _build(self, reconfigure) -> None:
         perl = typing.cast(str, path_search(env, "perl", "perl"))
 
         make_j_args = []
+        extra_args = []
         if self.build_opts.is_windows():
-            make = "nmake.exe"
+            # jom is compatible with nmake, adds the /j argument for parallel build
+            make = "jom.exe"
+            make_j_args = ["/j%s" % self.num_jobs]
             args = ["VC-WIN64A-masm", "-utf-8"]
+            # fixes "if multiple CL.EXE write to the same .PDB file, please use /FS"
+            extra_args = ["/FS"]
         elif self.build_opts.is_darwin():
             make = "make"
             make_j_args = ["-j%s" % self.num_jobs]
@@ -1119,11 +1206,14 @@ def _build(self, reconfigure) -> None:
                 "no-unit-test",
                 "no-tests",
             ]
+            + extra_args
         )
+        # show the config produced
+        self._run_cmd([perl, "configdata.pm", "--dump"], env=env)
         make_build = [make] + make_j_args
-        self._run_cmd(make_build)
+        self._run_cmd(make_build, env=env)
         make_install = [make, "install_sw", "install_ssldirs"]
-        self._run_cmd(make_install)
+        self._run_cmd(make_install, env=env)
 
 
 class Boost(BuilderBase):
@@ -1240,7 +1330,7 @@ def build(self, reconfigure: bool) -> None:
                     os.makedirs(dest_parent)
                 if os.path.isdir(full_src):
                     if not os.path.exists(full_dest):
-                        shutil.copytree(full_src, full_dest)
+                        simple_copytree(full_src, full_dest)
                 else:
                     shutil.copyfile(full_src, full_dest)
                     shutil.copymode(full_src, full_dest)
@@ -1252,7 +1342,7 @@ def build(self, reconfigure: bool) -> None:
                         os.chmod(full_dest, st.st_mode | stat.S_IXUSR)
         else:
             if not os.path.exists(self.inst_dir):
-                shutil.copytree(self.src_dir, self.inst_dir)
+                simple_copytree(self.src_dir, self.inst_dir)
 
 
 class SqliteBuilder(BuilderBase):

build/fbcode_builder/getdeps/buildopts.py

@@ -304,12 +304,14 @@ def compute_env_for_install_dirs(
             is_direct_dep = (
                 manifest is not None and m.name in manifest.get_dependencies(ctx)
             )
-            self.add_prefix_to_env(
-                loader.get_project_install_dir(m),
-                env,
-                append=False,
-                is_direct_dep=is_direct_dep,
-            )
+            d = loader.get_project_install_dir(m)
+            if os.path.exists(d):
+                self.add_prefix_to_env(
+                    d,
+                    env,
+                    append=False,
+                    is_direct_dep=is_direct_dep,
+                )
 
         # Linux is always system openssl
         system_openssl = self.is_linux()
@@ -521,7 +523,8 @@ def find_unused_drive_letter():
     return available[-1]
 
 
-def create_subst_path(path: str) -> str:
+def map_subst_path(path: str) -> str:
+    """find a short drive letter mapping for a path"""
     for _attempt in range(0, 24):
         drive = find_existing_win32_subst_for_path(
             path, subst_mapping=list_win32_subst_letters()
@@ -542,9 +545,11 @@ def create_subst_path(path: str) -> str:
         # other processes on the same host, so this may not succeed.
         try:
             subprocess.check_call(["subst", "%s:" % available, path])
-            return "%s:\\" % available
+            subst = "%s:\\" % available
+            print("Mapped scratch dir %s -> %s" % (path, subst), file=sys.stderr)
+            return subst
         except Exception:
-            print("Failed to map %s -> %s" % (available, path))
+            print("Failed to map %s -> %s" % (available, path), file=sys.stderr)
 
     raise Exception("failed to set up a subst path for %s" % path)
 
@@ -617,10 +622,7 @@ def setup_build_options(args, host_type=None) -> BuildOptions:
             os.makedirs(scratch_dir)
 
         if is_windows():
-            subst = create_subst_path(scratch_dir)
-            print(
-                "Mapping scratch dir %s -> %s" % (scratch_dir, subst), file=sys.stderr
-            )
+            subst = map_subst_path(scratch_dir)
             scratch_dir = subst
     else:
         if not os.path.exists(scratch_dir):