Merge branch 'main' into move_lz4_from_arrow_to_velox_parquet

.github/workflows/benchmark.yml

@@ -58,7 +58,7 @@ jobs:
           key: ccache-benchmark-${{ github.sha }}
           restore-keys: |
             ccache-benchmark-
-      
+
       - name: "Checkout Repo"
         if: ${{ github.event_name == 'pull_request' }}
         uses: actions/checkout@v3

build/deps/github_hashes/facebook/folly-rev.txt

@@ -1 +1 @@
-Subproject commit 97acbe4a62cd7ae620773affc45d9a6956e33cf2
+Subproject commit 34f110b7e9fafeb319fd65167f19e95c88e1f792

build/fbcode_builder/getdeps.py

@@ -795,6 +795,12 @@ def setup_project_cmd_parser(self, parser):
             action="store_true",
             default=False,
         )
+        parser.add_argument(
+            "--free-up-disk",
+            help="Remove unused tools and clean up intermediate files if possible to maximise space for the build",
+            action="store_true",
+            default=False,
+        )
 
 
 @cmd("fixup-dyn-deps", "Adjusts dynamic dependencies for packaging purposes")
@@ -933,7 +939,8 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
         # We do this by looking at the builder type in the manifest file
         # rather than creating a builder and checking its type because we
         # don't know enough to create the full builder instance here.
-        if manifest.get("build", "builder", ctx=manifest_ctx) == "nop":
+        builder_name = manifest.get("build", "builder", ctx=manifest_ctx)
+        if builder_name == "nop":
             return None
 
         # We want to be sure that we're running things with python 3
@@ -1015,6 +1022,19 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
 
             out.write("    - uses: actions/checkout@v2\n")
 
+            if build_opts.free_up_disk:
+                free_up_disk = "--free-up-disk "
+                if not build_opts.is_windows():
+                    out.write("    - name: Show disk space at start\n")
+                    out.write("      run: df -h\n")
+                    # remove the unused github supplied android dev tools
+                    out.write("    - name: Free up disk space\n")
+                    out.write("      run: sudo rm -rf /usr/local/lib/android\n")
+                    out.write("    - name: Show disk space after freeing up\n")
+                    out.write("      run: df -h\n")
+            else:
+                free_up_disk = ""
+
             allow_sys_arg = ""
             if (
                 build_opts.allow_system_packages
@@ -1039,18 +1059,30 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
             main_repo_url = manifest.get_repo_url(manifest_ctx)
             has_same_repo_dep = False
 
+            # Add the rust dep which doesn't have a manifest
             for m in projects:
-                if m != manifest:
-                    if m.name == "rust":
-                        out.write("    - name: Install Rust Stable\n")
-                        out.write("      uses: dtolnay/rust-toolchain@stable\n")
-                    else:
-                        ctx = loader.ctx_gen.get_context(m.name)
-                        if m.get_repo_url(ctx) != main_repo_url:
-                            out.write("    - name: Fetch %s\n" % m.name)
-                            out.write(
-                                f"      run: {getdepscmd}{allow_sys_arg} fetch --no-tests {m.name}\n"
-                            )
+                if m == manifest:
+                    continue
+                mbuilder_name = m.get("build", "builder", ctx=manifest_ctx)
+                if (
+                    m.name == "rust"
+                    or builder_name == "cargo"
+                    or mbuilder_name == "cargo"
+                ):
+                    out.write("    - name: Install Rust Stable\n")
+                    out.write("      uses: dtolnay/rust-toolchain@stable\n")
+                    break
+
+            # Normal deps that have manifests
+            for m in projects:
+                if m == manifest or m.name == "rust":
+                    continue
+                ctx = loader.ctx_gen.get_context(m.name)
+                if m.get_repo_url(ctx) != main_repo_url:
+                    out.write("    - name: Fetch %s\n" % m.name)
+                    out.write(
+                        f"      run: {getdepscmd}{allow_sys_arg} fetch --no-tests {m.name}\n"
+                    )
 
             for m in projects:
                 if m != manifest:
@@ -1065,7 +1097,7 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
                             has_same_repo_dep = True
                         out.write("    - name: Build %s\n" % m.name)
                         out.write(
-                            f"      run: {getdepscmd}{allow_sys_arg} build {src_dir_arg}--no-tests {m.name}\n"
+                            f"      run: {getdepscmd}{allow_sys_arg} build {src_dir_arg}{free_up_disk}--no-tests {m.name}\n"
                         )
 
             out.write("    - name: Build %s\n" % manifest.name)
@@ -1081,8 +1113,12 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
             if has_same_repo_dep:
                 no_deps_arg = "--no-deps "
 
+            no_tests_arg = ""
+            if not args.enable_tests:
+                no_tests_arg = "--no-tests "
+
             out.write(
-                f"      run: {getdepscmd}{allow_sys_arg} build {no_deps_arg}--src-dir=. {manifest.name} {project_prefix}\n"
+                f"      run: {getdepscmd}{allow_sys_arg} build {no_tests_arg}{no_deps_arg}--src-dir=. {manifest.name} {project_prefix}\n"
             )
 
             out.write("    - name: Copy artifacts\n")
@@ -1106,11 +1142,18 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
             out.write("        name: %s\n" % manifest.name)
             out.write("        path: _artifacts\n")
 
-            if manifest.get("github.actions", "run_tests", ctx=manifest_ctx) != "off":
+            if (
+                args.enable_tests
+                and manifest.get("github.actions", "run_tests", ctx=manifest_ctx)
+                != "off"
+            ):
                 out.write("    - name: Test %s\n" % manifest.name)
                 out.write(
                     f"      run: {getdepscmd}{allow_sys_arg} test --src-dir=. {manifest.name} {project_prefix}\n"
                 )
+            if build_opts.free_up_disk and not build_opts.is_windows():
+                out.write("    - name: Show disk space at end\n")
+                out.write("      run: df -h\n")
 
     def setup_project_cmd_parser(self, parser):
         parser.add_argument(
@@ -1155,6 +1198,12 @@ def setup_project_cmd_parser(self, parser):
             help="add a prefix to all job names",
             default=None,
         )
+        parser.add_argument(
+            "--free-up-disk",
+            help="Remove unused tools and clean up intermediate files if possible to maximise space for the build",
+            action="store_true",
+            default=False,
+        )
 
 
 def get_arg_var_name(args):

build/fbcode_builder/getdeps/builder.py

@@ -136,6 +136,16 @@ def build(self, install_dirs, reconfigure: bool) -> None:
         self._prepare(install_dirs=install_dirs, reconfigure=reconfigure)
         self._build(install_dirs=install_dirs, reconfigure=reconfigure)
 
+        if self.build_opts.free_up_disk:
+            # don't clean --src-dir=. case as user may want to build again or run tests on the build
+            if self.src_dir.startswith(self.build_opts.scratch_dir) and os.path.isdir(
+                self.build_dir
+            ):
+                if os.path.islink(self.build_dir):
+                    os.remove(self.build_dir)
+                else:
+                    shutil.rmtree(self.build_dir)
+
         # On Windows, emit a wrapper script that can be used to run build artifacts
         # directly from the build directory, without installing them.  On Windows $PATH
         # needs to be updated to include all of the directories containing the runtime

build/fbcode_builder/getdeps/buildopts.py

@@ -51,6 +51,7 @@ def __init__(
         lfs_path=None,
         shared_libs: bool = False,
         facebook_internal=None,
+        free_up_disk: bool = False,
     ) -> None:
         """fbcode_builder_dir - the path to either the in-fbsource fbcode_builder dir,
                              or for shipit-transformed repos, the build dir that
@@ -65,6 +66,7 @@ def __init__(
         use_shipit - use real shipit instead of the simple shipit transformer
         vcvars_path - Path to external VS toolchain's vsvarsall.bat
         shared_libs - whether to build shared libraries
+        free_up_disk - take extra actions to save runner disk space
         """
 
         if not install_dir:
@@ -103,6 +105,7 @@ def __init__(
         self.allow_system_packages = allow_system_packages
         self.lfs_path = lfs_path
         self.shared_libs = shared_libs
+        self.free_up_disk = free_up_disk
 
         lib_path = None
         if self.is_darwin():
@@ -602,6 +605,7 @@ def setup_build_options(args, host_type=None) -> BuildOptions:
             "allow_system_packages",
             "lfs_path",
             "shared_libs",
+            "free_up_disk",
         }
     }
 

build/fbcode_builder/getdeps/cargo.py

@@ -64,7 +64,10 @@ def manifest_dir(self, manifest):
 
     def recreate_dir(self, src, dst) -> None:
         if os.path.isdir(dst):
-            shutil.rmtree(dst)
+            if os.path.islink(dst):
+                os.remove(dst)
+            else:
+                shutil.rmtree(dst)
         shutil.copytree(src, dst)
 
     def cargo_config_file(self):
@@ -80,10 +83,18 @@ def _create_cargo_config(self):
         if not os.path.isdir(cargo_config_dir):
             os.mkdir(cargo_config_dir)
 
-        print(f"Writing cargo config for {self.manifest.name} to {cargo_config_file}")
-        with open(cargo_config_file, "w+") as f:
-            f.write(
-                """\
+        dep_to_git = self._resolve_dep_to_git()
+
+        if os.path.isfile(cargo_config_file):
+            with open(cargo_config_file, "r") as f:
+                print(f"Reading {cargo_config_file}")
+                cargo_content = f.read()
+        else:
+            cargo_content = ""
+
+        new_content = cargo_content
+        if "# Generated by getdeps.py" not in cargo_content:
+            new_content += """\
 # Generated by getdeps.py
 [build]
 target-dir = '''{}'''
@@ -92,24 +103,25 @@ def _create_cargo_config(self):
 debug = false
 incremental = false
 """.format(
-                    self.build_dir.replace("\\", "\\\\")
-                )
+                self.build_dir.replace("\\", "\\\\")
             )
 
         # Point to vendored sources from getdeps manifests
-        dep_to_git = self._resolve_dep_to_git()
         for _dep, git_conf in dep_to_git.items():
             if "cargo_vendored_sources" in git_conf:
-                with open(cargo_config_file, "a") as f:
-                    vendored_dir = git_conf["cargo_vendored_sources"].replace(
-                        "\\", "\\\\"
-                    )
-                    f.write(
-                        f"""
-[source."{git_conf["repo_url"]}"]
-directory = "{vendored_dir}"
-"""
-                    )
+                vendored_dir = git_conf["cargo_vendored_sources"].replace("\\", "\\\\")
+                override = (
+                    f'[source."{git_conf["repo_url"]}"]\ndirectory = "{vendored_dir}"\n'
+                )
+                if override not in cargo_content:
+                    new_content += override
+
+        if new_content != cargo_content:
+            with open(cargo_config_file, "w") as f:
+                print(
+                    f"Writing cargo config for {self.manifest.name} to {cargo_config_file}"
+                )
+                f.write(new_content)
 
         if self.build_opts.fbsource_dir:
             # Point to vendored crates.io if possible
@@ -198,45 +210,59 @@ def _patchup_workspace(self, dep_to_git) -> None:
         producing bad results.
         """
         workspace_dir = self.workspace_dir()
-        config = self._resolve_config(dep_to_git)
-        if config:
+        git_url_to_crates_and_paths = self._resolve_config(dep_to_git)
+        if git_url_to_crates_and_paths:
             patch_cargo = os.path.join(workspace_dir, "Cargo.toml")
-            print(f"writing patch to {patch_cargo}")
-            with open(patch_cargo, "r+") as f:
-                manifest_content = f.read()
-                if "[package]" not in manifest_content:
-                    # A fake manifest has to be crated to change the virtual
-                    # manifest into a non-virtual. The virtual manifests are limited
-                    # in many ways and the inability to define patches on them is
-                    # one. Check https://github.com/rust-lang/cargo/issues/4934 to
-                    # see if it is resolved.
-                    null_file = "/dev/null"
-                    if self.build_opts.is_windows():
-                        null_file = "nul"
-                    f.write(
-                        f"""
+            if os.path.isfile(patch_cargo):
+                with open(patch_cargo, "r") as f:
+                    manifest_content = f.read()
+            else:
+                manifest_content = ""
+
+            new_content = manifest_content
+            if "[package]" not in manifest_content:
+                # A fake manifest has to be crated to change the virtual
+                # manifest into a non-virtual. The virtual manifests are limited
+                # in many ways and the inability to define patches on them is
+                # one. Check https://github.com/rust-lang/cargo/issues/4934 to
+                # see if it is resolved.
+                null_file = "/dev/null"
+                if self.build_opts.is_windows():
+                    null_file = "nul"
+                new_content += f"""
 [package]
 name = "fake_manifest_of_{self.manifest.name}"
 version = "0.0.0"
 
 [lib]
 path = "{null_file}"
 """
+            config = []
+            for git_url, crates_to_patch_path in git_url_to_crates_and_paths.items():
+                crates_patches = [
+                    '{} = {{ path = "{}" }}'.format(
+                        crate,
+                        crates_to_patch_path[crate].replace("\\", "\\\\"),
                     )
-                else:
-                    f.write("\n")
-                f.write(config)
-
-    def _resolve_config(self, dep_to_git) -> str:
+                    for crate in sorted(crates_to_patch_path.keys())
+                ]
+                patch_key = f'[patch."{git_url}"]'
+                if patch_key not in manifest_content:
+                    config.append(f"\n{patch_key}\n" + "\n".join(crates_patches))
+            new_content += "\n".join(config)
+            if new_content != manifest_content:
+                with open(patch_cargo, "w") as f:
+                    print(f"writing patch to {patch_cargo}")
+                    f.write(new_content)
+
+    def _resolve_config(self, dep_to_git) -> typing.Dict[str, typing.Dict[str, str]]:
         """
         Returns a configuration to be put inside root Cargo.toml file which
         patches the dependencies git code with local getdeps versions.
         See https://doc.rust-lang.org/cargo/reference/manifest.html#the-patch-section
         """
         dep_to_crates = self._resolve_dep_to_crates(self.build_source_dir(), dep_to_git)
 
-        config = []
-
         git_url_to_crates_and_paths = {}
         for dep_name in sorted(dep_to_git.keys()):
             git_conf = dep_to_git[dep_name]
@@ -257,17 +283,7 @@ def _resolve_config(self, dep_to_git) -> str:
                 if crates_to_patch_path:
                     git_url_to_crates_and_paths[git_url] = crates_to_patch_path
 
-        for git_url, crates_to_patch_path in git_url_to_crates_and_paths.items():
-            crates_patches = [
-                '{} = {{ path = "{}" }}'.format(
-                    crate,
-                    crates_to_patch_path[crate].replace("\\", "\\\\"),
-                )
-                for crate in sorted(crates_to_patch_path.keys())
-            ]
-            config.append(f'\n[patch."{git_url}"]\n' + "\n".join(crates_patches))
-
-        return "\n".join(config)
+        return git_url_to_crates_and_paths
 
     def _resolve_dep_to_git(self):
         """
@@ -382,7 +398,7 @@ def _resolve_dep_to_crates(self, build_source_dir, dep_to_git):
                                 print(
                                     f"Patch {self.manifest.name} uses {dep_name} crate {crates}"
                                 )
-                                existing_crates.insert(c)
+                                existing_crates.add(c)
                         dep_to_crates.setdefault(name, set()).update(existing_crates)
         return dep_to_crates