From 9d99368ce8a2f87bff5cab67b2cbf47dd72d89e8 Mon Sep 17 00:00:00 2001 From: Jeremy Maitin-Shepard Date: Thu, 12 Aug 2021 14:56:03 -0700 Subject: [PATCH] Refactor third-party dependency configuration and quiet warnings PiperOrigin-RevId: 390467159 Change-Id: Iaf4f2cbf865191f2f762c2f91eb862deef5a0863 --- .bazelrc | 7 ++ external.bzl | 73 +------------------ tensorstore/internal/json_tuple.h | 14 ++-- third_party/bazel_skylib/workspace.bzl | 24 ++++++ .../build_bazel_rules_nodejs/workspace.bzl | 24 ++++++ third_party/com_google_absl/workspace.bzl | 6 +- .../com_google_boringssl/workspace.bzl | 6 +- third_party/jpeg/bundled.BUILD.bazel | 7 -- third_party/third_party.bzl | 48 ++++++++++++ third_party/update_third_party_bzl.py | 42 +++++++++++ third_party/update_versions.py | 1 + 11 files changed, 163 insertions(+), 89 deletions(-) create mode 100644 third_party/bazel_skylib/workspace.bzl create mode 100644 third_party/build_bazel_rules_nodejs/workspace.bzl create mode 100644 third_party/third_party.bzl create mode 100755 third_party/update_third_party_bzl.py mode change 100644 => 100755 third_party/update_versions.py diff --git a/.bazelrc b/.bazelrc index 0d3224921..18b8d185a 100644 --- a/.bazelrc +++ b/.bazelrc @@ -24,6 +24,13 @@ build:linux --copt='-Wno-sign-compare' build:macos --copt='-Wno-sign-compare' build:freebsd --copt='-Wno-sign-compare' +# disable spurious warnings from gcc +build:linux --copt='-Wno-unused-but-set-parameter' +build:linux --copt='-Wno-maybe-uninitialized' + +# disable warnings from clang about unknown warnings +build:linux --copt='-Wno-unknown-warning-option' + # Windows 10 RS1 required for rename/delete with POSIX semantics build:windows --copt=-D_WIN32_WINNT=0x0A000002 diff --git a/external.bzl b/external.bzl index 388543aee..4f26496f4 100644 --- a/external.bzl +++ b/external.bzl @@ -12,76 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") - -# Third-party repositories -load("//third_party:jpeg/workspace.bzl", repo_jpeg = "repo") -load("//third_party:com_google_absl/workspace.bzl", repo_com_google_absl = "repo") -load("//third_party:com_google_re2/workspace.bzl", repo_com_google_re2 = "repo") -load("//third_party:com_google_googletest/workspace.bzl", repo_com_google_googletest = "repo") -load("//third_party:com_google_brotli/workspace.bzl", repo_com_google_brotli = "repo") -load("//third_party:com_google_benchmark/workspace.bzl", repo_com_google_benchmark = "repo") -load("//third_party:net_zlib/workspace.bzl", repo_net_zlib = "repo") -load("//third_party:org_sourceware_bzip2/workspace.bzl", repo_org_sourceware_bzip2 = "repo") -load("//third_party:com_google_snappy/workspace.bzl", repo_com_google_snappy = "repo") -load("//third_party:com_github_nlohmann_json/workspace.bzl", repo_com_github_nlohmann_json = "repo") -load("//third_party:net_sourceforge_half/workspace.bzl", repo_net_sourceforge_half = "repo") -load("//third_party:org_nghttp2/workspace.bzl", repo_org_nghttp2 = "repo") -load("//third_party:se_curl/workspace.bzl", repo_se_curl = "repo") -load("//third_party:com_google_boringssl/workspace.bzl", repo_com_google_boringssl = "repo") -load("//third_party:org_lz4/workspace.bzl", repo_org_lz4 = "repo") -load("//third_party:com_facebook_zstd/workspace.bzl", repo_com_facebook_zstd = "repo") -load("//third_party:org_blosc_cblosc/workspace.bzl", repo_org_blosc_cblosc = "repo") -load("//third_party:nasm/workspace.bzl", repo_nasm = "repo") -load("//third_party:org_tukaani_xz/workspace.bzl", repo_org_tukaani_xz = "repo") +load("//third_party:third_party.bzl", "third_party_dependencies") load("//third_party:python/python_configure.bzl", "python_configure") -load("//third_party:com_github_pybind_pybind11/workspace.bzl", repo_com_github_pybind_pybind11 = "repo") -load("//third_party:pypa/workspace.bzl", repo_pypa = "repo") - -def _bazel_dependencies(): - maybe( - http_archive, - name = "bazel_skylib", - urls = ["https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz"], - sha256 = "1c531376ac7e5a180e0237938a2536de0c54d93f5c278634818e0efc952dd56c", - ) - - # Needed to build documentation - maybe( - http_archive, - name = "build_bazel_rules_nodejs", - sha256 = "f533eeefc8fe1ddfe93652ec50f82373d0c431f7faabd5e6323f6903195ef227", - urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/3.3.0/rules_nodejs-3.3.0.tar.gz"], - ) - -def _python_dependencies(): - python_configure(name = "local_config_python") - repo_com_github_pybind_pybind11() - repo_pypa() - -def _cc_dependencies(): - repo_com_google_absl() - repo_com_google_googletest() - repo_com_google_re2() - repo_com_google_brotli() - repo_com_google_benchmark() - repo_nasm() - repo_jpeg() - repo_com_google_boringssl() - repo_net_zlib() - repo_org_sourceware_bzip2() - repo_com_google_snappy() - repo_com_github_nlohmann_json() - repo_net_sourceforge_half() - repo_org_nghttp2() - repo_se_curl() - repo_org_lz4() - repo_com_facebook_zstd() - repo_org_blosc_cblosc() - repo_org_tukaani_xz() def tensorstore_dependencies(): - _bazel_dependencies() - _cc_dependencies() - _python_dependencies() + python_configure(name = "local_config_python") + third_party_dependencies() diff --git a/tensorstore/internal/json_tuple.h b/tensorstore/internal/json_tuple.h index 256e835c2..5d3ee3d38 100644 --- a/tensorstore/internal/json_tuple.h +++ b/tensorstore/internal/json_tuple.h @@ -158,12 +158,14 @@ constexpr auto HeterogeneousArray(ElementBinder... element_binder) { is_loading, j, sizeof...(ElementBinder))); absl::Status status; size_t i = 0; - (((status = element_binder(is_loading, options, obj, &(*array_ptr)[i++])) - .ok() || - ((status = internal_json::MaybeAnnotateArrayElementError(status, i - 1, - is_loading)), - false)) && - ...); + [[maybe_unused]] bool ok = + (((status = + element_binder(is_loading, options, obj, &(*array_ptr)[i++])) + .ok() || + ((status = internal_json::MaybeAnnotateArrayElementError( + status, i - 1, is_loading)), + false)) && + ...); return status; }; } diff --git a/third_party/bazel_skylib/workspace.bzl b/third_party/bazel_skylib/workspace.bzl new file mode 100644 index 000000000..30e88dc48 --- /dev/null +++ b/third_party/bazel_skylib/workspace.bzl @@ -0,0 +1,24 @@ +# Copyright 2021 The TensorStore Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") + +def repo(): + maybe( + http_archive, + name = "bazel_skylib", + urls = ["https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz"], + sha256 = "1c531376ac7e5a180e0237938a2536de0c54d93f5c278634818e0efc952dd56c", + ) diff --git a/third_party/build_bazel_rules_nodejs/workspace.bzl b/third_party/build_bazel_rules_nodejs/workspace.bzl new file mode 100644 index 000000000..5f4b927d3 --- /dev/null +++ b/third_party/build_bazel_rules_nodejs/workspace.bzl @@ -0,0 +1,24 @@ +# Copyright 2021 The TensorStore Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") + +def repo(): + maybe( + http_archive, + name = "build_bazel_rules_nodejs", + sha256 = "f533eeefc8fe1ddfe93652ec50f82373d0c431f7faabd5e6323f6903195ef227", + urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/3.3.0/rules_nodejs-3.3.0.tar.gz"], + ) diff --git a/third_party/com_google_absl/workspace.bzl b/third_party/com_google_absl/workspace.bzl index b013b71c6..79b465340 100644 --- a/third_party/com_google_absl/workspace.bzl +++ b/third_party/com_google_absl/workspace.bzl @@ -22,9 +22,9 @@ def repo(): maybe( third_party_http_archive, name = "com_google_absl", - sha256 = "b54c84452fc0786c1763eeb1a8d999272f0ecc9ec4ee74eb6cfe0a61adb6aa1c", - strip_prefix = "abseil-cpp-62ce712ecc887f669610a93efe18abecf70b47a0", + sha256 = "cda0ac8640183d1f1de3632354dd6a4dcfe2548d1aa66f3f099fd6b626c77f54", + strip_prefix = "abseil-cpp-b699707f0bfeae034e36cdfd909b66b0fcab696c", urls = [ - "https://github.com/abseil/abseil-cpp/archive/62ce712ecc887f669610a93efe18abecf70b47a0.tar.gz", # master(2021-01-09) + "https://github.com/abseil/abseil-cpp/archive/b699707f0bfeae034e36cdfd909b66b0fcab696c.tar.gz", # master(2021-08-12) ], ) diff --git a/third_party/com_google_boringssl/workspace.bzl b/third_party/com_google_boringssl/workspace.bzl index d1cdd1788..8bc2998f1 100644 --- a/third_party/com_google_boringssl/workspace.bzl +++ b/third_party/com_google_boringssl/workspace.bzl @@ -23,10 +23,10 @@ def repo(): third_party_http_archive, name = "com_google_boringssl", urls = [ - "https://github.com/google/boringssl/archive/bdbe37905216bea8dd4d0fdee93f6ee415d3aa15.tar.gz", # master-with-bazel(2021-01-09) + "https://github.com/google/boringssl/archive/fc44652a42b396e1645d5e72aba053349992136a.tar.gz", # master-with-bazel(2021-08-12) ], - sha256 = "ce183cb587c0a0f5982e441dff91cb5456d4c85cfa3fb12816e7a93f20645e51", - strip_prefix = "boringssl-bdbe37905216bea8dd4d0fdee93f6ee415d3aa15", + sha256 = "6f640262999cd1fb33cf705922e453e835d2d20f3f06fe0d77f6426c19257308", + strip_prefix = "boringssl-fc44652a42b396e1645d5e72aba053349992136a", system_build_file = Label("//third_party:com_google_boringssl/system.BUILD.bazel"), patches = [ # boringssl sets -Werror by default. That makes the build fragile diff --git a/third_party/jpeg/bundled.BUILD.bazel b/third_party/jpeg/bundled.BUILD.bazel index 84b74b541..7c11ee381 100644 --- a/third_party/jpeg/bundled.BUILD.bazel +++ b/third_party/jpeg/bundled.BUILD.bazel @@ -5,20 +5,13 @@ licenses(["notice"]) exports_files(["LICENSE.md"]) WIN_COPTS = [ - "/Ox", "-DWITH_SIMD", "-wd4996", ] libjpegturbo_copts = select({ - ":android": [ - "-O2", - "-fPIE", - "-w", - ], ":windows": WIN_COPTS, "//conditions:default": [ - "-O3", "-w", ], }) + select({ diff --git a/third_party/third_party.bzl b/third_party/third_party.bzl new file mode 100644 index 000000000..3408c4e7f --- /dev/null +++ b/third_party/third_party.bzl @@ -0,0 +1,48 @@ +load("//third_party:bazel_skylib/workspace.bzl", repo_bazel_skylib = "repo") +load("//third_party:build_bazel_rules_nodejs/workspace.bzl", repo_build_bazel_rules_nodejs = "repo") +load("//third_party:com_facebook_zstd/workspace.bzl", repo_com_facebook_zstd = "repo") +load("//third_party:com_github_nlohmann_json/workspace.bzl", repo_com_github_nlohmann_json = "repo") +load("//third_party:com_github_pybind_pybind11/workspace.bzl", repo_com_github_pybind_pybind11 = "repo") +load("//third_party:com_google_absl/workspace.bzl", repo_com_google_absl = "repo") +load("//third_party:com_google_benchmark/workspace.bzl", repo_com_google_benchmark = "repo") +load("//third_party:com_google_boringssl/workspace.bzl", repo_com_google_boringssl = "repo") +load("//third_party:com_google_brotli/workspace.bzl", repo_com_google_brotli = "repo") +load("//third_party:com_google_googletest/workspace.bzl", repo_com_google_googletest = "repo") +load("//third_party:com_google_re2/workspace.bzl", repo_com_google_re2 = "repo") +load("//third_party:com_google_snappy/workspace.bzl", repo_com_google_snappy = "repo") +load("//third_party:jpeg/workspace.bzl", repo_jpeg = "repo") +load("//third_party:nasm/workspace.bzl", repo_nasm = "repo") +load("//third_party:net_sourceforge_half/workspace.bzl", repo_net_sourceforge_half = "repo") +load("//third_party:net_zlib/workspace.bzl", repo_net_zlib = "repo") +load("//third_party:org_blosc_cblosc/workspace.bzl", repo_org_blosc_cblosc = "repo") +load("//third_party:org_lz4/workspace.bzl", repo_org_lz4 = "repo") +load("//third_party:org_nghttp2/workspace.bzl", repo_org_nghttp2 = "repo") +load("//third_party:org_sourceware_bzip2/workspace.bzl", repo_org_sourceware_bzip2 = "repo") +load("//third_party:org_tukaani_xz/workspace.bzl", repo_org_tukaani_xz = "repo") +load("//third_party:pypa/workspace.bzl", repo_pypa = "repo") +load("//third_party:se_curl/workspace.bzl", repo_se_curl = "repo") + +def third_party_dependencies(): + repo_bazel_skylib() + repo_build_bazel_rules_nodejs() + repo_com_facebook_zstd() + repo_com_github_nlohmann_json() + repo_com_github_pybind_pybind11() + repo_com_google_absl() + repo_com_google_benchmark() + repo_com_google_boringssl() + repo_com_google_brotli() + repo_com_google_googletest() + repo_com_google_re2() + repo_com_google_snappy() + repo_jpeg() + repo_nasm() + repo_net_sourceforge_half() + repo_net_zlib() + repo_org_blosc_cblosc() + repo_org_lz4() + repo_org_nghttp2() + repo_org_sourceware_bzip2() + repo_org_tukaani_xz() + repo_pypa() + repo_se_curl() diff --git a/third_party/update_third_party_bzl.py b/third_party/update_third_party_bzl.py new file mode 100755 index 000000000..6e98a0cb6 --- /dev/null +++ b/third_party/update_third_party_bzl.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +# Copyright 2021 The TensorStore Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Updates third_party.bzl file to import all third-party workspace files.""" + +import os + + +def main(): + script_dir = os.path.dirname(__file__) + + deps = [] + for dep in os.listdir(script_dir): + if not os.path.exists(os.path.join(script_dir, dep, 'workspace.bzl')): + continue + deps.append(dep) + + deps.sort() + + with open(os.path.join(script_dir, 'third_party.bzl'), 'w') as f: + for dep in deps: + f.write( + f'load("//third_party:{dep}/workspace.bzl", repo_{dep} = "repo")\n') + f.write('\n') + f.write('def third_party_dependencies():\n') + for dep in deps: + f.write(f' repo_{dep}()\n') + + +if __name__ == '__main__': + main() diff --git a/third_party/update_versions.py b/third_party/update_versions.py old mode 100644 new mode 100755 index 2dc5f2f74..dcaa866b4 --- a/third_party/update_versions.py +++ b/third_party/update_versions.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # Copyright 2021 The TensorStore Authors # # Licensed under the Apache License, Version 2.0 (the "License");