Upgrade protobuf dependency to v3.8.0 to support TensorFlow 2.1.0.

PiperOrigin-RevId: 340955598

.bazelrc

@@ -53,5 +53,11 @@ test --test_output=errors
 
 # Until https://github.com/bazelbuild/bazel/issues/4815 is fixed, we need
 # to force the use of our py_runtime.
-build --python_top=//:nucleus_python_runtime
-test --python_top=//:nucleus_python_runtime
+# Versions of bazel used by TensorFlow 2.1+ disable the use of --python_top by
+# default with the below error message, necessitating the additional flag.
+#  "ERROR: `--python_top` is disabled by `--incompatible_use_python_toolchains`.
+#    Instead of configuring the Python runtime directly, register a Python
+#    toolchain. See https://github.com/bazelbuild/bazel/issues/7899. You can
+#    temporarily revert to the legacy flag-based way of specifying toolchains by
+#    setting `--incompatible_use_python_toolchains=false`."
+build --python_top=//:nucleus_python_runtime --incompatible_use_python_toolchains=false --incompatible_depset_is_not_iterable=false

WORKSPACE

@@ -26,14 +26,14 @@ http_archive(
     ],
 )
 
-# Import tensorflow.  Note path.
+# Import tensorflow. Note path. (This is generated as part of ./install.sh).
 local_repository(
     name = "org_tensorflow",
     path = "../tensorflow",
 )
 
 # Required boilerplate for tf_workspace(), apparently.
-# This is copied from https://github.com/tensorflow/tensorflow/blob/v2.0.0/WORKSPACE.
+# This is copied from https://github.com/tensorflow/tensorflow/blob/v2.3.0/WORKSPACE.
 http_archive(
     name = "io_bazel_rules_closure",
     sha256 = "5b00383d08dd71f28503736db0500b6fb4dda47489ff5fc6bed42557c07c6ba9",
@@ -46,7 +46,7 @@ http_archive(
 
 # This needs to be in sync with the version of protobuf used by TensorFlow,
 # which is currently defined in @tensorflow/tensorflow/workspace.bzl.
-# We supply our # own BUILD file, though, so we can prevent ODR violations by
+# We supply our own BUILD file, though, so we can prevent ODR violations by
 # putting all of Nucleus's C++ binary dependencies into a single library.
 # That BUILD file must be kept in sync with the version of protobuf used.
 http_archive(
@@ -64,37 +64,11 @@ http_archive(
 # bazel_skylib is now a required dependency of com_google_protobuf.
 http_archive(
     name = "bazel_skylib",
-    sha256 = "bbccf674aa441c266df9894182d80de104cabd19be98be002f6d478aaa31574d",
-    strip_prefix = "bazel-skylib-2169ae1c374aab4a09aa90e65efe1a3aad4e279b",
-    urls = ["https://github.com/bazelbuild/bazel-skylib/archive/2169ae1c374aab4a09aa90e65efe1a3aad4e279b.tar.gz"],
-)
-
-# rules_cc is now a required dependency of com_google_protobuf.
-http_archive(
-    name = "rules_cc",
-    strip_prefix = "rules_cc-master",
-    urls = ["https://github.com/bazelbuild/rules_cc/archive/master.zip"],
-)
-
-# rules_java is now a required dependency of com_google_protobuf.
-http_archive(
-    name = "rules_java",
-    strip_prefix = "rules_java-master",
-    urls = ["https://github.com/bazelbuild/rules_java/archive/master.zip"],
-)
-
-# rules_proto is now a required dependency of com_google_protobuf.
-http_archive(
-    name = "rules_proto",
-    strip_prefix = "rules_proto-master",
-    urls = ["https://github.com/bazelbuild/rules_proto/archive/master.zip"],
-)
-
-# rules_python is now a required dependency of com_google_protobuf.
-http_archive(
-    name = "rules_python",
-    strip_prefix = "rules_python-master",
-    urls = ["https://github.com/bazelbuild/rules_python/archive/master.zip"],
+    sha256 = "1dde365491125a3db70731e25658dfdd3bc5dbdfd11b840b3e987ecf043c7ca0",
+    urls = [
+        "https://storage.googleapis.com/mirror.tensorflow.org/github.com/bazelbuild/bazel-skylib/releases/download/0.9.0/bazel_skylib-0.9.0.tar.gz",
+        "https://github.com/bazelbuild/bazel-skylib/releases/download/0.9.0/bazel_skylib-0.9.0.tar.gz",
+    ],
 )
 
 # six is now a required dependency of com_google_protobuf.

install.sh

@@ -30,8 +30,8 @@
 # Global setting for nucleus builds
 # ------------------------------------------------------------------------------
 
-NUCLEUS_BAZEL_VERSION="0.26.1"
-NUCLEUS_TENSORFLOW_VERSION="2.0.0"
+NUCLEUS_BAZEL_VERSION="0.29.1"
+NUCLEUS_TENSORFLOW_VERSION="2.1.0"
 
 function note_build_stage {
   echo "========== [$(date)] Stage '${1}' starting"
@@ -70,7 +70,7 @@ python3 -m pip install --user contextlib2
 python3 -m pip install --user 'sortedcontainers==2.1.0'
 python3 -m pip install --user 'intervaltree==3.0.2'
 python3 -m pip install --user 'mock>=2.0.0'
-python3 -m pip install --user 'numpy==1.14'
+python3 -m pip install --user 'numpy==1.18.5'
 python3 -m pip install --user 'six>=1.11.0'
 python3 -m pip install --user 'Pillow>=5.4.1'
 python3 -m pip install --user 'ipython>=7.9.0'
@@ -83,9 +83,8 @@ python3 -m pip install --user 'ipython>=7.9.0'
 python3 -m pip install --user 'setuptools==49.6.0'
 
 # These are required to build TensorFlow from source.
-python3 -m pip install --user 'keras_applications==1.0.6' --no-deps
-python3 -m pip install --user 'keras_preprocessing==1.0.5' --no-deps
-python3 -m pip install --user 'h5py==2.8.0'
+python3 -m pip install --user 'keras_preprocessing==1.1.2' --no-deps
+python3 -m pip install --user 'h5py==2.10.0'
 python3 -m pip install --user enum34
 python3 -m pip install --user 'protobuf==3.8.0'
 
@@ -171,6 +170,13 @@ export PYTHON_BIN_PATH=`which python3`
  echo | ./configure
  )
 
+# We use TensorFlow's .bazelrc as part of Nucleus's. In it they use a java
+# toolchain flag based on a definition in a BUILD file in the TF repo. This
+# causes that flag's usage to raise build errors when building Nucleus unless we
+# also include that BUILD file.
+mkdir -p third_party/toolchains/java
+cp ../tensorflow/third_party/toolchains/java/BUILD third_party/toolchains/java/
+
 echo "Done installing prereqs at $(date)!"
 
 if [[ "$#" -eq 0 ]] || [[ "$1" != "--prereqs_only" ]]; then
@@ -179,10 +185,17 @@ if [[ "$#" -eq 0 ]] || [[ "$1" != "--prereqs_only" ]]; then
   note_build_stage "Building Nucleus"
 
   COPT_FLAGS="--copt=-msse4.1 --copt=-msse4.2 --copt=-mavx --copt=-O3"
-  BAZEL_FLAGS="${COPT_FLAGS}"
+  # N.B. The --experimental_build_setting_api had to be added on protobuf
+  # upgrade to 3.9.2 to avoid error in bazel_skylib:
+  #   "parameter 'build_setting' is experimental and thus unavailable with the
+  #    current flags. It may be enabled by setting
+  #    --experimental_build_setting_api"
+  # Presumably it won't be needed at some later point when bazel_skylib is
+  # upgraded again.
+  BAZEL_FLAGS="${COPT_FLAGS} --experimental_build_setting_api"
   bazel build -c opt ${BAZEL_FLAGS} nucleus/...
 
-  bazel build :licenses_zip
+  bazel build ${BAZEL_FLAGS} :licenses_zip
 
 fi