Merge pull request #1 from thinker0/feature/Update-protobuf

Update protobuf-3.16.1
thinker0 · Mar 25, 2022 · 5a8746b · 5a8746b
2 parents 52ed112 + 4eabf69
commit 5a8746b
Show file tree

Hide file tree

Showing 9 changed files with 94 additions and 89 deletions.
diff --git a/WORKSPACE b/WORKSPACE
@@ -109,6 +109,7 @@ maven_install(
         "com.google.inject:guice:5.1.0",
         "com.google.inject.extensions:guice-assistedinject:5.1.0",
         "com.google.guava:guava:23.6-jre",
+        "com.google.protobuf:protobuf-java:3.16.1",
         "io.gsonfire:gson-fire:1.8.3",
         "org.apache.curator:curator-framework:" + curator_version,
         "org.apache.curator:curator-recipes:" + curator_version,
@@ -135,26 +136,26 @@ maven_install(
         "com.puppycrawl.tools:checkstyle:6.17",
         "com.googlecode.json-simple:json-simple:1.1",
         maven.artifact(
-            group = "org.apache.httpcomponents",
             artifact = "httpclient",
-            version = http_client_version,
             classifier = "tests",
+            group = "org.apache.httpcomponents",
             packaging = "test-jar",
+            version = http_client_version,
         ),
     ],
+    excluded_artifacts = [
+        "org.slf4j:slf4j-jdk14",
+        "org.slf4j:slf4j-log4j12",
+        "log4j:log4j",
+        "commons-logging:commons-logging",
+    ],
     fetch_sources = True,
     maven_install_json = "//:maven_install.json",
     repositories = [
         "https://jcenter.bintray.com",
         "https://maven.google.com",
         "https://repo1.maven.org/maven2",
     ],
-    excluded_artifacts = [
-        "org.slf4j:slf4j-jdk14",
-        "org.slf4j:slf4j-log4j12",
-        "log4j:log4j",
-        "commons-logging:commons-logging",
-    ],
     version_conflict_policy = "pinned",
 )
 
@@ -169,7 +170,7 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
 
 git_repository(
     name = "com_github_johnynek_bazel_jar_jar",
-    commit = "171f268569384c57c19474b04aebe574d85fde0d", # Latest commit SHA as at 2019/02/13
+    commit = "171f268569384c57c19474b04aebe574d85fde0d",  # Latest commit SHA as at 2019/02/13
     remote = "https://github.com/johnynek/bazel_jar_jar.git",
     shallow_since = "1594234634 -1000",
 )
@@ -178,6 +179,7 @@ load(
     "@com_github_johnynek_bazel_jar_jar//:jar_jar.bzl",
     "jar_jar_repositories",
 )
+
 jar_jar_repositories()
 
 http_archive(
@@ -395,6 +397,7 @@ load(
     "@io_bazel_rules_docker//repositories:repositories.bzl",
     container_repositories = "repositories",
 )
+
 container_repositories()
 
 load("@io_bazel_rules_docker//repositories:deps.bzl", container_deps = "deps")
@@ -421,22 +424,24 @@ container_pull(
 
 http_archive(
     name = "rules_pkg",
+    sha256 = "aeca78988341a2ee1ba097641056d168320ecc51372ef7ff8e64b139516a4937",
     urls = [
         "https://github.com/bazelbuild/rules_pkg/releases/download/0.2.6/rules_pkg-0.2.6.tar.gz",
         "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.2.6/rules_pkg-0.2.6.tar.gz",
     ],
-    sha256 = "aeca78988341a2ee1ba097641056d168320ecc51372ef7ff8e64b139516a4937",
 )
+
 load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies")
+
 rules_pkg_dependencies()
 
 # scala integration
 rules_scala_version = "358ab829626c6c2d34ec27f856485d3121e299c7"  # Jan 15 2020 - update this as needed
 
 http_archive(
     name = "io_bazel_rules_scala",
-    strip_prefix = "rules_scala-%s" % rules_scala_version,
     sha256 = "5abd638278de10ccccb0b4d614158f394278b828708ba990461334ecc01529a6",
+    strip_prefix = "rules_scala-%s" % rules_scala_version,
     type = "zip",
     url = "https://github.com/bazelbuild/rules_scala/archive/%s.zip" % rules_scala_version,
 )

diff --git a/heron/instance/src/cpp/boltimpl/bolt-output-collector-impl.cpp b/heron/instance/src/cpp/boltimpl/bolt-output-collector-impl.cpp
@@ -97,7 +97,7 @@ void BoltOutputCollectorImpl::ack(std::shared_ptr<api::tuple::Tuple> tup) {
       int64_t tupSize = 0;
       for (int i = 0; i < actualRepr->roots_size(); ++i) {
         ack->add_roots()->CopyFrom(actualRepr->roots(i));
-        tupSize += actualRepr->roots(i).ByteSize();
+        tupSize += actualRepr->roots(i).ByteSizeLong();
       }
       collector_->addAckTuple(ack, tupSize);
       int64_t currentTime = std::chrono::duration_cast<std::chrono::nanoseconds>(
@@ -118,7 +118,7 @@ void BoltOutputCollectorImpl::fail(std::shared_ptr<api::tuple::Tuple> tup) {
       int64_t tupSize = 0;
       for (int i = 0; i < actualRepr->roots_size(); ++i) {
         fl->add_roots()->CopyFrom(actualRepr->roots(i));
-        tupSize += actualRepr->roots(i).ByteSize();
+        tupSize += actualRepr->roots(i).ByteSizeLong();
       }
       collector_->addFailTuple(fl, tupSize);
       int64_t currentTime = std::chrono::duration_cast<std::chrono::nanoseconds>(

diff --git a/heron/proto/BUILD b/heron/proto/BUILD
@@ -181,7 +181,7 @@ java_library(
 pex_library(
     name = "proto-py",
     reqs = [
-        "protobuf==3.8.0",
+        "protobuf==3.16.0",
         "setuptools==46.1.3",
     ],
     deps = [

diff --git a/heron/stmgr/src/cpp/util/tuple-cache.cpp b/heron/stmgr/src/cpp/util/tuple-cache.cpp
@@ -191,7 +191,7 @@ void TupleCache::TupleList::add_ack_tuple(sp_int32 _src_task_id,
     current_->set_src_task_id(_src_task_id);
     current_size_ = 0;
   }
-  sp_int64 tuple_size = _tuple.ByteSize();
+  sp_int64 tuple_size = _tuple.ByteSizeLong();
   current_size_ += tuple_size;
   *_total_size += tuple_size;
   current_->mutable_control()->add_acks()->CopyFrom(_tuple);
@@ -209,7 +209,7 @@ void TupleCache::TupleList::add_fail_tuple(sp_int32 _src_task_id,
     current_->set_src_task_id(_src_task_id);
     current_size_ = 0;
   }
-  sp_int64 tuple_size = _tuple.ByteSize();
+  sp_int64 tuple_size = _tuple.ByteSizeLong();
   current_size_ += tuple_size;
   *_total_size += tuple_size;
   current_->mutable_control()->add_fails()->CopyFrom(_tuple);
@@ -228,7 +228,7 @@ void TupleCache::TupleList::add_emit_tuple(sp_int32 _src_task_id,
     current_->set_src_task_id(_src_task_id);
     current_size_ = 0;
   }
-  sp_int64 tuple_size = _tuple.ByteSize();
+  sp_int64 tuple_size = _tuple.ByteSizeLong();
   current_size_ += tuple_size;
   *_total_size += tuple_size;
   current_->mutable_control()->add_emits()->CopyFrom(_tuple);
@@ -242,7 +242,7 @@ void TupleCache::TupleList::add_checkpoint_tuple(
     current_ = NULL;
     current_size_ = 0;
   }
-  sp_int64 tuple_size = _message->ByteSize();
+  sp_int64 tuple_size = _message->ByteSizeLong();
   *_total_size += tuple_size;
   tuples_.push_front(_message);
 }

diff --git a/heron/stmgr/tests/cpp/server/checkpoint-gateway_unittest.cpp b/heron/stmgr/tests/cpp/server/checkpoint-gateway_unittest.cpp
@@ -400,7 +400,7 @@ TEST(CheckpointGateway, overflow) {
         // Now send another tuple from the upstreamer.
         auto tup = new heron::proto::system::HeronTupleSet2();
         tup->set_src_task_id(upstreamer);
-        sp_uint32 cached_size = tup->ByteSize();
+        sp_uint64 cached_size = tup->ByteSizeLong();
         gateway->SendToInstance(local_bolt, tup);
         if (upstream_map[local_bolt].empty()) {
           // They only have one upstreamer, so the tuple is passed thru
@@ -420,14 +420,14 @@ TEST(CheckpointGateway, overflow) {
             EXPECT_EQ(0, drainer3_markers.size());
             tup = new heron::proto::system::HeronTupleSet2();
             tup->set_src_task_id(upstreamer);
-            cached_size += tup->ByteSize();
+            cached_size += tup->ByteSizeLong();
             total_sent++;
             gateway->SendToInstance(local_bolt, tup);
           }
           // Send one more to tip over
           tup = new heron::proto::system::HeronTupleSet2();
           tup->set_src_task_id(upstreamer);
-          cached_size += tup->ByteSize();
+          cached_size += tup->ByteSizeLong();
           total_sent++;
           gateway->SendToInstance(local_bolt, tup);
           EXPECT_EQ(total_sent, drainer1_tuples.size());

diff --git a/heron/tools/tracker/src/python/BUILD b/heron/tools/tracker/src/python/BUILD
@@ -10,7 +10,7 @@ pex_library(
         "click==7.1.2",
         "javaobj-py3==0.4.1",
         "networkx==2.4",
-        "protobuf==3.8.0",
+        "protobuf==3.16.0",
         "tornado==4.0.2",
     ],
     deps = [

diff --git a/heronpy/proto/BUILD b/heronpy/proto/BUILD
@@ -27,7 +27,7 @@ pex_library(
     name = "proto-py",
     srcs = glob(["**/*.py"]),
     reqs = [
-        "protobuf==3.8.0",
+        "protobuf==3.16.0",
         "setuptools==46.1.3",
     ],
     deps = [
@@ -50,7 +50,7 @@ pex_binary(
     name = "proto-py-package",
     srcs = glob(["**/*.py"]),
     reqs = [
-        "protobuf==3.8.0",
+        "protobuf==3.16.0",
         "setuptools==18.8.1",
     ],
     deps = [