Improve memory watermark detection

.github/workflows/nightly.yml

@@ -128,7 +128,7 @@ jobs:
         timeout-minutes: 20
       - name: Setup cluster
         run: |
-          make ENABLE_SSL=true CA_SIGNED=true up
+          make CONTAINERIZED=true ENABLE_SSL=true CA_SIGNED=true up
         working-directory: tests/
         timeout-minutes: 2
       - name: Pytest

.github/workflows/pull_request.yml

@@ -52,11 +52,11 @@ jobs:
         exclude:
           - os: centos7
             compiler: clang-10
+    env:
+      CCACHE_DIR: /tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }}
+      CCACHE_MAXSIZE: 8G
     container:
       image: vesoft/nebula-dev:${{ matrix.os }}
-      env:
-        CCACHE_DIR: /tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }}
-        CCACHE_MAXSIZE: 8G
       volumes:
         - /tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }}:/tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }}
       options: --cap-add=SYS_PTRACE
@@ -132,17 +132,17 @@ jobs:
             case ${{ matrix.os }} in
             centos7)
               # normal cluster
-              make up
+              make CONTAINERIZED=true up
               ;;
             ubuntu2004)
               # ssl cluster
-              make ENABLE_SSL=true CA_SIGNED=true up
+              make CONTAINERIZED=true ENABLE_SSL=true CA_SIGNED=true up
               ;;
             esac
             ;;
           clang-*)
             # graph ssl only cluster
-            make ENABLE_SSL=false ENABLE_GRAPH_SSL=true up
+            make CONTAINERIZED=true ENABLE_SSL=false ENABLE_GRAPH_SSL=true up
             ;;
           esac
         working-directory: tests/

README-CN.md

@@ -5,14 +5,22 @@
 </p>
 
 <p align="center">
-  <a href="https://user-images.githubusercontent.com/38887077/67449282-4362b300-f64c-11e9-878f-7efc373e5e55.jpg"><img src="https://img.shields.io/badge/WeChat-%E5%BE%AE%E4%BF%A1-brightgreen" alt="WeiXin"></a>
-  <a href="https://www.zhihu.com/org/nebulagraph/activities"><img src="https://img.shields.io/badge/Zhihu-%E7%9F%A5%E4%B9%8E-blue" alt="Zhihu"></a>
-  <a href="https://segmentfault.com/t/nebula"><img src="https://img.shields.io/badge/SegmentFault-%E6%80%9D%E5%90%A6-green" alt="SegmentFault"></a>
-  <a href="https://weibo.com/p/1006067122684542/home?from=page_100606&mod=TAB#place"><img src="https://img.shields.io/badge/Weibo-%E5%BE%AE%E5%8D%9A-red" alt="Sina Weibo"></a>
-  <a href="http://githubbadges.com/star.svg?user=vesoft-inc&repo=nebula&style=default">
+  <a href="https://user-images.githubusercontent.com/38887077/67449282-4362b300-f64c-11e9-878f-7efc373e5e55.jpg">
+    <img src="https://img.shields.io/badge/WeChat-%E5%BE%AE%E4%BF%A1-brightgreen" alt="WeiXin">
+  </a>
+  <a href="https://www.zhihu.com/org/nebulagraph/activities">
+    <img src="https://img.shields.io/badge/Zhihu-%E7%9F%A5%E4%B9%8E-blue" alt="Zhihu">
+  </a>
+  <a href="https://segmentfault.com/t/nebula">
+    <img src="https://img.shields.io/badge/SegmentFault-%E6%80%9D%E5%90%A6-green" alt="SegmentFault">
+  </a>
+  <a href="https://weibo.com/p/1006067122684542/home?from=page_100606&mod=TAB#place">
+    <img src="https://img.shields.io/badge/Weibo-%E5%BE%AE%E5%8D%9A-red" alt="Sina Weibo">
+  </a>
+  <a href="https://github.com/vesoft-inc/nebula/stargazers">
     <img src="http://githubbadges.com/star.svg?user=vesoft-inc&repo=nebula&style=default" alt="nebula star"/>
   </a>
-  <a href="http://githubbadges.com/fork.svg?user=vesoft-inc&repo=nebula&style=default">
+  <a href="https://github.com/vesoft-inc/nebula/network/members">
     <img src="http://githubbadges.com/fork.svg?user=vesoft-inc&repo=nebula&style=default" alt="nebula fork"/>
   </a>
 </p>

README.md

@@ -4,13 +4,16 @@
   <br>A distributed, scalable, lightning-fast graph database<br>
 </p>
 <p align="center">
+  <a href="https://app.codecov.io/gh/vesoft-inc/nebula">
+    <img src="https://codecov.io/github/vesoft-inc/nebula/coverage.svg?branch=master" alt="code coverage"/>
+  </a>
   <a href="https://github.com/vesoft-inc/nebula/actions?workflow=nightly">
     <img src="https://github.com/vesoft-inc/nebula/workflows/nightly/badge.svg" alt="nightly build"/>
   </a>
-  <a href="http://githubbadges.com/star.svg?user=vesoft-inc&repo=nebula&style=default">
+  <a href="https://github.com/vesoft-inc/nebula/stargazers">
     <img src="http://githubbadges.com/star.svg?user=vesoft-inc&repo=nebula&style=default" alt="nebula star"/>
   </a>
-  <a href="http://githubbadges.com/fork.svg?user=vesoft-inc&repo=nebula&style=default">
+  <a href="https://github.com/vesoft-inc/nebula/network/members">
     <img src="http://githubbadges.com/fork.svg?user=vesoft-inc&repo=nebula&style=default" alt="nebula fork"/>
   </a>
   <br>

docker/Dockerfile

@@ -20,7 +20,7 @@ RUN rpm -ivh *.rpm \
 
 EXPOSE 9669 19669 19670
 
-ENTRYPOINT ["/usr/local/nebula/bin/nebula-graphd", "--flagfile=/usr/local/nebula/etc/nebula-graphd.conf", "--daemonize=false"]
+ENTRYPOINT ["/usr/local/nebula/bin/nebula-graphd", "--flagfile=/usr/local/nebula/etc/nebula-graphd.conf", "--daemonize=false", "--containerized=true"]
 
 FROM centos:7 as metad
 
@@ -35,7 +35,7 @@ RUN rpm -ivh *.rpm \
 
 EXPOSE 9559 9560 19559 19560
 
-ENTRYPOINT ["/usr/local/nebula/bin/nebula-metad", "--flagfile=/usr/local/nebula/etc/nebula-metad.conf", "--daemonize=false"]
+ENTRYPOINT ["/usr/local/nebula/bin/nebula-metad", "--flagfile=/usr/local/nebula/etc/nebula-metad.conf", "--daemonize=false", "--containerized=true"]
 
 FROM centos:7 as storaged
 
@@ -50,7 +50,7 @@ RUN rpm -ivh *.rpm \
 
 EXPOSE 9777 9778 9779 9780 19779 19780
 
-ENTRYPOINT ["/usr/local/nebula/bin/nebula-storaged", "--flagfile=/usr/local/nebula/etc/nebula-storaged.conf", "--daemonize=false"]
+ENTRYPOINT ["/usr/local/nebula/bin/nebula-storaged", "--flagfile=/usr/local/nebula/etc/nebula-storaged.conf", "--daemonize=false", "--containerized=true"]
 
 FROM centos:7 as tools
 

docker/Dockerfile.graphd

@@ -20,4 +20,4 @@ RUN rpm -ivh *.rpm \
 
 EXPOSE 9669 19669 19670
 
-ENTRYPOINT ["/usr/local/nebula/bin/nebula-graphd", "--flagfile=/usr/local/nebula/etc/nebula-graphd.conf", "--daemonize=false"]
+ENTRYPOINT ["/usr/local/nebula/bin/nebula-graphd", "--flagfile=/usr/local/nebula/etc/nebula-graphd.conf", "--daemonize=false", "--containerized=true"]

docker/Dockerfile.metad

@@ -20,4 +20,4 @@ RUN rpm -ivh *.rpm \
 
 EXPOSE 9559 9560 19559 19560
 
-ENTRYPOINT ["/usr/local/nebula/bin/nebula-metad", "--flagfile=/usr/local/nebula/etc/nebula-metad.conf", "--daemonize=false"]
+ENTRYPOINT ["/usr/local/nebula/bin/nebula-metad", "--flagfile=/usr/local/nebula/etc/nebula-metad.conf", "--daemonize=false", "--containerized=true"]

docker/Dockerfile.storaged

@@ -20,4 +20,4 @@ RUN rpm -ivh *.rpm \
 
 EXPOSE 9777 9778 9779 9780 19779 19780
 
-ENTRYPOINT ["/usr/local/nebula/bin/nebula-storaged", "--flagfile=/usr/local/nebula/etc/nebula-storaged.conf", "--daemonize=false"]
+ENTRYPOINT ["/usr/local/nebula/bin/nebula-storaged", "--flagfile=/usr/local/nebula/etc/nebula-storaged.conf", "--daemonize=false", "--containerized=true"]

resources/gflags.json

@@ -11,6 +11,7 @@
         "custom_filter_interval_secs",
         "accept_partial_success",
         "system_memory_high_watermark_ratio",
+        "num_rows_to_check_memory",
         "session_idle_timeout_secs",
         "session_reclaim_interval_secs",
         "max_allowed_connections",

src/clients/storage/StorageClientBase-inl.h

@@ -139,8 +139,8 @@ folly::SemiFuture<StorageRpcResponse<Response>> StorageClientBase<ClientType>::c
           // then-callback will be executed on the same IO thread
           .via(evb)
           .then([this, context, host, spaceId, start](folly::Try<Response>&& val) {
-            auto& r = context->findRequest(host);
             if (val.hasException()) {
+              auto& r = context->findRequest(host);
               LOG(ERROR) << "Request to " << host << " failed: " << val.exception().what();
               auto parts = getReqPartsId(r);
               context->resp.appendFailedParts(parts, nebula::cpp2::ErrorCode::E_RPC_FAILURE);

src/common/CMakeLists.txt

@@ -27,3 +27,4 @@ nebula_add_subdirectory(plugin)
 nebula_add_subdirectory(utils)
 nebula_add_subdirectory(ssl)
 nebula_add_subdirectory(geo)
+nebula_add_subdirectory(memory)

src/common/base/CMakeLists.txt

@@ -16,7 +16,6 @@ nebula_add_library(
     SignalHandler.cpp
     SlowOpTracker.cpp
     StringValue.cpp
-    Memory.cpp
     ${gdb_debug_script}
 )
 

src/common/base/test/CMakeLists.txt

@@ -94,13 +94,6 @@ nebula_add_test(
     LIBRARIES gtest gtest_main
 )
 
-nebula_add_test(
-    NAME memory_test
-    SOURCES MemoryTest.cpp
-    OBJECTS $<TARGET_OBJECTS:base_obj>
-    LIBRARIES gtest gtest_main
-)
-
 nebula_add_executable(
     NAME range_vs_transform_bm
     SOURCES RangeVsTransformBenchmark.cpp

src/common/geo/io/wkt/test/CMakeLists.txt

@@ -44,6 +44,7 @@ set(WKT_PARSER_TEST_LIBS
     $<TARGET_OBJECTS:idgenerator_obj>
     $<TARGET_OBJECTS:graph_obj>
     $<TARGET_OBJECTS:ssl_obj>
+    $<TARGET_OBJECTS:memory_obj>
 )
 
 nebula_add_test(

src/common/memory/CMakeLists.txt

@@ -0,0 +1,12 @@
+# Copyright (c) 2021 vesoft inc. All rights reserved.
+#
+# This source code is licensed under Apache 2.0 License,
+# attached with Common Clause Condition 1.0, found in the LICENSES directory.
+
+
+nebula_add_library(
+  memory_obj OBJECT
+  MemoryUtils.cpp
+)
+
+nebula_add_subdirectory(test)

src/common/memory/MemoryUtils.cpp

@@ -0,0 +1,83 @@
+/* Copyright (c) 2021 vesoft inc. All rights reserved.
+ *
+ * This source code is licensed under Apache 2.0 License,
+ * attached with Common Clause Condition 1.0, found in the LICENSES directory.
+ */
+
+#include "common/memory/MemoryUtils.h"
+
+#include <folly/String.h>
+#include <gflags/gflags.h>
+
+#include <cstdio>
+#include <fstream>
+#include <regex>
+
+#include "common/fs/FileUtils.h"
+
+DEFINE_bool(containerized, false, "Whether run this process inside the docker container");
+DEFINE_double(system_memory_high_watermark_ratio, 0.8, "high watermark ratio of system memory");
+
+using nebula::fs::FileUtils;
+
+namespace nebula {
+
+static const std::regex reMemAvailable(R"(^Mem(Available|Total):\s+(\d+)\skB$)");
+static const std::regex reTotalCache(R"(^total_(cache|inactive_file)\s+(\d+)$)");
+
+std::atomic_bool MemoryUtils::kHitMemoryHighWatermark{false};
+
+StatusOr<bool> MemoryUtils::hitsHighWatermark() {
+  double available = 0.0, total = 0.0;
+  if (FLAGS_containerized) {
+    FileUtils::FileLineIterator iter("/sys/fs/cgroup/memory/memory.stat", &reTotalCache);
+    uint64_t cacheSize = 0;
+    for (; iter.valid(); ++iter) {
+      auto& sm = iter.matched();
+      cacheSize += std::stoul(sm[2].str(), NULL);
+    }
+
+    auto limitStatus = MemoryUtils::readSysContents("/sys/fs/cgroup/memory/memory.limit_in_bytes");
+    NG_RETURN_IF_ERROR(limitStatus);
+    uint64_t limitInBytes = std::move(limitStatus).value();
+
+    auto usageStatus = MemoryUtils::readSysContents("/sys/fs/cgroup/memory/memory.usage_in_bytes");
+    NG_RETURN_IF_ERROR(usageStatus);
+    uint64_t usageInBytes = std::move(usageStatus).value();
+
+    total = static_cast<double>(limitInBytes);
+    available = static_cast<double>(limitInBytes - usageInBytes + cacheSize);
+  } else {
+    FileUtils::FileLineIterator iter("/proc/meminfo", &reMemAvailable);
+    std::vector<uint64_t> memorySize;
+    for (; iter.valid(); ++iter) {
+      auto& sm = iter.matched();
+      memorySize.emplace_back(std::stoul(sm[2].str(), NULL) << 10);
+    }
+    CHECK_EQ(memorySize.size(), 2U);
+    size_t i = 0, j = 1;
+    if (memorySize[0] < memorySize[1]) {
+      std::swap(i, j);
+    }
+    total = memorySize[i];
+    available = memorySize[j];
+  }
+
+  auto hits = (1 - available / total) > FLAGS_system_memory_high_watermark_ratio;
+  LOG_IF_EVERY_N(WARNING, hits, 100)
+      << "Memory usage has hit the high watermark of system, available: " << available
+      << " vs. total: " << total << " in bytes.";
+  return hits;
+}
+
+StatusOr<uint64_t> MemoryUtils::readSysContents(const std::string& path) {
+  std::ifstream ifs(path);
+  if (!ifs) {
+    return Status::Error("Could not open the file: %s", path.c_str());
+  }
+  uint64_t value = 0;
+  ifs >> value;
+  return value;
+}
+
+}  // namespace nebula