diff --git a/docs/core_plugins.md b/docs/core_plugins.md index a6552345..7383f131 100644 --- a/docs/core_plugins.md +++ b/docs/core_plugins.md @@ -39,19 +39,25 @@ otherwise. ### Arguments cgroup - threshold + threshold (optional) + threshold_anon (optional) duration ### Description `cgroup` has the same semantics and features as `pressure_rising_beyond`. -`threshold` takes both an absolute value (in MB) or a percentage of total -memory used. A percentage threshold must be in the format `N%`, where -`0 <= N <= 100`. +`threshold` and `threshold_anon` take both an absolute value (in MB) or +a percentage of total memory used. A percentage threshold must be in the +format `N%`, where `0 <= N <= 100`. Either one of these parameters must +be specified. When both are specified, only `threshold_anon` is +effective. -CONTINUE if 10s anonymous memory usage > `threshold` longer than `duration`, -STOP otherwise. +If `threshold` is specified, CONTINUE if 10s total memory usage > +`threshold` longer than `duration`, STOP otherwise. + +If `threshold_anon` is specified, CONTINUE if 10s anonymous memory +usage > `threshold_anon` longer than `duration`, STOP otherwise. ## pressure_above diff --git a/plugins/CorePluginsTest.cpp b/plugins/CorePluginsTest.cpp index ef48fd5a..07e79d85 100644 --- a/plugins/CorePluginsTest.cpp +++ b/plugins/CorePluginsTest.cpp @@ -363,7 +363,7 @@ TEST(MemoryAbove, DetectsHighMemUsage) { OomdContext ctx; ctx.setCgroupContext( CgroupPath(args["cgroup_fs"], "high_memory"), - CgroupContext{{}, {}, 0, 0, 0, 20, 2147483648}); + CgroupContext{{}, {}, 2147483648}); EXPECT_EQ(plugin->run(ctx), Engine::PluginRet::CONTINUE); } @@ -384,7 +384,7 @@ TEST(MemoryAbove, NoDetectLowMemUsage) { OomdContext ctx; ctx.setCgroupContext( CgroupPath(args["cgroup_fs"], "low_memory"), - CgroupContext{{}, {}, 0, 0, 0, 20, 1073741824}); + CgroupContext{{}, {}, 1073741824}); EXPECT_EQ(plugin->run(ctx), Engine::PluginRet::STOP); } @@ -405,7 +405,7 @@ TEST(MemoryAbove, DetectsHighMemUsagePercent) { OomdContext ctx; ctx.setCgroupContext( CgroupPath(args["cgroup_fs"], "high_memory"), - CgroupContext{{}, {}, 0, 0, 0, 20, 2147483648}); + CgroupContext{{}, {}, 2147483648}); EXPECT_EQ(plugin->run(ctx), Engine::PluginRet::CONTINUE); } @@ -427,10 +427,10 @@ TEST(MemoryAbove, NoDetectLowMemUsageMultiple) { OomdContext ctx; ctx.setCgroupContext( CgroupPath(args["cgroup_fs"], "low_memory"), - CgroupContext{{}, {}, 0, 0, 0, 20, 1073741824}); + CgroupContext{{}, {}, 1073741824}); ctx.setCgroupContext( CgroupPath(args["cgroup_fs"], "high_memory"), - CgroupContext{{}, {}, 0, 0, 0, 20, 2147483648}); + CgroupContext{{}, {}, 2147483648}); EXPECT_EQ(plugin->run(ctx), Engine::PluginRet::STOP); } @@ -452,10 +452,10 @@ TEST(MemoryAbove, DetectsHighMemUsageMultiple) { OomdContext ctx; ctx.setCgroupContext( CgroupPath(args["cgroup_fs"], "low_memory"), - CgroupContext{{}, {}, 0, 0, 0, 20, 1073741824}); + CgroupContext{{}, {}, 1073741824}); ctx.setCgroupContext( CgroupPath(args["cgroup_fs"], "high_memory"), - CgroupContext{{}, {}, 0, 0, 0, 20, 2147483648}); + CgroupContext{{}, {}, 2147483648}); EXPECT_EQ(plugin->run(ctx), Engine::PluginRet::CONTINUE); } @@ -473,6 +473,48 @@ TEST(MemoryAbove, NoDetectLowMemUsagePercent) { ASSERT_EQ(plugin->init(resources, std::move(args)), 0); + OomdContext ctx; + ctx.setCgroupContext( + CgroupPath(args["cgroup_fs"], "low_memory"), + CgroupContext{{}, {}, 1073741824}); + EXPECT_EQ(plugin->run(ctx), Engine::PluginRet::STOP); +} + +TEST(MemoryAbove, DetectsHighAnonUsage) { + auto plugin = createPlugin("memory_above"); + ASSERT_NE(plugin, nullptr); + + Engine::MonitoredResources resources; + Engine::PluginArgs args; + args["cgroup_fs"] = "oomd/fixtures/plugins/memory_above"; + args["cgroup"] = "high_memory"; + args["meminfo_location"] = "oomd/fixtures/plugins/memory_above/meminfo"; + args["threshold_anon"] = "1536"; // in MB + args["duration"] = "0"; + + ASSERT_EQ(plugin->init(resources, std::move(args)), 0); + + OomdContext ctx; + ctx.setCgroupContext( + CgroupPath(args["cgroup_fs"], "high_memory"), + CgroupContext{{}, {}, 0, 0, 0, 20, 2147483648}); + EXPECT_EQ(plugin->run(ctx), Engine::PluginRet::CONTINUE); +} + +TEST(MemoryAbove, NoDetectLowAnonUsage) { + auto plugin = createPlugin("memory_above"); + ASSERT_NE(plugin, nullptr); + + Engine::MonitoredResources resources; + Engine::PluginArgs args; + args["cgroup_fs"] = "oomd/fixtures/plugins/memory_above"; + args["cgroup"] = "low_memory"; + args["meminfo_location"] = "oomd/fixtures/plugins/memory_above/meminfo"; + args["threshold_anon"] = "1536"; // in MB + args["duration"] = "0"; + + ASSERT_EQ(plugin->init(resources, std::move(args)), 0); + OomdContext ctx; ctx.setCgroupContext( CgroupPath(args["cgroup_fs"], "low_memory"), @@ -480,6 +522,50 @@ TEST(MemoryAbove, NoDetectLowMemUsagePercent) { EXPECT_EQ(plugin->run(ctx), Engine::PluginRet::STOP); } +TEST(MemoryAbove, DetectsHighAnonUsageIgnoreLowMemUsage) { + auto plugin = createPlugin("memory_above"); + ASSERT_NE(plugin, nullptr); + + Engine::MonitoredResources resources; + Engine::PluginArgs args; + args["cgroup_fs"] = "oomd/fixtures/plugins/memory_above"; + args["cgroup"] = "high_memory"; + args["meminfo_location"] = "oomd/fixtures/plugins/memory_above/meminfo"; + args["threshold_anon"] = "1536"; // in MB + args["threshold"] = "1536"; // in MB + args["duration"] = "0"; + + ASSERT_EQ(plugin->init(resources, std::move(args)), 0); + + OomdContext ctx; + ctx.setCgroupContext( + CgroupPath(args["cgroup_fs"], "high_memory"), + CgroupContext{{}, {}, 1073741824, 0, 0, 20, 2147483648}); + EXPECT_EQ(plugin->run(ctx), Engine::PluginRet::CONTINUE); +} + +TEST(MemoryAbove, NoDetectLowAnonUsageIgnoreHighMemUsage) { + auto plugin = createPlugin("memory_above"); + ASSERT_NE(plugin, nullptr); + + Engine::MonitoredResources resources; + Engine::PluginArgs args; + args["cgroup_fs"] = "oomd/fixtures/plugins/memory_above"; + args["cgroup"] = "low_memory"; + args["meminfo_location"] = "oomd/fixtures/plugins/memory_above/meminfo"; + args["threshold_anon"] = "1536"; // in MB + args["threshold"] = "1536"; // in MB + args["duration"] = "0"; + + ASSERT_EQ(plugin->init(resources, std::move(args)), 0); + + OomdContext ctx; + ctx.setCgroupContext( + CgroupPath(args["cgroup_fs"], "low_memory"), + CgroupContext{{}, {}, 2147483648, 0, 0, 20, 1073741824}); + EXPECT_EQ(plugin->run(ctx), Engine::PluginRet::STOP); +} + TEST(MemoryReclaim, SingleCgroupReclaimSuccess) { auto plugin = createPlugin("memory_reclaim"); ASSERT_NE(plugin, nullptr); diff --git a/plugins/MemoryAbove.cpp b/plugins/MemoryAbove.cpp index fca3e1c4..21ba89fd 100644 --- a/plugins/MemoryAbove.cpp +++ b/plugins/MemoryAbove.cpp @@ -31,6 +31,21 @@ namespace Oomd { REGISTER_PLUGIN(memory_above, MemoryAbove::create); +namespace { +int64_t parse_threshold(const std::string& str, int64_t memtotal) { + if (str.size() > 0 && str.at(str.size() - 1) == '%') { + double pct = std::stod(str.substr(0, str.size() - 1)); + if (pct < 0 || pct > 100) { + OLOG << str << " is an illegal percentage value"; + return -1; + } + return memtotal * pct / 100; + } else { + return std::stoll(str) * 1024 * 1024; + } +} +} // namespace + int MemoryAbove::init( Engine::MonitoredResources& resources, const Engine::PluginArgs& args) { @@ -49,20 +64,24 @@ int MemoryAbove::init( return 1; } - if (args.find("threshold") != args.end()) { - auto threshold_str = args.at("threshold"); - if (threshold_str.size() > 0) { - if (threshold_str.compare(threshold_str.size() - 1, 1, "%") == 0) { - relative_ = true; - } + auto meminfo = args.find("meminfo_location") != args.end() + ? Fs::getMeminfo(args.at("meminfo_location")) + : Fs::getMeminfo(); + + if (args.find("threshold_anon") != args.end()) { + threshold_ = + parse_threshold(args.at("threshold_anon"), meminfo["MemTotal"]); + if (threshold_ < 0) { + return 1; } - if (relative_) { - threshold_ = std::stoi(threshold_str.substr(0, threshold_str.size() - 1)); - } else { - threshold_ = std::stoi(threshold_str); + is_anon_ = true; + } else if (args.find("threshold") != args.end()) { + threshold_ = parse_threshold(args.at("threshold"), meminfo["MemTotal"]); + if (threshold_ < 0) { + return 1; } } else { - OLOG << "Argument=threshold not present"; + OLOG << "Argument=[anon_]threshold not present"; return 1; } @@ -81,9 +100,6 @@ int MemoryAbove::init( } } - if (args.find("meminfo_location") != args.end()) { - meminfo_location_ = args.at("meminfo_location"); - } // Success return 0; } @@ -97,10 +113,12 @@ Engine::PluginRet MemoryAbove::run(OomdContext& ctx) { auto cgroup_ctx = ctx.getCgroupContext(cgroup); if (debug_) { OLOG << "cgroup \"" << cgroup.relativePath() << "\" " + << "memory.current=" << cgroup_ctx.current_usage << "memory.stat (anon)=" << cgroup_ctx.anon_usage; } - if (current_memory_usage < cgroup_ctx.anon_usage) { - current_memory_usage = cgroup_ctx.anon_usage; + auto usage = is_anon_ ? cgroup_ctx.anon_usage : cgroup_ctx.current_usage; + if (current_memory_usage < usage) { + current_memory_usage = usage; current_cgroup = cgroup.relativePath(); } } catch (const std::exception& ex) { @@ -109,40 +127,14 @@ Engine::PluginRet MemoryAbove::run(OomdContext& ctx) { continue; } } - auto meminfo = meminfo_location_.size() ? Fs::getMeminfo(meminfo_location_) - : Fs::getMeminfo(); - const int64_t memtotal = meminfo["MemTotal"]; - if (debug_) { - OLOG << "MemTotal=" << memtotal; - } - - bool threshold_broken{false}; - int64_t threshold_bytes; - - if (relative_) { - auto percent = current_memory_usage * 100 / memtotal; - threshold_bytes = (threshold_ * memtotal) / 100; - if (percent > threshold_) { - OLOG << "cgroup \"" << current_cgroup << "\" " - << "memory usage=" << current_memory_usage << " (" << percent - << "%) " - << "hit threshold=" << threshold_bytes << " (" << threshold_ - << "%) of MemTotal"; - threshold_broken = true; - } - } else { - threshold_bytes = threshold_ * 1024 * 1024; - if (current_memory_usage > threshold_bytes) { - OLOG << "cgroup \"" << current_cgroup << "\" " - << "memory usage=" << current_memory_usage << " " - << "hit threshold=" << threshold_bytes; - threshold_broken = true; - } - } const auto now = steady_clock::now(); - if (threshold_broken) { + if (current_memory_usage > threshold_) { + OLOG << "cgroup \"" << current_cgroup << "\" " + << (is_anon_ ? "anon usage=" : "memory usage=") << current_memory_usage + << " hit threshold=" << threshold_; + if (hit_thres_at_ == steady_clock::time_point()) { hit_thres_at_ = now; } @@ -156,7 +148,7 @@ Engine::PluginRet MemoryAbove::run(OomdContext& ctx) { oss << std::setprecision(2) << std::fixed; oss << "cgroup \"" << current_cgroup << "\" " << "current memory usage " << current_memory_usage / 1024 / 1024 - << "MB is over the threshold of " << threshold_bytes / 1024 / 1024 + << "MB is over the threshold of " << threshold_ / 1024 / 1024 << "MB for " << duration_ << " seconds"; OLOG << oss.str(); diff --git a/plugins/MemoryAbove.h b/plugins/MemoryAbove.h index 7ccd3819..33a3c354 100644 --- a/plugins/MemoryAbove.h +++ b/plugins/MemoryAbove.h @@ -41,11 +41,10 @@ class MemoryAbove : public Oomd::Engine::BasePlugin { private: std::unordered_set cgroups_; // Initialized to bogus values; init() will crash oomd if non-0 return - int threshold_; + int64_t threshold_; int duration_; - bool relative_{false}; + bool is_anon_{false}; bool debug_{false}; - std::string meminfo_location_; std::chrono::steady_clock::time_point hit_thres_at_{}; };