Skip to content

Commit

Permalink
Update memory_above detector to support total memory usage
Browse files Browse the repository at this point in the history 
Summary:
memory_above confusingly tests anon usage against the threshold.
Update it so that it supports both total and anon usages.

* MemoryAbove is updated to interpret threshold as the threshold for
  the total memory usage and threshold_anon the anon memory usage.
  To provide a transition from older config, when both are specified,
  threshold_anon has the priority so that configurations can specify
  both threshold and threshold_anon during transition.

* For simplicity, MemoryAbove now evaulates the thresholds into bytes
  during configuration instead of execution.

* All existing memory_above tests are converted to test against total
  memory usage.  Two anon usage tests added and two anon vs. memory
  priority tests added.

Reviewed By: danobi

Differential Revision: D14751533

fbshipit-source-id: 04a3a7e96cb4e6fa8cdf70f2292a7d2d69d1ceba
  • Loading branch information
@htejun @facebook-github-bot
htejun authored and facebook-github-bot committed Apr 4, 2019
1 parent 198f248 commit 3016f1f
Show file tree
Hide file tree
Showing 4 changed files with 147 additions and 64 deletions.
18 changes: 12 additions & 6 deletions docs/core_plugins.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,19 +39,25 @@ otherwise.
### Arguments

cgroup
threshold
threshold (optional)
threshold_anon (optional)
duration

### Description

`cgroup` has the same semantics and features as `pressure_rising_beyond`.

`threshold` takes both an absolute value (in MB) or a percentage of total
memory used. A percentage threshold must be in the format `N%`, where
`0 <= N <= 100`.
`threshold` and `threshold_anon` take both an absolute value (in MB) or
a percentage of total memory used. A percentage threshold must be in the
format `N%`, where `0 <= N <= 100`. Either one of these parameters must
be specified. When both are specified, only `threshold_anon` is
effective.

CONTINUE if 10s anonymous memory usage > `threshold` longer than `duration`,
STOP otherwise.
If `threshold` is specified, CONTINUE if 10s total memory usage >
`threshold` longer than `duration`, STOP otherwise.

If `threshold_anon` is specified, CONTINUE if 10s anonymous memory
usage > `threshold_anon` longer than `duration`, STOP otherwise.

## pressure_above

Expand Down
100 changes: 93 additions & 7 deletions plugins/CorePluginsTest.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -363,7 +363,7 @@ TEST(MemoryAbove, DetectsHighMemUsage) {
OomdContext ctx;
ctx.setCgroupContext(
CgroupPath(args["cgroup_fs"], "high_memory"),
CgroupContext{{}, {}, 0, 0, 0, 20, 2147483648});
CgroupContext{{}, {}, 2147483648});
EXPECT_EQ(plugin->run(ctx), Engine::PluginRet::CONTINUE);
}

Expand All @@ -384,7 +384,7 @@ TEST(MemoryAbove, NoDetectLowMemUsage) {
OomdContext ctx;
ctx.setCgroupContext(
CgroupPath(args["cgroup_fs"], "low_memory"),
CgroupContext{{}, {}, 0, 0, 0, 20, 1073741824});
CgroupContext{{}, {}, 1073741824});
EXPECT_EQ(plugin->run(ctx), Engine::PluginRet::STOP);
}

Expand All @@ -405,7 +405,7 @@ TEST(MemoryAbove, DetectsHighMemUsagePercent) {
OomdContext ctx;
ctx.setCgroupContext(
CgroupPath(args["cgroup_fs"], "high_memory"),
CgroupContext{{}, {}, 0, 0, 0, 20, 2147483648});
CgroupContext{{}, {}, 2147483648});
EXPECT_EQ(plugin->run(ctx), Engine::PluginRet::CONTINUE);
}

Expand All @@ -427,10 +427,10 @@ TEST(MemoryAbove, NoDetectLowMemUsageMultiple) {
OomdContext ctx;
ctx.setCgroupContext(
CgroupPath(args["cgroup_fs"], "low_memory"),
CgroupContext{{}, {}, 0, 0, 0, 20, 1073741824});
CgroupContext{{}, {}, 1073741824});
ctx.setCgroupContext(
CgroupPath(args["cgroup_fs"], "high_memory"),
CgroupContext{{}, {}, 0, 0, 0, 20, 2147483648});
CgroupContext{{}, {}, 2147483648});
EXPECT_EQ(plugin->run(ctx), Engine::PluginRet::STOP);
}

Expand All @@ -452,10 +452,10 @@ TEST(MemoryAbove, DetectsHighMemUsageMultiple) {
OomdContext ctx;
ctx.setCgroupContext(
CgroupPath(args["cgroup_fs"], "low_memory"),
CgroupContext{{}, {}, 0, 0, 0, 20, 1073741824});
CgroupContext{{}, {}, 1073741824});
ctx.setCgroupContext(
CgroupPath(args["cgroup_fs"], "high_memory"),
CgroupContext{{}, {}, 0, 0, 0, 20, 2147483648});
CgroupContext{{}, {}, 2147483648});
EXPECT_EQ(plugin->run(ctx), Engine::PluginRet::CONTINUE);
}

Expand All @@ -473,13 +473,99 @@ TEST(MemoryAbove, NoDetectLowMemUsagePercent) {

ASSERT_EQ(plugin->init(resources, std::move(args)), 0);

OomdContext ctx;
ctx.setCgroupContext(
CgroupPath(args["cgroup_fs"], "low_memory"),
CgroupContext{{}, {}, 1073741824});
EXPECT_EQ(plugin->run(ctx), Engine::PluginRet::STOP);
}

TEST(MemoryAbove, DetectsHighAnonUsage) {
auto plugin = createPlugin("memory_above");
ASSERT_NE(plugin, nullptr);

Engine::MonitoredResources resources;
Engine::PluginArgs args;
args["cgroup_fs"] = "oomd/fixtures/plugins/memory_above";
args["cgroup"] = "high_memory";
args["meminfo_location"] = "oomd/fixtures/plugins/memory_above/meminfo";
args["threshold_anon"] = "1536"; // in MB
args["duration"] = "0";

ASSERT_EQ(plugin->init(resources, std::move(args)), 0);

OomdContext ctx;
ctx.setCgroupContext(
CgroupPath(args["cgroup_fs"], "high_memory"),
CgroupContext{{}, {}, 0, 0, 0, 20, 2147483648});
EXPECT_EQ(plugin->run(ctx), Engine::PluginRet::CONTINUE);
}

TEST(MemoryAbove, NoDetectLowAnonUsage) {
auto plugin = createPlugin("memory_above");
ASSERT_NE(plugin, nullptr);

Engine::MonitoredResources resources;
Engine::PluginArgs args;
args["cgroup_fs"] = "oomd/fixtures/plugins/memory_above";
args["cgroup"] = "low_memory";
args["meminfo_location"] = "oomd/fixtures/plugins/memory_above/meminfo";
args["threshold_anon"] = "1536"; // in MB
args["duration"] = "0";

ASSERT_EQ(plugin->init(resources, std::move(args)), 0);

OomdContext ctx;
ctx.setCgroupContext(
CgroupPath(args["cgroup_fs"], "low_memory"),
CgroupContext{{}, {}, 0, 0, 0, 20, 1073741824});
EXPECT_EQ(plugin->run(ctx), Engine::PluginRet::STOP);
}

TEST(MemoryAbove, DetectsHighAnonUsageIgnoreLowMemUsage) {
auto plugin = createPlugin("memory_above");
ASSERT_NE(plugin, nullptr);

Engine::MonitoredResources resources;
Engine::PluginArgs args;
args["cgroup_fs"] = "oomd/fixtures/plugins/memory_above";
args["cgroup"] = "high_memory";
args["meminfo_location"] = "oomd/fixtures/plugins/memory_above/meminfo";
args["threshold_anon"] = "1536"; // in MB
args["threshold"] = "1536"; // in MB
args["duration"] = "0";

ASSERT_EQ(plugin->init(resources, std::move(args)), 0);

OomdContext ctx;
ctx.setCgroupContext(
CgroupPath(args["cgroup_fs"], "high_memory"),
CgroupContext{{}, {}, 1073741824, 0, 0, 20, 2147483648});
EXPECT_EQ(plugin->run(ctx), Engine::PluginRet::CONTINUE);
}

TEST(MemoryAbove, NoDetectLowAnonUsageIgnoreHighMemUsage) {
auto plugin = createPlugin("memory_above");
ASSERT_NE(plugin, nullptr);

Engine::MonitoredResources resources;
Engine::PluginArgs args;
args["cgroup_fs"] = "oomd/fixtures/plugins/memory_above";
args["cgroup"] = "low_memory";
args["meminfo_location"] = "oomd/fixtures/plugins/memory_above/meminfo";
args["threshold_anon"] = "1536"; // in MB
args["threshold"] = "1536"; // in MB
args["duration"] = "0";

ASSERT_EQ(plugin->init(resources, std::move(args)), 0);

OomdContext ctx;
ctx.setCgroupContext(
CgroupPath(args["cgroup_fs"], "low_memory"),
CgroupContext{{}, {}, 2147483648, 0, 0, 20, 1073741824});
EXPECT_EQ(plugin->run(ctx), Engine::PluginRet::STOP);
}

TEST(MemoryReclaim, SingleCgroupReclaimSuccess) {
auto plugin = createPlugin("memory_reclaim");
ASSERT_NE(plugin, nullptr);
Expand Down
88 changes: 40 additions & 48 deletions plugins/MemoryAbove.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,21 @@ namespace Oomd {

REGISTER_PLUGIN(memory_above, MemoryAbove::create);

namespace {
int64_t parse_threshold(const std::string& str, int64_t memtotal) {
if (str.size() > 0 && str.at(str.size() - 1) == '%') {
double pct = std::stod(str.substr(0, str.size() - 1));
if (pct < 0 || pct > 100) {
OLOG << str << " is an illegal percentage value";
return -1;
}
return memtotal * pct / 100;
} else {
return std::stoll(str) * 1024 * 1024;
}
}
} // namespace

int MemoryAbove::init(
Engine::MonitoredResources& resources,
const Engine::PluginArgs& args) {
Expand All @@ -49,20 +64,24 @@ int MemoryAbove::init(
return 1;
}

if (args.find("threshold") != args.end()) {
auto threshold_str = args.at("threshold");
if (threshold_str.size() > 0) {
if (threshold_str.compare(threshold_str.size() - 1, 1, "%") == 0) {
relative_ = true;
}
auto meminfo = args.find("meminfo_location") != args.end()
? Fs::getMeminfo(args.at("meminfo_location"))
: Fs::getMeminfo();

if (args.find("threshold_anon") != args.end()) {
threshold_ =
parse_threshold(args.at("threshold_anon"), meminfo["MemTotal"]);
if (threshold_ < 0) {
return 1;
}
if (relative_) {
threshold_ = std::stoi(threshold_str.substr(0, threshold_str.size() - 1));
} else {
threshold_ = std::stoi(threshold_str);
is_anon_ = true;
} else if (args.find("threshold") != args.end()) {
threshold_ = parse_threshold(args.at("threshold"), meminfo["MemTotal"]);
if (threshold_ < 0) {
return 1;
}
} else {
OLOG << "Argument=threshold not present";
OLOG << "Argument=[anon_]threshold not present";
return 1;
}

Expand All @@ -81,9 +100,6 @@ int MemoryAbove::init(
}
}

if (args.find("meminfo_location") != args.end()) {
meminfo_location_ = args.at("meminfo_location");
}
// Success
return 0;
}
Expand All @@ -97,10 +113,12 @@ Engine::PluginRet MemoryAbove::run(OomdContext& ctx) {
auto cgroup_ctx = ctx.getCgroupContext(cgroup);
if (debug_) {
OLOG << "cgroup \"" << cgroup.relativePath() << "\" "
<< "memory.current=" << cgroup_ctx.current_usage
<< "memory.stat (anon)=" << cgroup_ctx.anon_usage;
}
if (current_memory_usage < cgroup_ctx.anon_usage) {
current_memory_usage = cgroup_ctx.anon_usage;
auto usage = is_anon_ ? cgroup_ctx.anon_usage : cgroup_ctx.current_usage;
if (current_memory_usage < usage) {
current_memory_usage = usage;
current_cgroup = cgroup.relativePath();
}
} catch (const std::exception& ex) {
Expand All @@ -109,40 +127,14 @@ Engine::PluginRet MemoryAbove::run(OomdContext& ctx) {
continue;
}
}
auto meminfo = meminfo_location_.size() ? Fs::getMeminfo(meminfo_location_)
: Fs::getMeminfo();
const int64_t memtotal = meminfo["MemTotal"];
if (debug_) {
OLOG << "MemTotal=" << memtotal;
}

bool threshold_broken{false};
int64_t threshold_bytes;

if (relative_) {
auto percent = current_memory_usage * 100 / memtotal;
threshold_bytes = (threshold_ * memtotal) / 100;
if (percent > threshold_) {
OLOG << "cgroup \"" << current_cgroup << "\" "
<< "memory usage=" << current_memory_usage << " (" << percent
<< "%) "
<< "hit threshold=" << threshold_bytes << " (" << threshold_
<< "%) of MemTotal";
threshold_broken = true;
}
} else {
threshold_bytes = threshold_ * 1024 * 1024;
if (current_memory_usage > threshold_bytes) {
OLOG << "cgroup \"" << current_cgroup << "\" "
<< "memory usage=" << current_memory_usage << " "
<< "hit threshold=" << threshold_bytes;
threshold_broken = true;
}
}

const auto now = steady_clock::now();

if (threshold_broken) {
if (current_memory_usage > threshold_) {
OLOG << "cgroup \"" << current_cgroup << "\" "
<< (is_anon_ ? "anon usage=" : "memory usage=") << current_memory_usage
<< " hit threshold=" << threshold_;

if (hit_thres_at_ == steady_clock::time_point()) {
hit_thres_at_ = now;
}
Expand All @@ -156,7 +148,7 @@ Engine::PluginRet MemoryAbove::run(OomdContext& ctx) {
oss << std::setprecision(2) << std::fixed;
oss << "cgroup \"" << current_cgroup << "\" "
<< "current memory usage " << current_memory_usage / 1024 / 1024
<< "MB is over the threshold of " << threshold_bytes / 1024 / 1024
<< "MB is over the threshold of " << threshold_ / 1024 / 1024
<< "MB for " << duration_ << " seconds";
OLOG << oss.str();

Expand Down
5 changes: 2 additions & 3 deletions plugins/MemoryAbove.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,10 @@ class MemoryAbove : public Oomd::Engine::BasePlugin {
private:
std::unordered_set<CgroupPath> cgroups_;
// Initialized to bogus values; init() will crash oomd if non-0 return
int threshold_;
int64_t threshold_;
int duration_;
bool relative_{false};
bool is_anon_{false};
bool debug_{false};
std::string meminfo_location_;
std::chrono::steady_clock::time_point hit_thres_at_{};
};

Expand Down

0 comments on commit 3016f1f

Please sign in to comment.