Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[PoC] Simplify rule evaluation #218

Closed
wants to merge 7 commits into from
Closed

[PoC] Simplify rule evaluation #218

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
3ce813e
* Replace collections with multiset with specific ordering rules
Anilm3 Sep 26, 2023
262aca9
Remove disabled rules
Anilm3 Sep 26, 2023
6743b1d
Evaluate only necessary rules
Anilm3 Sep 26, 2023
9bb074d
Small changes
Anilm3 Sep 27, 2023
c077c76
Use uint32_t instead of std::size_t for rule index
Anilm3 Sep 27, 2023
5e498c6
Remove redundant target check for rules
Anilm3 Sep 27, 2023
dff59da
Merge branch 'master' into anilm3/simplify-rule-evaluation
Anilm3 Sep 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,6 @@ set(LIBDDWAF_SOURCE
${libddwaf_SOURCE_DIR}/src/event.cpp
${libddwaf_SOURCE_DIR}/src/object.cpp
${libddwaf_SOURCE_DIR}/src/object_store.cpp
${libddwaf_SOURCE_DIR}/src/collection.cpp
${libddwaf_SOURCE_DIR}/src/expression.cpp
${libddwaf_SOURCE_DIR}/src/rule.cpp
${libddwaf_SOURCE_DIR}/src/ruleset_info.cpp
Expand Down
104 changes: 0 additions & 104 deletions src/collection.cpp
View file
Open in desktop

This file was deleted.

72 changes: 0 additions & 72 deletions src/collection.hpp
View file
Open in desktop

This file was deleted.

130 changes: 91 additions & 39 deletions src/context.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,18 +59,16 @@ DDWAF_RET_CODE context::run(ddwaf_object &input, optional_ref<ddwaf_result> res,
eval_preprocessors(derived, deadline);

// If no rule targets are available, there is no point in evaluating them
bool eval_rules = should_eval_rules();
bool eval_filters = eval_rules || should_eval_filters();

if (eval_filters) {
const auto &rules = rules_to_eval();
if (!rules.empty() || should_eval_filters()) {
// Filters need to be evaluated even if rules don't, otherwise it'll
// break the current condition cache mechanism which requires knowing
// if an address is new to this run.
const auto &rules_to_exclude = filter_rules(deadline);
const auto &objects_to_exclude = filter_inputs(rules_to_exclude, deadline);

if (eval_rules) {
events = match(rules_to_exclude, objects_to_exclude, deadline);
if (!rules.empty()) {
events = match(rules, rules_to_exclude, objects_to_exclude, deadline);
}
}

Expand Down Expand Up @@ -197,48 +195,102 @@ const memory::unordered_map<rule *, context::object_set> &context::filter_inputs
return objects_to_exclude_;
}

memory::vector<event> context::match(
const memory::set<rule *, rule::greater_than> &context::rules_to_eval()
{
rules_.clear();

const auto &targets = store_.get_latest_targets();
for (auto target : targets) {
auto it = ruleset_->rules_by_targets.find(target);
if (it == ruleset_->rules_by_targets.end()) {
continue;
}

const auto &target_rules = it->second;
for (auto *rule : target_rules) { rules_.emplace(rule); }
}

return rules_;
}

memory::vector<event> context::match(const memory::set<rule *, rule::greater_than> &rules,
const memory::unordered_map<rule *, filter_mode> &rules_to_exclude,
const memory::unordered_map<rule *, object_set> &objects_to_exclude, ddwaf::timer &deadline)
{
memory::vector<ddwaf::event> events;

auto eval_collection = [&](const auto &type, const auto &collection) {
auto it = collection_cache_.find(type);
if (it == collection_cache_.end()) {
auto [new_it, res] = collection_cache_.emplace(type, collection_cache{});
it = new_it;
for (auto rule_it = rules.begin(); rule_it != rules.end();) {
auto type = (*rule_it)->get_type();
auto cache_it = collection_cache_.find(type);
if (cache_it == collection_cache_.end()) {
auto [new_it, res] = collection_cache_.emplace(type, collection_type::none);
cache_it = new_it;
}
collection.match(events, store_, it->second, rules_to_exclude, objects_to_exclude,
ruleset_->dynamic_matchers, deadline);
};

// Evaluate user priority collections first
for (auto &[type, collection] : ruleset_->user_priority_collections) {
DDWAF_DEBUG("Evaluating user priority collection '%.*s'", static_cast<int>(type.length()),
type.data());
eval_collection(type, collection);
}

// Evaluate priority collections first
for (auto &[type, collection] : ruleset_->base_priority_collections) {
DDWAF_DEBUG(
"Evaluating priority collection '%.*s'", static_cast<int>(type.length()), type.data());
eval_collection(type, collection);
}
do {
const auto &rule = *rule_it;

// Evaluate regular collection after
for (auto &[type, collection] : ruleset_->user_collections) {
DDWAF_DEBUG(
"Evaluating user collection '%.*s'", static_cast<int>(type.length()), type.data());
eval_collection(type, collection);
}
auto level = rule->has_actions() ? collection_type::priority : collection_type::regular;
if (cache_it->second >= level) {
// Skip to next type
while (++rule_it != rules.end() && (*rule_it)->get_type() == type) {}
break;
}

// Evaluate regular collection after
for (auto &[type, collection] : ruleset_->base_collections) {
DDWAF_DEBUG(
"Evaluating base collection '%.*s'", static_cast<int>(type.length()), type.data());
eval_collection(type, collection);
if (deadline.expired()) {
DDWAF_INFO("Ran out of time while evaluating rule '%s'", rule->get_id().c_str());
throw timeout_exception();
}

bool skip_actions = false;
auto exclude_it = rules_to_exclude.find(rule);
if (exclude_it != rules_to_exclude.end()) {
if (exclude_it->second == exclusion::filter_mode::bypass) {
DDWAF_DEBUG("Bypassing rule '%s'", rule->get_id().c_str());
continue;
}

DDWAF_DEBUG("Monitoring rule '%s'", rule->get_id().c_str());
skip_actions = true;
} else {
DDWAF_DEBUG("Evaluating rule '%s'", rule->get_id().c_str());
}

try {
auto it = rule_cache_.find(rule);
if (it == rule_cache_.end()) {
auto [new_it, res] = rule_cache_.emplace(rule, rule::cache_type{});
it = new_it;
}

const auto &dynamic_matchers = ruleset_->dynamic_matchers;

rule::cache_type &rule_cache = it->second;
std::optional<event> event;
auto exclude_it = objects_to_exclude.find(rule);
if (exclude_it != objects_to_exclude.end()) {
const auto &objects_excluded = exclude_it->second;
event = rule->match(
store_, rule_cache, objects_excluded, dynamic_matchers, deadline);
} else {
event = rule->match(store_, rule_cache, {}, dynamic_matchers, deadline);
}

if (event.has_value()) {
cache_it->second = level;
event->skip_actions = skip_actions;
events.emplace_back(std::move(*event));
DDWAF_DEBUG("Found event on rule %s", rule->get_id().c_str());

// Skip to next type
while (++rule_it != rules.end() && (*rule_it)->get_type() == type) {}
break;
}
} catch (const ddwaf::timeout_exception &) {
DDWAF_INFO("Ran out of time while evaluating rule '%s'", rule->get_id().c_str());
throw;
}
} while (++rule_it != rules.end() && (*rule_it)->get_type() == type);
}

return events;
Expand Down
21 changes: 8 additions & 13 deletions src/context.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ class context {
{
rule_filter_cache_.reserve(ruleset_->rule_filters.size());
input_filter_cache_.reserve(ruleset_->input_filters.size());
collection_cache_.reserve(ruleset_->collection_types.size());
}

context(const context &) = delete;
Expand All @@ -50,22 +49,15 @@ class context {
const memory::unordered_map<rule *, filter_mode> &filter_rules(ddwaf::timer &deadline);
const memory::unordered_map<rule *, object_set> &filter_inputs(
const memory::unordered_map<rule *, filter_mode> &rules_to_exclude, ddwaf::timer &deadline);
const memory::set<rule *, rule::greater_than> &rules_to_eval();

memory::vector<event> match(const memory::unordered_map<rule *, filter_mode> &rules_to_exclude,
memory::vector<event> match(const memory::set<rule *, rule::greater_than> &rules,
const memory::unordered_map<rule *, filter_mode> &rules_to_exclude,
const memory::unordered_map<rule *, object_set> &objects_to_exclude,
ddwaf::timer &deadline);

protected:
bool is_first_run() const { return collection_cache_.empty(); }
bool should_eval_rules() const
{
for (const auto &[target, str] : ruleset_->rule_addresses) {
if (store_.is_new_target(target)) {
return true;
}
}
return false;
}
bool should_eval_filters() const
{
for (const auto &[target, str] : ruleset_->filter_addresses) {
Expand All @@ -90,8 +82,11 @@ class context {
memory::unordered_map<rule *, filter_mode> rules_to_exclude_;
memory::unordered_map<rule *, object_set> objects_to_exclude_;

// Cache of collections to avoid processing once a result has been obtained
memory::unordered_map<std::string_view, collection::cache_type> collection_cache_;
memory::set<rule *, rule::greater_than> rules_;

enum class collection_type : uint8_t { none = 0, regular = 1, priority = 2 };
memory::unordered_map<rule::type_index, collection_type> collection_cache_;
memory::unordered_map<rule *, rule::cache_type> rule_cache_;
};

class context_wrapper {
Expand Down
Loading