[PoC] Global context rules

.github/workflows/test.yml

@@ -264,6 +264,9 @@ jobs:
       - name: Create directories
         run: mkdir Debug
 
+      - name: Install dependencies
+        run: sudo apt update ; sudo apt install -y libreadline-dev
+
       - name: CMake
         env:
           CC: gcc-12

cmake/objects.cmake

@@ -4,6 +4,7 @@ set(LIBDDWAF_SOURCE
     ${libddwaf_SOURCE_DIR}/src/parameter.cpp
     ${libddwaf_SOURCE_DIR}/src/interface.cpp
     ${libddwaf_SOURCE_DIR}/src/context.cpp
+    ${libddwaf_SOURCE_DIR}/src/global_context.cpp
     ${libddwaf_SOURCE_DIR}/src/context_allocator.cpp
     ${libddwaf_SOURCE_DIR}/src/event.cpp
     ${libddwaf_SOURCE_DIR}/src/object.cpp
@@ -21,6 +22,7 @@ set(LIBDDWAF_SOURCE
     ${libddwaf_SOURCE_DIR}/src/platform.cpp
     ${libddwaf_SOURCE_DIR}/src/sha256.cpp
     ${libddwaf_SOURCE_DIR}/src/uuid.cpp
+    ${libddwaf_SOURCE_DIR}/src/rule/threshold_rule.cpp
     ${libddwaf_SOURCE_DIR}/src/action_mapper.cpp
     ${libddwaf_SOURCE_DIR}/src/builder/processor_builder.cpp
     ${libddwaf_SOURCE_DIR}/src/tokenizer/sql_base.cpp
@@ -45,6 +47,7 @@ set(LIBDDWAF_SOURCE
     ${libddwaf_SOURCE_DIR}/src/parser/rule_override_parser.cpp
     ${libddwaf_SOURCE_DIR}/src/parser/scanner_parser.cpp
     ${libddwaf_SOURCE_DIR}/src/parser/exclusion_parser.cpp
+    ${libddwaf_SOURCE_DIR}/src/parser/global_rule_parser.cpp
     ${libddwaf_SOURCE_DIR}/src/processor/extract_schema.cpp
     ${libddwaf_SOURCE_DIR}/src/processor/fingerprint.cpp
     ${libddwaf_SOURCE_DIR}/src/condition/exists.cpp

src/collection.cpp

@@ -20,7 +20,7 @@
 #include "log.hpp"
 #include "matcher/base.hpp"
 #include "object_store.hpp"
-#include "rule.hpp"
+#include "rule/rule.hpp"
 
 namespace ddwaf {
 

src/collection.hpp

@@ -9,7 +9,7 @@
 #include "context_allocator.hpp"
 #include "event.hpp"
 #include "exclusion/rule_filter.hpp"
-#include "rule.hpp"
+#include "rule/rule.hpp"
 
 namespace ddwaf {
 

src/context.cpp

@@ -103,6 +103,10 @@ DDWAF_RET_CODE context::run(optional_ref<ddwaf_object> persistent,
     try {
         eval_preprocessors(derived, deadline);
 
+        if (ruleset_->gctx) {
+            ruleset_->gctx->eval(events, store_, gctx_cache_, deadline);
+        }
+
         // If no rule targets are available, there is no point in evaluating them
         const bool should_eval_rules = check_new_rule_targets();
         const bool should_eval_filters = should_eval_rules || check_new_filter_targets();
@@ -114,7 +118,7 @@ DDWAF_RET_CODE context::run(optional_ref<ddwaf_object> persistent,
             const auto &policy = eval_filters(deadline);
 
             if (should_eval_rules) {
-                events = eval_rules(policy, deadline);
+                eval_rules(events, policy, deadline);
                 if (!events.empty()) {
                     set_context_event_address(store_);
                 }
@@ -230,11 +234,9 @@ exclusion::context_policy &context::eval_filters(ddwaf::timer &deadline)
     return exclusion_policy_;
 }
 
-std::vector<event> context::eval_rules(
-    const exclusion::context_policy &policy, ddwaf::timer &deadline)
+void context::eval_rules(std::vector<ddwaf::event> &events, const exclusion::context_policy &policy,
+    ddwaf::timer &deadline)
 {
-    std::vector<ddwaf::event> events;
-
     auto eval_collection = [&](const auto &type, const auto &collection) {
         auto it = collection_cache_.find(type);
         if (it == collection_cache_.end()) {
@@ -267,8 +269,6 @@ std::vector<event> context::eval_rules(
         DDWAF_DEBUG("Evaluating base collection '{}'", type);
         eval_collection(type, collection);
     }
-
-    return events;
 }
 
 } // namespace ddwaf

src/context.hpp

@@ -17,7 +17,7 @@
 #include "exclusion/input_filter.hpp"
 #include "exclusion/rule_filter.hpp"
 #include "obfuscator.hpp"
-#include "rule.hpp"
+#include "rule/rule.hpp"
 #include "ruleset.hpp"
 #include "utils.hpp"
 
@@ -50,7 +50,8 @@ class context {
     // This function below returns a reference to an internal object,
     // however using them this way helps with testing
     exclusion::context_policy &eval_filters(ddwaf::timer &deadline);
-    std::vector<event> eval_rules(const exclusion::context_policy &policy, ddwaf::timer &deadline);
+    void eval_rules(std::vector<ddwaf::event> &events, const exclusion::context_policy &policy,
+        ddwaf::timer &deadline);
 
 protected:
     bool is_first_run() const { return collection_cache_.empty(); }
@@ -89,6 +90,8 @@ class context {
 
     // Cache of collections to avoid processing once a result has been obtained
     memory::unordered_map<std::string_view, collection::cache_type> collection_cache_{};
+
+    global_context::cache_type gctx_cache_{};
 };
 
 class context_wrapper {

src/event.cpp

@@ -14,7 +14,7 @@
 #include "ddwaf.h"
 #include "event.hpp"
 #include "obfuscator.hpp"
-#include "rule.hpp"
+#include "rule/rule.hpp" // IWYU pragma: keep
 #include "uuid.hpp"
 
 namespace ddwaf {
@@ -151,7 +151,7 @@ void add_action_to_tracker(action_tracker &actions, std::string_view id, action_
     }
 }
 
-void serialize_rule(const ddwaf::rule &rule, ddwaf_object &rule_map)
+void serialize_rule(const ddwaf::base_rule &rule, ddwaf_object &rule_map)
 {
     ddwaf_object tmp;
     ddwaf_object tags_map;
@@ -186,7 +186,7 @@ void serialize_empty_rule(ddwaf_object &rule_map)
     ddwaf_object_map_add(&rule_map, "tags", &tags_map);
 }
 
-void serialize_and_consolidate_rule_actions(const ddwaf::rule &rule, ddwaf_object &rule_map,
+void serialize_and_consolidate_rule_actions(const ddwaf::base_rule &rule, ddwaf_object &rule_map,
     std::string_view action_override, action_tracker &actions, ddwaf_object &stack_id)
 {
     const auto &rule_actions = rule.get_actions();

src/event.hpp

@@ -15,10 +15,10 @@
 
 namespace ddwaf {
 
-class rule;
+class base_rule;
 
 struct event {
-    const ddwaf::rule *rule{nullptr};
+    const ddwaf::base_rule *rule{nullptr};
     std::vector<condition_match> matches;
     bool ephemeral{false};
     std::string_view action_override;

src/exclusion/input_filter.cpp

@@ -19,7 +19,7 @@
 #include "log.hpp"
 #include "matcher/base.hpp"
 #include "object_store.hpp"
-#include "rule.hpp"
+#include "rule/rule.hpp"
 
 namespace ddwaf::exclusion {
 

src/exclusion/input_filter.hpp

@@ -13,7 +13,7 @@
 #include "clock.hpp"
 #include "exclusion/object_filter.hpp"
 #include "object_store.hpp"
-#include "rule.hpp"
+#include "rule/rule.hpp"
 
 namespace ddwaf::exclusion {
 

src/exclusion/rule_filter.cpp

@@ -18,7 +18,7 @@
 #include "log.hpp"
 #include "matcher/base.hpp"
 #include "object_store.hpp"
-#include "rule.hpp"
+#include "rule/rule.hpp"
 
 namespace ddwaf::exclusion {
 

src/exclusion/rule_filter.hpp

@@ -13,7 +13,7 @@
 #include "clock.hpp"
 #include "exclusion/common.hpp"
 #include "object_store.hpp"
-#include "rule.hpp"
+#include "rule/rule.hpp"
 
 namespace ddwaf::exclusion {
 

src/global_context.cpp

@@ -0,0 +1,40 @@
+// Unless explicitly stated otherwise all files in this repository are
+// dual-licensed under the Apache-2.0 License or BSD-3-Clause License.
+//
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2021 Datadog, Inc.
+#include <utility>
+#include <vector>
+
+#include "clock.hpp"
+#include "event.hpp"
+#include "global_context.hpp"
+#include "log.hpp"
+#include "object_store.hpp"
+#include "rule/base.hpp"
+
+namespace ddwaf {
+void global_context::eval(std::vector<event> &events, const object_store &store, cache_type &cache,
+    ddwaf::timer &deadline)
+{
+    auto timepoint = monotonic_clock::now();
+
+    DDWAF_DEBUG("Evaluating global rules");
+    for (const auto &rule : rules_) {
+        auto cache_it = cache.find(rule.get());
+        if (cache_it == cache.end()) {
+            auto [new_it, res] = cache.emplace(rule.get(), base_threshold_rule::cache_type{});
+            if (!res) {
+                continue;
+            }
+            cache_it = new_it;
+        }
+        DDWAF_DEBUG("Evaluating rule {}", rule->get_id());
+        auto opt_evt = rule->eval(store, cache_it->second, timepoint, deadline);
+        if (opt_evt.has_value()) {
+            events.emplace_back(std::move(opt_evt.value()));
+        }
+    }
+}
+
+} // namespace ddwaf

src/global_context.hpp

@@ -0,0 +1,35 @@
+// Unless explicitly stated otherwise all files in this repository are
+// dual-licensed under the Apache-2.0 License or BSD-3-Clause License.
+//
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2021 Datadog, Inc.
+
+#pragma once
+
+#include <unordered_map>
+
+#include "rule/rule.hpp"
+
+namespace ddwaf {
+
+class global_context {
+public:
+    using cache_type = std::unordered_map<base_threshold_rule *, base_threshold_rule::cache_type>;
+
+    explicit global_context(std::vector<std::unique_ptr<base_threshold_rule>> rules)
+        : rules_(std::move(rules))
+    {}
+    global_context(const global_context &) = delete;
+    global_context &operator=(const global_context &) = delete;
+    global_context(global_context &&) = default;
+    global_context &operator=(global_context &&) = delete;
+    ~global_context() = default;
+
+    void eval(std::vector<event> &events, const object_store &store, cache_type &lcache,
+        ddwaf::timer &deadline);
+
+protected:
+    std::vector<std::unique_ptr<base_threshold_rule>> rules_;
+};
+
+} // namespace ddwaf

src/lru_cache.hpp

@@ -0,0 +1,84 @@
+// Unless explicitly stated otherwise all files in this repository are
+// dual-licensed under the Apache-2.0 License or BSD-3-Clause License.
+//
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2021 Datadog, Inc.
+
+#pragma once
+
+#include <chrono>
+#include <iostream>
+#include <map>
+#include <mutex>
+#include <string>
+#include <type_traits>
+#include <vector>
+
+#include "traits.hpp"
+
+namespace ddwaf {
+
+template <typename KeyType, typename DurationType, typename DataType, typename ConstructorType>
+    requires is_duration<DurationType>
+class lru_cache {
+public:
+    explicit lru_cache(ConstructorType constructor, std::size_t max_index_size = 32)
+        : constructor_(std::move(constructor)), max_index_size_(max_index_size)
+    {}
+
+    template <typename CompatKeyType>
+        requires std::is_constructible_v<CompatKeyType, KeyType>
+    DataType &emplace_or_retrieve(CompatKeyType key, DurationType timepoint)
+    {
+        auto it = index_.find(key);
+        if (it == index_.end()) {
+            if (index_.size() == max_index_size_) {
+                remove_oldest_entry(timepoint);
+            }
+
+            auto [new_it, res] =
+                index_.emplace(KeyType{key}, cache_entry{timepoint, constructor_()});
+            if (!res) {
+                throw std::out_of_range("failed to add element to cache");
+            }
+            return new_it->second.data;
+        }
+
+        it->second.latest_timepoint = std::max(timepoint, it->second.latest_timepoint);
+
+        return it->second.data;
+    }
+
+protected:
+    struct cache_entry {
+        DurationType latest_timepoint;
+        DataType data;
+    };
+
+    void remove_oldest_entry(DurationType timepoint)
+    {
+        using iterator_type = typename decltype(index_)::iterator;
+
+        DurationType max_delta{0};
+        iterator_type oldest_it;
+        for (auto it = index_.begin(); it != index_.end(); ++it) {
+            auto window_last = it->second.latest_timepoint;
+            auto delta = timepoint - window_last;
+            if (delta > max_delta) {
+                max_delta = delta;
+                oldest_it = it;
+            }
+        }
+
+        index_.erase(oldest_it);
+    }
+
+    ConstructorType constructor_;
+    std::size_t max_index_size_{};
+    std::map<KeyType, cache_entry, std::less<>> index_;
+};
+
+template <typename DataType, typename ConstructorType>
+using lru_cache_ms = lru_cache<std::string, std::chrono::milliseconds, DataType, ConstructorType>;
+
+} // namespace ddwaf

src/monitor.hpp

@@ -0,0 +1,33 @@
+// Unless explicitly stated otherwise all files in this repository are
+// dual-licensed under the Apache-2.0 License or BSD-3-Clause License.
+//
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2021 Datadog, Inc.
+
+#pragma once
+
+#include <mutex>
+namespace ddwaf {
+
+template <typename T> class monitor {
+public:
+    template <typename... Args> explicit monitor(Args... args) : underlying_value_(args...) {}
+
+    class locked_monitor {
+    public:
+        explicit locked_monitor(monitor &m) : m_(m), lock_(m.mtx_) {}
+        T *operator->() { return &m_.underlying_value_; }
+
+    protected:
+        monitor &m_;
+        std::unique_lock<std::mutex> lock_;
+    };
+
+    locked_monitor operator->() { return locked_monitor{*this}; }
+
+protected:
+    std::mutex mtx_{};
+    T underlying_value_;
+};
+
+} // namespace ddwaf