From 98c9ddd1a60c82e95e053841f8cb9edbd4d64b92 Mon Sep 17 00:00:00 2001 From: ProBrian Date: Wed, 4 Sep 2024 16:24:16 +0800 Subject: [PATCH 01/16] use flat array for field indexing instead of hashmap --- Cargo.lock | 248 ++++++++++++++++++++++++++++++++--- Cargo.toml | 9 +- benches/atc_benchmark.rs | 92 +++++++++++++ benches/data.json | 55 ++++++++ lib/resty/router/cdefs.lua | 4 +- lib/resty/router/context.lua | 13 +- src/ast.rs | 1 + src/context.rs | 68 +++++++--- src/ffi.rs | 17 +-- src/interpreter.rs | 20 ++- src/parser.rs | 1 + src/router.rs | 145 ++++++++++++++++++-- src/semantics.rs | 96 +++++++++++++- t/01-sanity.t | 33 ++--- t/02-bugs.t | 14 +- t/03-contains.t | 12 +- t/04-rawstr.t | 12 +- t/05-equals.t | 18 +-- t/07-in_notin.t | 8 +- t/08-equals.t | 20 +-- t/09-not.t | 10 +- 21 files changed, 764 insertions(+), 132 deletions(-) create mode 100644 benches/atc_benchmark.rs create mode 100644 benches/data.json diff --git a/Cargo.lock b/Cargo.lock index fb34bf1f..13e8b955 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -35,15 +35,16 @@ dependencies = [ "pest_derive", "regex", "serde", + "serde_json", "serde_regex", "uuid", ] [[package]] name = "autocfg" -version = "1.4.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "block-buffer" @@ -99,6 +100,33 @@ dependencies = [ "half", ] +[[package]] +name = "cidr" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "cidr" version = "0.2.3" @@ -110,7 +138,32 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.20" +version = "4.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed6719fffa43d0d87e5fd8caeab59be1554fb028cd30edc88fc4369b17971019" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "216aec2b177652e3846684cbfe25c9964d18ec45234f0f5da5157b207ed1aab6" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" + +[[package]] +name = "cpufeatures" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" dependencies = [ @@ -251,6 +304,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "getrandom" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "half" version = "2.4.1" @@ -293,6 +357,57 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +[[package]] +name = "js-sys" +version = "0.3.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "cfg-if", + "crunchy", +] + +[[package]] +name = "hermit-abi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" + +[[package]] +name = "is-terminal" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + [[package]] name = "js-sys" version = "0.3.72" @@ -320,6 +435,12 @@ version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + [[package]] name = "memchr" version = "2.7.4" @@ -335,6 +456,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "once_cell" version = "1.20.2" @@ -347,6 +477,12 @@ version = "11.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" +[[package]] +name = "oorandom" +version = "11.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" + [[package]] name = "pest" version = "2.7.14" @@ -394,7 +530,35 @@ dependencies = [ [[package]] name = "plotters" -version = "0.3.7" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a15b6eccb8484002195a3e44fe65a4ce8e93a625797a063735536fd59cb01cf3" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "414cec62c6634ae900ea1c56128dfe87cf63e7caece0852ec76aba307cebadb7" + +[[package]] +name = "plotters-svg" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81b30686a7d9c3e010b84284bdd26a29f2138574f52f5eb6f794fc0ad924e705" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "proc-macro2" +version = "1.0.80" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" dependencies = [ @@ -458,6 +622,26 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "regex" +version = "1.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "regex" version = "1.11.0" @@ -502,6 +686,21 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "serde" version = "1.0.210" @@ -524,9 +723,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.128" +version = "1.0.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" +checksum = "8043c06d9f82bd7271361ed64f415fe5e12a77fdb52e573e7f06a516dea329ad" dependencies = [ "itoa", "memchr", @@ -618,13 +817,16 @@ checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" name = "uuid" version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" +checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" +dependencies = [ + "getrandom", +] [[package]] name = "version_check" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "walkdir" @@ -636,11 +838,17 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + [[package]] name = "wasm-bindgen" -version = "0.2.95" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" +checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" dependencies = [ "cfg-if", "once_cell", @@ -649,9 +857,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.95" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" +checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" dependencies = [ "bumpalo", "log", @@ -664,9 +872,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.95" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" +checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -674,9 +882,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.95" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" +checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", @@ -687,15 +895,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.95" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" +checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" [[package]] name = "web-sys" -version = "0.3.72" +version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" +checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index 5782964c..5f550e12 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,14 @@ serde_regex = { version = "1.1", optional = true } fnv = "1" [dev-dependencies] -criterion = "0.*" +criterion = { version = "0.5", features = ["html_reports"] } +serde_json = "1" +serde = "1" +uuid = {version = "1.8", features = ["v4"]} + +[[bench]] +name = "atc_benchmark" +harness = false [lib] crate-type = ["lib", "cdylib", "staticlib"] diff --git a/benches/atc_benchmark.rs b/benches/atc_benchmark.rs new file mode 100644 index 00000000..8f1fb9ba --- /dev/null +++ b/benches/atc_benchmark.rs @@ -0,0 +1,92 @@ +use std::{hint::black_box, str::FromStr}; +use criterion::{criterion_group, criterion_main, Criterion}; +use atc_router::{ast::Type, ast::Value, context::Context, router::Router, schema::{self, Schema}}; +use uuid::Uuid; +use std::net::{IpAddr, Ipv4Addr}; +use serde_json; +use std::fs; +use serde::{Deserialize, Serialize}; +use std::env; + + +#[derive(Serialize, Deserialize)] +struct TestData { + rules: Vec, + match_keys: Vec, + match_values: Vec, + not_match_values: Vec +} + +// prepare match rules, context keys, context values from data.json file +fn prepare_data() -> TestData { + let cwd = env::current_dir().unwrap(); + let file_str = fs::read_to_string(cwd.join("benches/data.json")).expect("unable to open data.json"); + serde_json::from_str(&file_str).unwrap() +} + +// setup Schema +fn setup_schema() -> schema::Schema { + let mut s = Schema::default(); + s.add_field("net.protocol", Type::String); + s.add_field("tls.sni", Type::String); + s.add_field("http.method", Type::String); + s.add_field("http.host", Type::String); + s.add_field("http.path", Type::String); + s.add_field("http.path.segments.*", Type::String); + s.add_field("http.path.segments.len", Type::Int); + s.add_field("http.headers.*", Type::String); + s.add_field("net.dst.port", Type::Int); + s.add_field("net.src.ip", Type::IpAddr); + s +} + +// setup matchers, which be added from priority 100 with descending order +fn setup_matchers(r: &mut Router, data: &TestData) { + let mut pri = 100; + for v in &data.rules { + let id = Uuid::new_v4(); + let _ = r.add_matcher(pri, id, v.as_str().unwrap()); + pri -= 1; + } +} + +// mock contexts with field values passed in from json data +fn setup_context(ctx: &mut Context, data: &TestData, test_match: bool) { + let values = if test_match {&data.match_values} else {&data.not_match_values}; + for (i, v) in values.iter().enumerate() { + match v { + serde_json::Value::String(s) => { + ctx.add_value(i, Value::String(s.to_string())); + }, + serde_json::Value::Number(n) => { + ctx.add_value(i, Value::Int(n.as_i64().unwrap())); + }, + serde_json::Value::Array(l) => { + ctx.add_value(i, Value::IpAddr(IpAddr::V4(Ipv4Addr::from_str(l[0].as_str().unwrap()).unwrap()))); + }, + _ => panic!("incorrect data type") + } + } +} + +fn router_match(router: &Router, ctx: &mut Context) -> bool { + router.execute(ctx) +} + +fn criterion_benchmark(c: &mut Criterion) { + let data = prepare_data(); + let s = setup_schema(); + let mut r = Router::new(&s); + setup_matchers(&mut r, &data); + + let mut ctx = Context::new(r.fields.len()); + setup_context(&mut ctx, &data, true); + c.bench_function("route match all", |b| b.iter(|| router_match(black_box(&r), black_box(&mut ctx)))); + + let mut ctx = Context::new(r.fields.len()); + setup_context(&mut ctx, &data, false); + c.bench_function("route mismatch all", |b| b.iter(|| router_match(black_box(&r), black_box(&mut ctx)))); +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); \ No newline at end of file diff --git a/benches/data.json b/benches/data.json new file mode 100644 index 00000000..3798e1f3 --- /dev/null +++ b/benches/data.json @@ -0,0 +1,55 @@ +{ + "rules": [ + "net.protocol == \"http\"", + "tls.sni == \"server1\"", + "http.method == \"GET\"", + "http.host == \"example.com\"", + "http.path == \"/foo\"", + "http.path.segments.1 == \"/bar\"", + "http.path.segments.0_1 == \"/foo/bar\"", + "http.path.segments.len == 2", + "http.headers.foo_bar == \"whatever\"", + "net.dst.port == 8443", + "net.src.ip == 192.168.1.1", + "net.src.ip in 192.168.1.0/24" + ], + "match_keys": [ + "net.protocol", + "tls.sni", + "http.method", + "http.host", + "http.path", + "http.path.segments.1", + "http.path.segments.0_1", + "http.path.segments.len", + "http.headers.foo_bar", + "net.dst.port", + "net.src.ip" + ], + "match_values": [ + "http", + "server1", + "GET", + "example.com", + "/foo", + "/bar\"", + "/foo/bar\"", + 2, + "whatever", + 8443, + ["192.168.1.1"] + ], + "not_match_values": [ + "https", + "server2", + "POST", + "example_foo.com", + "/fooo", + "/barr\"", + "/fooo/bar\"", + 3, + "whatever_wrong", + 18443, + ["192.168.2.1"] + ] +} \ No newline at end of file diff --git a/lib/resty/router/cdefs.lua b/lib/resty/router/cdefs.lua index 0d0349f0..94ae6e18 100644 --- a/lib/resty/router/cdefs.lua +++ b/lib/resty/router/cdefs.lua @@ -70,12 +70,12 @@ uintptr_t router_get_fields(const struct Router *router, const uint8_t **fields, uintptr_t *fields_len); -struct Context *context_new(const struct Schema *schema); +struct Context *context_new(uintptr_t fields_len); void context_free(struct Context *context); bool context_add_value(struct Context *context, - const int8_t *field, + uintptr_t field_idx, const struct CValue *value, uint8_t *errbuf, uintptr_t *errbuf_len); diff --git a/lib/resty/router/context.lua b/lib/resty/router/context.lua index c88d6552..3508dc6b 100644 --- a/lib/resty/router/context.lua +++ b/lib/resty/router/context.lua @@ -26,22 +26,27 @@ local clib = cdefs.clib local context_free = cdefs.context_free -function _M.new(schema) - local context = clib.context_new(schema.schema) +function _M.new(schema, size) + local context = clib.context_new(size) local c = setmetatable({ context = ffi_gc(context, context_free), schema = schema, + max_fields = size, }, _MT) return c end -function _M:add_value(field, value) +function _M:add_value(index, field, value) if not value then return true end + if index > self.max_fields then + return false, "context value index out of bound" + end + local typ, err = self.schema:get_field_type(field) if not typ then return nil, err @@ -65,7 +70,7 @@ function _M:add_value(field, value) local errbuf_len = get_size_ptr() errbuf_len[0] = ERR_BUF_MAX_LEN - if clib.context_add_value(self.context, field, CACHED_VALUE, errbuf, errbuf_len) == false then + if clib.context_add_value(self.context, index-1, CACHED_VALUE, errbuf, errbuf_len) == false then return nil, ffi_string(errbuf, errbuf_len[0]) end diff --git a/src/ast.rs b/src/ast.rs index 42a04752..a2cd68b2 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -104,6 +104,7 @@ pub enum Type { #[derive(Debug, Clone)] pub struct Lhs { pub var_name: String, + pub index: usize, pub transformations: Vec, } diff --git a/src/context.rs b/src/context.rs index 3d27305f..fc2d464d 100644 --- a/src/context.rs +++ b/src/context.rs @@ -1,5 +1,4 @@ use crate::ast::Value; -use crate::schema::Schema; use fnv::FnvHashMap; use uuid::Uuid; @@ -25,38 +24,73 @@ impl Default for Match { } } -pub struct Context<'a> { - schema: &'a Schema, - values: FnvHashMap>, +pub struct Context { + values: Vec>>, pub result: Option, } -impl<'a> Context<'a> { - pub fn new(schema: &'a Schema) -> Self { +impl Context { + pub fn new(fields_cnt: usize) -> Self { Context { - schema, - values: FnvHashMap::with_hasher(Default::default()), + values: vec![None; fields_cnt], result: None, } } - pub fn add_value(&mut self, field: &str, value: Value) { - if &value.my_type() != self.schema.type_of(field).unwrap() { - panic!("value provided does not match schema"); + pub fn add_value(&mut self, index: usize, value: Value) { + if index >= self.values.len() { + panic!("value provided does not match schema: index {}, max fields count {}", index, self.values.len()); } - self.values - .entry(field.to_string()) - .or_default() - .push(value); + if let Some(v) = &mut self.values[index] { + v.push(value); + } else { + self.values[index] = Some(vec![value]); + } } - pub fn value_of(&self, field: &str) -> Option<&[Value]> { - self.values.get(field).map(|v| v.as_slice()) + pub fn value_of(&self, index: usize) -> Option<&[Value]> { + if !self.values.is_empty() && self.values[index].is_some() { Some(self.values[index].as_ref().unwrap().as_slice()) } else {None} } pub fn reset(&mut self) { + let len = self.values.len(); + // reserve the capacity of values for reuse, avoid re-alloc self.values.clear(); + self.values.resize_with(len, Default::default); self.result = None; } } + +#[cfg(test)] +mod tests { + use crate::context::Context; + use crate::ast::Value; + #[test] + fn test_context() { + let fields_cnt = 3; + let mut ctx = Context::new(fields_cnt); + assert!(ctx.values.len() == fields_cnt); + assert_eq!(ctx.values, vec![None; fields_cnt]); + // access value with out of bound index + assert_eq!(ctx.value_of(0), None); + + // add value in bound + ctx.add_value(1, Value::String("foo".to_string())); + assert_eq!(ctx.value_of(0), None); + assert_eq!(ctx.value_of(1).unwrap().len(), 1); + assert_eq!(ctx.value_of(1).unwrap(), vec![Value::String("foo".to_string())].as_slice()); + + // reset context keeps values capacity with all None + ctx.reset(); + assert!(ctx.values.len() == fields_cnt); + assert_eq!(ctx.values, vec![None; fields_cnt]); + + // reuse this context + ctx.add_value(0, Value::String("bar".to_string())); + ctx.add_value(0, Value::String("foo".to_string())); + assert!(ctx.values.len() == fields_cnt); + assert_eq!(ctx.value_of(0).unwrap().len(), 2); + assert_eq!(ctx.value_of(0).unwrap(), vec![Value::String("bar".to_string()), Value::String("foo".to_string())].as_slice()); + } +} diff --git a/src/ffi.rs b/src/ffi.rs index 2a4aa8c2..7e7091de 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -327,9 +327,9 @@ pub unsafe extern "C" fn router_get_fields( let fields = from_raw_parts_mut(fields, *fields_len); let fields_len = from_raw_parts_mut(fields_len, *fields_len); - for (i, k) in router.fields.keys().enumerate() { - fields[i] = k.as_bytes().as_ptr(); - fields_len[i] = k.len() + for (i, (name, _)) in router.fields.iter().enumerate() { + fields[i] = name.as_bytes().as_ptr(); + fields_len[i] = name.len() } } @@ -348,8 +348,8 @@ pub unsafe extern "C" fn router_get_fields( /// /// - `schema` must be a valid pointer returned by [`schema_new`]. #[no_mangle] -pub unsafe extern "C" fn context_new(schema: &Schema) -> *mut Context { - Box::into_raw(Box::new(Context::new(schema))) +pub unsafe extern "C" fn context_new(fields_len: usize) -> *mut Context { + Box::into_raw(Box::new(Context::new(fields_len))) } /// Deallocate the context object. @@ -409,14 +409,11 @@ pub unsafe extern "C" fn context_free(context: *mut Context) { #[no_mangle] pub unsafe extern "C" fn context_add_value( context: &mut Context, - field: *const i8, + index: usize, value: &CValue, errbuf: *mut u8, errbuf_len: *mut usize, ) -> bool { - let field = ffi::CStr::from_ptr(field as *const c_char) - .to_str() - .unwrap(); let errbuf = from_raw_parts_mut(errbuf, ERR_BUF_MAX_LEN); let value: Result = value.try_into(); @@ -427,7 +424,7 @@ pub unsafe extern "C" fn context_add_value( return false; } - context.add_value(field, value.unwrap()); + context.add_value(index, value.unwrap()); true } diff --git a/src/interpreter.rs b/src/interpreter.rs index 2031fe87..5c092cc5 100644 --- a/src/interpreter.rs +++ b/src/interpreter.rs @@ -7,7 +7,7 @@ pub trait Execute { impl Execute for Predicate { fn execute(&self, ctx: &mut Context, m: &mut Match) -> bool { - let lhs_values = match ctx.value_of(&self.lhs.var_name) { + let lhs_values = match ctx.value_of(self.lhs.index) { None => return false, Some(v) => v, }; @@ -266,13 +266,15 @@ fn test_predicate() { let mut mat = Match::new(); let mut schema = schema::Schema::default(); schema.add_field("my_key", ast::Type::String); - let mut ctx = Context::new(&schema); + let mut ctx = Context::new(1); + let field_index: usize = 0; // check when value list is empty // check if all values match starts_with foo -- should be false let p = Predicate { lhs: ast::Lhs { var_name: "my_key".to_string(), + index: field_index, transformations: vec![], }, rhs: Value::String("foo".to_string()), @@ -285,6 +287,7 @@ fn test_predicate() { let p = Predicate { lhs: ast::Lhs { var_name: "my_key".to_string(), + index: field_index, transformations: vec![], }, rhs: Value::String("foo".to_string()), @@ -300,15 +303,16 @@ fn test_predicate() { Value::String("foocar".to_string()), Value::String("fooban".to_string()), ]; - + for v in lhs_values { - ctx.add_value("my_key", v); + ctx.add_value(field_index, v); } // check if all values match starts_with foo -- should be true let p = Predicate { lhs: ast::Lhs { var_name: "my_key".to_string(), + index: field_index, transformations: vec![], }, rhs: Value::String("foo".to_string()), @@ -321,6 +325,7 @@ fn test_predicate() { let p = Predicate { lhs: ast::Lhs { var_name: "my_key".to_string(), + index: field_index, transformations: vec![], }, rhs: Value::String("foo".to_string()), @@ -333,6 +338,7 @@ fn test_predicate() { let p = Predicate { lhs: ast::Lhs { var_name: "my_key".to_string(), + index: field_index, transformations: vec![ast::LhsTransformations::Any], }, rhs: Value::String("foo".to_string()), @@ -345,6 +351,7 @@ fn test_predicate() { let p = Predicate { lhs: ast::Lhs { var_name: "my_key".to_string(), + index: field_index, transformations: vec![ast::LhsTransformations::Any], }, rhs: Value::String("foo".to_string()), @@ -357,6 +364,7 @@ fn test_predicate() { let p = Predicate { lhs: ast::Lhs { var_name: "my_key".to_string(), + index: field_index, transformations: vec![ast::LhsTransformations::Any], }, rhs: Value::String("nar".to_string()), @@ -369,6 +377,7 @@ fn test_predicate() { let p = Predicate { lhs: ast::Lhs { var_name: "my_key".to_string(), + index: field_index, transformations: vec![ast::LhsTransformations::Any], }, rhs: Value::String("".to_string()), @@ -381,6 +390,7 @@ fn test_predicate() { let p = Predicate { lhs: ast::Lhs { var_name: "my_key".to_string(), + index: field_index, transformations: vec![ast::LhsTransformations::Any], }, rhs: Value::String("".to_string()), @@ -393,6 +403,7 @@ fn test_predicate() { let p = Predicate { lhs: ast::Lhs { var_name: "my_key".to_string(), + index: field_index, transformations: vec![ast::LhsTransformations::Any], }, rhs: Value::String("ob".to_string()), @@ -405,6 +416,7 @@ fn test_predicate() { let p = Predicate { lhs: ast::Lhs { var_name: "my_key".to_string(), + index: field_index, transformations: vec![ast::LhsTransformations::Any], }, rhs: Value::String("ok".to_string()), diff --git a/src/parser.rs b/src/parser.rs index 6d59d860..0e3a6986 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -88,6 +88,7 @@ fn parse_lhs(pair: Pair) -> ParseResult { let var = parse_ident(pair)?; Lhs { var_name: var, + index: Default::default(), transformations: Vec::new(), } } diff --git a/src/router.rs b/src/router.rs index 2c94aaa2..0849c87c 100644 --- a/src/router.rs +++ b/src/router.rs @@ -12,8 +12,9 @@ struct MatcherKey(usize, Uuid); pub struct Router<'a> { schema: &'a Schema, - matchers: BTreeMap, - pub fields: HashMap, + matchers: BTreeMap, + pub fields: Vec<(String, usize)>, // fileds array of tuple(name, count) + pub fields_map: HashMap, // field name -> index map } impl<'a> Router<'a> { @@ -21,7 +22,8 @@ impl<'a> Router<'a> { Self { schema, matchers: BTreeMap::new(), - fields: HashMap::new(), + fields: Vec::new(), + fields_map: HashMap::new(), } } @@ -31,12 +33,13 @@ impl<'a> Router<'a> { if self.matchers.contains_key(&key) { return Err("UUID already exists".to_string()); } + // lhs's index maybe changed in `ast.add_to_counter` + let mut ast = parse(atc).map_err(|e| e.to_string())?; - let ast = parse(atc).map_err(|e| e.to_string())?; ast.validate(self.schema)?; - let cir = ast.translate(); - cir.add_to_counter(&mut self.fields); - assert!(self.matchers.insert(key, cir).is_none()); + ast.add_to_counter(&mut self.fields, &mut self.fields_map); + + assert!(self.matchers.insert(key, ast).is_none()); Ok(()) } @@ -44,14 +47,25 @@ impl<'a> Router<'a> { pub fn remove_matcher(&mut self, priority: usize, uuid: Uuid) -> bool { let key = MatcherKey(priority, uuid); - if let Some(cir) = self.matchers.remove(&key) { - cir.remove_from_counter(&mut self.fields); + if let Some(mut ast) = self.matchers.remove(&key) { + let fields_cnt = self.fields.len(); + ast.remove_from_counter(&mut self.fields, &mut self.fields_map); + // if fields array changed, we need to reindex lhs in matchers + if self.fields.len() != fields_cnt { + self.reindexing_matchers(); + } return true; } false } + pub fn reindexing_matchers(&mut self) { + for (_, m) in self.matchers.iter_mut() { + m.fix_lhs_index(&self.fields_map); + } + } + pub fn execute(&self, context: &mut Context) -> bool { for (MatcherKey(_, id), m) in self.matchers.iter().rev() { let mut mat = Match::new(); @@ -66,3 +80,116 @@ impl<'a> Router<'a> { false } } + +#[cfg(test)] +mod tests { + use std::{collections::HashMap, net::{IpAddr, Ipv4Addr}}; + + use uuid::Uuid; + + use crate::{ast::{Expression, LogicalExpression, Type, Value}, context::Context, router::Router, schema::Schema}; + + type FieldsType = Vec<(String, usize)>; + type ContextValues<'a> = HashMap<&'a str, Value>; + + fn setup_matcher(r: &mut Router, priority: usize, expression: &str) -> (Uuid, usize) { + let id = Uuid::new_v4(); + r.add_matcher(priority, id, expression).ok().expect("failed to addd matcher"); + (id, priority) + } + + fn init_context(fields: &FieldsType, ctx_values: &ContextValues) -> Context { + let mut ctx = Context::new(fields.len()); + for (i, v) in fields.iter().enumerate() { + let key = &v.0; + if ctx_values.contains_key(key.as_str()) { + ctx.add_value(i, ctx_values.get(key.as_str()).unwrap().clone()); + } + } + ctx + } + + fn is_index_match(e: &Expression, rt: &Router) -> bool { + match e { + Expression::Logical(l) => match l.as_ref() { + LogicalExpression::And(l, r) | LogicalExpression::Or(l, r) => { + is_index_match(l, rt) && is_index_match(r, rt) + } + LogicalExpression::Not(r) => { + is_index_match(r, rt) + } + } + Expression::Predicate(p) => { + rt.fields[p.lhs.index].0 == p.lhs.var_name && *rt.fields_map.get(&p.lhs.var_name).unwrap() == p.lhs.index + } + } + } + + fn validate_index(r: &Router) -> bool { + for (_, e) in r.matchers.iter() { + if !is_index_match(e, r) { + return false; + } + } + true + } + + #[test] + fn test_router_execution() { + // init schema + let mut s = Schema::default(); + s.add_field("http.host", Type::String); + s.add_field("net.dst.port", Type::Int); + s.add_field("net.src.ip", Type::IpAddr); + + // init router + let mut r = Router::new(&s); + assert!(r.fields.len() == 0); + assert!(validate_index(&r)); + + // add matchers + let (id_0, pri_0) = setup_matcher(&mut r, 99, r#"http.host == "example.com""#); + let (id_1, pri_1) = setup_matcher(&mut r, 98, r#"net.dst.port == 8443 || net.dst.port == 443"#); + let (id_2, pri_2) = setup_matcher(&mut r, 97, r#"net.src.ip == 192.168.1.1"#); + assert!(r.fields.len() == 3); + assert!(validate_index(&r)); + + // mock context values + let mut ctx_values = HashMap::from([ + ("http.host", Value::String("example.com".to_string())), + ("net.src.ip", Value::IpAddr(IpAddr::V4(Ipv4Addr::new(192,168,1,2)))) + ]); + let mut ctx = init_context(&r.fields, &ctx_values); + + // match the first matcher + let res = r.execute(&mut ctx); + assert!(res); + + // delete matcher, no field match now + r.remove_matcher(pri_0, id_0); + assert!(r.fields.len() == 2); + assert!(validate_index(&r)); + ctx = init_context(&r.fields, &ctx_values); + assert!(!r.execute(&mut ctx)); + + // context value change, match again + *ctx_values.get_mut("net.src.ip").unwrap() = Value::IpAddr(IpAddr::V4(Ipv4Addr::new(192,168,1,1))); + ctx = init_context(&r.fields, &ctx_values); + assert!(r.execute(&mut ctx)); + + // delete all matchers + r.remove_matcher(pri_1, id_1); + r.remove_matcher(pri_2, id_2); + assert!(r.fields.len() == 0); + assert!(validate_index(&r)); + ctx = init_context(&r.fields, &ctx_values); + assert!(!r.execute(&mut ctx)); + + // add a new matcher + let (_, _) = setup_matcher(&mut r, 96, r#"net.src.ip == 192.168.1.1"#); + assert!(r.fields.len() == 1); + assert!(validate_index(&r)); + ctx = init_context(&r.fields, &ctx_values); + assert!(r.execute(&mut ctx)); + } +} diff --git a/src/semantics.rs b/src/semantics.rs index 9c4dc5b8..2ff00467 100644 --- a/src/semantics.rs +++ b/src/semantics.rs @@ -9,8 +9,100 @@ pub trait Validate { } pub trait FieldCounter { - fn add_to_counter(&self, map: &mut HashMap); - fn remove_from_counter(&self, map: &mut HashMap); + fn add_to_counter(&mut self, fields: &mut Vec<(String, usize)>, map: &mut HashMap); + fn remove_from_counter(&mut self, fields: &mut Vec<(String, usize)>, map: &mut HashMap); + fn fix_lhs_index(&mut self, map: &HashMap); +} + +impl FieldCounter for Expression { + fn add_to_counter(&mut self, fields: &mut Vec<(String, usize)>, map: &mut HashMap) { + match self { + Expression::Logical(l) => match l.as_mut() { + LogicalExpression::And(l, r) => { + l.add_to_counter(fields, map); + r.add_to_counter(fields, map); + } + LogicalExpression::Or(l, r) => { + l.add_to_counter(fields,map); + r.add_to_counter(fields, map); + } + LogicalExpression::Not(r) => { + r.add_to_counter(fields, map); + } + }, + Expression::Predicate(p) => { + // 1. fields: increment counter for field + // 2. lhs: assign field index to the LHS + // 3. map: maintain the fields map: {field_name : field_index} + if let Some(index) = map.get(&p.lhs.var_name) { + fields[*index].1 += 1; + p.lhs.index = *index; + } else { + fields.push((p.lhs.var_name.clone(), 1)); + map.insert(p.lhs.var_name.clone(), fields.len()-1); + p.lhs.index = fields.len()-1; + } + } + } + } + + fn remove_from_counter(&mut self, fields: &mut Vec<(String, usize)>, map: &mut HashMap) { + match self { + Expression::Logical(l) => match l.as_mut() { + LogicalExpression::And(l, r) => { + l.remove_from_counter(fields, map); + r.remove_from_counter(fields, map); + } + LogicalExpression::Or(l, r) => { + l.remove_from_counter(fields, map); + r.remove_from_counter(fields, map); + } + LogicalExpression::Not(r) => { + r.remove_from_counter(fields, map); + } + }, + Expression::Predicate(p) => { + // field must be in hashmap and fields array + let index: usize = *map.get(&p.lhs.var_name).unwrap(); + // 1. decrement counter of field + // 2. for field removing, swap another field to fill the slot. + // 3. re-assign index in fields_map for swapped field in array. + fields[index].1 -= 1; + if fields[index].1 == 0 { + fields.swap_remove(index); + assert!(map.remove(&p.lhs.var_name).is_some()); + // for field who been swapped to fill the empty slot in fields array, + // we need to fix its index in fields_map + if index < fields.len() { + *map.get_mut(&fields[index].0).unwrap() = index; + } + } + } + } + } + + // this may run only after remove_from_counter, since existing field in field array + // may change its postion after that. run this to fix lhs's field index. + fn fix_lhs_index(&mut self, map: &HashMap) { + match self { + Expression::Logical(l) => match l.as_mut() { + LogicalExpression::And(l, r) => { + l.fix_lhs_index(map); + r.fix_lhs_index(map); + } + LogicalExpression::Or(l, r) => { + l.fix_lhs_index(map); + r.fix_lhs_index(map); + } + LogicalExpression::Not(r) => { + r.fix_lhs_index(map); + } + }, + Expression::Predicate(p) => { + p.lhs.index = *map.get(&p.lhs.var_name).unwrap(); + } + } + } } impl Validate for Expression { diff --git a/t/01-sanity.t b/t/01-sanity.t index 70f007f9..5eb06abb 100644 --- a/t/01-sanity.t +++ b/t/01-sanity.t @@ -39,9 +39,9 @@ __DATA__ assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c", "http.path ^= \"/foo\" && tcp.port == 80")) - local c = context.new(s) - c:add_value("http.path", "/foo/bar") - c:add_value("tcp.port", 80) + local c = context.new(s, #r:get_fields()) + c:add_value(1, "http.path", "/foo/bar") + c:add_value(2, "tcp.port", 80) local matched = r:execute(c) ngx.say(matched) @@ -84,9 +84,9 @@ a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-1aa5583d150d", "http.path ^= \"/\"")) - local c = context.new(s) - c:add_value("http.path", "/foo/bar") - c:add_value("tcp.port", 80) + local c = context.new(s, #r:get_fields()) + c:add_value(1, "http.path", "/foo/bar") + c:add_value(2, "tcp.port", 80) local matched = r:execute(c) ngx.say(matched) @@ -126,9 +126,9 @@ uuid = a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c prefix = /foo assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c", "http.path ^= \"/foo\" && tcp.port == 80")) - local c = context.new(s) - c:add_value("http.path", "/foo/bar") - c:add_value("tcp.port", 80) + local c = context.new(s, #r:get_fields()) + c:add_value(1, "http.path", "/foo/bar") + c:add_value(2, "tcp.port", 80) local matched = r:execute(c) ngx.say(matched) @@ -138,10 +138,11 @@ uuid = a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c prefix = /foo ngx.say(prefix) assert(r:remove_matcher("a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c")) - - c = context.new(s) - c:add_value("http.path", "/foo/bar") - c:add_value("tcp.port", 80) + assert(#r:get_fields() == 0) + + c = context.new(s, #r:get_fields()) + c:add_value(1, "http.path", "/foo/bar") + c:add_value(2, "tcp.port", 80) matched = r:execute(c) ngx.say(matched) @@ -214,9 +215,9 @@ nil --> 1:11 assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c", "http.path ^= \"/foo\" && tcp.port == 80")) - local c = context.new(s) - c:add_value("http.path", "/foo/bar") - c:add_value("tcp.port", 80) + local c = context.new(s, #r:get_fields()) + c:add_value(1, "http.path", "/foo/bar") + c:add_value(2, "tcp.port", 80) local matched = r:execute(c) ngx.say(matched) diff --git a/t/02-bugs.t b/t/02-bugs.t index c10aae45..239db197 100644 --- a/t/02-bugs.t +++ b/t/02-bugs.t @@ -39,9 +39,9 @@ __DATA__ "\xfc\x80\x80\x80\x80\xaf", } - local c = context.new(s) + local c = context.new(s, #r:get_fields()) for _, v in ipairs(BAD_UTF8) do - local ok, err = c:add_value("http.path", v) + local ok, err = c:add_value(1, "http.path", v) ngx.say(err) end } @@ -71,8 +71,8 @@ invalid utf-8 sequence of 1 bytes from index 0 s:add_field("http.path", "String") - local c = context.new(s) - assert(c:add_value("http.path", "\x00")) + local c = context.new(s, #r:get_fields()) + assert(c:add_value(1, "http.path", "\x00")) ngx.say("ok") } } @@ -146,8 +146,8 @@ ok assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c", "http.body =^ \"world\"")) - local c = context.new(s) - c:add_value("http.body", "hello\x00world") + local c = context.new(s, #r:get_fields()) + c:add_value(1, "http.body", "hello\x00world") local matched = r:execute(c) ngx.say(matched) @@ -156,7 +156,7 @@ ok ngx.say(uuid) c:reset() - c:add_value("http.body", "world\x00hello") + c:add_value(1, "http.body", "world\x00hello") local matched = r:execute(c) ngx.say(matched) diff --git a/t/03-contains.t b/t/03-contains.t index 88e00be3..49d34816 100644 --- a/t/03-contains.t +++ b/t/03-contains.t @@ -39,9 +39,9 @@ __DATA__ assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c", "http.path contains \"keyword\" && tcp.port == 80")) - local c = context.new(s) - c:add_value("http.path", "/foo/keyword/bar") - c:add_value("tcp.port", 80) + local c = context.new(s, #r:get_fields()) + c:add_value(1, "http.path", "/foo/keyword/bar") + c:add_value(2, "tcp.port", 80) local matched = r:execute(c) ngx.say(matched) @@ -83,9 +83,9 @@ nil assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c", "http.path contains \"keyword\" && tcp.port == 80")) - local c = context.new(s) - c:add_value("http.path", "/foo/bar") - c:add_value("tcp.port", 80) + local c = context.new(s, #r:get_fields()) + c:add_value(1, "http.path", "/foo/bar") + c:add_value(2, "tcp.port", 80) local matched = r:execute(c) ngx.say(matched) diff --git a/t/04-rawstr.t b/t/04-rawstr.t index 80cc1d31..89405bb9 100644 --- a/t/04-rawstr.t +++ b/t/04-rawstr.t @@ -39,7 +39,7 @@ __DATA__ assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c", "http.path ^= r#\"/foo\"# && tcp.port == 80")) - local c = context.new(s) + local c = context.new(s, #r:get_fields()) c:add_value("http.path", "/foo/bar") c:add_value("tcp.port", 80) @@ -82,7 +82,7 @@ a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c", "http.path ^= r#\"/foo\"\'\"# && tcp.port == 80")) - local c = context.new(s) + local c = context.new(s, #r:get_fields()) c:add_value("http.path", "/foo\"\'/bar") c:add_value("tcp.port", 80) @@ -126,7 +126,7 @@ a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c", "http.path ~ r#\"^/\\d+/test$\"# && tcp.port == 80")) - local c = context.new(s) + local c = context.new(s, #r:get_fields()) c:add_value("http.path", "/123/test") c:add_value("tcp.port", 80) @@ -169,9 +169,9 @@ a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c", "http.path ~ r#\"^/\\D+/test$\"# && tcp.port == 80")) - local c = context.new(s) - c:add_value("http.path", "/123/test") - c:add_value("tcp.port", 80) + local c = context.new(s, #r:get_fields()) + c:add_value(1, "http.path", "/123/test") + c:add_value(2, "tcp.port", 80) local matched = r:execute(c) ngx.say(matched) diff --git a/t/05-equals.t b/t/05-equals.t index c41de0a8..9d6f5f9f 100644 --- a/t/05-equals.t +++ b/t/05-equals.t @@ -38,10 +38,10 @@ __DATA__ assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c", "http.headers.foo == \"bar\"")) - local c = context.new(s) - c:add_value("http.headers.foo", "bar") - c:add_value("http.headers.foo", "bar") - c:add_value("http.headers.foo", "bar") + local c = context.new(s, #r:get_fields()) + c:add_value(1, "http.headers.foo", "bar") + c:add_value(1, "http.headers.foo", "bar") + c:add_value(1, "http.headers.foo", "bar") local matched = r:execute(c) ngx.say(matched) @@ -82,10 +82,10 @@ bar assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c", "http.headers.foo == \"bar\"")) - local c = context.new(s) - c:add_value("http.headers.foo", "bar") - c:add_value("http.headers.foo", "bar") - c:add_value("http.headers.foo", "barX") + local c = context.new(s, #r:get_fields()) + c:add_value(1, "http.headers.foo", "bar") + c:add_value(1, "http.headers.foo", "bar") + c:add_value(1, "http.headers.foo", "barX") local matched = r:execute(c) ngx.say(matched) @@ -124,7 +124,7 @@ nil assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c", "http.headers.foo == \"bar\"")) - local c = context.new(s) + local c = context.new(s, #r:get_fields()) local matched = r:execute(c) ngx.say(matched) diff --git a/t/07-in_notin.t b/t/07-in_notin.t index d925072c..db4b0e72 100644 --- a/t/07-in_notin.t +++ b/t/07-in_notin.t @@ -76,14 +76,14 @@ nilIn/NotIn operators only supports IP in CIDR assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c", "l3.ip in 192.168.12.0/24")) - local c = context.new(s) - c:add_value("l3.ip", "192.168.12.1") + local c = context.new(s, #r:get_fields()) + c:add_value(1, "l3.ip", "192.168.12.1") local matched = r:execute(c) ngx.say(matched) - c = context.new(s) - c:add_value("l3.ip", "192.168.1.1") + c = context.new(s, #r:get_fields()) + c:add_value(1, "l3.ip", "192.168.1.1") local matched = r:execute(c) ngx.say(matched) diff --git a/t/08-equals.t b/t/08-equals.t index 3f8090a6..83dd41cf 100644 --- a/t/08-equals.t +++ b/t/08-equals.t @@ -40,15 +40,15 @@ __DATA__ assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-8aa5583d150c", "net.port != 8000")) - local c = context.new(s) - c:add_value("net.port", 8000) + local c = context.new(s, #r:get_fields()) + c:add_value(1, "net.port", 8000) local matched = r:execute(c) ngx.say(matched) ngx.say(c:get_result()) c = context.new(s) - c:add_value("net.port", 8001) + c:add_value(1, "net.port", 8001) matched = r:execute(c) ngx.say(matched) @@ -88,15 +88,15 @@ a921a9aa-ec0e-4cf3-a6cc-8aa5583d150cnilnil assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-8aa5583d150c", "http.path != \"/foo\"")) - local c = context.new(s) - c:add_value("http.path", "/foo") + local c = context.new(s, #r:get_fields()) + c:add_value(1, "http.path", "/foo") local matched = r:execute(c) ngx.say(matched) ngx.say(c:get_result()) c = context.new(s) - c:add_value("http.path", "/foo1") + c:add_value(1, "http.path", "/foo1") matched = r:execute(c) ngx.say(matched) @@ -136,15 +136,15 @@ a921a9aa-ec0e-4cf3-a6cc-8aa5583d150cnilnil assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-8aa5583d150c", "net.ip != 192.168.1.1")) - local c = context.new(s) - c:add_value("net.ip", "192.168.1.1") + local c = context.new(s, #r:get_fields()) + c:add_value(1, "net.ip", "192.168.1.1") local matched = r:execute(c) ngx.say(matched) ngx.say(c:get_result()) - c = context.new(s) - c:add_value("net.ip", "192.168.1.2") + c = context.new(s, #r:get_fields()) + c:add_value(1, "net.ip", "192.168.1.2") matched = r:execute(c) ngx.say(matched) diff --git a/t/09-not.t b/t/09-not.t index 518ebbdd..3e4432c8 100644 --- a/t/09-not.t +++ b/t/09-not.t @@ -38,15 +38,15 @@ __DATA__ assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c", [[!(http.path ^= "/abc")]])) - local c = context.new(s) - c:add_value("http.path", "/abc/d") + local c = context.new(s, #r:get_fields()) + c:add_value(1, "http.path", "/abc/d") local matched = r:execute(c) ngx.say(matched) c:reset() - c:add_value("http.path", "/abb/d") + c:add_value(1, "http.path", "/abb/d") local matched = r:execute(c) ngx.say(matched) @@ -57,13 +57,13 @@ __DATA__ c:reset() - c:add_value("http.path", "/abb/d/") + c:add_value(1, "http.path", "/abb/d/") local matched = r:execute(c) ngx.say(matched) c:reset() - c:add_value("http.path", "/abb/d") + c:add_value(1, "http.path", "/abb/d") local matched = r:execute(c) ngx.say(matched) } From decb148ca8d899ae27046a432c08ab07141c1f2d Mon Sep 17 00:00:00 2001 From: ProBrian Date: Wed, 4 Sep 2024 17:04:11 +0800 Subject: [PATCH 02/16] fix lua test case --- t/02-bugs.t | 4 ++-- t/08-equals.t | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/t/02-bugs.t b/t/02-bugs.t index 239db197..7bbf97dc 100644 --- a/t/02-bugs.t +++ b/t/02-bugs.t @@ -39,7 +39,7 @@ __DATA__ "\xfc\x80\x80\x80\x80\xaf", } - local c = context.new(s, #r:get_fields()) + local c = context.new(s, 1) for _, v in ipairs(BAD_UTF8) do local ok, err = c:add_value(1, "http.path", v) ngx.say(err) @@ -71,7 +71,7 @@ invalid utf-8 sequence of 1 bytes from index 0 s:add_field("http.path", "String") - local c = context.new(s, #r:get_fields()) + local c = context.new(s, 1) assert(c:add_value(1, "http.path", "\x00")) ngx.say("ok") } diff --git a/t/08-equals.t b/t/08-equals.t index 83dd41cf..9f430999 100644 --- a/t/08-equals.t +++ b/t/08-equals.t @@ -47,7 +47,7 @@ __DATA__ ngx.say(matched) ngx.say(c:get_result()) - c = context.new(s) + c = context.new(s, #r:get_fields()) c:add_value(1, "net.port", 8001) matched = r:execute(c) @@ -95,7 +95,7 @@ a921a9aa-ec0e-4cf3-a6cc-8aa5583d150cnilnil ngx.say(matched) ngx.say(c:get_result()) - c = context.new(s) + c = context.new(s, #r:get_fields()) c:add_value(1, "http.path", "/foo1") matched = r:execute(c) From 6c262d9ced6073eeb8cb866226488431baf3b461 Mon Sep 17 00:00:00 2001 From: ProBrian Date: Thu, 5 Sep 2024 09:40:14 +0800 Subject: [PATCH 03/16] format code --- benches/atc_benchmark.rs | 55 +++++++++++++++++++++++++++------------- src/context.rs | 36 +++++++++++++++++++------- src/interpreter.rs | 2 +- src/router.rs | 46 +++++++++++++++++++++------------ src/semantics.rs | 32 +++++++++++++++++------ 5 files changed, 119 insertions(+), 52 deletions(-) diff --git a/benches/atc_benchmark.rs b/benches/atc_benchmark.rs index 8f1fb9ba..efc74430 100644 --- a/benches/atc_benchmark.rs +++ b/benches/atc_benchmark.rs @@ -1,26 +1,32 @@ -use std::{hint::black_box, str::FromStr}; +use atc_router::{ + ast::Type, + ast::Value, + context::Context, + router::Router, + schema::{self, Schema}, +}; use criterion::{criterion_group, criterion_main, Criterion}; -use atc_router::{ast::Type, ast::Value, context::Context, router::Router, schema::{self, Schema}}; -use uuid::Uuid; -use std::net::{IpAddr, Ipv4Addr}; -use serde_json; -use std::fs; use serde::{Deserialize, Serialize}; +use serde_json; use std::env; - +use std::fs; +use std::net::{IpAddr, Ipv4Addr}; +use std::{hint::black_box, str::FromStr}; +use uuid::Uuid; #[derive(Serialize, Deserialize)] struct TestData { rules: Vec, match_keys: Vec, match_values: Vec, - not_match_values: Vec + not_match_values: Vec, } // prepare match rules, context keys, context values from data.json file fn prepare_data() -> TestData { let cwd = env::current_dir().unwrap(); - let file_str = fs::read_to_string(cwd.join("benches/data.json")).expect("unable to open data.json"); + let file_str = + fs::read_to_string(cwd.join("benches/data.json")).expect("unable to open data.json"); serde_json::from_str(&file_str).unwrap() } @@ -52,19 +58,28 @@ fn setup_matchers(r: &mut Router, data: &TestData) { // mock contexts with field values passed in from json data fn setup_context(ctx: &mut Context, data: &TestData, test_match: bool) { - let values = if test_match {&data.match_values} else {&data.not_match_values}; + let values = if test_match { + &data.match_values + } else { + &data.not_match_values + }; for (i, v) in values.iter().enumerate() { match v { serde_json::Value::String(s) => { ctx.add_value(i, Value::String(s.to_string())); - }, + } serde_json::Value::Number(n) => { ctx.add_value(i, Value::Int(n.as_i64().unwrap())); - }, + } serde_json::Value::Array(l) => { - ctx.add_value(i, Value::IpAddr(IpAddr::V4(Ipv4Addr::from_str(l[0].as_str().unwrap()).unwrap()))); - }, - _ => panic!("incorrect data type") + ctx.add_value( + i, + Value::IpAddr(IpAddr::V4( + Ipv4Addr::from_str(l[0].as_str().unwrap()).unwrap(), + )), + ); + } + _ => panic!("incorrect data type"), } } } @@ -81,12 +96,16 @@ fn criterion_benchmark(c: &mut Criterion) { let mut ctx = Context::new(r.fields.len()); setup_context(&mut ctx, &data, true); - c.bench_function("route match all", |b| b.iter(|| router_match(black_box(&r), black_box(&mut ctx)))); + c.bench_function("route match all", |b| { + b.iter(|| router_match(black_box(&r), black_box(&mut ctx))) + }); let mut ctx = Context::new(r.fields.len()); setup_context(&mut ctx, &data, false); - c.bench_function("route mismatch all", |b| b.iter(|| router_match(black_box(&r), black_box(&mut ctx)))); + c.bench_function("route mismatch all", |b| { + b.iter(|| router_match(black_box(&r), black_box(&mut ctx))) + }); } criterion_group!(benches, criterion_benchmark); -criterion_main!(benches); \ No newline at end of file +criterion_main!(benches); diff --git a/src/context.rs b/src/context.rs index fc2d464d..98e3b5cf 100644 --- a/src/context.rs +++ b/src/context.rs @@ -30,7 +30,7 @@ pub struct Context { } impl Context { - pub fn new(fields_cnt: usize) -> Self { + pub fn new(fields_cnt: usize) -> Self { Context { values: vec![None; fields_cnt], result: None, @@ -39,18 +39,26 @@ impl Context { pub fn add_value(&mut self, index: usize, value: Value) { if index >= self.values.len() { - panic!("value provided does not match schema: index {}, max fields count {}", index, self.values.len()); + panic!( + "value provided does not match schema: index {}, max fields count {}", + index, + self.values.len() + ); } if let Some(v) = &mut self.values[index] { v.push(value); } else { - self.values[index] = Some(vec![value]); + self.values[index] = Some(vec![value]); } } pub fn value_of(&self, index: usize) -> Option<&[Value]> { - if !self.values.is_empty() && self.values[index].is_some() { Some(self.values[index].as_ref().unwrap().as_slice()) } else {None} + if !self.values.is_empty() && self.values[index].is_some() { + Some(self.values[index].as_ref().unwrap().as_slice()) + } else { + None + } } pub fn reset(&mut self) { @@ -64,8 +72,8 @@ impl Context { #[cfg(test)] mod tests { - use crate::context::Context; use crate::ast::Value; + use crate::context::Context; #[test] fn test_context() { let fields_cnt = 3; @@ -74,14 +82,17 @@ mod tests { assert_eq!(ctx.values, vec![None; fields_cnt]); // access value with out of bound index assert_eq!(ctx.value_of(0), None); - + // add value in bound ctx.add_value(1, Value::String("foo".to_string())); assert_eq!(ctx.value_of(0), None); assert_eq!(ctx.value_of(1).unwrap().len(), 1); - assert_eq!(ctx.value_of(1).unwrap(), vec![Value::String("foo".to_string())].as_slice()); + assert_eq!( + ctx.value_of(1).unwrap(), + vec![Value::String("foo".to_string())].as_slice() + ); - // reset context keeps values capacity with all None + // reset context keeps values capacity with all None ctx.reset(); assert!(ctx.values.len() == fields_cnt); assert_eq!(ctx.values, vec![None; fields_cnt]); @@ -91,6 +102,13 @@ mod tests { ctx.add_value(0, Value::String("foo".to_string())); assert!(ctx.values.len() == fields_cnt); assert_eq!(ctx.value_of(0).unwrap().len(), 2); - assert_eq!(ctx.value_of(0).unwrap(), vec![Value::String("bar".to_string()), Value::String("foo".to_string())].as_slice()); + assert_eq!( + ctx.value_of(0).unwrap(), + vec![ + Value::String("bar".to_string()), + Value::String("foo".to_string()) + ] + .as_slice() + ); } } diff --git a/src/interpreter.rs b/src/interpreter.rs index 5c092cc5..63f457bb 100644 --- a/src/interpreter.rs +++ b/src/interpreter.rs @@ -303,7 +303,7 @@ fn test_predicate() { Value::String("foocar".to_string()), Value::String("fooban".to_string()), ]; - + for v in lhs_values { ctx.add_value(field_index, v); } diff --git a/src/router.rs b/src/router.rs index 0849c87c..7532aa13 100644 --- a/src/router.rs +++ b/src/router.rs @@ -63,7 +63,7 @@ impl<'a> Router<'a> { pub fn reindexing_matchers(&mut self) { for (_, m) in self.matchers.iter_mut() { m.fix_lhs_index(&self.fields_map); - } + } } pub fn execute(&self, context: &mut Context) -> bool { @@ -83,18 +83,28 @@ impl<'a> Router<'a> { #[cfg(test)] mod tests { - use std::{collections::HashMap, net::{IpAddr, Ipv4Addr}}; + use std::{ + collections::HashMap, + net::{IpAddr, Ipv4Addr}, + }; use uuid::Uuid; - use crate::{ast::{Expression, LogicalExpression, Type, Value}, context::Context, router::Router, schema::Schema}; + use crate::{ + ast::{Expression, LogicalExpression, Type, Value}, + context::Context, + router::Router, + schema::Schema, + }; type FieldsType = Vec<(String, usize)>; type ContextValues<'a> = HashMap<&'a str, Value>; fn setup_matcher(r: &mut Router, priority: usize, expression: &str) -> (Uuid, usize) { let id = Uuid::new_v4(); - r.add_matcher(priority, id, expression).ok().expect("failed to addd matcher"); + r.add_matcher(priority, id, expression) + .ok() + .expect("failed to addd matcher"); (id, priority) } @@ -115,12 +125,11 @@ mod tests { LogicalExpression::And(l, r) | LogicalExpression::Or(l, r) => { is_index_match(l, rt) && is_index_match(r, rt) } - LogicalExpression::Not(r) => { - is_index_match(r, rt) - } - } + LogicalExpression::Not(r) => is_index_match(r, rt), + }, Expression::Predicate(p) => { - rt.fields[p.lhs.index].0 == p.lhs.var_name && *rt.fields_map.get(&p.lhs.var_name).unwrap() == p.lhs.index + rt.fields[p.lhs.index].0 == p.lhs.var_name + && *rt.fields_map.get(&p.lhs.var_name).unwrap() == p.lhs.index } } } @@ -130,10 +139,10 @@ mod tests { if !is_index_match(e, r) { return false; } - } + } true } - + #[test] fn test_router_execution() { // init schema @@ -146,10 +155,11 @@ mod tests { let mut r = Router::new(&s); assert!(r.fields.len() == 0); assert!(validate_index(&r)); - + // add matchers let (id_0, pri_0) = setup_matcher(&mut r, 99, r#"http.host == "example.com""#); - let (id_1, pri_1) = setup_matcher(&mut r, 98, r#"net.dst.port == 8443 || net.dst.port == 443"#); + let (id_1, pri_1) = + setup_matcher(&mut r, 98, r#"net.dst.port == 8443 || net.dst.port == 443"#); let (id_2, pri_2) = setup_matcher(&mut r, 97, r#"net.src.ip == 192.168.1.1"#); assert!(r.fields.len() == 3); assert!(validate_index(&r)); @@ -157,10 +167,13 @@ mod tests { // mock context values let mut ctx_values = HashMap::from([ ("http.host", Value::String("example.com".to_string())), - ("net.src.ip", Value::IpAddr(IpAddr::V4(Ipv4Addr::new(192,168,1,2)))) + ( + "net.src.ip", + Value::IpAddr(IpAddr::V4(Ipv4Addr::new(192, 168, 1, 2))), + ), ]); let mut ctx = init_context(&r.fields, &ctx_values); - + // match the first matcher let res = r.execute(&mut ctx); assert!(res); @@ -173,7 +186,8 @@ mod tests { assert!(!r.execute(&mut ctx)); // context value change, match again - *ctx_values.get_mut("net.src.ip").unwrap() = Value::IpAddr(IpAddr::V4(Ipv4Addr::new(192,168,1,1))); + *ctx_values.get_mut("net.src.ip").unwrap() = + Value::IpAddr(IpAddr::V4(Ipv4Addr::new(192, 168, 1, 1))); ctx = init_context(&r.fields, &ctx_values); assert!(r.execute(&mut ctx)); diff --git a/src/semantics.rs b/src/semantics.rs index 2ff00467..1e7953c4 100644 --- a/src/semantics.rs +++ b/src/semantics.rs @@ -9,21 +9,33 @@ pub trait Validate { } pub trait FieldCounter { - fn add_to_counter(&mut self, fields: &mut Vec<(String, usize)>, map: &mut HashMap); - fn remove_from_counter(&mut self, fields: &mut Vec<(String, usize)>, map: &mut HashMap); + fn add_to_counter( + &mut self, + fields: &mut Vec<(String, usize)>, + map: &mut HashMap, + ); + fn remove_from_counter( + &mut self, + fields: &mut Vec<(String, usize)>, + map: &mut HashMap, + ); fn fix_lhs_index(&mut self, map: &HashMap); } impl FieldCounter for Expression { - fn add_to_counter(&mut self, fields: &mut Vec<(String, usize)>, map: &mut HashMap) { - match self { + fn add_to_counter( + &mut self, + fields: &mut Vec<(String, usize)>, + map: &mut HashMap, + ) { + match self { Expression::Logical(l) => match l.as_mut() { LogicalExpression::And(l, r) => { l.add_to_counter(fields, map); r.add_to_counter(fields, map); } LogicalExpression::Or(l, r) => { - l.add_to_counter(fields,map); + l.add_to_counter(fields, map); r.add_to_counter(fields, map); } LogicalExpression::Not(r) => { @@ -39,14 +51,18 @@ impl FieldCounter for Expression { p.lhs.index = *index; } else { fields.push((p.lhs.var_name.clone(), 1)); - map.insert(p.lhs.var_name.clone(), fields.len()-1); - p.lhs.index = fields.len()-1; + map.insert(p.lhs.var_name.clone(), fields.len() - 1); + p.lhs.index = fields.len() - 1; } } } } - fn remove_from_counter(&mut self, fields: &mut Vec<(String, usize)>, map: &mut HashMap) { + fn remove_from_counter( + &mut self, + fields: &mut Vec<(String, usize)>, + map: &mut HashMap, + ) { match self { Expression::Logical(l) => match l.as_mut() { LogicalExpression::And(l, r) => { From dae678a920ae32b0b4a1d346297a8748dfef945c Mon Sep 17 00:00:00 2001 From: ProBrian Date: Mon, 9 Sep 2024 14:46:39 +0800 Subject: [PATCH 04/16] keep add_value interface --- benches/atc_benchmark.rs | 38 +++++++--- benches/data.json | 8 +- lib/resty/router/cdefs.lua | 13 +++- lib/resty/router/context.lua | 34 ++++++--- lib/resty/router/router.lua | 9 +-- src/context.rs | 99 ++++++++++++++++++++++--- src/ffi.rs | 49 ++++++++++--- src/interpreter.rs | 12 ++- src/router.rs | 138 ++++++++++++++++++++++++++--------- src/semantics.rs | 110 +++++++++------------------- t/01-sanity.t | 61 ++++++++++++---- t/02-bugs.t | 20 +++-- t/03-contains.t | 12 +-- t/04-rawstr.t | 12 +-- t/05-equals.t | 18 ++--- t/07-in_notin.t | 8 +- t/08-equals.t | 24 +++--- t/09-not.t | 6 +- 18 files changed, 442 insertions(+), 229 deletions(-) diff --git a/benches/atc_benchmark.rs b/benches/atc_benchmark.rs index efc74430..11092247 100644 --- a/benches/atc_benchmark.rs +++ b/benches/atc_benchmark.rs @@ -66,13 +66,13 @@ fn setup_context(ctx: &mut Context, data: &TestData, test_match: bool) { for (i, v) in values.iter().enumerate() { match v { serde_json::Value::String(s) => { - ctx.add_value(i, Value::String(s.to_string())); + ctx.add_value_by_index(i, Value::String(s.to_string())); } serde_json::Value::Number(n) => { - ctx.add_value(i, Value::Int(n.as_i64().unwrap())); + ctx.add_value_by_index(i, Value::Int(n.as_i64().unwrap())); } serde_json::Value::Array(l) => { - ctx.add_value( + ctx.add_value_by_index( i, Value::IpAddr(IpAddr::V4( Ipv4Addr::from_str(l[0].as_str().unwrap()).unwrap(), @@ -84,8 +84,24 @@ fn setup_context(ctx: &mut Context, data: &TestData, test_match: bool) { } } -fn router_match(router: &Router, ctx: &mut Context) -> bool { - router.execute(ctx) +fn router_match(router: &Router, ctx: &mut Context, expected: bool) { + assert_eq!(router.execute(ctx), expected); +} + +fn matchers_batch_handling(s: &Schema) { + let mut r = Router::new(&s); + let pri_max = 10000; + let mut ids = vec![]; + for pri in 0..pri_max { + let id: Uuid = Uuid::new_v4(); + let exp = format!(r#"http.path.segments.{} == "/bar""#, pri.to_string()); + assert!(r.add_matcher(pri, id, exp.as_str()).is_ok()); + ids.push((pri, id)); + } + + for (pri, id) in ids { + assert!(r.remove_matcher(pri, id)); + } } fn criterion_benchmark(c: &mut Criterion) { @@ -94,16 +110,20 @@ fn criterion_benchmark(c: &mut Criterion) { let mut r = Router::new(&s); setup_matchers(&mut r, &data); - let mut ctx = Context::new(r.fields.len()); + let mut ctx = Context::new(&r); setup_context(&mut ctx, &data, true); c.bench_function("route match all", |b| { - b.iter(|| router_match(black_box(&r), black_box(&mut ctx))) + b.iter(|| router_match(black_box(&r), black_box(&mut ctx), black_box(true))) }); - let mut ctx = Context::new(r.fields.len()); + let mut ctx = Context::new(&r); setup_context(&mut ctx, &data, false); c.bench_function("route mismatch all", |b| { - b.iter(|| router_match(black_box(&r), black_box(&mut ctx))) + b.iter(|| router_match(black_box(&r), black_box(&mut ctx), black_box(false))) + }); + + c.bench_function("route matchers batch create and delete", |b| { + b.iter(|| matchers_batch_handling(black_box(&s))) }); } diff --git a/benches/data.json b/benches/data.json index 3798e1f3..2d43d68e 100644 --- a/benches/data.json +++ b/benches/data.json @@ -5,8 +5,8 @@ "http.method == \"GET\"", "http.host == \"example.com\"", "http.path == \"/foo\"", - "http.path.segments.1 == \"/bar\"", - "http.path.segments.0_1 == \"/foo/bar\"", + "http.path.segments.1 == \"bar\"", + "http.path.segments.0_1 == \"foo/bar\"", "http.path.segments.len == 2", "http.headers.foo_bar == \"whatever\"", "net.dst.port == 8443", @@ -32,8 +32,8 @@ "GET", "example.com", "/foo", - "/bar\"", - "/foo/bar\"", + "bar\"", + "foo/bar\"", 2, "whatever", 8443, diff --git a/lib/resty/router/cdefs.lua b/lib/resty/router/cdefs.lua index 94ae6e18..5d0824fb 100644 --- a/lib/resty/router/cdefs.lua +++ b/lib/resty/router/cdefs.lua @@ -68,18 +68,25 @@ bool router_execute(const struct Router *router, struct Context *context); uintptr_t router_get_fields(const struct Router *router, const uint8_t **fields, - uintptr_t *fields_len); + uintptr_t *fields_len, + uintptr_t *indexes); -struct Context *context_new(uintptr_t fields_len); +struct Context *context_new(const struct Router *router); void context_free(struct Context *context); bool context_add_value(struct Context *context, - uintptr_t field_idx, + const int8_t *field, const struct CValue *value, uint8_t *errbuf, uintptr_t *errbuf_len); +bool context_add_value_by_index(struct Context *context, + uintptr_t index, + const struct CValue *value, + uint8_t *errbuf, + uintptr_t *errbuf_len); + void context_reset(struct Context *context); intptr_t context_get_result(const struct Context *context, diff --git a/lib/resty/router/context.lua b/lib/resty/router/context.lua index 3508dc6b..5a8fc049 100644 --- a/lib/resty/router/context.lua +++ b/lib/resty/router/context.lua @@ -26,27 +26,22 @@ local clib = cdefs.clib local context_free = cdefs.context_free -function _M.new(schema, size) - local context = clib.context_new(size) +function _M.new(router) + local context = clib.context_new(router.router) local c = setmetatable({ context = ffi_gc(context, context_free), - schema = schema, - max_fields = size, + schema = router.schema, }, _MT) return c end -function _M:add_value(index, field, value) +function _M:add_value_impl(field, value, index) if not value then return true end - if index > self.max_fields then - return false, "context value index out of bound" - end - local typ, err = self.schema:get_field_type(field) if not typ then return nil, err @@ -68,9 +63,14 @@ function _M:add_value(index, field, value) local errbuf = get_string_buf(ERR_BUF_MAX_LEN) local errbuf_len = get_size_ptr() - errbuf_len[0] = ERR_BUF_MAX_LEN - - if clib.context_add_value(self.context, index-1, CACHED_VALUE, errbuf, errbuf_len) == false then + errbuf_len[0] = ERR_BUF_MAX_LEN + local res + if index ~= nil then + res = clib.context_add_value_by_index(self.context, index, CACHED_VALUE, errbuf, errbuf_len) + else + res = clib.context_add_value(self.context, field, CACHED_VALUE, errbuf, errbuf_len) + end + if res == false then return nil, ffi_string(errbuf, errbuf_len[0]) end @@ -78,6 +78,16 @@ function _M:add_value(index, field, value) end +function _M:add_value(field, value) + return _M.add_value_impl(self, field, value) +end + + +function _M:add_value_by_index(field, value, index) + return _M.add_value_impl(self, field, value, index) +end + + function _M:get_result(matched_field) local captures_len = tonumber(clib.context_get_result( self.context, nil, nil, nil, nil, nil, nil, nil, nil)) diff --git a/lib/resty/router/router.lua b/lib/resty/router/router.lua index a914a2db..19d522a5 100644 --- a/lib/resty/router/router.lua +++ b/lib/resty/router/router.lua @@ -78,20 +78,19 @@ function _M:get_fields() local out_n = 0 local router = self.router - local total = tonumber(clib.router_get_fields(router, nil, nil)) + local total = tonumber(clib.router_get_fields(router, nil, nil, nil)) if total == 0 then return out end local fields = ffi_new("const uint8_t *[?]", total) local fields_len = ffi_new("size_t [?]", total) + local indexes = ffi_new("size_t [?]", total) fields_len[0] = total - clib.router_get_fields(router, fields, fields_len) - + clib.router_get_fields(router, fields, fields_len, indexes) for i = 0, total - 1 do - out_n = out_n + 1 - out[out_n] = ffi_string(fields[i], fields_len[i]) + out[ffi_string(fields[i], fields_len[i])] = indexes[i] end return out diff --git a/src/context.rs b/src/context.rs index 98e3b5cf..68e14323 100644 --- a/src/context.rs +++ b/src/context.rs @@ -1,4 +1,4 @@ -use crate::ast::Value; +use crate::{ast::Value, router::Router}; use fnv::FnvHashMap; use uuid::Uuid; @@ -24,20 +24,34 @@ impl Default for Match { } } -pub struct Context { +pub struct Context<'a> { + router: &'a Router<'a>, values: Vec>>, pub result: Option, } -impl Context { - pub fn new(fields_cnt: usize) -> Self { +impl<'a> Context<'a> { + pub fn new(router: &'a Router) -> Self { Context { - values: vec![None; fields_cnt], + router, + values: vec![None; router.fields.list.len()], result: None, } } - pub fn add_value(&mut self, index: usize, value: Value) { + pub fn add_value(&mut self, field: &str, value: Value) { + if let Some(index) = self.router.fields.map.get(field) { + if let Some(v) = &mut self.values[*index] { + v.push(value); + } else { + self.values[*index] = Some(vec![value]); + } + } else { + panic!("value provided does not match field: {}", field,); + } + } + + pub fn add_value_by_index(&mut self, index: usize, value: Value) { if index >= self.values.len() { panic!( "value provided does not match schema: index {}, max fields count {}", @@ -72,19 +86,80 @@ impl Context { #[cfg(test)] mod tests { - use crate::ast::Value; + use crate::ast::{Type, Value}; use crate::context::Context; + use crate::router::Router; + use crate::schema::Schema; + use uuid::Uuid; + + fn setup_matcher(r: &mut Router) -> usize { + let fields_cnt = 3; + for i in 0..fields_cnt { + let id: Uuid = Uuid::new_v4(); + let exp = format!(r#"http.path.segments.{} == "/bar""#, i.to_string()); + let pri = i; + assert!(r.add_matcher(pri, id, exp.as_str()).is_ok()); + } + fields_cnt + } + #[test] fn test_context() { - let fields_cnt = 3; - let mut ctx = Context::new(fields_cnt); + let mut s = Schema::default(); + s.add_field("http.path.segments.*", Type::String); + let mut r = Router::new(&s); + let fields_cnt = setup_matcher(&mut r); + + let mut ctx = Context::new(&r); + assert!(ctx.values.len() == fields_cnt); + assert_eq!(ctx.values, vec![None; fields_cnt]); + // access value with out of bound index + assert_eq!(ctx.value_of(0), None); + + // add value in bound + ctx.add_value("http.path.segments.1", Value::String("foo".to_string())); + assert_eq!(ctx.value_of(0), None); + assert_eq!(ctx.value_of(1).unwrap().len(), 1); + assert_eq!( + ctx.value_of(1).unwrap(), + vec![Value::String("foo".to_string())].as_slice() + ); + + // reset context keeps values capacity with all None + ctx.reset(); + assert!(ctx.values.len() == fields_cnt); + assert_eq!(ctx.values, vec![None; fields_cnt]); + + // reuse this context + ctx.add_value("http.path.segments.0", Value::String("bar".to_string())); + ctx.add_value("http.path.segments.0", Value::String("foo".to_string())); + assert!(ctx.values.len() == fields_cnt); + assert_eq!(ctx.value_of(0).unwrap().len(), 2); + assert_eq!( + ctx.value_of(0).unwrap(), + vec![ + Value::String("bar".to_string()), + Value::String("foo".to_string()) + ] + .as_slice() + ); + } + + #[test] + fn test_context_by_index() { + let mut s = Schema::default(); + s.add_field("http.path.segments.*", Type::String); + let mut r = Router::new(&s); + let fields_cnt = setup_matcher(&mut r); + + let mut ctx = Context::new(&r); assert!(ctx.values.len() == fields_cnt); assert_eq!(ctx.values, vec![None; fields_cnt]); // access value with out of bound index assert_eq!(ctx.value_of(0), None); // add value in bound - ctx.add_value(1, Value::String("foo".to_string())); + ctx.add_value_by_index(1, Value::String("foo".to_string())); assert_eq!(ctx.value_of(0), None); assert_eq!(ctx.value_of(1).unwrap().len(), 1); assert_eq!( @@ -98,8 +173,8 @@ mod tests { assert_eq!(ctx.values, vec![None; fields_cnt]); // reuse this context - ctx.add_value(0, Value::String("bar".to_string())); - ctx.add_value(0, Value::String("foo".to_string())); + ctx.add_value_by_index(0, Value::String("bar".to_string())); + ctx.add_value_by_index(0, Value::String("foo".to_string())); assert!(ctx.values.len() == fields_cnt); assert_eq!(ctx.value_of(0).unwrap().len(), 2); assert_eq!( diff --git a/src/ffi.rs b/src/ffi.rs index 7e7091de..10a92ae5 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -319,21 +319,26 @@ pub unsafe extern "C" fn router_get_fields( router: &Router, fields: *mut *const u8, fields_len: *mut usize, + indexes: *mut usize, ) -> usize { if !fields.is_null() { assert!(!fields_len.is_null()); - assert!(*fields_len >= router.fields.len()); + assert!(*fields_len >= router.fields.map.len()); let fields = from_raw_parts_mut(fields, *fields_len); + let indexes = from_raw_parts_mut(indexes, *fields_len); let fields_len = from_raw_parts_mut(fields_len, *fields_len); - for (i, (name, _)) in router.fields.iter().enumerate() { - fields[i] = name.as_bytes().as_ptr(); - fields_len[i] = name.len() + let mut i = 0; + for (field, pos) in router.fields.map.iter() { + fields[i] = field.as_bytes().as_ptr(); + fields_len[i] = field.len(); + indexes[i] = *pos; + i += 1; } } - router.fields.len() + router.fields.map.len() } /// Allocate a new context object associated with the schema. @@ -348,8 +353,8 @@ pub unsafe extern "C" fn router_get_fields( /// /// - `schema` must be a valid pointer returned by [`schema_new`]. #[no_mangle] -pub unsafe extern "C" fn context_new(fields_len: usize) -> *mut Context { - Box::into_raw(Box::new(Context::new(fields_len))) +pub unsafe extern "C" fn context_new<'a>(router: &'a Router<'a>) -> *mut Context<'a> { + Box::into_raw(Box::new(Context::new(router))) } /// Deallocate the context object. @@ -407,7 +412,32 @@ pub unsafe extern "C" fn context_free(context: *mut Context) { /// * `errbuf_len` must be vlaid to read and write for `size_of::()` bytes, /// and it must be properly aligned. #[no_mangle] -pub unsafe extern "C" fn context_add_value( +pub extern "C" fn context_add_value( + context: &mut Context, + field: *const i8, + value: &CValue, + errbuf: *mut u8, + errbuf_len: *mut usize, +) -> bool { + let field = unsafe { ffi::CStr::from_ptr(field).to_str().unwrap() }; + let errbuf = unsafe { from_raw_parts_mut(errbuf, ERR_BUF_MAX_LEN) }; + + let value: Result = value.try_into(); + if let Err(e) = value { + errbuf[..e.len()].copy_from_slice(e.as_bytes()); + unsafe { + *errbuf_len = e.len(); + } + return false; + } + + context.add_value(field, value.unwrap()); + + true +} + +#[no_mangle] +pub unsafe extern "C" fn context_add_value_by_index( context: &mut Context, index: usize, value: &CValue, @@ -424,8 +454,7 @@ pub unsafe extern "C" fn context_add_value( return false; } - context.add_value(index, value.unwrap()); - + context.add_value_by_index(index, value.unwrap()); true } diff --git a/src/interpreter.rs b/src/interpreter.rs index 63f457bb..ce574f41 100644 --- a/src/interpreter.rs +++ b/src/interpreter.rs @@ -261,12 +261,20 @@ impl Execute for Predicate { #[test] fn test_predicate() { use crate::ast; + use crate::router::Router; use crate::schema; + use uuid::Uuid; let mut mat = Match::new(); let mut schema = schema::Schema::default(); schema.add_field("my_key", ast::Type::String); - let mut ctx = Context::new(1); + let mut r = Router::new(&schema); + // expression here is not practical, just used to setup context + assert!(r + .add_matcher(1, Uuid::new_v4(), r#"my_key=="whatever""#) + .is_ok()); + + let mut ctx = Context::new(&r); let field_index: usize = 0; // check when value list is empty @@ -305,7 +313,7 @@ fn test_predicate() { ]; for v in lhs_values { - ctx.add_value(field_index, v); + ctx.add_value_by_index(field_index, v); } // check if all values match starts_with foo -- should be true diff --git a/src/router.rs b/src/router.rs index 7532aa13..38041dce 100644 --- a/src/router.rs +++ b/src/router.rs @@ -10,11 +10,16 @@ use uuid::Uuid; #[derive(PartialEq, Eq, PartialOrd, Ord)] struct MatcherKey(usize, Uuid); +pub struct Fields { + pub list: Vec>, // fileds list of tuple(name, count) + pub slots: Vec, // slots in list to be reused + pub map: HashMap, // 'name' to 'list index' maping +} + pub struct Router<'a> { schema: &'a Schema, matchers: BTreeMap, - pub fields: Vec<(String, usize)>, // fileds array of tuple(name, count) - pub fields_map: HashMap, // field name -> index map + pub fields: Fields, } impl<'a> Router<'a> { @@ -22,8 +27,11 @@ impl<'a> Router<'a> { Self { schema, matchers: BTreeMap::new(), - fields: Vec::new(), - fields_map: HashMap::new(), + fields: Fields { + list: Vec::new(), + slots: Vec::new(), + map: HashMap::new(), + }, } } @@ -37,7 +45,7 @@ impl<'a> Router<'a> { let mut ast = parse(atc).map_err(|e| e.to_string())?; ast.validate(self.schema)?; - ast.add_to_counter(&mut self.fields, &mut self.fields_map); + ast.add_to_counter(&mut self.fields); assert!(self.matchers.insert(key, ast).is_none()); @@ -48,24 +56,13 @@ impl<'a> Router<'a> { let key = MatcherKey(priority, uuid); if let Some(mut ast) = self.matchers.remove(&key) { - let fields_cnt = self.fields.len(); - ast.remove_from_counter(&mut self.fields, &mut self.fields_map); - // if fields array changed, we need to reindex lhs in matchers - if self.fields.len() != fields_cnt { - self.reindexing_matchers(); - } + ast.remove_from_counter(&mut self.fields); return true; } false } - pub fn reindexing_matchers(&mut self) { - for (_, m) in self.matchers.iter_mut() { - m.fix_lhs_index(&self.fields_map); - } - } - pub fn execute(&self, context: &mut Context) -> bool { for (MatcherKey(_, id), m) in self.matchers.iter().rev() { let mut mat = Match::new(); @@ -84,6 +81,7 @@ impl<'a> Router<'a> { #[cfg(test)] mod tests { use std::{ + cmp::max, collections::HashMap, net::{IpAddr, Ipv4Addr}, }; @@ -97,7 +95,6 @@ mod tests { schema::Schema, }; - type FieldsType = Vec<(String, usize)>; type ContextValues<'a> = HashMap<&'a str, Value>; fn setup_matcher(r: &mut Router, priority: usize, expression: &str) -> (Uuid, usize) { @@ -108,12 +105,15 @@ mod tests { (id, priority) } - fn init_context(fields: &FieldsType, ctx_values: &ContextValues) -> Context { - let mut ctx = Context::new(fields.len()); - for (i, v) in fields.iter().enumerate() { - let key = &v.0; + fn init_context<'a>(r: &'a Router, ctx_values: &'a ContextValues<'a>) -> Context<'a> { + let mut ctx = Context::new(r); + for (i, v) in r.fields.list.iter().enumerate() { + if v.is_none() { + continue; + } + let key = &v.as_ref().unwrap().0; if ctx_values.contains_key(key.as_str()) { - ctx.add_value(i, ctx_values.get(key.as_str()).unwrap().clone()); + ctx.add_value_by_index(i, ctx_values.get(key.as_str()).unwrap().clone()); } } ctx @@ -128,8 +128,8 @@ mod tests { LogicalExpression::Not(r) => is_index_match(r, rt), }, Expression::Predicate(p) => { - rt.fields[p.lhs.index].0 == p.lhs.var_name - && *rt.fields_map.get(&p.lhs.var_name).unwrap() == p.lhs.index + rt.fields.list[p.lhs.index].as_ref().unwrap().0 == p.lhs.var_name + && *rt.fields.map.get(&p.lhs.var_name).unwrap() == p.lhs.index } } } @@ -153,7 +153,7 @@ mod tests { // init router let mut r = Router::new(&s); - assert!(r.fields.len() == 0); + assert!(r.fields.list.len() == 0); assert!(validate_index(&r)); // add matchers @@ -161,7 +161,8 @@ mod tests { let (id_1, pri_1) = setup_matcher(&mut r, 98, r#"net.dst.port == 8443 || net.dst.port == 443"#); let (id_2, pri_2) = setup_matcher(&mut r, 97, r#"net.src.ip == 192.168.1.1"#); - assert!(r.fields.len() == 3); + assert!(r.fields.list.len() == 3); + assert!(r.fields.slots.len() == 0); assert!(validate_index(&r)); // mock context values @@ -172,7 +173,7 @@ mod tests { Value::IpAddr(IpAddr::V4(Ipv4Addr::new(192, 168, 1, 2))), ), ]); - let mut ctx = init_context(&r.fields, &ctx_values); + let mut ctx = init_context(&r, &ctx_values); // match the first matcher let res = r.execute(&mut ctx); @@ -180,30 +181,97 @@ mod tests { // delete matcher, no field match now r.remove_matcher(pri_0, id_0); - assert!(r.fields.len() == 2); + assert!(r.fields.list.len() == 3); + assert!(r.fields.slots.len() == 1); assert!(validate_index(&r)); - ctx = init_context(&r.fields, &ctx_values); + ctx = init_context(&r, &ctx_values); assert!(!r.execute(&mut ctx)); // context value change, match again *ctx_values.get_mut("net.src.ip").unwrap() = Value::IpAddr(IpAddr::V4(Ipv4Addr::new(192, 168, 1, 1))); - ctx = init_context(&r.fields, &ctx_values); + ctx = init_context(&r, &ctx_values); assert!(r.execute(&mut ctx)); // delete all matchers r.remove_matcher(pri_1, id_1); r.remove_matcher(pri_2, id_2); - assert!(r.fields.len() == 0); + assert!(r.fields.list.len() == 3); + assert!(r.fields.slots.len() == 3); assert!(validate_index(&r)); - ctx = init_context(&r.fields, &ctx_values); + ctx = init_context(&r, &ctx_values); assert!(!r.execute(&mut ctx)); // add a new matcher let (_, _) = setup_matcher(&mut r, 96, r#"net.src.ip == 192.168.1.1"#); - assert!(r.fields.len() == 1); + assert!(r.fields.list.len() == 3); + assert!(r.fields.slots.len() == 2); assert!(validate_index(&r)); - ctx = init_context(&r.fields, &ctx_values); + ctx = init_context(&r, &ctx_values); assert!(r.execute(&mut ctx)); } + + #[test] + fn test_fields_list() { + let mut s = Schema::default(); + s.add_field("http.path.segments.*", Type::String); + let mut r = Router::new(&s); + let i_max = 1000; + let mut ids = vec![]; + for i in 0..i_max { + let id: Uuid = Uuid::new_v4(); + let exp = format!(r#"http.path.segments.{} == "/bar""#, i.to_string()); + let pri = i; + assert!(r.add_matcher(pri, id, exp.as_str()).is_ok()); + assert!(r.fields.list.len() == i + 1); + assert!(r.fields.slots.len() == 0); + assert!(r.fields.map.len() == i + 1); + ids.push((pri, id)); + } + + // delete 100 fields + let mut valid_cnt = i_max; + for (idx, id) in &ids[100..200] { + let pri = idx; + assert!(r.remove_matcher(*pri, *id)); + valid_cnt -= 1; + assert!(r.fields.list.len() == i_max); + assert!(r.fields.slots.len() == i_max - valid_cnt); + assert!(r.fields.map.len() == valid_cnt); + } + + // deleted fields leave None in fields list + for i in 100..200 { + assert!(r.fields.list[i] == None); + } + + // adds 200 fields back + let fields_len = r.fields.list.len(); + let mut slot_cnt = r.fields.slots.len(); + for i in 0..200 { + let id: Uuid = Uuid::new_v4(); + let exp = format!( + r#"http.path.segments.{} == "/bar""#, + (i_max + i).to_string() + ); + let pri = i; + if slot_cnt > 0 { + slot_cnt -= 1; + } + assert!(r.add_matcher(pri, id, exp.as_str()).is_ok()); + assert!(r.fields.list.len() == max(fields_len, r.fields.map.len())); + assert!(r.fields.slots.len() == slot_cnt); + assert!(r.fields.map.len() == r.fields.list.len() - slot_cnt); + } + + // 100 slot deleted before should be reused + for i in 100..200 { + assert!(r.fields.list[i].is_some()); + } + + // 100 slot newly added should be valid + for i in i_max..i_max + 100 { + assert!(r.fields.list[i].is_some()); + } + } } diff --git a/src/semantics.rs b/src/semantics.rs index 1e7953c4..8467e7d7 100644 --- a/src/semantics.rs +++ b/src/semantics.rs @@ -1,6 +1,6 @@ use crate::ast::{BinaryOperator, Expression, LogicalExpression, Type, Value}; +use crate::router::Fields; use crate::schema::Schema; -use std::collections::HashMap; type ValidationResult = Result<(), String>; @@ -9,113 +9,75 @@ pub trait Validate { } pub trait FieldCounter { - fn add_to_counter( - &mut self, - fields: &mut Vec<(String, usize)>, - map: &mut HashMap, - ); - fn remove_from_counter( - &mut self, - fields: &mut Vec<(String, usize)>, - map: &mut HashMap, - ); - fn fix_lhs_index(&mut self, map: &HashMap); + fn add_to_counter(&mut self, fields: &mut Fields); + fn remove_from_counter(&mut self, fields: &mut Fields); } impl FieldCounter for Expression { - fn add_to_counter( - &mut self, - fields: &mut Vec<(String, usize)>, - map: &mut HashMap, - ) { + fn add_to_counter(&mut self, fields: &mut Fields) { match self { Expression::Logical(l) => match l.as_mut() { LogicalExpression::And(l, r) => { - l.add_to_counter(fields, map); - r.add_to_counter(fields, map); + l.add_to_counter(fields); + r.add_to_counter(fields); } LogicalExpression::Or(l, r) => { - l.add_to_counter(fields, map); - r.add_to_counter(fields, map); + l.add_to_counter(fields); + r.add_to_counter(fields); } LogicalExpression::Not(r) => { - r.add_to_counter(fields, map); + r.add_to_counter(fields); } }, Expression::Predicate(p) => { // 1. fields: increment counter for field // 2. lhs: assign field index to the LHS // 3. map: maintain the fields map: {field_name : field_index} - if let Some(index) = map.get(&p.lhs.var_name) { - fields[*index].1 += 1; + if let Some(index) = fields.map.get(&p.lhs.var_name) { + fields.list[*index].as_mut().unwrap().1 += 1; p.lhs.index = *index; } else { - fields.push((p.lhs.var_name.clone(), 1)); - map.insert(p.lhs.var_name.clone(), fields.len() - 1); - p.lhs.index = fields.len() - 1; - } - } - } - } - - fn remove_from_counter( - &mut self, - fields: &mut Vec<(String, usize)>, - map: &mut HashMap, - ) { - match self { - Expression::Logical(l) => match l.as_mut() { - LogicalExpression::And(l, r) => { - l.remove_from_counter(fields, map); - r.remove_from_counter(fields, map); - } - LogicalExpression::Or(l, r) => { - l.remove_from_counter(fields, map); - r.remove_from_counter(fields, map); - } - LogicalExpression::Not(r) => { - r.remove_from_counter(fields, map); - } - }, - Expression::Predicate(p) => { - // field must be in hashmap and fields array - let index: usize = *map.get(&p.lhs.var_name).unwrap(); - // 1. decrement counter of field - // 2. for field removing, swap another field to fill the slot. - // 3. re-assign index in fields_map for swapped field in array. - fields[index].1 -= 1; - if fields[index].1 == 0 { - fields.swap_remove(index); - assert!(map.remove(&p.lhs.var_name).is_some()); - // for field who been swapped to fill the empty slot in fields array, - // we need to fix its index in fields_map - if index < fields.len() { - *map.get_mut(&fields[index].0).unwrap() = index; + // reuse slots in queue if possible + let new_idx: usize; + if fields.slots.is_empty() { + fields.list.push(Some((p.lhs.var_name.clone(), 1))); + new_idx = fields.list.len() - 1; + } else { + new_idx = fields.slots.pop().unwrap(); + fields.list[new_idx] = Some((p.lhs.var_name.clone(), 1)); } + fields.map.insert(p.lhs.var_name.clone(), new_idx); + p.lhs.index = new_idx; } } } } - // this may run only after remove_from_counter, since existing field in field array - // may change its postion after that. run this to fix lhs's field index. - fn fix_lhs_index(&mut self, map: &HashMap) { + fn remove_from_counter(&mut self, fields: &mut Fields) { match self { Expression::Logical(l) => match l.as_mut() { LogicalExpression::And(l, r) => { - l.fix_lhs_index(map); - r.fix_lhs_index(map); + l.remove_from_counter(fields); + r.remove_from_counter(fields); } LogicalExpression::Or(l, r) => { - l.fix_lhs_index(map); - r.fix_lhs_index(map); + l.remove_from_counter(fields); + r.remove_from_counter(fields); } LogicalExpression::Not(r) => { - r.fix_lhs_index(map); + r.remove_from_counter(fields); } }, Expression::Predicate(p) => { - p.lhs.index = *map.get(&p.lhs.var_name).unwrap(); + let index: usize = p.lhs.index; + // decrement counter of field + fields.list[index].as_mut().unwrap().1 -= 1; + // for field removing, reserve the slot for resue and remove it in map + if fields.list[index].as_mut().unwrap().1 == 0 { + fields.list[index] = None; + fields.slots.push(index); + assert!(fields.map.remove(&p.lhs.var_name).is_some()); + } } } } diff --git a/t/01-sanity.t b/t/01-sanity.t index 5eb06abb..6c6084dd 100644 --- a/t/01-sanity.t +++ b/t/01-sanity.t @@ -39,9 +39,9 @@ __DATA__ assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c", "http.path ^= \"/foo\" && tcp.port == 80")) - local c = context.new(s, #r:get_fields()) - c:add_value(1, "http.path", "/foo/bar") - c:add_value(2, "tcp.port", 80) + local c = context.new(r) + c:add_value("http.path", "/foo/bar") + c:add_value("tcp.port", 80) local matched = r:execute(c) ngx.say(matched) @@ -49,6 +49,18 @@ __DATA__ local uuid, prefix = c:get_result("http.path") ngx.say(uuid) ngx.say(prefix) + + -- context set by index + c:reset() + c:add_value_by_index("http.path", "/foo/bar", 0) + c:add_value_by_index("tcp.port", 80, 1) + + matched = r:execute(c) + ngx.say(matched) + + uuid, prefix = c:get_result("http.path") + ngx.say(uuid) + ngx.say(prefix) } } --- request @@ -57,6 +69,9 @@ GET /t true a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c /foo +true +a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c +/foo --- no_error_log [error] [warn] @@ -84,9 +99,9 @@ a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-1aa5583d150d", "http.path ^= \"/\"")) - local c = context.new(s, #r:get_fields()) - c:add_value(1, "http.path", "/foo/bar") - c:add_value(2, "tcp.port", 80) + local c = context.new(r) + c:add_value("http.path", "/foo/bar") + c:add_value("tcp.port", 80) local matched = r:execute(c) ngx.say(matched) @@ -94,6 +109,18 @@ a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c local uuid, prefix = c:get_result("http.path") ngx.say("uuid = " .. uuid .. " prefix = " .. prefix) + + -- context set by index + c:reset() + c:add_value_by_index("http.path", "/foo/bar", 0) + c:add_value_by_index("tcp.port", 80, 1) + + matched = r:execute(c) + ngx.say(matched) + + uuid, prefix = c:get_result("http.path") + ngx.say(uuid) + ngx.say("uuid = " .. uuid .. " prefix = " .. prefix) } } --- request @@ -101,6 +128,8 @@ GET /t --- response_body true uuid = a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c prefix = /foo +true +uuid = a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c prefix = /foo --- no_error_log [error] [warn] @@ -126,9 +155,9 @@ uuid = a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c prefix = /foo assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c", "http.path ^= \"/foo\" && tcp.port == 80")) - local c = context.new(s, #r:get_fields()) - c:add_value(1, "http.path", "/foo/bar") - c:add_value(2, "tcp.port", 80) + local c = context.new(r) + c:add_value("http.path", "/foo/bar") + c:add_value("tcp.port", 80) local matched = r:execute(c) ngx.say(matched) @@ -138,11 +167,11 @@ uuid = a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c prefix = /foo ngx.say(prefix) assert(r:remove_matcher("a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c")) - assert(#r:get_fields() == 0) + assert(r:get_fields().size == 0) - c = context.new(s, #r:get_fields()) - c:add_value(1, "http.path", "/foo/bar") - c:add_value(2, "tcp.port", 80) + c = context.new(r) + c:add_value("http.path", "/foo/bar") + c:add_value("tcp.port", 80) matched = r:execute(c) ngx.say(matched) @@ -215,9 +244,9 @@ nil --> 1:11 assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c", "http.path ^= \"/foo\" && tcp.port == 80")) - local c = context.new(s, #r:get_fields()) - c:add_value(1, "http.path", "/foo/bar") - c:add_value(2, "tcp.port", 80) + local c = context.new(r) + c:add_value("http.path", "/foo/bar") + c:add_value("tcp.port", 80) local matched = r:execute(c) ngx.say(matched) diff --git a/t/02-bugs.t b/t/02-bugs.t index 7bbf97dc..3255e22f 100644 --- a/t/02-bugs.t +++ b/t/02-bugs.t @@ -28,10 +28,14 @@ __DATA__ content_by_lua_block { local schema = require("resty.router.schema") local context = require("resty.router.context") + local router = require("resty.router.router") local s = schema.new() s:add_field("http.path", "String") + local r = router.new() + assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c", + "http.path ^= \"/foo\" && tcp.port == 80")) local BAD_UTF8 = { "\x80", @@ -39,9 +43,9 @@ __DATA__ "\xfc\x80\x80\x80\x80\xaf", } - local c = context.new(s, 1) + local c = context.new(r) for _, v in ipairs(BAD_UTF8) do - local ok, err = c:add_value(1, "http.path", v) + local ok, err = c:add_value("http.path", v) ngx.say(err) end } @@ -66,13 +70,15 @@ invalid utf-8 sequence of 1 bytes from index 0 content_by_lua_block { local schema = require("resty.router.schema") local context = require("resty.router.context") + local router = require("resty.router.router") local s = schema.new() s:add_field("http.path", "String") + local r = router.new() - local c = context.new(s, 1) - assert(c:add_value(1, "http.path", "\x00")) + local c = context.new(r) + assert(c:add_value("http.path", "\x00")) ngx.say("ok") } } @@ -146,8 +152,8 @@ ok assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c", "http.body =^ \"world\"")) - local c = context.new(s, #r:get_fields()) - c:add_value(1, "http.body", "hello\x00world") + local c = context.new(r) + c:add_value("http.body", "hello\x00world") local matched = r:execute(c) ngx.say(matched) @@ -156,7 +162,7 @@ ok ngx.say(uuid) c:reset() - c:add_value(1, "http.body", "world\x00hello") + c:add_value("http.body", "world\x00hello") local matched = r:execute(c) ngx.say(matched) diff --git a/t/03-contains.t b/t/03-contains.t index 49d34816..c419551b 100644 --- a/t/03-contains.t +++ b/t/03-contains.t @@ -39,9 +39,9 @@ __DATA__ assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c", "http.path contains \"keyword\" && tcp.port == 80")) - local c = context.new(s, #r:get_fields()) - c:add_value(1, "http.path", "/foo/keyword/bar") - c:add_value(2, "tcp.port", 80) + local c = context.new(r) + c:add_value("http.path", "/foo/keyword/bar") + c:add_value("tcp.port", 80) local matched = r:execute(c) ngx.say(matched) @@ -83,9 +83,9 @@ nil assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c", "http.path contains \"keyword\" && tcp.port == 80")) - local c = context.new(s, #r:get_fields()) - c:add_value(1, "http.path", "/foo/bar") - c:add_value(2, "tcp.port", 80) + local c = context.new(r) + c:add_value("http.path", "/foo/bar") + c:add_value("tcp.port", 80) local matched = r:execute(c) ngx.say(matched) diff --git a/t/04-rawstr.t b/t/04-rawstr.t index 89405bb9..eada3558 100644 --- a/t/04-rawstr.t +++ b/t/04-rawstr.t @@ -39,7 +39,7 @@ __DATA__ assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c", "http.path ^= r#\"/foo\"# && tcp.port == 80")) - local c = context.new(s, #r:get_fields()) + local c = context.new(r) c:add_value("http.path", "/foo/bar") c:add_value("tcp.port", 80) @@ -82,7 +82,7 @@ a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c", "http.path ^= r#\"/foo\"\'\"# && tcp.port == 80")) - local c = context.new(s, #r:get_fields()) + local c = context.new(r) c:add_value("http.path", "/foo\"\'/bar") c:add_value("tcp.port", 80) @@ -126,7 +126,7 @@ a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c", "http.path ~ r#\"^/\\d+/test$\"# && tcp.port == 80")) - local c = context.new(s, #r:get_fields()) + local c = context.new(r) c:add_value("http.path", "/123/test") c:add_value("tcp.port", 80) @@ -169,9 +169,9 @@ a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c", "http.path ~ r#\"^/\\D+/test$\"# && tcp.port == 80")) - local c = context.new(s, #r:get_fields()) - c:add_value(1, "http.path", "/123/test") - c:add_value(2, "tcp.port", 80) + local c = context.new(r) + c:add_value("http.path", "/123/test") + c:add_value("tcp.port", 80) local matched = r:execute(c) ngx.say(matched) diff --git a/t/05-equals.t b/t/05-equals.t index 9d6f5f9f..23878a31 100644 --- a/t/05-equals.t +++ b/t/05-equals.t @@ -38,10 +38,10 @@ __DATA__ assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c", "http.headers.foo == \"bar\"")) - local c = context.new(s, #r:get_fields()) - c:add_value(1, "http.headers.foo", "bar") - c:add_value(1, "http.headers.foo", "bar") - c:add_value(1, "http.headers.foo", "bar") + local c = context.new(r) + c:add_value("http.headers.foo", "bar") + c:add_value("http.headers.foo", "bar") + c:add_value("http.headers.foo", "bar") local matched = r:execute(c) ngx.say(matched) @@ -82,10 +82,10 @@ bar assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c", "http.headers.foo == \"bar\"")) - local c = context.new(s, #r:get_fields()) - c:add_value(1, "http.headers.foo", "bar") - c:add_value(1, "http.headers.foo", "bar") - c:add_value(1, "http.headers.foo", "barX") + local c = context.new(r) + c:add_value("http.headers.foo", "bar") + c:add_value("http.headers.foo", "bar") + c:add_value("http.headers.foo", "barX") local matched = r:execute(c) ngx.say(matched) @@ -124,7 +124,7 @@ nil assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c", "http.headers.foo == \"bar\"")) - local c = context.new(s, #r:get_fields()) + local c = context.new(r) local matched = r:execute(c) ngx.say(matched) diff --git a/t/07-in_notin.t b/t/07-in_notin.t index db4b0e72..b4d3ce49 100644 --- a/t/07-in_notin.t +++ b/t/07-in_notin.t @@ -76,14 +76,14 @@ nilIn/NotIn operators only supports IP in CIDR assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c", "l3.ip in 192.168.12.0/24")) - local c = context.new(s, #r:get_fields()) - c:add_value(1, "l3.ip", "192.168.12.1") + local c = context.new(r) + c:add_value("l3.ip", "192.168.12.1") local matched = r:execute(c) ngx.say(matched) - c = context.new(s, #r:get_fields()) - c:add_value(1, "l3.ip", "192.168.1.1") + c = context.new(r) + c:add_value("l3.ip", "192.168.1.1") local matched = r:execute(c) ngx.say(matched) diff --git a/t/08-equals.t b/t/08-equals.t index 9f430999..e6039848 100644 --- a/t/08-equals.t +++ b/t/08-equals.t @@ -40,15 +40,15 @@ __DATA__ assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-8aa5583d150c", "net.port != 8000")) - local c = context.new(s, #r:get_fields()) - c:add_value(1, "net.port", 8000) + local c = context.new(r) + c:add_value("net.port", 8000) local matched = r:execute(c) ngx.say(matched) ngx.say(c:get_result()) - c = context.new(s, #r:get_fields()) - c:add_value(1, "net.port", 8001) + c = context.new(r) + c:add_value("net.port", 8001) matched = r:execute(c) ngx.say(matched) @@ -88,15 +88,15 @@ a921a9aa-ec0e-4cf3-a6cc-8aa5583d150cnilnil assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-8aa5583d150c", "http.path != \"/foo\"")) - local c = context.new(s, #r:get_fields()) - c:add_value(1, "http.path", "/foo") + local c = context.new(r) + c:add_value("http.path", "/foo") local matched = r:execute(c) ngx.say(matched) ngx.say(c:get_result()) - c = context.new(s, #r:get_fields()) - c:add_value(1, "http.path", "/foo1") + c = context.new(r) + c:add_value("http.path", "/foo1") matched = r:execute(c) ngx.say(matched) @@ -136,15 +136,15 @@ a921a9aa-ec0e-4cf3-a6cc-8aa5583d150cnilnil assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-8aa5583d150c", "net.ip != 192.168.1.1")) - local c = context.new(s, #r:get_fields()) - c:add_value(1, "net.ip", "192.168.1.1") + local c = context.new(r) + c:add_value("net.ip", "192.168.1.1") local matched = r:execute(c) ngx.say(matched) ngx.say(c:get_result()) - c = context.new(s, #r:get_fields()) - c:add_value(1, "net.ip", "192.168.1.2") + c = context.new(r) + c:add_value("net.ip", "192.168.1.2") matched = r:execute(c) ngx.say(matched) diff --git a/t/09-not.t b/t/09-not.t index 3e4432c8..af7dd51c 100644 --- a/t/09-not.t +++ b/t/09-not.t @@ -38,15 +38,15 @@ __DATA__ assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c", [[!(http.path ^= "/abc")]])) - local c = context.new(s, #r:get_fields()) - c:add_value(1, "http.path", "/abc/d") + local c = context.new(r) + c:add_value("http.path", "/abc/d") local matched = r:execute(c) ngx.say(matched) c:reset() - c:add_value(1, "http.path", "/abb/d") + c:add_value("http.path", "/abb/d") local matched = r:execute(c) ngx.say(matched) From bbf0a6306df92ce08e230b59d847abefb2cdfd73 Mon Sep 17 00:00:00 2001 From: ProBrian Date: Mon, 9 Sep 2024 14:59:13 +0800 Subject: [PATCH 05/16] fix test assert --- t/01-sanity.t | 1 - 1 file changed, 1 deletion(-) diff --git a/t/01-sanity.t b/t/01-sanity.t index 6c6084dd..6b1c1694 100644 --- a/t/01-sanity.t +++ b/t/01-sanity.t @@ -167,7 +167,6 @@ uuid = a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c prefix = /foo ngx.say(prefix) assert(r:remove_matcher("a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c")) - assert(r:get_fields().size == 0) c = context.new(r) c:add_value("http.path", "/foo/bar") From 51d548affaae97675b0ccff122ba75871b5380aa Mon Sep 17 00:00:00 2001 From: ProBrian Date: Mon, 9 Sep 2024 15:04:53 +0800 Subject: [PATCH 06/16] fix more test cases --- t/02-bugs.t | 4 ++-- t/09-not.t | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/t/02-bugs.t b/t/02-bugs.t index 3255e22f..7293356d 100644 --- a/t/02-bugs.t +++ b/t/02-bugs.t @@ -33,7 +33,7 @@ __DATA__ local s = schema.new() s:add_field("http.path", "String") - local r = router.new() + local r = router.new(s) assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c", "http.path ^= \"/foo\" && tcp.port == 80")) @@ -75,7 +75,7 @@ invalid utf-8 sequence of 1 bytes from index 0 local s = schema.new() s:add_field("http.path", "String") - local r = router.new() + local r = router.new(s) local c = context.new(r) assert(c:add_value("http.path", "\x00")) diff --git a/t/09-not.t b/t/09-not.t index af7dd51c..142ae9c2 100644 --- a/t/09-not.t +++ b/t/09-not.t @@ -57,13 +57,13 @@ __DATA__ c:reset() - c:add_value(1, "http.path", "/abb/d/") + c:add_value("http.path", "/abb/d/") local matched = r:execute(c) ngx.say(matched) c:reset() - c:add_value(1, "http.path", "/abb/d") + c:add_value("http.path", "/abb/d") local matched = r:execute(c) ngx.say(matched) } From 49f3ecd27753e53b5fc746b51b7b21463d40fec0 Mon Sep 17 00:00:00 2001 From: ProBrian Date: Mon, 9 Sep 2024 15:32:21 +0800 Subject: [PATCH 07/16] fix more test cases --- lib/resty/router/router.lua | 1 - src/context.rs | 10 ++-------- src/router.rs | 4 ++++ t/01-sanity.t | 1 - 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/lib/resty/router/router.lua b/lib/resty/router/router.lua index 19d522a5..536c8e82 100644 --- a/lib/resty/router/router.lua +++ b/lib/resty/router/router.lua @@ -75,7 +75,6 @@ end function _M:get_fields() local out = {} - local out_n = 0 local router = self.router local total = tonumber(clib.router_get_fields(router, nil, nil, nil)) diff --git a/src/context.rs b/src/context.rs index 68e14323..7d1ddefb 100644 --- a/src/context.rs +++ b/src/context.rs @@ -40,14 +40,8 @@ impl<'a> Context<'a> { } pub fn add_value(&mut self, field: &str, value: Value) { - if let Some(index) = self.router.fields.map.get(field) { - if let Some(v) = &mut self.values[*index] { - v.push(value); - } else { - self.values[*index] = Some(vec![value]); - } - } else { - panic!("value provided does not match field: {}", field,); + if &value.my_type() != self.router.schema().type_of(field).unwrap() { + panic!("value provided does not match schema"); } } diff --git a/src/router.rs b/src/router.rs index 38041dce..95e5c2c2 100644 --- a/src/router.rs +++ b/src/router.rs @@ -76,6 +76,10 @@ impl<'a> Router<'a> { false } + + pub fn schema(&self) -> &Schema { + &self.schema + } } #[cfg(test)] diff --git a/t/01-sanity.t b/t/01-sanity.t index 6b1c1694..d334863f 100644 --- a/t/01-sanity.t +++ b/t/01-sanity.t @@ -119,7 +119,6 @@ a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c ngx.say(matched) uuid, prefix = c:get_result("http.path") - ngx.say(uuid) ngx.say("uuid = " .. uuid .. " prefix = " .. prefix) } } From 84f167991b4e93cdcce5648b68e5e08ca2ed0397 Mon Sep 17 00:00:00 2001 From: ProBrian Date: Mon, 9 Sep 2024 15:39:32 +0800 Subject: [PATCH 08/16] fix add value --- src/context.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/context.rs b/src/context.rs index 7d1ddefb..e13e8e75 100644 --- a/src/context.rs +++ b/src/context.rs @@ -43,6 +43,13 @@ impl<'a> Context<'a> { if &value.my_type() != self.router.schema().type_of(field).unwrap() { panic!("value provided does not match schema"); } + if let Some(index) = self.router.fields.map.get(field) { + if let Some(v) = &mut self.values[*index] { + v.push(value); + } else { + self.values[*index] = Some(vec![value]); + } + } } pub fn add_value_by_index(&mut self, index: usize, value: Value) { From 23f5754301a292bdfa6b01c0ae20ba180132d896 Mon Sep 17 00:00:00 2001 From: ProBrian Date: Mon, 9 Sep 2024 16:31:51 +0800 Subject: [PATCH 09/16] fix two test suite --- t/02-bugs.t | 1 + t/06-validate.t | 6 ++---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/t/02-bugs.t b/t/02-bugs.t index 7293356d..7f855622 100644 --- a/t/02-bugs.t +++ b/t/02-bugs.t @@ -75,6 +75,7 @@ invalid utf-8 sequence of 1 bytes from index 0 local s = schema.new() s:add_field("http.path", "String") + s:add_field("tcp.port", "Int") local r = router.new(s) local c = context.new(r) diff --git a/t/06-validate.t b/t/06-validate.t index a4a9b2a7..466477ec 100644 --- a/t/06-validate.t +++ b/t/06-validate.t @@ -37,8 +37,7 @@ __DATA__ ngx.say(type(r)) ngx.say(err) - ngx.say(#r) - ngx.say(r[1]) + ngx.say(r[http.headers.foo]) } } --- request @@ -46,8 +45,7 @@ GET /t --- response_body table nil -1 -http.headers.foo +0 --- no_error_log [error] [warn] From bf641313f94321a56affd184e84ab5e0d3a4c82c Mon Sep 17 00:00:00 2001 From: ProBrian Date: Mon, 9 Sep 2024 16:41:33 +0800 Subject: [PATCH 10/16] fix two test suite --- t/02-bugs.t | 1 + t/06-validate.t | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/t/02-bugs.t b/t/02-bugs.t index 7f855622..e74bf2e7 100644 --- a/t/02-bugs.t +++ b/t/02-bugs.t @@ -33,6 +33,7 @@ __DATA__ local s = schema.new() s:add_field("http.path", "String") + s:add_field("tcp.port", "Int") local r = router.new(s) assert(r:add_matcher(0, "a921a9aa-ec0e-4cf3-a6cc-1aa5583d150c", "http.path ^= \"/foo\" && tcp.port == 80")) diff --git a/t/06-validate.t b/t/06-validate.t index 466477ec..8af59b5a 100644 --- a/t/06-validate.t +++ b/t/06-validate.t @@ -37,7 +37,7 @@ __DATA__ ngx.say(type(r)) ngx.say(err) - ngx.say(r[http.headers.foo]) + ngx.say(r["http.headers.foo"]) } } --- request @@ -45,7 +45,7 @@ GET /t --- response_body table nil -0 +0ULL --- no_error_log [error] [warn] From 0721a4a039fd3c9b2caa2da0d407b21c48cf8c60 Mon Sep 17 00:00:00 2001 From: ProBrian Date: Mon, 9 Sep 2024 16:56:42 +0800 Subject: [PATCH 11/16] fix 06 test --- t/06-validate.t | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/t/06-validate.t b/t/06-validate.t index 8af59b5a..5c9a69f9 100644 --- a/t/06-validate.t +++ b/t/06-validate.t @@ -37,7 +37,11 @@ __DATA__ ngx.say(type(r)) ngx.say(err) - ngx.say(r["http.headers.foo"]) + + for k, v in pairs(r) do + ngx.say(k) + ngx.say(v) + end } } --- request @@ -45,7 +49,8 @@ GET /t --- response_body table nil -0ULL +"http.headers.foo" +0 --- no_error_log [error] [warn] From 9b478c1ef39ddb0b050cb7623dfed82cdcee8b32 Mon Sep 17 00:00:00 2001 From: ProBrian Date: Mon, 9 Sep 2024 21:09:38 +0800 Subject: [PATCH 12/16] fix ngx.say does not support cdata --- t/06-validate.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/06-validate.t b/t/06-validate.t index 5c9a69f9..775a53fb 100644 --- a/t/06-validate.t +++ b/t/06-validate.t @@ -40,7 +40,7 @@ __DATA__ for k, v in pairs(r) do ngx.say(k) - ngx.say(v) + ngx.say(tonumber(v)) end } } From 8c3fd20962c70764081924eb1601193b4d7bab25 Mon Sep 17 00:00:00 2001 From: ProBrian Date: Mon, 9 Sep 2024 21:13:18 +0800 Subject: [PATCH 13/16] fix expected result --- t/06-validate.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/06-validate.t b/t/06-validate.t index 775a53fb..ef63990a 100644 --- a/t/06-validate.t +++ b/t/06-validate.t @@ -49,7 +49,7 @@ GET /t --- response_body table nil -"http.headers.foo" +http.headers.foo 0 --- no_error_log [error] From f7187b92907aab7c62daf1de5a7f482fd5b9f6dc Mon Sep 17 00:00:00 2001 From: ProBrian Date: Tue, 10 Sep 2024 09:59:10 +0800 Subject: [PATCH 14/16] update interface description --- README.md | 32 +++++++++++++++++++++++++++----- lib/resty/router/context.lua | 12 ++++++------ src/ffi.rs | 6 +++--- 3 files changed, 36 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index cc441902..5ccf5c94 100644 --- a/README.md +++ b/README.md @@ -186,9 +186,9 @@ none of the matcher matched. **context:** *any* -Returns the currently used field names by all matchers inside the router as -an Lua array. It can help reduce unnecessarily producing values that are not -actually used by the user supplied matchers. +Returns the currently used {field name: field index} map by all matchers inside +the router as an Lua table. It can help reduce unnecessarily producing values +that are not actually used by the user supplied matchers. [Back to TOC](#table-of-contents) @@ -209,12 +209,12 @@ Returns the fields used in the provided expression when the expression is valid. ### new -**syntax:** *c = context.new(schema)* +**syntax:** *c = context.new(router)* **context:** *any* Create a new context instance that can later be used for storing contextual information. -for router matches. `schema` must refer to an existing schema instance. +for router matches. `router` must refer to an existing router instance. [Back to TOC](#table-of-contents) @@ -232,6 +232,28 @@ If an error occurred, `nil` and a string describing the error will be returned. [Back to TOC](#table-of-contents) +### add\_value\_by\_index + +**syntax:** *res, err = c:add_value(field, value, index)* + +**context:** *any* + +Provides `value` for `field` inside the context. + +Use `index` got from `r:get_fields()` to provide `value` for `field`. + +This method is faster than `add_value`, but notice that users should gurantee + +the index got from `r:get_fields()` is not stale. e.g. `router` fields changed + +by `remove_matcher` may cause field indexes got previously invalid. + +Returns `true` if field exists and value has successfully been provided. + +If an error occurred, `nil` and a string describing the error will be returned. + +[Back to TOC](#table-of-contents) + ### get\_result **syntax:** *uuid, matched_value, captures = c:get_result(matched_field)* diff --git a/lib/resty/router/context.lua b/lib/resty/router/context.lua index 5a8fc049..bd709894 100644 --- a/lib/resty/router/context.lua +++ b/lib/resty/router/context.lua @@ -37,12 +37,12 @@ function _M.new(router) end -function _M:add_value_impl(field, value, index) +local function add_value_impl(ctx, field, value, index) if not value then return true end - local typ, err = self.schema:get_field_type(field) + local typ, err = ctx.schema:get_field_type(field) if not typ then return nil, err end @@ -66,9 +66,9 @@ function _M:add_value_impl(field, value, index) errbuf_len[0] = ERR_BUF_MAX_LEN local res if index ~= nil then - res = clib.context_add_value_by_index(self.context, index, CACHED_VALUE, errbuf, errbuf_len) + res = clib.context_add_value_by_index(ctx.context, index, CACHED_VALUE, errbuf, errbuf_len) else - res = clib.context_add_value(self.context, field, CACHED_VALUE, errbuf, errbuf_len) + res = clib.context_add_value(ctx.context, field, CACHED_VALUE, errbuf, errbuf_len) end if res == false then return nil, ffi_string(errbuf, errbuf_len[0]) @@ -79,12 +79,12 @@ end function _M:add_value(field, value) - return _M.add_value_impl(self, field, value) + return add_value_impl(self, field, value) end function _M:add_value_by_index(field, value, index) - return _M.add_value_impl(self, field, value, index) + return add_value_impl(self, field, value, index) end diff --git a/src/ffi.rs b/src/ffi.rs index 10a92ae5..e4dc4e7f 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -412,15 +412,15 @@ pub unsafe extern "C" fn context_free(context: *mut Context) { /// * `errbuf_len` must be vlaid to read and write for `size_of::()` bytes, /// and it must be properly aligned. #[no_mangle] -pub extern "C" fn context_add_value( +pub unsafe extern "C" fn context_add_value( context: &mut Context, field: *const i8, value: &CValue, errbuf: *mut u8, errbuf_len: *mut usize, ) -> bool { - let field = unsafe { ffi::CStr::from_ptr(field).to_str().unwrap() }; - let errbuf = unsafe { from_raw_parts_mut(errbuf, ERR_BUF_MAX_LEN) }; + let field = ffi::CStr::from_ptr(field).to_str().unwrap(); + let errbuf = from_raw_parts_mut(errbuf, ERR_BUF_MAX_LEN); let value: Result = value.try_into(); if let Err(e) = value { From e3c2d31fba061bd4c6b142815cbc9a13332e2a76 Mon Sep 17 00:00:00 2001 From: ProBrian Date: Tue, 29 Oct 2024 17:04:12 +0800 Subject: [PATCH 15/16] rebase main branch --- Cargo.lock | 269 +++----------------- Cargo.toml | 4 +- benches/build.rs | 3 - benches/match_mix.rs | 2 +- benches/{atc_benchmark.rs => misc_match.rs} | 5 + benches/not_match_mix.rs | 2 +- benches/string.rs | 2 +- benches/test.rs | 2 +- src/cir.rs | 146 ++++++++--- src/router.rs | 34 ++- 10 files changed, 179 insertions(+), 290 deletions(-) rename benches/{atc_benchmark.rs => misc_match.rs} (97%) diff --git a/Cargo.lock b/Cargo.lock index 13e8b955..3986b55e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -19,9 +19,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstyle" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" +checksum = "8365de52b16c035ff4fcafe0092ba9390540e3e352870ac09933bebcaa2c8c56" [[package]] name = "atc-router" @@ -42,9 +42,9 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "block-buffer" @@ -100,33 +100,6 @@ dependencies = [ "half", ] -[[package]] -name = "cidr" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" -dependencies = [ - "ciborium-io", - "ciborium-ll", - "serde", -] - -[[package]] -name = "ciborium-io" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" - -[[package]] -name = "ciborium-ll" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" -dependencies = [ - "ciborium-io", - "half", -] - [[package]] name = "cidr" version = "0.2.3" @@ -138,32 +111,7 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed6719fffa43d0d87e5fd8caeab59be1554fb028cd30edc88fc4369b17971019" -dependencies = [ - "clap_builder", -] - -[[package]] -name = "clap_builder" -version = "4.5.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216aec2b177652e3846684cbfe25c9964d18ec45234f0f5da5157b207ed1aab6" -dependencies = [ - "anstyle", - "clap_lex", -] - -[[package]] -name = "clap_lex" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" - -[[package]] -name = "cpufeatures" -version = "0.2.12" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" dependencies = [ @@ -306,9 +254,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "libc", @@ -357,57 +305,6 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" -[[package]] -name = "js-sys" -version = "0.3.70" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" -dependencies = [ - "wasm-bindgen", -] - -[[package]] -name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" -dependencies = [ - "cfg-if", - "crunchy", -] - -[[package]] -name = "hermit-abi" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" - -[[package]] -name = "is-terminal" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys 0.52.0", -] - -[[package]] -name = "itertools" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" -dependencies = [ - "either", -] - -[[package]] -name = "itoa" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" - [[package]] name = "js-sys" version = "0.3.72" @@ -425,15 +322,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.159" +version = "0.2.161" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" - -[[package]] -name = "log" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" [[package]] name = "log" @@ -456,15 +347,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "num-traits" -version = "0.2.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" -dependencies = [ - "autocfg", -] - [[package]] name = "once_cell" version = "1.20.2" @@ -477,12 +359,6 @@ version = "11.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" -[[package]] -name = "oorandom" -version = "11.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" - [[package]] name = "pest" version = "2.7.14" @@ -530,35 +406,7 @@ dependencies = [ [[package]] name = "plotters" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a15b6eccb8484002195a3e44fe65a4ce8e93a625797a063735536fd59cb01cf3" -dependencies = [ - "num-traits", - "plotters-backend", - "plotters-svg", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "plotters-backend" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "414cec62c6634ae900ea1c56128dfe87cf63e7caece0852ec76aba307cebadb7" - -[[package]] -name = "plotters-svg" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81b30686a7d9c3e010b84284bdd26a29f2138574f52f5eb6f794fc0ad924e705" -dependencies = [ - "plotters-backend", -] - -[[package]] -name = "proc-macro2" -version = "1.0.80" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" dependencies = [ @@ -586,9 +434,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.87" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3e4daa0dcf6feba26f985457cdf104d4b4256fc5a09547140f3631bb076b19a" +checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" dependencies = [ "unicode-ident", ] @@ -624,29 +472,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.4" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - -[[package]] -name = "regex" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", @@ -686,35 +514,20 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "ryu" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" - -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - [[package]] name = "serde" -version = "1.0.210" +version = "1.0.214" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.210" +version = "1.0.214" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" dependencies = [ "proc-macro2", "quote", @@ -723,9 +536,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.127" +version = "1.0.132" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8043c06d9f82bd7271361ed64f415fe5e12a77fdb52e573e7f06a516dea329ad" +checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" dependencies = [ "itoa", "memchr", @@ -756,9 +569,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.79" +version = "2.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" +checksum = "5023162dfcd14ef8f32034d8bcd4cc5ddc61ef7a247c024a33e24e1f24d21b56" dependencies = [ "proc-macro2", "quote", @@ -767,18 +580,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.64" +version = "1.0.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" +checksum = "5d11abd9594d9b38965ef50805c5e469ca9cc6f197f883f717e0269a3057b3d5" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.64" +version = "1.0.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" +checksum = "ae71770322cbd277e69d762a16c444af02aa0575ac0d174f0b9562d3b37f8602" dependencies = [ "proc-macro2", "quote", @@ -815,9 +628,9 @@ checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "uuid" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" +checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" dependencies = [ "getrandom", ] @@ -826,7 +639,7 @@ dependencies = [ name = "version_check" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "walkdir" @@ -846,9 +659,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" +checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" dependencies = [ "cfg-if", "once_cell", @@ -857,9 +670,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" +checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" dependencies = [ "bumpalo", "log", @@ -872,9 +685,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" +checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -882,9 +695,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" +checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", @@ -895,15 +708,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" +checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" [[package]] name = "web-sys" -version = "0.3.70" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0" +checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index 5f550e12..f7fef601 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,13 +23,13 @@ serde_regex = { version = "1.1", optional = true } fnv = "1" [dev-dependencies] -criterion = { version = "0.5", features = ["html_reports"] } +criterion = "0.*" serde_json = "1" serde = "1" uuid = {version = "1.8", features = ["v4"]} [[bench]] -name = "atc_benchmark" +name = "misc_match" harness = false [lib] diff --git a/benches/build.rs b/benches/build.rs index b42fc8c4..229c39e4 100644 --- a/benches/build.rs +++ b/benches/build.rs @@ -20,9 +20,6 @@ fn criterion_benchmark(c: &mut Criterion) { let mut schema = Schema::default(); schema.add_field("a", Type::Int); - let mut context = Context::new(&schema); - context.add_value("a", Value::Int(N as i64)); - c.bench_function("Build Router", |b| { b.iter_with_large_drop(|| { let mut router = Router::new(&schema); diff --git a/benches/match_mix.rs b/benches/match_mix.rs index 056c7f69..d7a6a535 100644 --- a/benches/match_mix.rs +++ b/benches/match_mix.rs @@ -34,7 +34,7 @@ fn criterion_benchmark(c: &mut Criterion) { router.add_matcher(N - i, uuid, &expr).unwrap(); } - let mut ctx_match = Context::new(&schema); + let mut ctx_match = Context::new(&router); ctx_match.add_value( "http.path", atc_router::ast::Value::String("hello49999".to_string()), diff --git a/benches/atc_benchmark.rs b/benches/misc_match.rs similarity index 97% rename from benches/atc_benchmark.rs rename to benches/misc_match.rs index 11092247..fdcff753 100644 --- a/benches/atc_benchmark.rs +++ b/benches/misc_match.rs @@ -14,6 +14,11 @@ use std::net::{IpAddr, Ipv4Addr}; use std::{hint::black_box, str::FromStr}; use uuid::Uuid; +// To run this benchmark, execute the following command: +// ```shell +// cargo bench --bench misc_match +// ``` + #[derive(Serialize, Deserialize)] struct TestData { rules: Vec, diff --git a/benches/not_match_mix.rs b/benches/not_match_mix.rs index bd93432a..53a07527 100644 --- a/benches/not_match_mix.rs +++ b/benches/not_match_mix.rs @@ -34,7 +34,7 @@ fn criterion_benchmark(c: &mut Criterion) { router.add_matcher(N - i, uuid, &expr).unwrap(); } - let mut ctx_match = Context::new(&schema); + let mut ctx_match = Context::new(&router); ctx_match.add_value( "http.path", atc_router::ast::Value::String("hello49999".to_string()), diff --git a/benches/string.rs b/benches/string.rs index 3516ff38..542d8d5a 100644 --- a/benches/string.rs +++ b/benches/string.rs @@ -34,7 +34,7 @@ fn criterion_benchmark(c: &mut Criterion) { router.add_matcher(N_MATCHER - i, uuid, &expr).unwrap(); } - let mut context = Context::new(&schema); + let mut context = Context::new(&router); context.add_value("http.path.segments.0_1", "test/run".to_string().into()); context.add_value("http.path.segments.3", "bar".to_string().into()); context.add_value("http.path.segments.len", Value::Int(3 as i64)); diff --git a/benches/test.rs b/benches/test.rs index e361af73..adcc8ac3 100644 --- a/benches/test.rs +++ b/benches/test.rs @@ -29,7 +29,7 @@ fn criterion_benchmark(c: &mut Criterion) { router.add_matcher(N - i, uuid, &expr).unwrap(); } - let mut context = Context::new(&schema); + let mut context = Context::new(&router); context.add_value("a", Value::Int(N as i64)); c.bench_function("Doesn't Match", |b| { diff --git a/src/cir.rs b/src/cir.rs index da1fecbb..f77e19e6 100644 --- a/src/cir.rs +++ b/src/cir.rs @@ -2,8 +2,8 @@ use crate::ast::{Expression, LogicalExpression, Predicate}; use crate::context::{Context, Match}; use crate::interpreter::Execute; +use crate::router::Fields; use crate::semantics::FieldCounter; -use std::collections::HashMap; #[derive(Debug)] pub struct CirProgram { @@ -178,61 +178,103 @@ impl Execute for CirProgram { } impl FieldCounter for CirOperand { - fn add_to_counter(&self, map: &mut HashMap) { - if let CirOperand::Predicate(p) = &self { - *map.entry(p.lhs.var_name.clone()).or_default() += 1 + fn add_to_counter(&mut self, fields: &mut Fields) { + if let CirOperand::Predicate(p) = self { + // 1. fields: increment counter for field + // 2. lhs: assign field index to the LHS + // 3. map: maintain the fields map: {field_name : field_index} + if let Some(index) = fields.map.get(&p.lhs.var_name) { + fields.list[*index].as_mut().unwrap().1 += 1; + p.lhs.index = *index; + } else { + // reuse slots in queue if possible + let new_idx: usize; + if fields.slots.is_empty() { + fields.list.push(Some((p.lhs.var_name.clone(), 1))); + new_idx = fields.list.len() - 1; + } else { + new_idx = fields.slots.pop().unwrap(); + fields.list[new_idx] = Some((p.lhs.var_name.clone(), 1)); + } + fields.map.insert(p.lhs.var_name.clone(), new_idx); + p.lhs.index = new_idx; + } } } - fn remove_from_counter(&self, map: &mut HashMap) { + fn remove_from_counter(&mut self, fields: &mut Fields) { if let CirOperand::Predicate(p) = &self { - let val = map.get_mut(&p.lhs.var_name).unwrap(); - *val -= 1; - - if *val == 0 { - assert!(map.remove(&p.lhs.var_name).is_some()); + let index: usize = p.lhs.index; + // decrement counter of field + fields.list[index].as_mut().unwrap().1 -= 1; + // for field removing, reserve the slot for resue and remove it in map + if fields.list[index].as_mut().unwrap().1 == 0 { + fields.list[index] = None; + fields.slots.push(index); + assert!(fields.map.remove(&p.lhs.var_name).is_some()); } } } } impl FieldCounter for CirInstruction { - fn add_to_counter(&self, map: &mut HashMap) { + fn add_to_counter(&mut self, fields: &mut Fields) { match self { CirInstruction::AndIns(and) => { - and.left.add_to_counter(map); - and.right.add_to_counter(map); + and.left.add_to_counter(fields); + and.right.add_to_counter(fields); } CirInstruction::OrIns(or) => { - or.left.add_to_counter(map); - or.right.add_to_counter(map); + or.left.add_to_counter(fields); + or.right.add_to_counter(fields); } CirInstruction::NotIns(not) => { - not.right.add_to_counter(map); + not.right.add_to_counter(fields); } CirInstruction::Predicate(p) => { - *map.entry(p.lhs.var_name.clone()).or_default() += 1; + // 1. fields: increment counter for field + // 2. lhs: assign field index to the LHS + // 3. map: maintain the fields map: {field_name : field_index} + if let Some(index) = fields.map.get(&p.lhs.var_name) { + fields.list[*index].as_mut().unwrap().1 += 1; + p.lhs.index = *index; + } else { + // reuse slots in queue if possible + let new_idx: usize; + if fields.slots.is_empty() { + fields.list.push(Some((p.lhs.var_name.clone(), 1))); + new_idx = fields.list.len() - 1; + } else { + new_idx = fields.slots.pop().unwrap(); + fields.list[new_idx] = Some((p.lhs.var_name.clone(), 1)); + } + fields.map.insert(p.lhs.var_name.clone(), new_idx); + p.lhs.index = new_idx; + } } } } - fn remove_from_counter(&self, map: &mut HashMap) { + fn remove_from_counter(&mut self, fields: &mut Fields) { match self { CirInstruction::AndIns(and) => { - and.left.remove_from_counter(map); - and.right.remove_from_counter(map); + and.left.remove_from_counter(fields); + and.right.remove_from_counter(fields); } CirInstruction::OrIns(or) => { - or.left.remove_from_counter(map); - or.right.remove_from_counter(map); + or.left.remove_from_counter(fields); + or.right.remove_from_counter(fields); } CirInstruction::NotIns(not) => { - not.right.remove_from_counter(map); + not.right.remove_from_counter(fields); } CirInstruction::Predicate(p) => { - let val = map.get_mut(&p.lhs.var_name).unwrap(); - *val -= 1; - - if *val == 0 { - assert!(map.remove(&p.lhs.var_name).is_some()); + let index: usize = p.lhs.index; + // decrement counter of field + fields.list[index].as_mut().unwrap().1 -= 1; + // for field removing, reserve the slot for resue and remove it in map + if fields.list[index].as_mut().unwrap().1 == 0 { + fields.list[index] = None; + fields.slots.push(index); + assert!(fields.map.remove(&p.lhs.var_name).is_some()); } } } @@ -240,19 +282,51 @@ impl FieldCounter for CirInstruction { } impl FieldCounter for CirProgram { - fn add_to_counter(&self, map: &mut HashMap) { + fn add_to_counter(&mut self, fields: &mut Fields) { self.instructions - .iter() - .for_each(|instruction: &CirInstruction| instruction.add_to_counter(map)); + .iter_mut() + .for_each(|instruction: &mut CirInstruction| instruction.add_to_counter(fields)); } - fn remove_from_counter(&self, map: &mut HashMap) { + fn remove_from_counter(&mut self, fields: &mut Fields) { self.instructions - .iter() - .for_each(|instruction: &CirInstruction| instruction.remove_from_counter(map)); + .iter_mut() + .for_each(|instruction: &mut CirInstruction| instruction.remove_from_counter(fields)); } } +#[cfg(test)] +pub fn get_predicates(cir: &CirProgram) -> Vec<&Predicate> { + let mut predicates = Vec::new(); + cir.instructions.iter().for_each(|ins| match ins { + CirInstruction::AndIns(and) => { + if let CirOperand::Predicate(predicate) = &and.left { + predicates.push(predicate); + } + if let CirOperand::Predicate(predicate) = &and.right { + predicates.push(predicate); + } + } + CirInstruction::OrIns(or) => { + if let CirOperand::Predicate(predicate) = &or.left { + predicates.push(predicate); + } + if let CirOperand::Predicate(predicate) = &or.right { + predicates.push(predicate); + } + } + CirInstruction::NotIns(not) => { + if let CirOperand::Predicate(predicate) = ¬.right { + predicates.push(predicate); + } + } + CirInstruction::Predicate(predicate) => { + predicates.push(predicate); + } + }); + predicates +} + #[cfg(test)] mod tests { use super::*; @@ -260,6 +334,7 @@ mod tests { use crate::ast::Value; use crate::context::Match; use crate::interpreter::Execute; + use crate::router::Router; use crate::schema::Schema; impl Execute for Expression { @@ -292,7 +367,8 @@ mod tests { r#"http.path == "hello" && http.version == "1.1""#, ]; - let mut context = crate::context::Context::new(&schema); + let r = Router::new(&schema); + let mut context = crate::context::Context::new(&r); context.add_value("http.path", crate::ast::Value::String("hello".to_string())); context.add_value("http.version", crate::ast::Value::String("1.1".to_string())); context.add_value("a", Value::Int(3 as i64)); diff --git a/src/router.rs b/src/router.rs index 95e5c2c2..13445847 100644 --- a/src/router.rs +++ b/src/router.rs @@ -18,7 +18,7 @@ pub struct Fields { pub struct Router<'a> { schema: &'a Schema, - matchers: BTreeMap, + matchers: BTreeMap, pub fields: Fields, } @@ -42,12 +42,11 @@ impl<'a> Router<'a> { return Err("UUID already exists".to_string()); } // lhs's index maybe changed in `ast.add_to_counter` - let mut ast = parse(atc).map_err(|e| e.to_string())?; - + let ast = parse(atc).map_err(|e| e.to_string())?; ast.validate(self.schema)?; - ast.add_to_counter(&mut self.fields); - - assert!(self.matchers.insert(key, ast).is_none()); + let mut cir = ast.translate(); + cir.add_to_counter(&mut self.fields); + assert!(self.matchers.insert(key, cir).is_none()); Ok(()) } @@ -93,7 +92,8 @@ mod tests { use uuid::Uuid; use crate::{ - ast::{Expression, LogicalExpression, Type, Value}, + ast::{Type, Value}, + cir::{get_predicates, CirProgram}, context::Context, router::Router, schema::Schema, @@ -123,19 +123,17 @@ mod tests { ctx } - fn is_index_match(e: &Expression, rt: &Router) -> bool { - match e { - Expression::Logical(l) => match l.as_ref() { - LogicalExpression::And(l, r) | LogicalExpression::Or(l, r) => { - is_index_match(l, rt) && is_index_match(r, rt) - } - LogicalExpression::Not(r) => is_index_match(r, rt), - }, - Expression::Predicate(p) => { - rt.fields.list[p.lhs.index].as_ref().unwrap().0 == p.lhs.var_name - && *rt.fields.map.get(&p.lhs.var_name).unwrap() == p.lhs.index + fn is_index_match(cir: &CirProgram, rt: &Router) -> bool { + let predicates = get_predicates(cir); + for p in predicates { + if rt.fields.list[p.lhs.index].as_ref().unwrap().0 == p.lhs.var_name + && *rt.fields.map.get(&p.lhs.var_name).unwrap() == p.lhs.index + { + continue; } + return false; } + true } fn validate_index(r: &Router) -> bool { From 0600b83596430cdd7a4093f6461f1e381bbbcf06 Mon Sep 17 00:00:00 2001 From: ProBrian Date: Wed, 30 Oct 2024 09:29:02 +0800 Subject: [PATCH 16/16] update readme --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5ccf5c94..89105f80 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ ATC Router library for Kong. * [resty.router.context](#restyroutercontext) * [new](#new) * [add\_value](#add_value) + * [add\_value\_by\_index](#add_value_by_index) * [get\_result](#get_result) * [reset](#reset) * [Copyright and license](#copyright-and-license) @@ -234,7 +235,7 @@ If an error occurred, `nil` and a string describing the error will be returned. ### add\_value\_by\_index -**syntax:** *res, err = c:add_value(field, value, index)* +**syntax:** *res, err = c:add_value_by_index(field, value, index)* **context:** *any*