diff --git a/logprep/filter/lucene_filter.py b/logprep/filter/lucene_filter.py index 31f16c7ed..37405eb4a 100644 --- a/logprep/filter/lucene_filter.py +++ b/logprep/filter/lucene_filter.py @@ -61,8 +61,8 @@ RegEx-Filter ------------ -It is possible use regex expressions to match values. -To be recognized as a regular expression the filter field has to be start and end with +It is possible to use regex expressions to match values. +To be recognized as a regular expression, the filter field has to start with :code:`/`. diff --git a/tests/unit/connector/test_http_input.py b/tests/unit/connector/test_http_input.py index 2b945530d..66828c08f 100644 --- a/tests/unit/connector/test_http_input.py +++ b/tests/unit/connector/test_http_input.py @@ -16,6 +16,8 @@ import uvicorn from requests.auth import HTTPBasicAuth +from falcon import testing + from logprep.abc.input import FatalInputError from logprep.connector.http.input import HttpInput from logprep.factory import Factory @@ -57,7 +59,9 @@ def setup_method(self): super().setup_method() self.object.pipeline_index = 1 self.object.setup() + self.app = self.object.http_server.server.config.app self.target = self.object.target + self.client = testing.TestClient(self.app) CONFIG: dict = { "type": "http_input", @@ -109,89 +113,83 @@ def test_not_first_pipeline(self): assert connector.http_server is None def test_get_method_returns_200(self): - resp = requests.get(url=f"{self.target}/json", timeout=0.5) + resp = self.client.simulate_get("/json") assert resp.status_code == 200 def test_get_method_returns_200_with_authentication(self): - resp = requests.get(url=f"{self.target}/auth-json-secret", timeout=0.5) + resp = self.client.simulate_get("/auth-json-secret") assert resp.status_code == 200 def test_get_method_returns_429_if_queue_is_full(self): self.object.messages.full = mock.MagicMock() self.object.messages.full.return_value = True - resp = requests.get(url=f"{self.target}/json", timeout=20) + resp = self.client.simulate_get("/json") assert resp.status_code == 429 def test_get_error_code_too_many_requests(self): data = {"message": "my log message"} self.object.messages.put = mock.MagicMock() self.object.messages.put.side_effect = queue.Full() - session = requests.Session() - resp = session.post(url=f"{self.target}/json", json=data, timeout=0.5) + resp = self.client.simulate_post("/json", json=data) assert resp.status_code == 429 def test_json_endpoint_accepts_post_request(self): data = {"message": "my log message"} - resp = requests.post(url=f"{self.target}/json", json=data, timeout=0.5) + resp = self.client.simulate_post("/json", json=data) assert resp.status_code == 200 def test_json_endpoint_match_wildcard_route(self): data = {"message": "my log message"} - resp = requests.post(url=f"{self.target}/api/wildcard_path/json", json=data, timeout=0.5) + resp = self.client.simulate_post("/json", json=data) assert resp.status_code == 200 def test_json_endpoint_not_match_wildcard_route(self): data = {"message": "my log message"} - resp = requests.post( - url=f"{self.target}/api/wildcard_path/json/another_path", json=data, timeout=0.5 - ) + resp = self.client.simulate_post("/api/wildcard_path/json/another_path", json=data) assert resp.status_code == 404 data = {"message": "my log message"} - resp = requests.post(url=f"{self.target}/json", json=data, timeout=0.5) + resp = self.client.simulate_post("/json", json=data) assert resp.status_code == 200 + event_from_queue = self.object.messages.get(timeout=0.001) assert event_from_queue == data def test_plaintext_endpoint_accepts_post_request(self): data = "my log message" - resp = requests.post(url=f"{self.target}/plaintext", json=data, timeout=0.5) + resp = self.client.simulate_post("/plaintext", json=data) assert resp.status_code == 200 def test_plaintext_message_is_put_in_queue(self): data = "my log message" - resp = requests.post(url=f"{self.target}/plaintext", data=data, timeout=0.5) + resp = self.client.simulate_post("/plaintext", body=data) assert resp.status_code == 200 event_from_queue = self.object.messages.get(timeout=0.001) assert event_from_queue.get("message") == data def test_jsonl_endpoint_match_regex_route(self): data = {"message": "my log message"} - resp = requests.post(url=f"{self.target}/first/jsonl", json=data, timeout=0.5) + resp = self.client.simulate_post("/first/jsonl", json=data) assert resp.status_code == 200 def test_jsonl_endpoint_not_match_regex_route(self): data = {"message": "my log message"} - resp = requests.post(url=f"{self.target}/firs/jsonl", json=data, timeout=0.5) + resp = self.client.simulate_post("/firs/jsonl", json=data) assert resp.status_code == 404 def test_jsonl_endpoint_not_match_before_start_regex(self): data = {"message": "my log message"} - resp = requests.post(url=f"{self.target}/api/first/jsonl", json=data, timeout=0.5) + resp = self.client.simulate_post("/api/first/jsonl", json=data) assert resp.status_code == 404 def test_jsonl_endpoint_match_wildcard_regex_mix_route(self): data = {"message": "my log message"} - resp = requests.post( - url=f"{self.target}/third/jsonl/another_path/last_path", json=data, timeout=0.5 - ) + resp = self.client.simulate_post("/third/jsonl/another_path/last_path", json=data) assert resp.status_code == 200 def test_jsonl_endpoint_not_match_wildcard_regex_mix_route(self): data = {"message": "my log message"} - resp = requests.post( - url=f"{self.target}/api/third/jsonl/another_path", json=data, timeout=0.5 - ) + resp = self.client.simulate_post("/api/third/jsonl/another_path", json=data) assert resp.status_code == 404 def test_jsonl_messages_are_put_in_queue(self): @@ -200,7 +198,7 @@ def test_jsonl_messages_are_put_in_queue(self): {"message": "my second log message"} {"message": "my third log message"} """ - resp = requests.post(url=f"{self.target}/jsonl", data=data, timeout=0.5) + resp = self.client.simulate_post("/jsonl", body=data) assert resp.status_code == 200 assert self.object.messages.qsize() == 3 event_from_queue = self.object.messages.get(timeout=1) @@ -212,7 +210,7 @@ def test_jsonl_messages_are_put_in_queue(self): def test_get_next_returns_message_from_queue(self): data = {"message": "my log message"} - requests.post(url=f"{self.target}/json", json=data, timeout=0.5) + self.client.simulate_post("/json", json=data) assert self.object.get_next(0.001) == data def test_get_next_returns_first_in_first_out(self): @@ -236,9 +234,9 @@ def test_get_next_returns_first_in_first_out_for_mixed_endpoints(self): for message in data: endpoint, post_data = message.values() if endpoint == "json": - requests.post(url=self.target + "/json", json=post_data, timeout=0.5) + self.client.simulate_post("/json", json=post_data) if endpoint == "plaintext": - requests.post(url=self.target + "/plaintext", data=post_data, timeout=0.5) + self.client.simulate_post("/plaintext", body=post_data) assert self.object.get_next(0.001) == data[0].get("data") assert self.object.get_next(0.001) == {"message": data[1].get("data")} assert self.object.get_next(0.001) == data[2].get("data") @@ -253,7 +251,7 @@ def test_server_starts_threaded_server(self): message = {"message": "my message"} for i in range(100): message["message"] = f"message number {i}" - requests.post(url=f"{self.target}/json", json=message, timeout=0.5) # nosemgrep + self.client.simulate_post("/json", json=message) assert self.object.messages.qsize() == 100, "messages are put to queue" def test_get_metadata(self): @@ -265,7 +263,7 @@ def test_get_metadata(self): connector.pipeline_index = 1 connector.setup() target = connector.target - resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) # nosemgrep + resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) assert resp.status_code == 200 message = connector.messages.get(timeout=0.5) assert message["custom"]["url"] == target + "/json" @@ -280,11 +278,11 @@ def test_server_multiple_config_changes(self): connector.pipeline_index = 1 connector.setup() target = connector.target - resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) # nosemgrep + resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) assert resp.status_code == 200 target = target.replace(":9001", ":9000") try: - resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) # nosemgrep + resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) except requests.exceptions.ConnectionError as e: assert e.response is None connector_config = deepcopy(self.CONFIG) @@ -292,7 +290,7 @@ def test_server_multiple_config_changes(self): connector.pipeline_index = 1 connector.setup() target = connector.target - resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) # nosemgrep + resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) assert resp.status_code == 200 def test_get_next_with_hmac_of_raw_message(self): @@ -312,7 +310,7 @@ def test_get_next_with_hmac_of_raw_message(self): connector.pipeline_index = 1 connector.setup() test_event = "the content" - requests.post(url=f"{self.target}/plaintext", data=test_event, timeout=0.5) # nosemgrep + self.client.simulate_post("/plaintext", body=test_event) expected_event = { "message": "the content", @@ -401,17 +399,17 @@ def test_messages_is_multiprocessing_queue(self): def test_all_endpoints_share_the_same_queue(self): data = {"message": "my log message"} - requests.post(url=f"{self.target}/json", json=data, timeout=0.5) + self.client.simulate_post("/json", json=data) assert self.object.messages.qsize() == 1 data = "my log message" - requests.post(url=f"{self.target}/plaintext", json=data, timeout=0.5) + self.client.simulate_post("/plaintext", json=data) assert self.object.messages.qsize() == 2 data = """ {"message": "my first log message"} {"message": "my second log message"} {"message": "my third log message"} """ - requests.post(url=f"{self.target}/jsonl", data=data, timeout=0.5) + self.client.simulate_post("/jsonl", body=data) assert self.object.messages.qsize() == 5 def test_sets_target_to_https_schema_if_ssl_options(self): @@ -438,9 +436,7 @@ def test_enpoints_count_requests(self): self.object.setup() random_number = random.randint(1, 100) for number in range(random_number): - requests.post( - url=f"{self.target}/json", json={"message": f"my message{number}"}, timeout=0.5 - ) + self.client.simulate_post("/json", json={"message": f"my message{number}"}) assert self.object.metrics.number_of_http_requests == random_number @pytest.mark.parametrize("endpoint", ["json", "plaintext", "jsonl"]) @@ -448,24 +444,20 @@ def test_endpoint_handles_gzip_compression(self, endpoint): data = {"message": "my log message"} data = gzip.compress(json.dumps(data).encode()) headers = {"Content-Encoding": "gzip"} - resp = requests.post( - url=f"{self.target}/{endpoint}", - data=data, - headers=headers, - timeout=0.5, - ) + resp = self.client.simulate_post(f"/{endpoint}", body=data, headers=headers) assert resp.status_code == 200 @pytest.mark.parametrize("endpoint", ["json", "jsonl"]) def test_raises_http_bad_request_on_decode_error(self, endpoint): data = "this is not a valid json nor jsonl" - resp = requests.post(url=f"{self.target}/{endpoint}", data=data, timeout=0.5) + resp = self.client.simulate_post(f"/{endpoint}", body=data) assert resp.status_code == 400 @responses.activate def test_health_endpoint_is_ready_if_all_endpoints_are_successful(self): for endpoint in self.object.health_endpoints: responses.get(f"http://127.0.0.1:9000{endpoint}", status=200) + assert self.object.health(), "Health endpoint should be ready" @responses.activate @@ -504,6 +496,7 @@ def test_health_endpoint_is_not_ready_if_one_endpoint_has_read_timeout(self): def test_health_check_logs_error(self): endpoint = self.object.health_endpoints[0] responses.get(f"http://127.0.0.1:9000{endpoint}", body=requests.Timeout("bad")) + with mock.patch("logging.Logger.error") as mock_logger: assert not self.object.health(), "Health endpoint should not be ready" mock_logger.assert_called()