From 943a92fd468dedf38108b694ddd029be222be725 Mon Sep 17 00:00:00 2001 From: Levi <141682181+levisingularity@users.noreply.github.com> Date: Thu, 25 Jan 2024 16:12:27 -0300 Subject: [PATCH 1/3] tests: add integration test for get incoming links (#63) * Add successful, unexpected, unknown and malformed * Create function to deep compare two jsons not caring about the order --- das-query-engine/actions.py | 1 - .../integration/handle/base_test_action.py | 25 +- .../handle/test_get_incoming_links.py | 272 ++++++++++++++++++ das-query-engine/validators/actions.py | 4 +- 4 files changed, 297 insertions(+), 5 deletions(-) create mode 100644 das-query-engine/tests/integration/handle/test_get_incoming_links.py diff --git a/das-query-engine/actions.py b/das-query-engine/actions.py index a21fedf..021d7e7 100644 --- a/das-query-engine/actions.py +++ b/das-query-engine/actions.py @@ -107,7 +107,6 @@ def get_incoming_links( atom_handle, **kwargs, ) - @execution_time_tracker @remove_none_args diff --git a/das-query-engine/tests/integration/handle/base_test_action.py b/das-query-engine/tests/integration/handle/base_test_action.py index 87def7e..c7cc493 100644 --- a/das-query-engine/tests/integration/handle/base_test_action.py +++ b/das-query-engine/tests/integration/handle/base_test_action.py @@ -26,10 +26,33 @@ def malformed_event(self): def unknown_action_event(self): return {"body": {"action": "unknown_action", "input": {}}} + @staticmethod + def _sorted_nested(obj): + if isinstance(obj, list): + return sorted(str(BaseTestHandlerAction._sorted_nested(item)) for item in obj) + elif isinstance(obj, dict): + return sorted( + (key, BaseTestHandlerAction._sorted_nested(value)) for key, value in obj.items() + ) + elif isinstance(obj, (int, float, str, bool, type(None))): + return obj + else: + return str(obj) + + @staticmethod + def _compare_nested(obj1, obj2): + return BaseTestHandlerAction._sorted_nested(obj1) == BaseTestHandlerAction._sorted_nested( + obj2 + ) + def assert_successful_execution(self, valid_event, expected_output): response = handle(valid_event, context={}) + body = json.loads(response["body"]) assert response["statusCode"] == 200 - assert response["body"] == json.dumps(expected_output), f"Assertion failed:\nResponse Body: {response['body']}\nExpected: {json.dumps(expected_output)}" + assert BaseTestHandlerAction._compare_nested( + body, + expected_output, + ), f"Assertion failed:\nResponse Body: {response['body']}\nExpected: {json.dumps(expected_output)}" def assert_payload_malformed(self, malformed_event): response = handle(malformed_event, context={}) diff --git a/das-query-engine/tests/integration/handle/test_get_incoming_links.py b/das-query-engine/tests/integration/handle/test_get_incoming_links.py new file mode 100644 index 0000000..edfed73 --- /dev/null +++ b/das-query-engine/tests/integration/handle/test_get_incoming_links.py @@ -0,0 +1,272 @@ +import pytest +from actions import ActionType +from tests.integration.handle.base_test_action import BaseTestHandlerAction + + +class TestGetIncomingLinksAction(BaseTestHandlerAction): + @pytest.fixture + def action_type(self): + return ActionType.GET_INCOMING_LINKS + + @pytest.fixture + def valid_event(self, action_type): + return { + "body": { + "action": action_type, + "input": { + "atom_handle": "af12f10f9ae2002a1607ba0b47ba8407", + "kwargs": { + "targets_document": True, + }, + }, + } + } + + @pytest.fixture + def expected_output(self): + return [ + [ + { + "handle": "16f7e407087bfa0b35b13d13a1aadcae", + "composite_type_hash": "ed73ea081d170e1d89fc950820ce1cee", + "is_toplevel": True, + "composite_type": [ + "a9dea78180588431ec64d6bc4872fdbc", + "d99a604c79ce3c2e76a2f43488d5d4c3", + "d99a604c79ce3c2e76a2f43488d5d4c3", + ], + "named_type": "Similarity", + "named_type_hash": "a9dea78180588431ec64d6bc4872fdbc", + "targets": [ + "af12f10f9ae2002a1607ba0b47ba8407", + "4e8e26e3276af8a5c2ac2cc2dc95c6d2", + ], + }, + [ + { + "handle": "af12f10f9ae2002a1607ba0b47ba8407", + "composite_type_hash": "d99a604c79ce3c2e76a2f43488d5d4c3", + "name": "human", + "named_type": "Concept", + }, + { + "handle": "4e8e26e3276af8a5c2ac2cc2dc95c6d2", + "composite_type_hash": "d99a604c79ce3c2e76a2f43488d5d4c3", + "name": "ent", + "named_type": "Concept", + }, + ], + ], + [ + { + "handle": "a45af31b43ee5ea271214338a5a5bd61", + "composite_type_hash": "ed73ea081d170e1d89fc950820ce1cee", + "is_toplevel": True, + "composite_type": [ + "a9dea78180588431ec64d6bc4872fdbc", + "d99a604c79ce3c2e76a2f43488d5d4c3", + "d99a604c79ce3c2e76a2f43488d5d4c3", + ], + "named_type": "Similarity", + "named_type_hash": "a9dea78180588431ec64d6bc4872fdbc", + "targets": [ + "4e8e26e3276af8a5c2ac2cc2dc95c6d2", + "af12f10f9ae2002a1607ba0b47ba8407", + ], + }, + [ + { + "handle": "4e8e26e3276af8a5c2ac2cc2dc95c6d2", + "composite_type_hash": "d99a604c79ce3c2e76a2f43488d5d4c3", + "name": "ent", + "named_type": "Concept", + }, + { + "handle": "af12f10f9ae2002a1607ba0b47ba8407", + "composite_type_hash": "d99a604c79ce3c2e76a2f43488d5d4c3", + "name": "human", + "named_type": "Concept", + }, + ], + ], + [ + { + "handle": "2c927fdc6c0f1272ee439ceb76a6d1a4", + "composite_type_hash": "ed73ea081d170e1d89fc950820ce1cee", + "is_toplevel": True, + "composite_type": [ + "a9dea78180588431ec64d6bc4872fdbc", + "d99a604c79ce3c2e76a2f43488d5d4c3", + "d99a604c79ce3c2e76a2f43488d5d4c3", + ], + "named_type": "Similarity", + "named_type_hash": "a9dea78180588431ec64d6bc4872fdbc", + "targets": [ + "5b34c54bee150c04f9fa584b899dc030", + "af12f10f9ae2002a1607ba0b47ba8407", + ], + }, + [ + { + "handle": "5b34c54bee150c04f9fa584b899dc030", + "composite_type_hash": "d99a604c79ce3c2e76a2f43488d5d4c3", + "name": "chimp", + "named_type": "Concept", + }, + { + "handle": "af12f10f9ae2002a1607ba0b47ba8407", + "composite_type_hash": "d99a604c79ce3c2e76a2f43488d5d4c3", + "name": "human", + "named_type": "Concept", + }, + ], + ], + [ + { + "handle": "b5459e299a5c5e8662c427f7e01b3bf1", + "composite_type_hash": "ed73ea081d170e1d89fc950820ce1cee", + "is_toplevel": True, + "composite_type": [ + "a9dea78180588431ec64d6bc4872fdbc", + "d99a604c79ce3c2e76a2f43488d5d4c3", + "d99a604c79ce3c2e76a2f43488d5d4c3", + ], + "named_type": "Similarity", + "named_type_hash": "a9dea78180588431ec64d6bc4872fdbc", + "targets": [ + "af12f10f9ae2002a1607ba0b47ba8407", + "5b34c54bee150c04f9fa584b899dc030", + ], + }, + [ + { + "handle": "af12f10f9ae2002a1607ba0b47ba8407", + "composite_type_hash": "d99a604c79ce3c2e76a2f43488d5d4c3", + "name": "human", + "named_type": "Concept", + }, + { + "handle": "5b34c54bee150c04f9fa584b899dc030", + "composite_type_hash": "d99a604c79ce3c2e76a2f43488d5d4c3", + "name": "chimp", + "named_type": "Concept", + }, + ], + ], + [ + { + "handle": "c93e1e758c53912638438e2a7d7f7b7f", + "composite_type_hash": "41c082428b28d7e9ea96160f7fd614ad", + "is_toplevel": True, + "composite_type": [ + "e40489cd1e7102e35469c937e05c8bba", + "d99a604c79ce3c2e76a2f43488d5d4c3", + "d99a604c79ce3c2e76a2f43488d5d4c3", + ], + "named_type": "Inheritance", + "named_type_hash": "e40489cd1e7102e35469c937e05c8bba", + "targets": [ + "af12f10f9ae2002a1607ba0b47ba8407", + "bdfe4e7a431f73386f37c6448afe5840", + ], + }, + [ + { + "handle": "af12f10f9ae2002a1607ba0b47ba8407", + "composite_type_hash": "d99a604c79ce3c2e76a2f43488d5d4c3", + "name": "human", + "named_type": "Concept", + }, + { + "handle": "bdfe4e7a431f73386f37c6448afe5840", + "composite_type_hash": "d99a604c79ce3c2e76a2f43488d5d4c3", + "name": "mammal", + "named_type": "Concept", + }, + ], + ], + [ + { + "handle": "2a8a69c01305563932b957de4b3a9ba6", + "composite_type_hash": "ed73ea081d170e1d89fc950820ce1cee", + "is_toplevel": True, + "composite_type": [ + "a9dea78180588431ec64d6bc4872fdbc", + "d99a604c79ce3c2e76a2f43488d5d4c3", + "d99a604c79ce3c2e76a2f43488d5d4c3", + ], + "named_type": "Similarity", + "named_type_hash": "a9dea78180588431ec64d6bc4872fdbc", + "targets": [ + "1cdffc6b0b89ff41d68bec237481d1e1", + "af12f10f9ae2002a1607ba0b47ba8407", + ], + }, + [ + { + "handle": "1cdffc6b0b89ff41d68bec237481d1e1", + "composite_type_hash": "d99a604c79ce3c2e76a2f43488d5d4c3", + "name": "monkey", + "named_type": "Concept", + }, + { + "handle": "af12f10f9ae2002a1607ba0b47ba8407", + "composite_type_hash": "d99a604c79ce3c2e76a2f43488d5d4c3", + "name": "human", + "named_type": "Concept", + }, + ], + ], + [ + { + "handle": "bad7472f41a0e7d601ca294eb4607c3a", + "composite_type_hash": "ed73ea081d170e1d89fc950820ce1cee", + "is_toplevel": True, + "composite_type": [ + "a9dea78180588431ec64d6bc4872fdbc", + "d99a604c79ce3c2e76a2f43488d5d4c3", + "d99a604c79ce3c2e76a2f43488d5d4c3", + ], + "named_type": "Similarity", + "named_type_hash": "a9dea78180588431ec64d6bc4872fdbc", + "targets": [ + "af12f10f9ae2002a1607ba0b47ba8407", + "1cdffc6b0b89ff41d68bec237481d1e1", + ], + }, + [ + { + "handle": "af12f10f9ae2002a1607ba0b47ba8407", + "composite_type_hash": "d99a604c79ce3c2e76a2f43488d5d4c3", + "name": "human", + "named_type": "Concept", + }, + { + "handle": "1cdffc6b0b89ff41d68bec237481d1e1", + "composite_type_hash": "d99a604c79ce3c2e76a2f43488d5d4c3", + "name": "monkey", + "named_type": "Concept", + }, + ], + ], + ] + + def test_incoming_links_action( + self, + valid_event, + expected_output, + ): + self.assert_successful_execution(valid_event, expected_output) + + def test_malformed_payload(self, malformed_event): + self.assert_payload_malformed(malformed_event) + + def test_unknown_action(self, unknown_action_event): + self.assert_unknown_action_dispatcher(unknown_action_event) + + def test_unexpected_exception( + self, + mocker, + valid_event, + ): + self.assert_unexpected_exception(mocker, valid_event) diff --git a/das-query-engine/validators/actions.py b/das-query-engine/validators/actions.py index 637eded..c5567d8 100644 --- a/das-query-engine/validators/actions.py +++ b/das-query-engine/validators/actions.py @@ -34,9 +34,7 @@ class GetIncomingLinksValidator(PayloadValidator): atom_handle = datatypes.String(required=True) kwargs = datatypes.Function( - lambda value, *args, **kwargs: isinstance(value, dict) - if value is not None - else True, + lambda value, *args, **kwargs: isinstance(value, dict) if value is not None else True, required=False, ) From 09d13e1eafcc186dc3d3e658987da0e8c0684705 Mon Sep 17 00:00:00 2001 From: Levi <141682181+levisingularity@users.noreply.github.com> Date: Thu, 25 Jan 2024 16:17:45 -0300 Subject: [PATCH 2/3] Setup atomdb and query engine from the host machine (#61) * make local run * feat: setup pythonpath from host machine * Set up environment variable ATOMDB_PACKAGE_PATH and QUERY_ENGINE_PACKAGE_PATH to bind the packages from the host machine into the container * docs: update readme * Add a section explaining how to setup the variables ATOMDB_PACKAGE_PATH and QUERY_ENGINE_PACKAGE_PATH * Update the commands to start and stop the function * Update commands to run tests * fix: set missing environment variables --- .dockerignore | 7 +++++++ .env.example | 4 +++- Makefile | 8 +++++++- README.md | 34 +++++++++++++++++++++++++++++----- docker-compose.yaml | 14 +++++++++++++- scripts/initd.sh | 18 +++++++++++++++++- 6 files changed, 76 insertions(+), 9 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..f81c736 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +env/ +.vscode +docs +build +.github +__pycache__ +.pytest_cache diff --git a/.env.example b/.env.example index 2900ca5..22064a7 100644 --- a/.env.example +++ b/.env.example @@ -10,6 +10,8 @@ DAS_MONGODB_USERNAME=${MONGO_INITDB_ROOT_USERNAME} DAS_MONGODB_PASSWORD=${MONGO_INITDB_ROOT_PASSWORD} DAS_DATABASE_USERNAME=${MONGO_INITDB_ROOT_USERNAME} DAS_DATABASE_PASSWORD=${MONGO_INITDB_ROOT_PASSWORD} -PYTHONPATH=/app + +ATOMDB_PACKAGE_PATH= +QUERY_ENGINE_PACKAGE_PATH= DAS_KNOWLEDGE_BASE=/data/samples/animals.metta diff --git a/Makefile b/Makefile index 0130ce4..a10d52d 100644 --- a/Makefile +++ b/Makefile @@ -18,4 +18,10 @@ unit-tests-coverage: integration-tests: ./scripts/run-tests.sh integration -pre-commit: unit-tests-coverage lint \ No newline at end of file +pre-commit: unit-tests-coverage lint + +serve: + @docker compose up --build --force-recreate -d + +stop: + @docker compose down diff --git a/README.md b/README.md index 18acb40..355a121 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ OpenFaaS, or Open Functions as a Service, is an open-source platform simplifying ## Running an OpenFaaS Function Locally ### Requirements + - Docker - Docker Compose @@ -33,11 +34,10 @@ The project architecture consists of: - **das-query-engine**: Operates on the same network as the 'openfaas' container. - **Port 8080**: Exposed for connection to the host machine. - ### Step-by-Step Guide 1. **Cloning the Project** - + Clone the project repository to your local environment. ```bash @@ -58,7 +58,7 @@ The project architecture consists of: Execute the following command at the root of the project to start the required services: ```bash - docker compose up -d + make serve ``` This will start containers for Redis, MongoDB, and a temporary container named 'canonical-load'. The latter is used to load initial data into the Redis and MongoDB databases. After its execution, a container named 'openfaas' will start, which includes the faas-cli. @@ -72,7 +72,7 @@ The project architecture consists of: To shut down the containers and clean up the environment, execute the following command: ```bash - docker compose down + make stop ``` ### Testing @@ -80,12 +80,36 @@ The project architecture consists of: To run automated tests for the project, use the following script: ```bash -./scripts/run-tests.sh +make integration-tests +``` + +```bash +make unit-tests ``` This script will set up the necessary environment for testing. +### Use the hyperon-das and hyperon-das-atomdb from the host machine + +This feature has been implemented to allow developers to test the integration of AtomDB and Query Engine packages locally, even before publishing them on PyPI. This facilitates efficient testing during the development phase. +1. Open the `.env` file at the root of your project. + +2. Add the following environment variables, adjusting the paths as necessary: + + ```dotenv + ATOMDB_PACKAGE_PATH=/path/to/your/atomdb/package + QUERY_ENGINE_PACKAGE_PATH=/path/to/your/query/engine/package + ``` + + Make sure to replace `/path/to/your/atomdb/package` and `/path/to/your/query/engine/package` with the actual paths of the AtomDB and Query Engine packages on your system. + +3. If you prefer to use the latest versions of the AtomDB and Query Engine packages published on PyPI, leave the variables empty: + + ```dotenv + ATOMDB_PACKAGE_PATH= + QUERY_ENGINE_PACKAGE_PATH= + ``` ## Obtaining Function Logs in faasd Using the `ctr` Command diff --git a/docker-compose.yaml b/docker-compose.yaml index 7489dcb..88ae876 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -41,6 +41,8 @@ services: command: python3 scripts/load_das.py --knowledge-base ${DAS_KNOWLEDGE_BASE} volumes: - /tmp:/tmp + environment: + - PYTHONPATH=/app env_file: - .env depends_on: @@ -52,7 +54,17 @@ services: privileged: true build: . volumes: - - ./das-query-engine:/usr/local/function/das-query-engine + - type: bind + source: ./das-query-engine + target: /usr/local/function/das-query-engine + + - type: bind + source: ${ATOMDB_PACKAGE_PATH:-/tmp} + target: /opt/repos/hyperon_das_atomdb + + - type: bind + source: ${QUERY_ENGINE_PACKAGE_PATH:-/tmp} + target: /opt/repos/hyperon_das ports: - 8080:8080 env_file: diff --git a/scripts/initd.sh b/scripts/initd.sh index 9688573..7746283 100755 --- a/scripts/initd.sh +++ b/scripts/initd.sh @@ -13,5 +13,21 @@ else done fi -faas-cli local-run --network host --watch +faas-cli build + +FUNCTION_NAME=$(grep -A6 "^functions:" "./stack.yml" | grep -E "^ +query-engine:" -A8 | awk '/^ +image:/ {print $2}') + +docker run --rm --name query-engine \ + --network host \ + -v /opt/repos:/opt/repos \ + -e PYTHONPATH=/opt/repos \ + -e DAS_MONGODB_NAME \ + -e DAS_MONGODB_HOSTNAME \ + -e DAS_MONGODB_PORT \ + -e DAS_REDIS_HOSTNAME \ + -e DAS_REDIS_PORT \ + -e DAS_MONGODB_USERNAME \ + -e DAS_MONGODB_PASSWORD \ + $FUNCTION_NAME + #tail -f /dev/null From 243b2678a7592df5ba516c3ca6ebe519c664cd4a Mon Sep 17 00:00:00 2001 From: levisingularity Date: Thu, 8 Feb 2024 20:15:42 -0300 Subject: [PATCH 3/3] Add changelog file --- CHANGELOG | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 CHANGELOG diff --git a/CHANGELOG b/CHANGELOG new file mode 100644 index 0000000..9caab98 --- /dev/null +++ b/CHANGELOG @@ -0,0 +1,3 @@ +[#57] Enable Running Integration Tests Locally with AtomDB Source Code and Local Query Engine +[#62] Create integration tests for get_incoming_links +[#64] Shutdown containers before run tests \ No newline at end of file