test_: run functional tests on host (no container) (#6159)

* test_: run on host

_assets/build/Dockerfile

@@ -30,6 +30,7 @@ LABEL source="https://github.com/status-im/status-go"
 LABEL description="status-go is an underlying part of Status - a browser, messenger, and gateway to a decentralized world."
 
 RUN apk add --no-cache ca-certificates bash libgcc libstdc++ curl
+RUN mkdir -p /usr/status-user && chmod -R 777 /usr/status-user
 RUN mkdir -p /static/keys
 RUN mkdir -p /static/configs
 

_assets/scripts/run_functional_tests.sh

@@ -26,15 +26,35 @@ mkdir -p "${test_results_path}"
 all_compose_files="-f ${root_path}/docker-compose.anvil.yml -f ${root_path}/docker-compose.test.status-go.yml"
 project_name="status-go-func-tests-$(date +%s)"
 
-export STATUS_BACKEND_COUNT=10
 export STATUS_BACKEND_URLS=$(eval echo http://${project_name}-status-backend-{1..${STATUS_BACKEND_COUNT}}:3333 | tr ' ' ,)
 
-# Run functional tests
+# Remove orphans
+docker ps -a --filter "name=status-go-func-tests-*-status-backend-*" --filter "status=exited" -q | xargs -r docker rm
+
+# Run docker
 echo -e "${GRN}Running tests${RST}, HEAD: $(git rev-parse HEAD)"
-docker compose -p ${project_name} ${all_compose_files} up -d --build --scale status-backend=${STATUS_BACKEND_COUNT} --remove-orphans
+docker compose -p ${project_name} ${all_compose_files} up -d --build --remove-orphans
+
+# Set up virtual environment
+venv_path="${root_path}/.venv"
+
+if [[ -d "${venv_path}" ]]; then
+    echo -e "${GRN}Using existing virtual environment${RST}"
+else
+    echo -e "${GRN}Creating new virtual environment${RST}"
+    python3 -m venv "${venv_path}"
+fi
 
-echo -e "${GRN}Running tests-rpc${RST}" # Follow the logs, wait for them to finish
-docker compose -p ${project_name} ${all_compose_files} logs -f tests-rpc > "${root_path}/tests-rpc.log"
+source "${venv_path}/bin/activate"
+
+# Upgrade pip and install requirements
+echo -e "${GRN}Installing dependencies${RST}"
+pip install --upgrade pip
+pip install -r "${root_path}/requirements.txt"
+
+# Run functional tests
+pytest -m rpc --docker_project_name=${project_name} --codecov_dir=${binary_coverage_reports_path} --junitxml=${test_results_path}/report.xml
+exit_code=$?
 
 # Stop containers
 echo -e "${GRN}Stopping docker containers${RST}"
@@ -45,9 +65,6 @@ echo -e "${GRN}Saving logs${RST}"
 docker compose -p ${project_name} ${all_compose_files} logs status-go > "${root_path}/statusd.log"
 docker compose -p ${project_name} ${all_compose_files} logs status-backend > "${root_path}/status-backend.log"
 
-# Retrieve exit code
-exit_code=$(docker inspect ${project_name}-tests-rpc-1 -f '{{.State.ExitCode}}');
-
 # Cleanup containers
 echo -e "${GRN}Removing docker containers${RST}"
 docker compose -p ${project_name} ${all_compose_files} down

tests-functional/README.MD

@@ -25,12 +25,13 @@ Functional tests for status-go
     * Status-im contracts will be deployed to the network
 
 ### Run tests
-- In `./tests-functional` run `docker compose -f docker-compose.anvil.yml -f docker-compose.test.status-go.yml -f docker-compose.status-go.local.yml up --build --scale status-backend=10 --remove-orphans`, as result:
-    * a container with [status-go as daemon](https://github.com/status-im/status-go/issues/5175) will be created with APIModules exposed on `0.0.0.0:3333`
+- In `./tests-functional` run `docker compose -f docker-compose.anvil.yml -f docker-compose.test.status-go.yml -f docker-compose.status-go.local.yml up --build --remove-orphans`, as result:
+    * a container with [status-backend](https://github.com/status-im/status-go/pull/5847) will be created with endpoint exposed on `0.0.0.0:3333`
     * status-go will use [anvil](https://book.getfoundry.sh/reference/anvil/) as RPCURL with ChainID 31337 
-    * all Status-im contracts will be deployed to the network
+    * Status-im contracts will be deployed to the network
 
-* In `./tests-functional/tests` directory run `pytest -m wallet` 
+* In `./tests-functional/tests` directory run `pytest -m rpc`
+* To run tests against binary run `pytest -m <your mark> --url=http:<binary_url>:<binary_port> --user_dir=/<path>`
 
 ## Implementation details
 

tests-functional/clients/status_backend.py

@@ -4,23 +4,31 @@
 import random
 import threading
 import requests
-from tenacity import retry, stop_after_delay, wait_fixed
+import docker
+import os
 
+from tenacity import retry, stop_after_delay, wait_fixed
 from clients.signals import SignalClient
 from clients.rpc import RpcClient
 from datetime import datetime
 from conftest import option
-from constants import user_1, DEFAULT_DISPLAY_NAME
+from constants import user_1, DEFAULT_DISPLAY_NAME, USER_DIR
 
 
 class StatusBackend(RpcClient, SignalClient):
 
-    def __init__(self, await_signals=[], url=None):
-        try:
-            url = url if url else random.choice(option.status_backend_urls)
-        except IndexError:
-            raise Exception("Not enough status-backend containers, please add more")
-        option.status_backend_urls.remove(url)
+    def __init__(self, await_signals=[]):
+
+        if option.status_backend_url:
+            url = option.status_backend_url
+        else:
+            self.docker_client = docker.from_env()
+            host_port = random.choice(option.status_backend_port_range)
+
+            self.container = self._start_container(host_port)
+            url = f"http://127.0.0.1:{host_port}"
+            option.status_backend_port_range.remove(host_port)
+
 
         self.api_url = f"{url}/statusgo"
         self.ws_url = f"{url}".replace("http", "ws")
@@ -29,14 +37,70 @@ def __init__(self, await_signals=[], url=None):
         RpcClient.__init__(self, self.rpc_url)
         SignalClient.__init__(self, self.ws_url, await_signals)
 
+        self._health_check()
+
         websocket_thread = threading.Thread(target=self._connect)
         websocket_thread.daemon = True
         websocket_thread.start()
 
+    def _start_container(self, host_port):
+        docker_project_name = option.docker_project_name
+
+        timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
+        image_name = f"{docker_project_name}-status-backend:latest"
+        container_name = f"{docker_project_name}-status-backend-{timestamp}"
+
+        coverage_path = option.codecov_dir if option.codecov_dir else os.path.abspath("./coverage/binary")
+
+        container_args = {
+            "image": image_name,
+            "detach": True,
+            "name": container_name,
+            "labels": {"com.docker.compose.project": docker_project_name},
+            "entrypoint": [
+                "status-backend",
+                "--address", "0.0.0.0:3333",
+            ],
+            "ports": {"3333/tcp": host_port},
+            "environment": {
+                "GOCOVERDIR": "/coverage/binary",
+            },
+            "volumes": {
+                coverage_path: {
+                    "bind": "/coverage/binary",
+                    "mode": "rw",
+                }
+            },
+        }
+
+        if "FUNCTIONAL_TESTS_DOCKER_UID" in os.environ:
+            container_args["user"] = os.environ["FUNCTIONAL_TESTS_DOCKER_UID"]
+
+        container = self.docker_client.containers.run(**container_args)
+
+        network = self.docker_client.networks.get(
+            f"{docker_project_name}_default")
+        network.connect(container)
+
+        option.status_backend_containers.append(container.id)
+        return container
+
+    def _health_check(self):
+        start_time = time.time()
+        while True:
+            try:
+                self.api_valid_request(method="Fleets", data=[])
+                break
+            except Exception as e:
+                if time.time() - start_time > 20:
+                    raise Exception(e)
+                time.sleep(1)
+
     def api_request(self, method, data, url=None):
         url = url if url else self.api_url
         url = f"{url}/{method}"
-        logging.info(f"Sending POST request to url {url} with data: {json.dumps(data, sort_keys=True, indent=4)}")
+        logging.info(
+            f"Sending POST request to url {url} with data: {json.dumps(data, sort_keys=True, indent=4)}")
         response = requests.post(url, json=data)
         logging.info(f"Got response: {response.content}")
         return response
@@ -46,7 +110,8 @@ def verify_is_valid_api_response(self, response):
         assert response.content
         logging.info(f"Got response: {response.content}")
         try:
-            assert not response.json()["error"]
+            error = response.json()["error"]
+            assert not error, f"Error: {error}"
         except json.JSONDecodeError:
             raise AssertionError(
                 f"Invalid JSON in response: {response.content}")
@@ -58,7 +123,7 @@ def api_valid_request(self, method, data):
         self.verify_is_valid_api_response(response)
         return response
 
-    def init_status_backend(self, data_dir="/"):
+    def init_status_backend(self, data_dir=USER_DIR):
         method = "InitializeApplication"
         data = {
             "dataDir": data_dir,
@@ -68,7 +133,7 @@ def init_status_backend(self, data_dir="/"):
         }
         return self.api_valid_request(method, data)
 
-    def create_account_and_login(self, data_dir="/", display_name=DEFAULT_DISPLAY_NAME, password=user_1.password):
+    def create_account_and_login(self, data_dir=USER_DIR, display_name=DEFAULT_DISPLAY_NAME, password=user_1.password):
         method = "CreateAccountAndLogin"
         data = {
             "rootDataDir": data_dir,
@@ -81,7 +146,7 @@ def create_account_and_login(self, data_dir="/", display_name=DEFAULT_DISPLAY_NA
         }
         return self.api_valid_request(method, data)
 
-    def restore_account_and_login(self, data_dir="/",display_name=DEFAULT_DISPLAY_NAME, user=user_1,
+    def restore_account_and_login(self, data_dir=USER_DIR, display_name=DEFAULT_DISPLAY_NAME, user=user_1,
                                   network_id=31337):
         method = "RestoreAccountAndLogin"
         data = {
@@ -136,7 +201,8 @@ def restore_account_and_wait_for_rpc_client_to_start(self, timeout=60):
                 return
             except AssertionError:
                 time.sleep(3)
-        raise TimeoutError(f"RPC client was not started after {timeout} seconds")
+        raise TimeoutError(
+            f"RPC client was not started after {timeout} seconds")
 
     @retry(stop=stop_after_delay(10), wait=wait_fixed(0.5), reraise=True)
     def start_messenger(self, params=[]):
@@ -173,7 +239,8 @@ def get_pubkey(self, display_name):
         for account in accounts:
             if account.get("name") == display_name:
                 return account.get("public-key")
-        raise ValueError(f"Public key not found for display name: {display_name}")
+        raise ValueError(
+            f"Public key not found for display name: {display_name}")
 
     def send_contact_request(self, params=[]):
         method = "wakuext_sendContactRequest"

tests-functional/conftest.py

@@ -1,44 +1,46 @@
 import os
-import threading
-from dataclasses import dataclass
-
+import docker
 import pytest as pytest
 
+from dataclasses import dataclass
+
 
 def pytest_addoption(parser):
     parser.addoption(
-        "--rpc_url_statusd",
+        "--status_backend_url",
         action="store",
         help="",
-        default="http://0.0.0.0:3333",
+        default=None,
     )
     parser.addoption(
-        "--ws_url_statusd",
+        "--anvil_url",
         action="store",
         help="",
-        default="ws://0.0.0.0:8354",
+        default="http://0.0.0.0:8545",
     )
     parser.addoption(
-        "--status_backend_urls",
+        "--password",
         action="store",
         help="",
-        default=[
-            f"http://0.0.0.0:{3314 + i}" for i in range(
-                int(os.getenv("STATUS_BACKEND_COUNT", 10))
-            )
-        ],
+        default="Strong12345",
     )
     parser.addoption(
-        "--anvil_url",
+        "--docker_project_name",
         action="store",
         help="",
-        default="http://0.0.0.0:8545",
+        default="tests-functional",
     )
     parser.addoption(
-        "--password",
+        "--codecov_dir",
         action="store",
         help="",
-        default="Strong12345",
+        default=None,
+    )
+    parser.addoption(
+        "--user_dir",
+        action="store",
+        help="",
+        default=None,
     )
 
 @dataclass
@@ -52,6 +54,24 @@ class Option:
 def pytest_configure(config):
     global option
     option = config.option
-    if type(option.status_backend_urls) is str:
-        option.status_backend_urls = option.status_backend_urls.split(",")
+
+    executor_number = int(os.getenv('EXECUTOR_NUMBER', 5))
+    base_port = 7000
+    range_size = 100
+
+    start_port = base_port + (executor_number * range_size)
+
+    option.status_backend_port_range = list(range(start_port, start_port + range_size - 1))
+    option.status_backend_containers = []
+
     option.base_dir = os.path.dirname(os.path.abspath(__file__))
+
+def pytest_unconfigure():
+    docker_client = docker.from_env()
+    for container_id in option.status_backend_containers:
+        try:
+            container = docker_client.containers.get(container_id)
+            container.stop(timeout=30)
+            container.remove()
+        except Exception as e:
+            print(e)

tests-functional/constants.py

@@ -1,4 +1,5 @@
 from dataclasses import dataclass
+from conftest import option
 import os
 
 
@@ -26,4 +27,5 @@ class Account:
 PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../"))
 TESTS_DIR = os.path.join(PROJECT_ROOT, "tests-functional")
 SIGNALS_DIR = os.path.join(TESTS_DIR, "signals")
-LOG_SIGNALS_TO_FILE = False # used for debugging purposes
+LOG_SIGNALS_TO_FILE = False # used for debugging purposes
+USER_DIR = option.user_dir if option.user_dir else "/usr/status-user"

tests-functional/docker-compose.status-go.local.yml

@@ -2,10 +2,6 @@ services:
   anvil:
     ports:
       - 8545:8545
-  status-go:
-    ports:
-      - 3333:3333
-      - 8354:8354
   status-backend:
     ports:
       - 3314-3324:3333