diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 4fd2f543..78a1bf08 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -29,6 +29,10 @@ on: description: 'Worker image [semvar]' required: false default: 'latest' + worker_api_ver: + description: 'Worker image run mode [v1, v2]' + required: false + default: 'v2' debug_mode: description: 'Enable verbose logging for containers [0, 1]' required: false @@ -70,6 +74,10 @@ on: description: 'Worker image [semvar]' required: false type: string + worker_api_ver: + description: 'Worker image run mode [v1, v2]' + required: false + type: string debug_mode: description: 'Enable verbose logging for containers [0, 1]' required: false @@ -119,6 +127,7 @@ jobs: SERVER_TAG: 'latest' WORKER_IMG: 'coreoasis/model_worker' WORKER_TAG: 'latest' + WORKER_API_VER: 'v2' DEBUG: 1 steps: @@ -138,6 +147,7 @@ jobs: [[ -z "${{ inputs.server_tag }}" ]] || echo "SERVER_TAG=${{ inputs.server_tag }}" >> $GITHUB_ENV [[ -z "${{ inputs.worker_image }}" ]] || echo "WORKER_IMG=${{ inputs.worker_image }}" >> $GITHUB_ENV [[ -z "${{ inputs.worker_tag }}" ]] || echo "WORKER_TAG=${{ inputs.worker_tag }}" >> $GITHUB_ENV + [[ -z "${{ inputs.worker_api_ver }}" ]] || echo "WORKER_API_VER=${{ inputs.worker_api_ver }}" >> $GITHUB_ENV [[ -z "${{ inputs.debug_mode }}" ]] || echo "DEBUG=${{ inputs.debug_mode }}" >> $GITHUB_ENV - name: clone PiWind repository diff --git a/docker/plat2-v2.docker-compose.yml b/docker/plat2-v2.docker-compose.yml new file mode 100644 index 00000000..78f2426a --- /dev/null +++ b/docker/plat2-v2.docker-compose.yml @@ -0,0 +1,174 @@ +version: '3.4' +volumes: + server-db-OasisData: + celery-db-OasisData: + filestore-OasisData: + +x-server-env: &server-env + OASIS_DEBUG: 1 + OASIS_URL_SUB_PATH: 0 + OASIS_CELERY_BROKER_URL: "amqp://rabbit:rabbit@broker:5672" + OASIS_SERVER_DB_HOST: server-db + OASIS_SERVER_DB_PASS: oasis + OASIS_SERVER_DB_USER: oasis + OASIS_SERVER_DB_NAME: oasis + OASIS_SERVER_DB_PORT: 5432 + OASIS_SERVER_CHANNEL_LAYER_HOST: channel-layer + OASIS_SERVER_DB_ENGINE: django.db.backends.postgresql_psycopg2 + OASIS_CELERY_DB_ENGINE: db+postgresql+psycopg2 + OASIS_CELERY_DB_HOST: celery-db + OASIS_CELERY_DB_PASS: password + OASIS_CELERY_DB_USER: celery + OASIS_CELERY_DB_NAME: celery + OASIS_CELERY_DB_PORT: 5432 + OASIS_INPUT_GENERATION_CONTROLLER_QUEUE: task-controller + OASIS_LOSSES_GENERATION_CONTROLLER_QUEUE: task-controller + +x-worker-env: &worker-env + OASIS_DEBUG: 1 + OASIS_CELERY_BROKER_URL: "amqp://rabbit:rabbit@broker:5672" + OASIS_CELERY_DB_ENGINE: db+postgresql+psycopg2 + OASIS_CELERY_DB_HOST: celery-db + OASIS_CELERY_DB_PASS: password + OASIS_CELERY_DB_USER: celery + OASIS_CELERY_DB_NAME: celery + OASIS_CELERY_DB_PORT: 5432 + # V1 + OASIS_RABBIT_HOST: broker + OASIS_RABBIT_PORT: 5672 + OASIS_RABBIT_USER: rabbit + OASIS_RABBIT_PASS: rabbit + + +x-volumes: &shared-volumes + - filestore-OasisData:/shared-fs:rw + +services: + server: + restart: always + image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-2.1.0-dev} + command: ["./wsgi/run-wsgi.sh"] + ports: + - 8000:8000 + links: + - server-db + - celery-db + - broker + environment: + <<: *server-env + STARTUP_RUN_MIGRATIONS: "true" + OASIS_ADMIN_USER: admin + OASIS_ADMIN_PASS: password + volumes: + - filestore-OasisData:/shared-fs:rw + websocket: + restart: always + image: coreoasis/api_server:dev + image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-2.1.0-dev} + command: ["./asgi/run-asgi.sh"] + links: + - server-db + - celery-db + - broker + ports: + - 8001:8001 + environment: + <<: *server-env + volumes: + - filestore-OasisData:/shared-fs:rw + v1-worker-monitor: + restart: always + image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-2.1.0-dev} + command: [celery, -A, src.server.oasisapi.celery_app_v1, worker, --loglevel=INFO, -Q, celery] + links: + - server-db + - celery-db + - broker + environment: + <<: *server-env + v2-worker-monitor: + restart: always + image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-2.1.0-dev} + command: [celery, -A, src.server.oasisapi.celery_app_v2, worker, --loglevel=INFO, -Q, celery-v2] + links: + - server-db + - celery-db + - broker + environment: + <<: *server-env + volumes: + - filestore-OasisData:/shared-fs:rw + task-controller: + restart: always + image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-2.1.0-dev} + command: [celery, -A, src.server.oasisapi.celery_app_v2, worker, --loglevel=INFO, -Q, task-controller] + links: + - server-db + - celery-db + - broker + environment: + <<: *server-env + volumes: + - filestore-OasisData:/shared-fs:rw + celery-beat: + restart: always + image: ${SERVER_IMG:-coreoasis/api_server}:${SERVER_TAG:-2.1.0-dev} + command: [celery, -A, src.server.oasisapi.celery_app_v2, beat, --loglevel=INFO] + links: + - server-db + - celery-db + - broker + environment: + <<: *server-env + volumes: *shared-volumes + worker: + restart: always + image: ${WORKER_IMG:-coreoasis/model_worker}:${WORKER_TAG:-2.1.0-dev} + links: + - celery-db + - broker:mybroker + environment: + <<: *worker-env + OASIS_MODEL_SUPPLIER_ID: OasisLMF + OASIS_MODEL_ID: PiWind + OASIS_MODEL_VERSION_ID: 1 + OASIS_API_VERSION: ${WORKER_API_VER:-v2} + volumes: + - ../:/home/worker/model + - filestore-OasisData:/shared-fs:rw + server-db: + restart: always + image: postgres + environment: + - POSTGRES_DB=oasis + - POSTGRES_USER=oasis + - POSTGRES_PASSWORD=oasis + volumes: + - server-db-OasisData:/var/lib/postgresql/data:rw + ports: + - 33307:3306 + celery-db: + restart: always + image: postgres + environment: + - POSTGRES_DB=celery + - POSTGRES_USER=celery + - POSTGRES_PASSWORD=password + volumes: + - celery-db-OasisData:/var/lib/postgresql/data:rw + ports: + - 33306:5432 + broker: + restart: always + image: rabbitmq:3.8.14-management + environment: + - RABBITMQ_DEFAULT_USER=rabbit + - RABBITMQ_DEFAULT_PASS=rabbit + ports: + - 5672:5672 + - 15672:15672 + channel-layer: + restart: always + image: redis:5.0.7 + ports: + - 6379:6379 diff --git a/tests/conftest.py b/tests/conftest.py index 731e1dac..7ede5cd4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,11 +1,17 @@ """pytest configuration""" +from os import getenv from .modelcheck import wait_for_api, _class_server_conn + def pytest_addoption(parser): parser.addoption( '--generate-expected', action='store_true', default=False, # dest='generate_expected', - help='If True, it generates the expected files instead of running the tests. Default: False.' + help='If True, it generates the expected files instead of running the tests. Default: False.', + ) + parser.addoption( + '--api-version', type=str, default=getenv('WORKER_API_VER', default='v1'), + help='Set the OasisAPI clients api version [v1, v2]' ) pytest_plugins = [ diff --git a/tests/modelcheck.py b/tests/modelcheck.py index 9ab2a5ff..bb068b18 100644 --- a/tests/modelcheck.py +++ b/tests/modelcheck.py @@ -51,7 +51,7 @@ @pytest.fixture(scope='module') -def wait_for_api(module_scoped_container_getter): +def wait_for_api(module_scoped_container_getter, request): """ Fixture that waits for the docker container running the OasisAPI to become responsive. see https://github.com/pytest-docker-compose/pytest-docker-compose for details on using @@ -76,6 +76,7 @@ def wait_for_api(module_scoped_container_getter): # Wait for server request_session.mount('http://', HTTPAdapter(max_retries=retries)) server = module_scoped_container_getter.get("server").network_info[0] + api_ver = request.config.getoption('--api-version', 'v1') api_url = f"http://{server.hostname}:{server.host_port}" assert request_session.get(f"{api_url}/healthcheck/") @@ -85,13 +86,12 @@ def wait_for_api(module_scoped_container_getter): assert request_session.get(localstack_url) # Wait for Model - oasis_client = APIClient(api_url=api_url) + oasis_client = APIClient(api_url=api_url, api_ver=api_ver) oasis_client.api.mount('http://', HTTPAdapter(max_retries=retries)) model_headers = {'authorization': f"Bearer {oasis_client.api.tkn_access}"} - model_url = f"{api_url}/V1/models/1/" + model_url = f"{api_url}/{api_ver}/models/1/" assert request_session.get(model_url, headers=model_headers) - #assert oasis_client.models.get(1) return oasis_client @@ -193,7 +193,7 @@ def setUpClass(cls, expected_dir=None, input_tar=None, results_tar=None, params= # skip run if paramters are missing if not cls.params: - pytest.skip(f"Skipping TestClass={cls.__name__}, no input files set in params") + pytest.skip(f"Skipping TestClass={cls.__name__}, no input files set in params") # Find PiWind's model id cls.model_id = cls._get_model_id(cls)