From 95f9b072fd22688cfec17c2c53f05ddd0796d0f0 Mon Sep 17 00:00:00 2001 From: artem-astafev Date: Fri, 6 Dec 2024 15:33:01 +0700 Subject: [PATCH] Added docker compose example for MultimodalQnA deploy on AMD ROCm systems Signed-off-by: artem-astafev --- .../docker_compose/amd/gpu/rocm/README.md | 346 ++++++++++++++++++ .../docker_compose/amd/gpu/rocm/compose.yaml | 156 ++++++++ .../docker_compose/amd/gpu/rocm/set_env.sh | 32 ++ MultimodalQnA/tests/test_compose_on_rocm.sh | 305 +++++++++++++++ 4 files changed, 839 insertions(+) create mode 100644 MultimodalQnA/docker_compose/amd/gpu/rocm/README.md create mode 100644 MultimodalQnA/docker_compose/amd/gpu/rocm/compose.yaml create mode 100644 MultimodalQnA/docker_compose/amd/gpu/rocm/set_env.sh create mode 100644 MultimodalQnA/tests/test_compose_on_rocm.sh diff --git a/MultimodalQnA/docker_compose/amd/gpu/rocm/README.md b/MultimodalQnA/docker_compose/amd/gpu/rocm/README.md new file mode 100644 index 000000000..485d23012 --- /dev/null +++ b/MultimodalQnA/docker_compose/amd/gpu/rocm/README.md @@ -0,0 +1,346 @@ +# Build Mega Service of MultimodalQnA for AMD ROCm + +This document outlines the deployment process for a MultimodalQnA application utilizing the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline on AMD server wit ROCm GPUs. The steps include Docker image creation, container deployment via Docker Compose, and service execution to integrate microservices such as `multimodal_embedding` that employs [BridgeTower](https://huggingface.co/BridgeTower/bridgetower-large-itm-mlm-gaudi) model as embedding model, `multimodal_retriever`, `lvm`, and `multimodal-data-prep`. We will publish the Docker images to Docker Hub soon, it will simplify the deployment process for this service. + +For detailed information about these instance types, you can refer to this [link](https://aws.amazon.com/ec2/instance-types/m7i/). Once you've chosen the appropriate instance type, proceed with configuring your instance settings, including network configurations, security groups, and storage options. + +After launching your instance, you can connect to it using SSH (for Linux instances) or Remote Desktop Protocol (RDP) (for Windows instances). From there, you'll have full access to your Xeon server, allowing you to install, configure, and manage your applications as needed. + + +## Setup Environment Variables + +Since the `compose.yaml` will consume some environment variables, you need to setup them in advance as below. + +**Export the value of the public IP address of your Xeon server to the `host_ip` environment variable** + +> Change the External_Public_IP below with the actual IPV4 value + +``` +export host_ip="External_Public_IP" +``` + +**Append the value of the public IP address to the no_proxy list** + +```bash +export your_no_proxy=${your_no_proxy},"External_Public_IP" +``` + +```bash +export no_proxy=${your_no_proxy} +export http_proxy=${your_http_proxy} +export https_proxy=${your_http_proxy} +export EMBEDDER_PORT=6006 +export MMEI_EMBEDDING_ENDPOINT="http://${host_ip}:$EMBEDDER_PORT/v1/encode" +export MM_EMBEDDING_PORT_MICROSERVICE=6000 +export REDIS_URL="redis://${host_ip}:6379" +export REDIS_HOST=${host_ip} +export INDEX_NAME="mm-rag-redis" +export LLAVA_SERVER_PORT=8399 +export LVM_ENDPOINT="http://${host_ip}:8399" +export EMBEDDING_MODEL_ID="BridgeTower/bridgetower-large-itm-mlm-itc" +export LVM_MODEL_ID="llava-hf/llava-1.5-7b-hf" +export WHISPER_MODEL="base" +export MM_EMBEDDING_SERVICE_HOST_IP=${host_ip} +export MM_RETRIEVER_SERVICE_HOST_IP=${host_ip} +export LVM_SERVICE_HOST_IP=${host_ip} +export MEGA_SERVICE_HOST_IP=${host_ip} +export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8888/v1/multimodalqna" +export DATAPREP_INGEST_SERVICE_ENDPOINT="http://${host_ip}:6007/v1/ingest_with_text" +export DATAPREP_GEN_TRANSCRIPT_SERVICE_ENDPOINT="http://${host_ip}:6007/v1/generate_transcripts" +export DATAPREP_GEN_CAPTION_SERVICE_ENDPOINT="http://${host_ip}:6007/v1/generate_captions" +export DATAPREP_GET_FILE_ENDPOINT="http://${host_ip}:6007/v1/dataprep/get_files" +export DATAPREP_DELETE_FILE_ENDPOINT="http://${host_ip}:6007/v1/dataprep/delete_files" +``` + +Note: Please replace with `host_ip` with you external IP address, do not use localhost. + +## 🚀 Build Docker Images + +### 1. Build embedding-multimodal-bridgetower Image + +Build embedding-multimodal-bridgetower docker image + +```bash +git clone https://github.com/opea-project/GenAIComps.git +cd GenAIComps +docker build --no-cache -t opea/embedding-multimodal-bridgetower:latest --build-arg EMBEDDER_PORT=$EMBEDDER_PORT --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/multimodal/bridgetower/Dockerfile . +``` + +Build embedding-multimodal microservice image + +```bash +docker build --no-cache -t opea/embedding-multimodal:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/multimodal/multimodal_langchain/Dockerfile . +``` + +### 2. Build retriever-multimodal-redis Image + +```bash +docker build --no-cache -t opea/retriever-multimodal-redis:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/retrievers/multimodal/redis/langchain/Dockerfile . +``` + +### 3. Build LVM Images + +Build lvm-llava image + +```bash +docker build --no-cache -t opea/lvm-llava:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/lvms/llava/dependency/Dockerfile . +``` + +### 4. Build dataprep-multimodal-redis Image + +```bash +docker build --no-cache -t opea/dataprep-multimodal-redis:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/multimodal/redis/langchain/Dockerfile . +``` + +### 5. Build MegaService Docker Image + +To construct the Mega Service, we utilize the [GenAIComps](https://github.com/opea-project/GenAIComps.git) microservice pipeline within the [multimodalqna.py](../../../../multimodalqna.py) Python script. Build MegaService Docker image via below command: + +```bash +git clone https://github.com/opea-project/GenAIExamples.git +cd GenAIExamples/MultimodalQnA +docker build --no-cache -t opea/multimodalqna:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile . +cd ../.. +``` + +### 6. Build UI Docker Image + +Build frontend Docker image via below command: + +```bash +cd GenAIExamples/MultimodalQnA/ui/ +docker build --no-cache -t opea/multimodalqna-ui:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f ./docker/Dockerfile . +cd ../../../ +``` + +### 7. Pull TGI AMD ROCm Image + +```bash +docker pull ghcr.io/huggingface/text-generation-inference:2.4.1-rocm +``` + +Then run the command `docker images`, you will have the following 8 Docker Images: + +1. `opea/dataprep-multimodal-redis:latest` +2. `ghcr.io/huggingface/text-generation-inference:2.4.1-rocm` +3. `opea/lvm-llava:latest` +4. `opea/retriever-multimodal-redis:latest` +5. `opea/embedding-multimodal:latest` +6. `opea/embedding-multimodal-bridgetower:latest` +7. `opea/multimodalqna:latest` +8. `opea/multimodalqna-ui:latest` + +## 🚀 Start Microservices + +### Required Models + +By default, the multimodal-embedding and LVM models are set to a default value as listed below: + +| Service | Model | +| -------------------- | ------------------------------------------- | +| embedding-multimodal | BridgeTower/bridgetower-large-itm-mlm-gaudi | +| LVM | llava-hf/llava-1.5-7b-hf | +| LVM | Xkev/Llama-3.2V-11B-cot | + + +Note: + +For AMD ROCm System "Xkev/Llama-3.2V-11B-cot" is recomended to run on ghcr.io/huggingface/text-generation-inference:2.4.1-rocm + +### Start all the services Docker Containers + +> Before running the docker compose command, you need to be in the folder that has the docker compose yaml file + +```bash +cd GenAIExamples/MultimodalQnA/docker_compose/amd/gpu/rocm +docker compose -f compose.yaml up -d +``` + +Note: Please replace with `host_ip` with your external IP address, do not use localhost. + +Note: In order to limit access to a subset of GPUs, please pass each device individually using one or more -device /dev/dri/rendered, where is the card index, starting from 128. (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) + +Example for set isolation for 1 GPU + +``` + - /dev/dri/card0:/dev/dri/card0 + - /dev/dri/renderD128:/dev/dri/renderD128 +``` + +Example for set isolation for 2 GPUs + +``` + - /dev/dri/card0:/dev/dri/card0 + - /dev/dri/renderD128:/dev/dri/renderD128 + - /dev/dri/card1:/dev/dri/card1 + - /dev/dri/renderD129:/dev/dri/renderD129 +``` + +Please find more information about accessing and restricting AMD GPUs in the link (https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html#docker-restrict-gpus) + +### Validate Microservices + +1. embedding-multimodal-bridgetower + +```bash +curl http://${host_ip}:${EMBEDDER_PORT}/v1/encode \ + -X POST \ + -H "Content-Type:application/json" \ + -d '{"text":"This is example"}' +``` + +```bash +curl http://${host_ip}:${EMBEDDER_PORT}/v1/encode \ + -X POST \ + -H "Content-Type:application/json" \ + -d '{"text":"This is example", "img_b64_str": "iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFUlEQVR42mP8/5+hnoEIwDiqkL4KAcT9GO0U4BxoAAAAAElFTkSuQmCC"}' +``` + +2. embedding-multimodal + +```bash +curl http://${host_ip}:$MM_EMBEDDING_PORT_MICROSERVICE/v1/embeddings \ + -X POST \ + -H "Content-Type: application/json" \ + -d '{"text" : "This is some sample text."}' +``` + +```bash +curl http://${host_ip}:$MM_EMBEDDING_PORT_MICROSERVICE/v1/embeddings \ + -X POST \ + -H "Content-Type: application/json" \ + -d '{"text": {"text" : "This is some sample text."}, "image" : {"url": "https://github.com/docarray/docarray/blob/main/tests/toydata/image-data/apple.png?raw=true"}}' +``` + +3. retriever-multimodal-redis + +```bash +export your_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(512)]; print(embedding)") +curl http://${host_ip}:7000/v1/multimodal_retrieval \ + -X POST \ + -H "Content-Type: application/json" \ + -d "{\"text\":\"test\",\"embedding\":${your_embedding}}" +``` + +4. lvm-llava + +```bash +curl http://${host_ip}:${LLAVA_SERVER_PORT}/generate \ + -X POST \ + -H "Content-Type:application/json" \ + -d '{"prompt":"Describe the image please.", "img_b64_str": "iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFUlEQVR42mP8/5+hnoEIwDiqkL4KAcT9GO0U4BxoAAAAAElFTkSuQmCC"}' +``` + +5. lvm-llava-svc + +```bash +curl http://${host_ip}:9399/v1/lvm \ + -X POST \ + -H 'Content-Type: application/json' \ + -d '{"retrieved_docs": [], "initial_query": "What is this?", "top_n": 1, "metadata": [{"b64_img_str": "iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFUlEQVR42mP8/5+hnoEIwDiqkL4KAcT9GO0U4BxoAAAAAElFTkSuQmCC", "transcript_for_inference": "yellow image", "video_id": "8c7461df-b373-4a00-8696-9a2234359fe0", "time_of_frame_ms":"37000000", "source_video":"WeAreGoingOnBullrun_8c7461df-b373-4a00-8696-9a2234359fe0.mp4"}], "chat_template":"The caption of the image is: '\''{context}'\''. {question}"}' +``` + +```bash +curl http://${host_ip}:9399/v1/lvm \ + -X POST \ + -H 'Content-Type: application/json' \ + -d '{"image": "iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFUlEQVR42mP8/5+hnoEIwDiqkL4KAcT9GO0U4BxoAAAAAElFTkSuQmCC", "prompt":"What is this?"}' +``` + +Also, validate LVM Microservice with empty retrieval results + +```bash +curl http://${host_ip}:9399/v1/lvm \ + -X POST \ + -H 'Content-Type: application/json' \ + -d '{"retrieved_docs": [], "initial_query": "What is this?", "top_n": 1, "metadata": [], "chat_template":"The caption of the image is: '\''{context}'\''. {question}"}' +``` + +6. dataprep-multimodal-redis + +Download a sample video, image, and audio file and create a caption + +```bash +export video_fn="WeAreGoingOnBullrun.mp4" +wget http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/WeAreGoingOnBullrun.mp4 -O ${video_fn} + +export image_fn="apple.png" +wget https://github.com/docarray/docarray/blob/main/tests/toydata/image-data/apple.png?raw=true -O ${image_fn} + +export caption_fn="apple.txt" +echo "This is an apple." > ${caption_fn} + +export audio_fn="AudioSample.wav" +wget https://github.com/intel/intel-extension-for-transformers/raw/main/intel_extension_for_transformers/neural_chat/assets/audio/sample.wav -O ${audio_fn} +``` + +Test dataprep microservice with generating transcript. This command updates a knowledge base by uploading a local video .mp4 and an audio .wav file. + +```bash +curl --silent --write-out "HTTPSTATUS:%{http_code}" \ + ${DATAPREP_GEN_TRANSCRIPT_SERVICE_ENDPOINT} \ + -H 'Content-Type: multipart/form-data' \ + -X POST \ + -F "files=@./${video_fn}" \ + -F "files=@./${audio_fn}" +``` + +Also, test dataprep microservice with generating an image caption using lvm microservice + +```bash +curl --silent --write-out "HTTPSTATUS:%{http_code}" \ + ${DATAPREP_GEN_CAPTION_SERVICE_ENDPOINT} \ + -H 'Content-Type: multipart/form-data' \ + -X POST -F "files=@./${image_fn}" +``` + +Now, test the microservice with posting a custom caption along with an image + +```bash +curl --silent --write-out "HTTPSTATUS:%{http_code}" \ + ${DATAPREP_INGEST_SERVICE_ENDPOINT} \ + -H 'Content-Type: multipart/form-data' \ + -X POST -F "files=@./${image_fn}" -F "files=@./${caption_fn}" +``` + +Also, you are able to get the list of all files that you uploaded: + +```bash +curl -X POST \ + -H "Content-Type: application/json" \ + ${DATAPREP_GET_FILE_ENDPOINT} +``` + +Then you will get the response python-style LIST like this. Notice the name of each uploaded file e.g., `videoname.mp4` will become `videoname_uuid.mp4` where `uuid` is a unique ID for each uploaded file. The same files that are uploaded twice will have different `uuid`. + +```bash +[ + "WeAreGoingOnBullrun_7ac553a1-116c-40a2-9fc5-deccbb89b507.mp4", + "WeAreGoingOnBullrun_6d13cf26-8ba2-4026-a3a9-ab2e5eb73a29.mp4", + "apple_fcade6e6-11a5-44a2-833a-3e534cbe4419.png", + "AudioSample_976a85a6-dc3e-43ab-966c-9d81beef780c.wav +] +``` + +To delete all uploaded files along with data indexed with `$INDEX_NAME` in REDIS. + +```bash +curl -X POST \ + -H "Content-Type: application/json" \ + ${DATAPREP_DELETE_FILE_ENDPOINT} +``` + +7. MegaService + +```bash +curl http://${host_ip}:8888/v1/multimodalqna \ + -H "Content-Type: application/json" \ + -X POST \ + -d '{"messages": "What is the revenue of Nike in 2023?"}' +``` + +```bash +curl http://${host_ip}:8888/v1/multimodalqna \ + -H "Content-Type: application/json" \ + -d '{"messages": [{"role": "user", "content": [{"type": "text", "text": "hello, "}, {"type": "image_url", "image_url": {"url": "https://www.ilankelman.org/stopsigns/australia.jpg"}}]}, {"role": "assistant", "content": "opea project! "}, {"role": "user", "content": "chao, "}], "max_tokens": 10}' +``` diff --git a/MultimodalQnA/docker_compose/amd/gpu/rocm/compose.yaml b/MultimodalQnA/docker_compose/amd/gpu/rocm/compose.yaml new file mode 100644 index 000000000..2d566f427 --- /dev/null +++ b/MultimodalQnA/docker_compose/amd/gpu/rocm/compose.yaml @@ -0,0 +1,156 @@ +# Copyright (C) 2024 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: Apache-2.0 + +services: + redis-vector-db: + image: redis/redis-stack:7.2.0-v9 + container_name: redis-vector-db + ports: + - "6379:6379" + - "8001:8001" + dataprep-multimodal-redis: + image: ${REGISTRY:-opea}/dataprep-multimodal-redis:${TAG:-latest} + container_name: dataprep-multimodal-redis + depends_on: + - redis-vector-db + - lvm-tgi + ports: + - "6007:6007" + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + REDIS_URL: ${REDIS_URL} + REDIS_HOST: ${REDIS_HOST} + INDEX_NAME: ${INDEX_NAME} + LVM_ENDPOINT: "http://${LVM_SERVICE_HOST_IP}:9399/v1/lvm" + HUGGINGFACEHUB_API_TOKEN: ${MULTIMODAL_HUGGINGFACEHUB_API_TOKEN} + restart: unless-stopped + embedding-multimodal-bridgetower: + image: ${REGISTRY:-opea}/embedding-multimodal-bridgetower:${TAG:-latest} + container_name: embedding-multimodal-bridgetower + ports: + - ${EMBEDDER_PORT}:${EMBEDDER_PORT} + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + PORT: ${EMBEDDER_PORT} + entrypoint: ["python", "bridgetower_server.py", "--device", "cpu", "--model_name_or_path", $EMBEDDING_MODEL_ID] + restart: unless-stopped + embedding-multimodal: + image: ${REGISTRY:-opea}/embedding-multimodal:${TAG:-latest} + container_name: embedding-multimodal + depends_on: + - embedding-multimodal-bridgetower + ports: + - ${MM_EMBEDDING_PORT_MICROSERVICE}:${MM_EMBEDDING_PORT_MICROSERVICE} + ipc: host + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + MMEI_EMBEDDING_ENDPOINT: ${MMEI_EMBEDDING_ENDPOINT} + MM_EMBEDDING_PORT_MICROSERVICE: ${MM_EMBEDDING_PORT_MICROSERVICE} + restart: unless-stopped + retriever-multimodal-redis: + image: ${REGISTRY:-opea}/retriever-multimodal-redis:${TAG:-latest} + container_name: retriever-multimodal-redis + depends_on: + - redis-vector-db + ports: + - "7000:7000" + ipc: host + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + REDIS_URL: ${REDIS_URL} + INDEX_NAME: ${INDEX_NAME} + restart: unless-stopped + tgi-rocm: + image: ghcr.io/huggingface/text-generation-inference:2.4.1-rocm + container_name: tgi-llava-rocm-server + ports: + - "8399:80" + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + TGI_LLM_ENDPOINT: "http://${HOST_IP}:${MULTIMODAL_TGI_SERVICE_PORT}" + HUGGINGFACEHUB_API_TOKEN: ${MULTIMODAL_HUGGINGFACEHUB_API_TOKEN} + HUGGING_FACE_HUB_TOKEN: ${MULTIMODAL_HUGGINGFACEHUB_API_TOKEN} + volumes: + - "/var/opea/multimodalqna-service/data:/data" + shm_size: 64g + devices: + - /dev/kfd:/dev/kfd + - /dev/dri/:/dev/dri/ + cap_add: + - SYS_PTRACE + group_add: + - video + security_opt: + - seccomp:unconfined + ipc: host + command: --model-id ${LVM_MODEL_ID} --max-input-tokens 3048 --max-total-tokens 4096 + restart: unless-stopped +lvm-tgi: + image: ${REGISTRY:-opea}/lvm-tgi:${TAG:-latest} + container_name: lvm-tgi + depends_on: + - tgi-rocm + ports: + - "9399:9399" + ipc: host + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + LVM_ENDPOINT: ${LVM_ENDPOINT} + HF_HUB_DISABLE_PROGRESS_BARS: 1 + HF_HUB_ENABLE_HF_TRANSFER: 0 + restart: unless-stopped + multimodalqna: + image: ${REGISTRY:-opea}/multimodalqna:${TAG:-latest} + container_name: multimodalqna-backend-server + depends_on: + - redis-vector-db + - dataprep-multimodal-redis + - embedding-multimodal + - retriever-multimodal-redis + - lvm-tgi + ports: + - "8888:8888" + environment: + no_proxy: ${no_proxy} + https_proxy: ${https_proxy} + http_proxy: ${http_proxy} + MEGA_SERVICE_HOST_IP: ${MEGA_SERVICE_HOST_IP} + MM_EMBEDDING_SERVICE_HOST_IP: ${MM_EMBEDDING_SERVICE_HOST_IP} + MM_EMBEDDING_PORT_MICROSERVICE: ${MM_EMBEDDING_PORT_MICROSERVICE} + MM_RETRIEVER_SERVICE_HOST_IP: ${MM_RETRIEVER_SERVICE_HOST_IP} + LVM_SERVICE_HOST_IP: ${LVM_SERVICE_HOST_IP} + ipc: host + restart: always + multimodalqna-ui: + image: ${REGISTRY:-opea}/multimodalqna-ui:${TAG:-latest} + container_name: multimodalqna-gradio-ui-server + depends_on: + - multimodalqna + ports: + - "5173:5173" + environment: + - no_proxy=${no_proxy} + - https_proxy=${https_proxy} + - http_proxy=${http_proxy} + - BACKEND_SERVICE_ENDPOINT=${BACKEND_SERVICE_ENDPOINT} + - DATAPREP_INGEST_SERVICE_ENDPOINT=${DATAPREP_INGEST_SERVICE_ENDPOINT} + - DATAPREP_GEN_TRANSCRIPT_SERVICE_ENDPOINT=${DATAPREP_GEN_TRANSCRIPT_SERVICE_ENDPOINT} + - DATAPREP_GEN_CAPTION_SERVICE_ENDPOINT=${DATAPREP_GEN_CAPTION_SERVICE_ENDPOINT} + ipc: host + restart: always + +networks: + default: + driver: bridge diff --git a/MultimodalQnA/docker_compose/amd/gpu/rocm/set_env.sh b/MultimodalQnA/docker_compose/amd/gpu/rocm/set_env.sh new file mode 100644 index 000000000..6c73b901d --- /dev/null +++ b/MultimodalQnA/docker_compose/amd/gpu/rocm/set_env.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +# Copyright (C) 2024 Advanced Micro Devices, Inc. +# SPDX-License-Identifier: Apache-2.0 + +export HOST_IP=${your_host_ip_address} +export MULTIMODAL_HUGGINGFACEHUB_API_TOKEN=${your_huggingfacehub_token} +export MULTIMODAL_TGI_SERVICE_PORT="8399" +export no_proxy=${your_no_proxy} +export http_proxy=${your_http_proxy} +export https_proxy=${your_http_proxy} +export EMBEDDER_PORT=6006 +export MMEI_EMBEDDING_ENDPOINT="http://${HOST_IP}:$EMBEDDER_PORT/v1/encode" +export MM_EMBEDDING_PORT_MICROSERVICE=6000 +export REDIS_URL="redis://${HOST_IP}:6379" +export REDIS_HOST=${HOST_IP} +export INDEX_NAME="mm-rag-redis" +export LLAVA_SERVER_PORT=8399 +export LVM_ENDPOINT="http://${HOST_IP}:8399" +export EMBEDDING_MODEL_ID="BridgeTower/bridgetower-large-itm-mlm-itc" +export LVM_MODEL_ID="Xkev/Llama-3.2V-11B-cot" +export WHISPER_MODEL="base" +export MM_EMBEDDING_SERVICE_HOST_IP=${HOST_IP} +export MM_RETRIEVER_SERVICE_HOST_IP=${HOST_IP} +export LVM_SERVICE_HOST_IP=${HOST_IP} +export MEGA_SERVICE_HOST_IP=${HOST_IP} +export BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:8888/v1/multimodalqna" +export DATAPREP_INGEST_SERVICE_ENDPOINT="http://${HOST_IP}:6007/v1/ingest_with_text" +export DATAPREP_GEN_TRANSCRIPT_SERVICE_ENDPOINT="http://${HOST_IP}:6007/v1/generate_transcripts" +export DATAPREP_GEN_CAPTION_SERVICE_ENDPOINT="http://${HOST_IP}:6007/v1/generate_captions" +export DATAPREP_GET_FILE_ENDPOINT="http://${HOST_IP}:6007/v1/dataprep/get_files" +export DATAPREP_DELETE_FILE_ENDPOINT="http://${HOST_IP}:6007/v1/dataprep/delete_files" diff --git a/MultimodalQnA/tests/test_compose_on_rocm.sh b/MultimodalQnA/tests/test_compose_on_rocm.sh new file mode 100644 index 000000000..b508109f3 --- /dev/null +++ b/MultimodalQnA/tests/test_compose_on_rocm.sh @@ -0,0 +1,305 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -ex +IMAGE_REPO=${IMAGE_REPO:-"opea"} +IMAGE_TAG=${IMAGE_TAG:-"latest"} +echo "REGISTRY=IMAGE_REPO=${IMAGE_REPO}" +echo "TAG=IMAGE_TAG=${IMAGE_TAG}" +export REGISTRY=${IMAGE_REPO} +export TAG=${IMAGE_TAG} + +WORKPATH=$(dirname "$PWD") +LOG_PATH="$WORKPATH/tests" +ip_address=$(hostname -I | awk '{print $1}') + +export image_fn="apple.png" +export video_fn="WeAreGoingOnBullrun.mp4" +export caption_fn="apple.txt" + +function build_docker_images() { + cd $WORKPATH/docker_image_build + git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout "${opea_branch:-"main"}" && cd ../ + + echo "Build all the images with --no-cache, check docker_image_build.log for details..." + service_list="multimodalqna multimodalqna-ui embedding-multimodal-bridgetower embedding-multimodal retriever-multimodal-redis lvm-llava lvm-llava-svc dataprep-multimodal-redis" + docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log + + docker images && sleep 1m +} + +function setup_env() { + export HOST_IP=${ip_address} + export host_ip=${ip_address} + export MULTIMODAL_HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} + export MULTIMODAL_TGI_SERVICE_PORT="8399" + export no_proxy=${your_no_proxy} + export http_proxy=${your_http_proxy} + export https_proxy=${your_http_proxy} + export EMBEDDER_PORT=6006 + export MMEI_EMBEDDING_ENDPOINT="http://${HOST_IP}:$EMBEDDER_PORT/v1/encode" + export MM_EMBEDDING_PORT_MICROSERVICE=6000 + export REDIS_URL="redis://${HOST_IP}:6379" + export REDIS_HOST=${HOST_IP} + export INDEX_NAME="mm-rag-redis" + export LLAVA_SERVER_PORT=8399 + export LVM_ENDPOINT="http://${HOST_IP}:8399" + export EMBEDDING_MODEL_ID="BridgeTower/bridgetower-large-itm-mlm-itc" + export LVM_MODEL_ID="Xkev/Llama-3.2V-11B-cot" + export WHISPER_MODEL="base" + export MM_EMBEDDING_SERVICE_HOST_IP=${HOST_IP} + export MM_RETRIEVER_SERVICE_HOST_IP=${HOST_IP} + export LVM_SERVICE_HOST_IP=${HOST_IP} + export MEGA_SERVICE_HOST_IP=${HOST_IP} + export BACKEND_SERVICE_ENDPOINT="http://${HOST_IP}:8888/v1/multimodalqna" + export DATAPREP_INGEST_SERVICE_ENDPOINT="http://${HOST_IP}:6007/v1/ingest_with_text" + export DATAPREP_GEN_TRANSCRIPT_SERVICE_ENDPOINT="http://${HOST_IP}:6007/v1/generate_transcripts" + export DATAPREP_GEN_CAPTION_SERVICE_ENDPOINT="http://${HOST_IP}:6007/v1/generate_captions" + export DATAPREP_GET_FILE_ENDPOINT="http://${HOST_IP}:6007/v1/dataprep/get_files" + export DATAPREP_DELETE_FILE_ENDPOINT="http://${HOST_IP}:6007/v1/dataprep/delete_files" +} + +function start_services() { + cd $WORKPATH/docker_compose/amd/gpu/rocm + + + # Start Docker Containers + docker compose -f compose.yaml up -d > ${LOG_PATH}/start_services_with_compose.log + sleep 2m +} + +function prepare_data() { + cd $LOG_PATH + echo "Downloading image and video" + wget https://github.com/docarray/docarray/blob/main/tests/toydata/image-data/apple.png?raw=true -O ${image_fn} + wget http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/WeAreGoingOnBullrun.mp4 -O ${video_fn} + echo "Writing caption file" + echo "This is an apple." > ${caption_fn} + + sleep 30s +} + +function validate_service() { + local URL="$1" + local EXPECTED_RESULT="$2" + local SERVICE_NAME="$3" + local DOCKER_NAME="$4" + local INPUT_DATA="$5" + + if [[ $SERVICE_NAME == *"dataprep-multimodal-redis-transcript"* ]]; then + cd $LOG_PATH + HTTP_RESPONSE=$(curl --silent --write-out "HTTPSTATUS:%{http_code}" -X POST -F "files=@./${video_fn}" -H 'Content-Type: multipart/form-data' "$URL") + elif [[ $SERVICE_NAME == *"dataprep-multimodal-redis-caption"* ]]; then + cd $LOG_PATH + HTTP_RESPONSE=$(curl --silent --write-out "HTTPSTATUS:%{http_code}" -X POST -F "files=@./${image_fn}" -H 'Content-Type: multipart/form-data' "$URL") + elif [[ $SERVICE_NAME == *"dataprep-multimodal-redis-ingest"* ]]; then + cd $LOG_PATH + HTTP_RESPONSE=$(curl --silent --write-out "HTTPSTATUS:%{http_code}" -X POST -F "files=@./${image_fn}" -F "files=@./apple.txt" -H 'Content-Type: multipart/form-data' "$URL") + elif [[ $SERVICE_NAME == *"dataprep_get"* ]]; then + HTTP_RESPONSE=$(curl --silent --write-out "HTTPSTATUS:%{http_code}" -X POST -H 'Content-Type: application/json' "$URL") + elif [[ $SERVICE_NAME == *"dataprep_del"* ]]; then + HTTP_RESPONSE=$(curl --silent --write-out "HTTPSTATUS:%{http_code}" -X POST -H 'Content-Type: application/json' "$URL") + else + HTTP_RESPONSE=$(curl --silent --write-out "HTTPSTATUS:%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL") + fi + HTTP_STATUS=$(echo $HTTP_RESPONSE | tr -d '\n' | sed -e 's/.*HTTPSTATUS://') + RESPONSE_BODY=$(echo $HTTP_RESPONSE | sed -e 's/HTTPSTATUS\:.*//g') + + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + + # check response status + if [ "$HTTP_STATUS" -ne "200" ]; then + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + exit 1 + else + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + fi + # check response body + if [[ "$RESPONSE_BODY" != *"$EXPECTED_RESULT"* ]]; then + echo "[ $SERVICE_NAME ] Content does not match the expected result: $RESPONSE_BODY" + exit 1 + else + echo "[ $SERVICE_NAME ] Content is as expected." + fi + + sleep 1s +} + +function validate_microservices() { + # Check if the microservices are running correctly. + + # Bridgetower Embedding Server + echo "Validating embedding-multimodal-bridgetower" + validate_service \ + "http://${host_ip}:${EMBEDDER_PORT}/v1/encode" \ + '"embedding":[' \ + "embedding-multimodal-bridgetower" \ + "embedding-multimodal-bridgetower" \ + '{"text":"This is example"}' + + validate_service \ + "http://${host_ip}:${EMBEDDER_PORT}/v1/encode" \ + '"embedding":[' \ + "embedding-multimodal-bridgetower" \ + "embedding-multimodal-bridgetower" \ + '{"text":"This is example", "img_b64_str": "iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFUlEQVR42mP8/5+hnoEIwDiqkL4KAcT9GO0U4BxoAAAAAElFTkSuQmCC"}' + + # embedding microservice + echo "Validating embedding-multimodal" + validate_service \ + "http://${host_ip}:$MM_EMBEDDING_PORT_MICROSERVICE/v1/embeddings" \ + '"embedding":[' \ + "embedding-multimodal" \ + "embedding-multimodal" \ + '{"text" : "This is some sample text."}' + + validate_service \ + "http://${host_ip}:$MM_EMBEDDING_PORT_MICROSERVICE/v1/embeddings" \ + '"embedding":[' \ + "embedding-multimodal" \ + "embedding-multimodal" \ + '{"text": {"text" : "This is some sample text."}, "image" : {"url": "https://github.com/docarray/docarray/blob/main/tests/toydata/image-data/apple.png?raw=true"}}' + + sleep 1m # retrieval can't curl as expected, try to wait for more time + + # test data prep + echo "Data Prep with Generating Transcript for Video" + validate_service \ + "${DATAPREP_GEN_TRANSCRIPT_SERVICE_ENDPOINT}" \ + "Data preparation succeeded" \ + "dataprep-multimodal-redis-transcript" \ + "dataprep-multimodal-redis" + + echo "Data Prep with Image & Caption Ingestion" + validate_service \ + "${DATAPREP_INGEST_SERVICE_ENDPOINT}" \ + "Data preparation succeeded" \ + "dataprep-multimodal-redis-ingest" \ + "dataprep-multimodal-redis" + + echo "Validating get file returns mp4" + validate_service \ + "${DATAPREP_GET_FILE_ENDPOINT}" \ + '.mp4' \ + "dataprep_get" \ + "dataprep-multimodal-redis" + + echo "Validating get file returns png" + validate_service \ + "${DATAPREP_GET_FILE_ENDPOINT}" \ + '.png' \ + "dataprep_get" \ + "dataprep-multimodal-redis" + + sleep 1m + + # multimodal retrieval microservice + echo "Validating retriever-multimodal-redis" + your_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(512)]; print(embedding)") + validate_service \ + "http://${host_ip}:7000/v1/multimodal_retrieval" \ + "retrieved_docs" \ + "retriever-multimodal-redis" \ + "retriever-multimodal-redis" \ + "{\"text\":\"test\",\"embedding\":${your_embedding}}" + + sleep 3m + + # llava server + echo "Evaluating LLAVA tgi-rocm" + validate_service \ + "http://${host_ip}:${LLAVA_SERVER_PORT}/generate" \ + '"generated_text":' \ + "tgi-llava-rocm-server" \ + "tgi-llava-rocm-server" \ + '{"inputs":"![](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/rabbit.png)What is this a picture of?\n\n","parameters":{"max_new_tokens":16, "seed": 42}}' + + # lvm + echo "Evaluating lvm-tgi" + validate_service \ + "http://${host_ip}:9399/v1/lvm" \ + '"text":"' \ + "lvm-tgi" \ + "lvm-tgi" \ + '{"retrieved_docs": [], "initial_query": "What is this?", "top_n": 1, "metadata": [{"b64_img_str": "iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFUlEQVR42mP8/5+hnoEIwDiqkL4KAcT9GO0U4BxoAAAAAElFTkSuQmCC", "transcript_for_inference": "yellow i> + + # data prep requiring lvm + echo "Data Prep with Generating Caption for Image" + validate_service \ + "${DATAPREP_GEN_CAPTION_SERVICE_ENDPOINT}" \ + "Data preparation succeeded" \ + "dataprep-multimodal-redis-caption" \ + "dataprep-multimodal-redis" + + sleep 1m +} + +function validate_megaservice() { + # Curl the Mega Service with retrieval + echo "Validate megaservice with first query" + validate_service \ + "http://${host_ip}:8888/v1/multimodalqna" \ + '"time_of_frame_ms":' \ + "multimodalqna" \ + "multimodalqna-backend-server" \ + '{"messages": "What is the revenue of Nike in 2023?"}' + + echo "Validate megaservice with follow-up query" + validate_service \ + "http://${host_ip}:8888/v1/multimodalqna" \ + '"content":"' \ + "multimodalqna" \ + "multimodalqna-backend-server" \ + '{"messages": [{"role": "user", "content": [{"type": "text", "text": "hello, "}, {"type": "image_url", "image_url": {"url": "https://www.ilankelman.org/stopsigns/australia.jpg"}}]}, {"role": "assistant", "content": "opea project! "}, {"role": "> + +} + +function validate_delete { + echo "Validate data prep delete files" + validate_service \ + "${DATAPREP_DELETE_FILE_ENDPOINT}" \ + '{"status":true}' \ + "dataprep_del" \ + "dataprep-multimodal-redis" +} + +function delete_data() { + cd $LOG_PATH + echo "Deleting image, video, and caption" + rm -rf ${image_fn} + rm -rf ${video_fn} + rm -rf ${caption_fn} +} + +function stop_docker() { + cd $WORKPATH/docker_compose/amd/gpu/rocm + docker compose -f compose.yaml stop && docker compose -f compose.yaml rm -f +} + +function main() { + + setup_env + stop_docker + if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi + start_time=$(date +%s) + start_services + end_time=$(date +%s) + duration=$((end_time-start_time)) + echo "Mega service start duration is $duration s" && sleep 1s + prepare_data + + validate_microservices + echo "==== microservices validated ====" + validate_megaservice + echo "==== megaservice validated ====" + validate_delete + echo "==== delete validated ====" + + delete_data + stop_docker + echo y | docker system prune + +} + +main