Merge branch 'main' into manual_workflow

.github/workflows/_get-test-matrix.yml

@@ -38,7 +38,10 @@ jobs:
         run: |
           set -xe
           if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ github.event_name }}" == "pull_request_target" ]; then
-            base_commit=${{ github.event.pull_request.base.sha }}
+            LATEST_COMMIT_SHA=$(curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+            "https://api.github.com/repos/opea-project/GenAIComps/commits?sha=main" | jq -r '.[0].sha')
+            echo "Latest commit SHA is $LATEST_COMMIT_SHA"
+            base_commit=$LATEST_COMMIT_SHA
           else
             base_commit=$(git rev-parse HEAD~1) # push event
           fi

.github/workflows/manual-freeze-requirements.yml

@@ -34,5 +34,5 @@ jobs:
       - name: Commit changes
         run: |
           git add .
-          git commit -m "Freeze requirements"
+          git commit -s -m "Freeze requirements"
           git push

.github/workflows/pr-examples-test.yml

@@ -16,6 +16,7 @@ on:
       - comps/llms/text-generation/tgi/**
       - comps/dataprep/redis/langchain/**
       - requirements.txt
+      - "!**.md"
 
 # If there is a new commit, the previous jobs will be canceled
 concurrency:
@@ -41,26 +42,25 @@ jobs:
         env:
           HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}
         run: |
+          cd ../ && sudo rm -rf GenAIExamples
           git clone https://github.com/opea-project/GenAIExamples.git
-          cd ${{ github.workspace }}/GenAIExamples/ChatQnA/docker/gaudi
-          sed -i "s#:latest#:comps#g" compose.yaml
-          cat compose.yaml
+          cd GenAIExamples/ChatQnA/docker
+          cp -r ${{ github.workspace }}/../GenAIComps .
 
-          cd ${{ github.workspace }}/GenAIExamples/ChatQnA/tests
-          GenAIComps_dir=${{github.workspace}}
+          cd ../tests
           sed -i '/GenAIComps.git/d' test_chatqna_on_gaudi.sh
-          sed -i "s#cd GenAIComps#cd ${GenAIComps_dir}#g" test_chatqna_on_gaudi.sh
-          sed -i "s#docker build -t#docker build --no-cache -q -t#g" test_chatqna_on_gaudi.sh
-          sed -i "s#:latest#:comps#g" test_chatqna_on_gaudi.sh
           cat test_chatqna_on_gaudi.sh
 
           echo "Run test..."
+          export IMAGE_TAG="comps"
           timeout 50m bash test_chatqna_on_gaudi.sh
 
+          echo "LOG_PATH=$(pwd)/*.log" >> $GITHUB_ENV
+
       - name: Clean up container
         if: cancelled() || failure()
         run: |
-          cd ${{ github.workspace }}/GenAIExamples/ChatQnA/docker/gaudi
+          cd ${{ github.workspace }}/../GenAIExamples/ChatQnA/docker/gaudi
           docker compose stop && docker compose rm -f
           docker system prune -f
 
@@ -69,4 +69,4 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: "Examples-Test-Logs"
-          path: ${{ github.workspace }}/GenAIExamples/ChatQnA/tests/*.log
+          path: ${{ env.LOG_PATH }}

.github/workflows/scripts/freeze_requirements.sh

@@ -15,16 +15,20 @@ function freeze() {
         --output-file "$folder/freeze.txt" \
         "$file"
     echo "::endgroup::"
+
     if [[ -e "$folder/freeze.txt" ]]; then
         if [[ "$keep_origin_packages" == "true" ]]; then
-            sed -i '/^\s*#/d; s/#.*//; /^\s*$/d' "$file"
-            sed -i '/^\s*#/d; s/#.*//; /^\s*$/d' "$folder/freeze.txt"
-
-            packages1=$(cut -d'=' -f1 "$file" | tr '[:upper:]' '[:lower:]' | sed 's/[-_]/-/g')
+            # fix corner cases
+            sed -i '/^\s*#/d; s/#.*//; /^\s*$/d; s/ //g' "$file"
+            sed -i '/^\s*#/d; s/#.*//; /^\s*$/d; s/ //g; s/huggingface-hub\[inference\]/huggingface-hub/g; s/uvicorn\[standard\]/uvicorn/g' "$folder/freeze.txt"
+            if grep -q '^transformers$' $file && ! grep -q '^transformers\[sentencepiece\]$' $file; then
+                sed -i "s/transformers\[sentencepiece\]/transformers/" "$folder/freeze.txt"
+            fi
+            packages1=$(tr '><' '=' <"$file" | cut -d'=' -f1 | tr '[:upper:]' '[:lower:]' | sed 's/[-_]/-/g')
             packages2=$(cut -d'=' -f1 "$folder/freeze.txt" | tr '[:upper:]' '[:lower:]' | sed 's/[-_]/-/g')
             common_packages=$(comm -12 <(echo "$packages2" | sort) <(echo "$packages1" | sort))
-
-            rm "$file"
+            grep '^git\+' "$file" >temp_file || touch temp_file
+            rm -rf "$file" && mv temp_file "$file"
             while IFS= read -r line; do
                 package=$(echo "$line" | cut -d'=' -f1)
                 package_transformed=$(echo "$package" | tr '[:upper:]' '[:lower:]' | sed 's/[_-]/-/g')

README.md

@@ -173,7 +173,6 @@ A `Microservices` can be created by using the decorator `register_microservice`.
 
 ```python
 from langchain_community.embeddings import HuggingFaceHubEmbeddings
-from langsmith import traceable
 
 from comps import register_microservice, EmbedDoc, ServiceType, TextDoc
 
@@ -187,7 +186,6 @@ from comps import register_microservice, EmbedDoc, ServiceType, TextDoc
     input_datatype=TextDoc,
     output_datatype=EmbedDoc,
 )
-@traceable(run_type="embedding")
 def embedding(input: TextDoc) -> EmbedDoc:
     embed_vector = embeddings.embed_query(input.text)
     res = EmbedDoc(text=input.text, embedding=embed_vector)

comps/agent/langchain/README.md

@@ -4,32 +4,32 @@ The langchain agent model refers to a framework that integrates the reasoning ca
 
 ![Architecture Overview](agent_arch.jpg)
 
-# 🚀1. Start Microservice with Python（Option 1）
+## 🚀1. Start Microservice with Python（Option 1）
 
-## 1.1 Install Requirements
+### 1.1 Install Requirements
 
 ```bash
 cd comps/agent/langchain/
 pip install -r requirements.txt
 ```
 
-## 1.2 Start Microservice with Python Script
+### 1.2 Start Microservice with Python Script
 
 ```bash
 cd comps/agent/langchain/
 python agent.py
 ```
 
-# 🚀2. Start Microservice with Docker (Option 2)
+## 🚀2. Start Microservice with Docker (Option 2)
 
-## Build Microservices
+### Build Microservices
 
 ```bash
 cd GenAIComps/ # back to GenAIComps/ folder
 docker build -t opea/comps-agent-langchain:latest -f comps/agent/langchain/docker/Dockerfile .
 ```
 
-## start microservices
+### start microservices
 
 ```bash
 export ip_address=$(hostname -I | awk '{print $1}')
@@ -56,7 +56,7 @@ docker logs comps-langchain-agent-endpoint
 > docker run --rm --runtime=runc --name="comps-langchain-agent-endpoint" -v ./comps/agent/langchain/:/home/user/comps/agent/langchain/ -p 9090:9090 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} --env-file ${agent_env} opea/comps-agent-langchain:latest
 > ```
 
-# 🚀3. Validate Microservice
+## 🚀3. Validate Microservice
 
 Once microservice starts, user can use below script to invoke.
 
@@ -73,7 +73,7 @@ data: [DONE]
 
 ```
 
-# 🚀4. Provide your own tools
+## 🚀4. Provide your own tools
 
 - Define tools
 

comps/agent/langchain/requirements.txt

@@ -11,7 +11,6 @@ langchain-openai
 langchain_community
 langchainhub
 langgraph
-langsmith
 numpy
 
 # used by cloud native

comps/asr/README.md

@@ -2,17 +2,17 @@
 
 ASR (Audio-Speech-Recognition) microservice helps users convert speech to text. When building a talking bot with LLM, users will need to convert their audio inputs (What they talk, or Input audio from other sources) to text, so the LLM is able to tokenize the text and generate an answer. This microservice is built for that conversion stage.
 
-# 🚀1. Start Microservice with Python (Option 1)
+## 🚀1. Start Microservice with Python (Option 1)
 
 To start the ASR microservice with Python, you need to first install python packages.
 
-## 1.1 Install Requirements
+### 1.1 Install Requirements
 
 ```bash
 pip install -r requirements.txt
 ```
 
-## 1.2 Start Whisper Service/Test
+### 1.2 Start Whisper Service/Test
 
 - Xeon CPU
 
@@ -40,7 +40,7 @@ nohup python whisper_server.py --device=hpu &
 python check_whisper_server.py
 ```
 
-## 1.3 Start ASR Service/Test
+### 1.3 Start ASR Service/Test
 
 ```bash
 cd ../
@@ -54,13 +54,13 @@ While the Whisper service is running, you can start the ASR service. If the ASR
 {'id': '0e686efd33175ce0ebcf7e0ed7431673', 'text': 'who is pat gelsinger'}
 ```
 
-# 🚀2. Start Microservice with Docker (Option 2)
+## 🚀2. Start Microservice with Docker (Option 2)
 
 Alternatively, you can also start the ASR microservice with Docker.
 
-## 2.1 Build Images
+### 2.1 Build Images
 
-### 2.1.1 Whisper Server Image
+#### 2.1.1 Whisper Server Image
 
 - Xeon CPU
 
@@ -76,15 +76,15 @@ cd ../..
 docker build -t opea/whisper-gaudi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/asr/whisper/Dockerfile_hpu .
 ```
 
-### 2.1.2 ASR Service Image
+#### 2.1.2 ASR Service Image
 
 ```bash
 docker build -t opea/asr:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/asr/Dockerfile .
 ```
 
-## 2.2 Start Whisper and ASR Service
+### 2.2 Start Whisper and ASR Service
 
-### 2.2.1 Start Whisper Server
+#### 2.2.1 Start Whisper Server
 
 - Xeon
 
@@ -98,15 +98,15 @@ docker run -p 7066:7066 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$htt
 docker run -p 7066:7066 --runtime=habana -e HABANA_VISIBLE_DEVICES=all -e OMPI_MCA_btl_vader_single_copy_mechanism=none --cap-add=sys_nice --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy opea/whisper-gaudi:latest
 ```
 
-### 2.2.2 Start ASR service
+#### 2.2.2 Start ASR service
 
 ```bash
 ip_address=$(hostname -I | awk '{print $1}')
 
 docker run -d -p 9099:9099 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e ASR_ENDPOINT=http://$ip_address:7066 opea/asr:latest
 ```
 
-### 2.2.3 Test
+#### 2.2.3 Test
 
 ```bash
 # Use curl or python

comps/chathistory/mongo/README.md

@@ -17,16 +17,16 @@ export DB_NAME=${DB_NAME}
 export COLLECTION_NAME=${COLLECTION_NAME}
 ```
 
-# 🚀Start Microservice with Docker
+## 🚀Start Microservice with Docker
 
-## Build Docker Image
+### Build Docker Image
 
 ```bash
 cd ../../../../
 docker build -t opea/chathistory-mongo-server:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/chathistory/mongo/docker/Dockerfile .
 ```
 
-## Run Docker with CLI
+### Run Docker with CLI
 
 - Run mongoDB image
 
@@ -40,7 +40,7 @@ docker run -d -p 27017:27017 --name=mongo mongo:latest
 docker run -d --name="chathistory-mongo-server" -p 6013:6013 -p 6012:6012 -p 6014:6014 -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e no_proxy=$no_proxy -e MONGO_HOST=${MONGO_HOST} -e MONGO_PORT=${MONGO_PORT} -e DB_NAME=${DB_NAME} -e COLLECTION_NAME=${COLLECTION_NAME} opea/chathistory-mongo-server:latest
 ```
 
-# Invoke Microservice
+## Invoke Microservice
 
 Once chathistory service is up and running, users can update the database by using the below API endpoint. The API returns a unique UUID for the saved conversation.
 

comps/chathistory/mongo/requirements.txt

@@ -1 +1 @@
-motor==3.4.0
+motor==3.4.0

comps/dataprep/README.md

@@ -17,22 +17,22 @@ Occasionally unstructured data will contain image data, to convert the image dat
 export SUMMARIZE_IMAGE_VIA_LVM=1
 ```
 
-# Dataprep Microservice with Redis
+## Dataprep Microservice with Redis
 
 For details, please refer to this [readme](redis/README.md)
 
-# Dataprep Microservice with Milvus
+## Dataprep Microservice with Milvus
 
 For details, please refer to this [readme](milvus/README.md)
 
-# Dataprep Microservice with Qdrant
+## Dataprep Microservice with Qdrant
 
 For details, please refer to this [readme](qdrant/README.md)
 
-# Dataprep Microservice with Pinecone
+## Dataprep Microservice with Pinecone
 
 For details, please refer to this [readme](pinecone/README.md)
 
-# Dataprep Microservice with PGVector
+## Dataprep Microservice with PGVector
 
 For details, please refer to this [readme](pgvector/README.md)

comps/dataprep/milvus/README.md

@@ -1,8 +1,8 @@
 # Dataprep Microservice with Milvus
 
-# 🚀Start Microservice with Python
+## 🚀Start Microservice with Python
 
-## Install Requirements
+### Install Requirements
 
 ```bash
 pip install -r requirements.txt
@@ -11,11 +11,11 @@ apt-get install libtesseract-dev -y
 apt-get install poppler-utils -y
 ```
 
-## Start Milvus Server
+### Start Milvus Server
 
 Please refer to this [readme](../../../vectorstores/langchain/milvus/README.md).
 
-## Setup Environment Variables
+### Setup Environment Variables
 
 ```bash
 export no_proxy=${your_no_proxy}
@@ -27,30 +27,30 @@ export COLLECTION_NAME=${your_collection_name}
 export MOSEC_EMBEDDING_ENDPOINT=${your_embedding_endpoint}
 ```
 
-## Start Document Preparation Microservice for Milvus with Python Script
+### Start Document Preparation Microservice for Milvus with Python Script
 
 Start document preparation microservice for Milvus with below command.
 
 ```bash
 python prepare_doc_milvus.py
 ```
 
-# 🚀Start Microservice with Docker
+## 🚀Start Microservice with Docker
 
-## Build Docker Image
+### Build Docker Image
 
 ```bash
 cd ../../../../
 docker build -t opea/dataprep-milvus:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy --build-arg no_proxy=$no_proxy -f comps/dataprep/milvus/docker/Dockerfile .
 ```
 
-## Run Docker with CLI
+### Run Docker with CLI
 
 ```bash
 docker run -d --name="dataprep-milvus-server" -p 6010:6010 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e no_proxy=$no_proxy -e MOSEC_EMBEDDING_ENDPOINT=${your_embedding_endpoint} -e MILVUS=${your_milvus_host_ip} opea/dataprep-milvus:latest
 ```
 
-# Invoke Microservice
+## Invoke Microservice
 
 Once document preparation microservice for Milvus is started, user can use below command to invoke the microservice to convert the document to embedding and save to the database.