python-package.yml

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python

name: Python package

on:
  workflow_dispatch:
  push:
    branches: [ "master" ]
  pull_request_target:
    branches: [ "master" ]

jobs:
  Pylint:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.12.5"]

    steps:
    - uses: actions/checkout@v3
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v3
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        python -m pip install flake8 pytest
        python -m pip install chainlit~=1.0.200 flask~=2.3.2 flask-restful==0.3.9
        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
    - name: Lint with flake8
      run: |
        # stop the build if there are Python syntax errors or undefined names
        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics

  Linux-CI-Tests:
    runs-on: ubuntu-latest
    env:
      APPBUILDER_TOKEN: ${{ secrets.APPBUILDER_TOKEN }}
      APPBUILDER_TOKEN_V2: ${{ secrets.APPBUILDER_TOKEN_V2 }}
      BAIDU_VDB_API_KEY: ${{ secrets.BAIDU_VDB_API_KEY }}
      INSTANCE_ID: ${{ secrets.INSTANCE_ID }}
      DATASET_ID: ${{ secrets.DATASET_ID }}
      APPBUILDER_TOKEN_DOC_FORMAT: ${{ secrets.APPBUILDER_TOKEN_DOC_FORMAT }}
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.9","3.12"]
    
    steps:
    - uses: actions/checkout@v4
      with:
        ref: ${{ github.event.pull_request.head.sha }}
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v3
      with:
        python-version: ${{ matrix.python-version }}
    - name: Print ENV
      run: |
        env
        ls
        pwd
    - name: Git Clone And Checkout Branch
      run: |
        fork_repo="https://github.com/$GITHUB_ACTOR/app-builder.git"
        mkdir cicd
        cd cicd
        git clone $fork_repo
        cd app-builder
        git checkout $GITHUB_HEAD_REF
        git remote add upstream https://github.com/baidubce/app-builder.git 
        git fetch upstream
        git remote -v
        git status

        # 找到当前分支与 upstream/master 的共同祖先提交
        merge_base=$(git merge-base HEAD upstream/master)
        echo "merge_base=$merge_base"

        # 比较当前分支与 merge_base 之间的差异
        changed_files=$(git diff --name-only --diff-filter=ACMRT $merge_base)
        changed_files_py_sh=$(git diff --name-only --diff-filter=ACMRT $merge_base -- $(find python -type f \( -name '*.py' -o -name '*.sh' \)))
        
        echo "发生更改的文件为:"
        echo "$changed_files"

        echo "发生更改的py/sh文件为:"

        if [ -n "$changed_files_py_sh" ]; then
          export APPBUILDER_PYTHON_TESTS=True
          echo "$changed_files_py_sh"
        else
          export APPBUILDER_PYTHON_TESTS=False
          echo "没有检测到Python或Shell文件被更改"
        fi

        echo "APPBUILDER_PYTHON_TESTS=$APPBUILDER_PYTHON_TESTS" >> $GITHUB_ENV
        pwd
    - name: Install dependencies
      run: |
        echo $APPBUILDER_PYTHON_TESTS
        if [ "$APPBUILDER_PYTHON_TESTS" = "False" ]; then
          echo "环境变量APPBUILDER_PYTHON_TESTS为False,没有检测到Python或Shell文件被更改,跳过Install dependencies部分。"
        else
          echo "检测到Python或Shell文件被更改(根据环境变量APPBUILDER_PYTHON_TESTS),准备启动单元测试Install dependencies部分..."
          cd cicd/app-builder
          pwd
          sudo apt-get update
          sudo apt-get install ffmpeg libavcodec-extra
          python3 -m ensurepip --upgrade
          python3 -m pip install --upgrade pip
          python3 -m pip install --upgrade setuptools
          python3 -m pip install wheel
          python3 -m pip install coverage
          python3 -m pip install diff-cover
          python3 -m pip install pydub
          python3 -m pip install SQLAlchemy==2.0.31
          python3 -m pip install chainlit~=1.0.200 flask~=2.3.2 flask-restful==0.3.9
          python3 -m pip install opentelemetry-exporter-otlp==1.23.0 opentelemetry-instrumentation==0.44b0 opentelemetry-sdk==1.23.0 opentelemetry-api==1.23.0
          python3 -m pip install pandas==2.2.2
        fi
    - name: Build whl
      run: |
        echo $APPBUILDER_PYTHON_TESTS
        if [ "$APPBUILDER_PYTHON_TESTS" = "False" ]; then
          echo "环境变量APPBUILDER_PYTHON_TESTS为False,没有检测到Python或Shell文件被更改,跳过Build whl部分。"
        else
          echo "检测到Python或Shell文件被更改(根据环境变量APPBUILDER_PYTHON_TESTS),准备启动单元测试Build whl部分..."
          cd cicd/app-builder     
          pwd
          python3 setup.py bdist_wheel
          python3 -m pip uninstall -y appbuilder-sdk
          python3 -m pip install dist/*.whl
        fi
    - name: Test with unittest
      run: |
        echo $APPBUILDER_PYTHON_TESTS
        if [ "$APPBUILDER_PYTHON_TESTS" = "False" ]; then
          echo "环境变量APPBUILDER_PYTHON_TESTS为False,没有检测到Python或Shell文件被更改,跳过Test with unittest部分。"
        else
          echo "检测到Python或Shell文件被更改(根据环境变量APPBUILDER_PYTHON_TESTS),准备启动单元测试Test with unittest部分..."
          cd cicd/app-builder
          pwd
          sh python/tests/run_python_test.sh
        fi
  Go-Test-CI:
    runs-on: ubuntu-latest
    env:
      APPBUILDER_TOKEN: ${{ secrets.APPBUILDER_TOKEN }}
      APPBUILDER_TOKEN_V2: ${{ secrets.APPBUILDER_TOKEN_V2 }}
      BAIDU_VDB_API_KEY: ${{ secrets.BAIDU_VDB_API_KEY }}
      INSTANCE_ID: ${{ secrets.INSTANCE_ID }}
      DATASET_ID: ${{ secrets.DATASET_ID }}
      APPBUILDER_TOKEN_DOC_FORMAT: ${{ secrets.APPBUILDER_TOKEN_DOC_FORMAT }}
 
      APPBUILDER_TOKEN_V3: ${{ secrets.APPBUILDER_TOKEN_V3 }}
      DATASET_ID_V3: ${{ secrets.DATASET_ID_V3 }}
      DOCUMENT_ID_V3: ${{ secrets.DOCUMENT_ID_V3 }}
    strategy:
      fail-fast: false
      matrix:
        go-version: ['1.18']
      max-parallel: 2

    steps:
    - name: Checkout code
      uses: actions/checkout@v3
      with:
        ref: ${{ github.event.pull_request.head.sha }}
    - name: Set up Go
      uses: actions/setup-go@v3
      with:
        go-version: ${{ matrix.go-version }}
    - name: Git Clone And Checkout Branch
      run: |
        fork_repo="https://github.com/$GITHUB_ACTOR/app-builder.git"
        mkdir cicd
        cd cicd
        git clone $fork_repo
        cd app-builder
        echo "github_head的值为："
        echo $GITHUB_HEAD_REF
        git checkout $GITHUB_HEAD_REF
        git remote add upstream https://github.com/baidubce/app-builder.git 
        git fetch upstream
        git remote -v
        git status

        # 找到当前分支与 upstream/master 的共同祖先提交
        merge_base=$(git merge-base HEAD upstream/master)
        echo "merge_base=$merge_base"

        # 比较当前分支与 merge_base 之间的差异
        changed_files=$(git diff --name-only --diff-filter=ACMRT $merge_base)
        changed_files_go=$(git diff --name-only --diff-filter=ACMRT $merge_base -- 'go/**')

        echo "发生更改的文件为:"
        echo "$changed_files"
        echo "发生更改的Go相关文件为:"

        if [ -n "$changed_files_go" ]; then
          export APPBUILDER_GO_TESTS=True
          echo "$changed_files_go"
        else
          export APPBUILDER_GO_TESTS=False
          echo "没有检测到Go或Shell文件被更改"
        fi

        echo "APPBUILDER_GO_TESTS=$APPBUILDER_GO_TESTS" >> $GITHUB_ENV
    - name: Install dependencies and tools
      run: |
        echo $APPBUILDER_GO_TESTS
        if [ "$APPBUILDER_GO_TESTS" = "False" ]; then
          echo "环境变量APPBUILDER_GO_TESTS为False,没有检测到Go文件被更改，跳过Install dependencies and tools部分。"
        else
          echo "检测到Go文件被更改，准备启动Install dependencies and tools部分..."
          cd cicd/app-builder/go/appbuilder
          go mod tidy

          go get github.com/axw/gocov/gocov  # 安装增量覆盖率工具

          # 安装 golangci-lint
          curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.53.3
          echo "$GOPATH/bin" >> $GITHUB_PATH

        fi

    - name: Run tests and calculate incremental coverage
      run: |
        echo $APPBUILDER_GO_TESTS
        if [ "$APPBUILDER_GO_TESTS" = "False" ]; then
          echo "环境变量APPBUILDER_GO_TESTS为False，没有检测到Go文件被更改，跳过Run tests部分。"
        else
          echo "检测到Go文件被更改，准备启动Run tests部分..."
          cd cicd/app-builder/go
          # 运行测试并生成覆盖率报告
          chmod +x run_go_test.sh
          ./run_go_test.sh
        fi
      shell: bash
  Java-CI-Tests:
    runs-on: ubuntu-latest
    env:
      APPBUILDER_TOKEN: ${{ secrets.APPBUILDER_TOKEN }}
      APPBUILDER_TOKEN_V2: ${{ secrets.APPBUILDER_TOKEN_V2 }}
      BAIDU_VDB_API_KEY: ${{ secrets.BAIDU_VDB_API_KEY }}
      INSTANCE_ID: ${{ secrets.INSTANCE_ID }}
      DATASET_ID: ${{ secrets.DATASET_ID }}
      APPBUILDER_TOKEN_DOC_FORMAT: ${{ secrets.APPBUILDER_TOKEN_DOC_FORMAT }}
 
      APPBUILDER_TOKEN_V3: ${{ secrets.APPBUILDER_TOKEN_V3 }}
      DATASET_ID_V3: ${{ secrets.DATASET_ID_V3 }}
      DOCUMENT_ID_V3: ${{ secrets.DOCUMENT_ID_V3 }}
    strategy:
      fail-fast: false
      matrix:
        java-version: ['8']  # 使用 Java 11 和 17 作为示例

    steps:
    - name: Checkout code
      uses: actions/checkout@v3
      with:
        ref: ${{ github.event.pull_request.head.sha }}
    
    - name: Set up JDK ${{ matrix.java-version }}
      uses: actions/setup-java@v3
      with:
        java-version: ${{ matrix.java-version }}
        distribution: 'temurin'

    - name: Print ENV
      run: |
        env
        ls
        pwd

    - name: Git Clone And Checkout Branch
      run: |
        fork_repo="https://github.com/$GITHUB_ACTOR/app-builder.git"
        mkdir cicd
        cd cicd
        git clone $fork_repo
        cd app-builder
        git checkout $GITHUB_HEAD_REF
        git remote add upstream https://github.com/baidubce/app-builder.git 
        git fetch upstream
        git remote -v
        git status

        # 找到当前分支与 upstream/master 的共同祖先提交
        merge_base=$(git merge-base HEAD upstream/master)
        echo "merge_base=$merge_base"

        # 比较当前分支与 merge_base 之间的差异
        changed_files=$(git diff --name-only --diff-filter=ACMRT $merge_base)
        changed_files_java=$(git diff --name-only --diff-filter=ACMRT $merge_base -- 'java/**')
        
        echo "发生更改的文件为:"
        echo "$changed_files"

        echo "发生更改的Java相关文件为:"

        if [ -n "$changed_files_java" ]; then
          export APPBUILDER_JAVA_TESTS=True
          echo "$changed_files_java"
        else
          export APPBUILDER_JAVA_TESTS=False
          echo "没有检测到Java文件被更改"
        fi

        echo "APPBUILDER_JAVA_TESTS=$APPBUILDER_JAVA_TESTS" >> $GITHUB_ENV
        pwd

    - name: Install Python dependencies
      run: |
        python -m pip install --upgrade pip
        pip install gitpython lxml
    - name: Install dependencies
      run: |
        echo $APPBUILDER_JAVA_TESTS
        if [ "$APPBUILDER_JAVA_TESTS" = "False" ]; then
          echo "环境变量APPBUILDER_JAVA_TESTS为False,没有检测到Java文件被更改,跳过Install dependencies部分。"
        else
          echo "检测到Java文件被更改(根据环境变量APPBUILDER_JAVA_TESTS),准备启动单元测试Install dependencies部分..."
          cd cicd/app-builder/java
          mvn clean install -DskipTests
        fi
 
    - name: Run Java tests with Jacoco
      if: env.APPBUILDER_JAVA_TESTS == 'True'
      run: |
        echo "检测到Java文件被更改(根据环境变量APPBUILDER_JAVA_TESTS),准备启动Run Java tests with Jacoco部分..."
        cd cicd/app-builder/java
        bash print_coverage.sh