Merge branch 'main' into enable_llamacpp_lm_eval

.github/CODEOWNERS

@@ -35,6 +35,7 @@
 /intel_extension_for_transformers/neural_chat [email protected]
 /workflows/chatbot [email protected]
 /intel_extension_for_transformers/transformers/llm/quantization [email protected]
+/intel_extension_for_transformers/transformers/modeling/modeling_gaudi [email protected]
 /intel_extension_for_transformers/transformers [email protected]
 /intel_extension_for_transformers/utils [email protected]
 /docs [email protected]

.github/workflows/Scaner_BDBA.yaml

@@ -0,0 +1,19 @@
+name: Scanner BDBA
+
+on:
+  workflow_dispatch:
+
+permissions: write-all
+jobs:
+  bdba_job:
+    name: BDBA Scan
+    uses: intel-innersource/frameworks.ai.infrastructure.code-scan-tools/.github/workflows/Scanner_Bdba.yml@one-ci-cd
+    with:
+      org: "intel-innersource"
+      repo: ${{ github.event.repository.name }}
+      refs: ${{ github.ref_name }}
+      group: "22"
+      runners: "['inner-source']"
+    secrets:
+      token: ${{ secrets.GITHUB_TOKEN }}
+      BDBA_TOKEN: ${{ secrets.BDBA_TOKEN }}

.github/workflows/Scaner_Coverity.yaml

@@ -0,0 +1,22 @@
+name: Scanner Coverity PYTHON
+
+on:
+    workflow_dispatch:
+
+permissions: write-all
+jobs:
+  coverity_job:
+    name: Coverity
+    runs-on: inner-source
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: "recursive"
+      - name: Scan
+        uses: intel-innersource/frameworks.devops.github.actions.coverity@main
+        with:
+          server: 'https://coverityent.devtools.intel.com/prod4'
+          username: ${{secrets.COVERITY_NAME }}
+          password: ${{secrets.COVERITY }}
+          project: IntelExtensionForTransformers
+          stream: 'IntelExtensionForTransformers12cpp'

.github/workflows/Scaner_Trivy.yaml

@@ -0,0 +1,12 @@
+name: Trivy Scan for Containers
+
+on:
+    workflow_dispatch:
+permissions: write-all
+jobs:
+  trivy_container_job:
+    uses: "intel-innersource/frameworks.ai.infrastructure.code-scan-tools/.github/workflows/Scanner_Trivy.yml@one-ci-cd"
+    with:
+      container: ${{ vars.TRIVY_CONTAINER_NAME }}
+      runners: "['inner-source']"
+      lmc: false

.github/workflows/docker/codeScan.dockerfile

@@ -29,7 +29,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing \
     build-essential \
     cloc \
     python3.10-venv \
-    git
+    git \
+    wget
 
 RUN ln -sf $(which python3) /usr/bin/python
 
@@ -38,4 +39,8 @@ RUN python -m pip install --no-cache-dir pylint==2.17.5\
     pyspelling\
     pydocstyle
 
+RUN python -m pip install --no-cache-dir bandit==1.7.8
+RUN wget -O /bin/hadolint https://github.com/hadolint/hadolint/releases/download/v2.12.0/hadolint-Linux-x86_64
+RUN chmod +x /bin/hadolint
+
 WORKDIR /

.github/workflows/format_scan.yml

@@ -37,6 +37,7 @@ jobs:
             "cpplint",
             # "pydocstyle",
             #"pyspelling",
+            "hadolint"
           ]
       fail-fast: false
     steps:

.github/workflows/llm-test.yml

@@ -51,9 +51,8 @@ jobs:
         run: |
           cd ${{ github.workspace }}
           conda activate llm-test || source activate llm-test
-          pip install build --upgrade
-          pip install -r requirements.txt
-          pip install torch==2.0 --extra-index-url https://download.pytorch.org/whl/cpu
+          compiler_version=11.1.0
+          conda install --update-deps -c conda-forge gxx==${compiler_version} gcc==${compiler_version} gxx_linux-64==${compiler_version} libstdcxx-ng sysroot_linux-64 libxcrypt -y
           python setup.py sdist bdist_wheel
           pip install dist/intel_extension_for_transformers*.whl
           pip list
@@ -127,7 +126,7 @@ jobs:
           cd ${{ env.OUT_SCRIPT_PATH }}
           /usr/bin/bash generate_report.sh --workflow=deploy
         env:
-          RUN_DISPLAY_URL: https://github.com/VincyZhang/intel-extension-for-transformers/actions/runs/${{ github.run_id }}
+          RUN_DISPLAY_URL: https://github.com/intel/intel-extension-for-transformers/actions/runs/${{ github.run_id }}
           BUILD_NUMBER: ${{ github.run_id }}
           JOB_STATUS: succeed
           MR_source_branch: ${{ github.event.pull_request._links.html.href }}

.github/workflows/optimize-test.yml

@@ -183,7 +183,7 @@ jobs:
           cd ${{ env.OUT_SCRIPT_PATH }}
           /usr/bin/bash generate_report.sh --workflow=optimize
         env:
-          RUN_DISPLAY_URL: https://github.com/VincyZhang/intel-extension-for-transformers/actions/runs/${{ github.run_id }}
+          RUN_DISPLAY_URL: https://github.com/intel/intel-extension-for-transformers/actions/runs/${{ github.run_id }}
           BUILD_NUMBER: ${{ github.run_id }}
           JOB_STATUS: succeed
           MR_source_branch: ${{ github.event.pull_request._links.html.href }}

.github/workflows/probot.yml

@@ -2,6 +2,7 @@ name: CI-Summary
 
 on:
   pull_request:
+    branches: [main]
     types: [opened, reopened, ready_for_review, synchronize] # added `ready_for_review` since draft is skipped
   workflow_dispatch:
 

.github/workflows/script/SparseLibCI/generate_sparse_lib.sh

@@ -42,7 +42,7 @@ function generate_html_overview {
 <body>
     <div id="main">
         <h1 align="center">Sparse Lib Tests ${PR_TITLE}
-        [ <a href="https://github.com/VincyZhang/intel-extension-for-transformers/actions/runs/${BUILD_ID}">Job-${BUILD_NUMBER}</a> ]</h1>
+        [ <a href="https://github.com/intel/intel-extension-for-transformers/actions/runs/${BUILD_ID}">Job-${BUILD_NUMBER}</a> ]</h1>
       <h1 align="center">Test Status: ${github_actions_job_status}</h1>
         <h2>Summary</h2>
         ${pr_comment_opt}

.github/workflows/script/formatScan/hadolint.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+source /intel-extension-for-transformers/.github/workflows/script/change_color.sh
+log_dir=/intel-extension-for-transformers/.github/workflows/script/formatScan
+
+find . -type f \( -name "Dockerfile*" \) -print -exec hadolint --ignore DL3006 --ignore DL3007 --ignore DL3008 {} \; 2>&1 | tee ${log_dir}/hadolint.log
+
+if [[ $(grep -c "error" ${log_dir}/hadolint.log) != 0 ]]; then
+    $BOLD_RED && echo "Error!! Please Click on the artifact button to download and check error details." && $RESET
+    exit 1
+fi
+
+$BOLD_PURPLE && echo "Congratulations, Hadolint check passed!" && $LIGHT_PURPLE && echo " You can click on the artifact button to see the log details." && $RESET
+exit 0

.github/workflows/script/formatScan/pylint.sh

@@ -31,6 +31,8 @@ fi
 pip install lm-eval
 pip install accelerate nlpaug nltk schema optimum-intel optimum peft
 pip install --upgrade --force-reinstall transformers==4.36.2
+pip install optimum-habana
+pip install deepspeed
 
 echo "[DEBUG] list pipdeptree..."
 pip install pipdeptree
@@ -41,25 +43,15 @@ python -m pylint -f json --disable=R,C,W,E1129 \
     --max-line-length=120 \
     --extension-pkg-whitelist=numpy,nltk \
     --ignored-classes=TensorProto,NodeProto \
-    --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime,neural_compressor,neural_compressor.benchmark,intel_extension_for_transformers.neural_engine_py,intel_extension_for_transformers.qbits,cv2,PIL.Image \
+    --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime,neural_compressor,neural_compressor.benchmark,intel_extension_for_transformers.neural_engine_py,intel_extension_for_transformers.qbits,intel_extension_for_transformers.transformers.modeling.modeling_gaudi.generation.utils,intel_extension_for_transformers.transformers.modeling.modeling_gaudi.models.mixtral.modeling_mixtral,cv2,PIL.Image \
     /intel-extension-for-transformers/intel_extension_for_transformers >${log_dir}/pylint.json
-exit_code1=$?
-
-python -m pylint -f json --disable=R,C,W,E1129 \
-    --enable=line-too-long \
-    --max-line-length=120 \
-    --disable=no-name-in-module,import-error,no-member,undefined-variable,no-value-for-parameter,unexpected-keyword-arg,not-callable,no-self-argument,too-many-format-args,invalid-unary-operand-type,too-many-function-args \
-    --extension-pkg-whitelist=numpy,nltk \
-    --ignored-classes=TensorProto,NodeProto \
-    --ignored-modules=tensorflow,torch,torch.quantization,torch.tensor,torchvision,mxnet,onnx,onnxruntime,neural_compressor,neural_compressor.benchmark,intel_extension_for_transformers.neural_engine_py,intel_extension_for_transformers.qbits,cv2,PIL.Image \
-    /intel-extension-for-transformers/intel_extension_for_transformers >> ${log_dir}/pylint.json
-exit_code2=$?
+exit_code=$?
 
 $BOLD_YELLOW && echo " -----------------  Current log file output start --------------------------" && $RESET
 cat ${log_dir}/pylint.json
 $BOLD_YELLOW && echo " -----------------  Current log file output end --------------------------" && $RESET
 
-if [ ${exit_code1} -ne 0 ] || [ ${exit_code2} -ne 0 ]; then
+if [ ${exit_code} -ne 0 ]; then
     $BOLD_RED && echo "Error!! Please Click on the artifact button to download and view Pylint error details." && $RESET
     exit 1
 fi