Update ORT training dockerfiles (#1445) #721
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build main documentation | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - doc-builder* | |
| - v*-release | |
| workflow_dispatch: | |
| jobs: | |
| build_documentation: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| repository: 'huggingface/doc-builder' | |
| path: doc-builder | |
| - uses: actions/checkout@v2 | |
| with: | |
| repository: 'huggingface/doc-build' | |
| path: doc-build | |
| token: ${{ secrets.HUGGINGFACE_PUSH }} | |
| - uses: actions/checkout@v2 | |
| with: | |
| repository: 'huggingface/optimum' | |
| path: optimum | |
| - uses: actions/checkout@v2 | |
| with: | |
| repository: 'huggingface/optimum-habana' | |
| path: optimum-habana | |
| - uses: actions/checkout@v2 | |
| with: | |
| repository: 'huggingface/optimum-graphcore' | |
| path: optimum-graphcore | |
| - uses: actions/checkout@v2 | |
| with: | |
| repository: 'huggingface/optimum-intel' | |
| path: optimum-intel | |
| - uses: actions/checkout@v2 | |
| with: | |
| repository: 'huggingface/optimum-furiosa' | |
| path: optimum-furiosa | |
| - name: Set environment variables | |
| run: | | |
| cd optimum | |
| version=`echo "$(grep '^__version__ =' optimum/version.py | cut -d '=' -f 2- | xargs)"` | |
| if [[ $version == *.dev0 ]] | |
| then | |
| echo "VERSION=main" >> $GITHUB_ENV | |
| else | |
| echo "VERSION=v$version" >> $GITHUB_ENV | |
| fi | |
| cd .. | |
| - name: Setup environment | |
| run: | | |
| pip uninstall -y doc-builder | |
| cd doc-builder | |
| git pull origin main | |
| pip install . | |
| pip install black | |
| cd .. | |
| - name: Make Graphcore documentation | |
| run: | | |
| cd optimum-graphcore | |
| make doc BUILD_DIR=graphcore-doc-build VERSION=${{ env.VERSION }} | |
| sudo mv graphcore-doc-build ../optimum | |
| cd .. | |
| - name: Make Habana documentation | |
| run: | | |
| sudo docker system prune -a -f | |
| cd optimum-habana | |
| make doc BUILD_DIR=habana-doc-build VERSION=${{ env.VERSION }} | |
| sudo mv habana-doc-build ../optimum | |
| cd .. | |
| - name: Make Intel documentation | |
| run: | | |
| sudo docker system prune -a -f | |
| cd optimum-intel | |
| make doc BUILD_DIR=intel-doc-build VERSION=${{ env.VERSION }} | |
| sudo mv intel-doc-build ../optimum | |
| cd .. | |
| - name: Make Furiosa documentation | |
| run: | | |
| cd optimum-furiosa | |
| pip install . | |
| sudo apt update | |
| sudo apt install -y ca-certificates apt-transport-https gnupg | |
| sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-key 5F03AFA423A751913F249259814F888B20B09A7E | |
| sudo tee -a /etc/apt/auth.conf.d/furiosa.conf > /dev/null <<EOT | |
| machine archive.furiosa.ai | |
| login ${{ secrets.FURIOSA_ACCESS_KEY }} | |
| password ${{ secrets.FURIOSA_SECRET_ACCESS_KEY }} | |
| EOT | |
| sudo chmod 400 /etc/apt/auth.conf.d/furiosa.conf | |
| sudo tee -a /etc/apt/sources.list.d/furiosa.list <<EOT | |
| deb [arch=amd64] https://archive.furiosa.ai/ubuntu jammy restricted | |
| EOT | |
| sudo apt update && sudo apt install -y furiosa-libnux | |
| doc-builder build optimum.furiosa docs/source/ --build_dir furiosa-doc-build --version pr_$PR_NUMBER --version_tag_suffix "" --html --clean | |
| mv furiosa-doc-build ../optimum | |
| cd .. | |
| - name: Make Optimum documentation | |
| run: | | |
| sudo docker system prune -a -f | |
| cd optimum | |
| mkdir -p optimum-doc-build/optimum && cd optimum-doc-build/optimum | |
| wget https://huggingface.co/datasets/hf-doc-build/doc-build/raw/main/optimum/_versions.yml | |
| cd ../.. | |
| make doc BUILD_DIR=optimum-doc-build VERSION=${{ env.VERSION }} COMMIT_SHA_OPTIMUM=${{ env.VERSION }} | |
| cd .. | |
| - name: Combine subpackage documentation | |
| run: | | |
| cd optimum | |
| sudo python docs/combine_docs.py --subpackages graphcore habana intel neuron furiosa --version ${{ env.VERSION }} | |
| cd .. | |
| - name: Push to repositories | |
| run: | | |
| cd optimum/optimum-doc-build | |
| sudo chmod -R ugo+rwx optimum | |
| doc-builder push optimum --doc_build_repo_id "hf-doc-build/doc-build" --token "${{ secrets.HF_DOC_BUILD_PUSH }}" --commit_msg "Updated with commit ${{ github.sha }} See: https://github.com/huggingface/optimum/commit/${{ github.sha }}" --n_retries 5 --upload_version_yml | |
| shell: bash |