Skip to content
Build main documentation

Update the habana extra (#2077) #1040

Sign in to view logs

Update the habana extra (#2077)

Update the habana extra (#2077) #1040

Workflow file for this run

.github/workflows/build_main_documentation.yml at c5bc378
name: Build main documentation
on:
push:
branches:
- main
- doc-builder*
- v*-release
workflow_dispatch:
jobs:
build_documentation:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v2
with:
repository: 'huggingface/doc-builder'
path: doc-builder
- uses: actions/checkout@v2
with:
repository: 'huggingface/doc-build'
path: doc-build
token: ${{ secrets.HUGGINGFACE_PUSH }}
- uses: actions/checkout@v2
with:
repository: 'huggingface/optimum'
path: optimum
- uses: actions/checkout@v2
with:
repository: 'huggingface/optimum-habana'
path: optimum-habana
- uses: actions/checkout@v2
with:
repository: 'huggingface/optimum-intel'
path: optimum-intel
- uses: actions/checkout@v2
with:
repository: 'huggingface/optimum-furiosa'
path: optimum-furiosa
- uses: actions/checkout@v2
with:
repository: 'huggingface/optimum-amd'
path: optimum-amd
- uses: actions/checkout@v2
with:
repository: 'huggingface/optimum-tpu'
path: optimum-tpu
- name: Free disk space
run: |
df -h
sudo apt-get update
sudo apt-get purge -y '^apache.*'
sudo apt-get purge -y '^imagemagick.*'
sudo apt-get purge -y '^dotnet.*'
sudo apt-get purge -y '^aspnetcore.*'
sudo apt-get purge -y 'php.*'
sudo apt-get purge -y '^temurin.*'
sudo apt-get purge -y '^mysql.*'
sudo apt-get purge -y '^java.*'
sudo apt-get purge -y '^openjdk.*'
sudo apt-get purge -y microsoft-edge-stable azure-cli google-chrome-stable firefox mono-devel
df -h
sudo apt-get autoremove -y >/dev/null 2>&1
sudo apt-get clean
df -h
echo "https://github.com/actions/virtual-environments/issues/709"
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
df -h
echo "remove big /usr/local"
sudo rm -rf "/usr/local/share/boost"
sudo rm -rf /usr/local/lib/android >/dev/null 2>&1
df -h
echo "remove /usr/share leftovers"
sudo rm -rf /usr/share/dotnet/sdk > /dev/null 2>&1
sudo rm -rf /usr/share/dotnet/shared > /dev/null 2>&1
sudo rm -rf /usr/share/swift > /dev/null 2>&1
df -h
echo "remove other leftovers"
sudo rm -rf /var/lib/mysql > /dev/null 2>&1
sudo rm -rf /home/runner/.dotnet > /dev/null 2>&1
sudo rm -rf /home/runneradmin/.dotnet > /dev/null 2>&1
sudo rm -rf /etc/skel/.dotnet > /dev/null 2>&1
sudo rm -rf /usr/local/.ghcup > /dev/null 2>&1
sudo rm -rf /usr/local/aws-cli > /dev/null 2>&1
sudo rm -rf /usr/lib/heroku > /dev/null 2>&1
sudo rm -rf /usr/local/share/chromium > /dev/null 2>&1
df -h
- name: Set environment variables
run: |
cd optimum
version=`echo "$(grep '^__version__ =' optimum/version.py | cut -d '=' -f 2- | xargs)"`
if [[ $version == *.dev0 ]]
then
echo "VERSION=main" >> $GITHUB_ENV
else
echo "VERSION=v$version" >> $GITHUB_ENV
fi
cd ..
- name: Setup environment
run: |
python -m venv venv-doc
source venv-doc/bin/activate
pip uninstall -y doc-builder
cd doc-builder
git pull origin main
pip install .
pip install black
cd ..
- name: Make Habana documentation
run: |
sudo docker system prune -a -f
cd optimum-habana
make doc BUILD_DIR=habana-doc-build VERSION=${{ env.VERSION }}
sudo mv habana-doc-build ../optimum
cd ..
- name: Make Intel documentation
run: |
sudo docker system prune -a -f
cd optimum-intel
make doc BUILD_DIR=intel-doc-build VERSION=${{ env.VERSION }}
sudo mv intel-doc-build ../optimum
cd ..
- name: Make Furiosa documentation
run: |
source venv-doc/bin/activate
cd optimum-furiosa
pip install .
sudo apt install software-properties-common
sudo add-apt-repository --remove https://packages.microsoft.com/ubuntu/22.04/prod
sudo apt update
sudo apt install -y ca-certificates apt-transport-https gnupg
sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-key 5F03AFA423A751913F249259814F888B20B09A7E
sudo tee -a /etc/apt/auth.conf.d/furiosa.conf > /dev/null <<EOT
machine archive.furiosa.ai
login ${{ secrets.FURIOSA_ACCESS_KEY }}
password ${{ secrets.FURIOSA_SECRET_ACCESS_KEY }}
EOT
sudo chmod 400 /etc/apt/auth.conf.d/furiosa.conf
sudo tee -a /etc/apt/sources.list.d/furiosa.list <<EOT
deb [arch=amd64] https://archive.furiosa.ai/ubuntu jammy restricted
EOT
sudo apt update && sudo apt install -y furiosa-libnux
doc-builder build optimum.furiosa docs/source/ --build_dir furiosa-doc-build --version pr_$PR_NUMBER --version_tag_suffix "" --html --clean
mv furiosa-doc-build ../optimum
cd ..
- name: Make TPU documentation
run: |
sudo docker system prune -a -f
source venv-doc/bin/activate
cd optimum-tpu
pip install -U pip
pip install . -f https://storage.googleapis.com/libtpu-releases/index.html
doc-builder build optimum.tpu docs/source/ --build_dir tpu-doc-build --version pr_$PR_NUMBER --version_tag_suffix "" --html --clean
mv tpu-doc-build ../optimum
cd ..
- name: Make AMD documentation
run: |
sudo docker system prune -a -f
cd optimum-amd
make doc BUILD_DIR=amd-doc-build VERSION=${{ env.VERSION }}
sudo mv amd-doc-build ../optimum
cd ..
- name: Make Optimum documentation
run: |
sudo docker system prune -a -f
cd optimum
mkdir -p optimum-doc-build/optimum && cd optimum-doc-build/optimum
wget https://huggingface.co/datasets/hf-doc-build/doc-build/raw/main/optimum/_versions.yml
cd ../..
make doc BUILD_DIR=optimum-doc-build VERSION=${{ env.VERSION }} COMMIT_SHA_OPTIMUM=${{ env.VERSION }}
cd ..
- name: Combine subpackage documentation
run: |
cd optimum
sudo python docs/combine_docs.py --subpackages nvidia amd intel neuron tpu habana furiosa --version ${{ env.VERSION }}
cd ..
- name: Push to repositories
run: |
source venv-doc/bin/activate
cd optimum/optimum-doc-build
sudo chmod -R ugo+rwx optimum
doc-builder push optimum --doc_build_repo_id "hf-doc-build/doc-build" --token "${{ secrets.HF_DOC_BUILD_PUSH }}" --commit_msg "Updated with commit ${{ github.sha }} See: https://github.com/huggingface/optimum/commit/${{ github.sha }}" --n_retries 5 --upload_version_yml
shell: bash