aitests #951
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "aitests" | |
| on: | |
| # push: | |
| # branches: | |
| # - 'release-*' | |
| # paths-ignore: | |
| # - 'docs/**' | |
| # pull_request: | |
| # #The branches below must be a subset of the branches above | |
| # branches: | |
| # - 'release-*' | |
| # paths-ignore: | |
| # - 'docs/**' | |
| schedule: | |
| - cron: '30 20 * * *' | |
| workflow_dispatch: | |
| jobs: | |
| aitests: | |
| timeout-minutes: 90 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| type: [ 'normal', 'cache'] | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 1 | |
| - name: Build | |
| timeout-minutes: 10 | |
| uses: ./.github/actions/build | |
| - name: Run Redis | |
| run: | | |
| sudo docker run -d --name redis -v redis-data:/data \ | |
| -p 6379:6379 redis redis-server --appendonly yes | |
| - name: Setup minio | |
| run: | | |
| docker run -d -p 9000:9000 --name minio \ | |
| -e "MINIO_ACCESS_KEY=minioadmin" \ | |
| -e "MINIO_SECRET_KEY=minioadmin" \ | |
| -v /tmp/data:/data \ | |
| -v /tmp/config:/root/.minio \ | |
| minio/minio server /data | |
| sleep 5 | |
| - name: Juicefs Format | |
| run: | | |
| sudo ./juicefs format --trash-days 0 --storage minio --bucket http://127.0.0.1:9000/mypics \ | |
| --access-key minioadmin \ | |
| --secret-key minioadmin \ | |
| redis://127.0.0.1:6379/1 pics | |
| - name: Juicefs Mount | |
| run: | | |
| if [ "${{matrix.type}}" == "normal" ]; then | |
| sudo ./juicefs mount -d redis://127.0.0.1:6379/1 /jfs/ --enable-xattr --no-usage-report | |
| elif [ "${{matrix.type}}" == "cache" ]; then | |
| sudo ./juicefs mount -d redis://127.0.0.1:6379/1 /jfs/ --enable-xattr --cache-size 3072 --no-usage-report | |
| else | |
| echo "matrix.type error: ${{matrix.type}} " | |
| fi | |
| python --version | |
| - name: Set up Conda | |
| uses: s-weigand/setup-conda@v1 | |
| with: | |
| python-version: 2.7 | |
| - name: Install Dependency | |
| run: | | |
| conda --version | |
| python --version | |
| conda install tensorflow=1.12.0 | |
| sudo .github/scripts/apt_install.sh libhdf5-dev | |
| pip install h5py | |
| pip install pillow | |
| - name: Prepare Data | |
| run: | | |
| stat /jfs/ | |
| cd /jfs/ | |
| git clone https://github.com/sanwan/SVHNClassifier | |
| cd SVHNClassifier | |
| git checkout 1a8c6f1 | |
| mkdir data | |
| cd data | |
| wget -O train.tar.gz -nv http://ufldl.stanford.edu/housenumbers/train.tar.gz | |
| wget -O test.tar.gz -nv http://ufldl.stanford.edu/housenumbers/test.tar.gz | |
| ls -lah | |
| time tar -zxf train.tar.gz | |
| time tar -zxf test.tar.gz | |
| mkdir extra | |
| mv test/1.png extra/ | |
| cp test/digitStruct.mat extra/ | |
| cp test/see_bboxes.m extra/ | |
| rm train.tar.gz | |
| rm test.tar.gz | |
| ls -lah | |
| du -sh | |
| cd .. | |
| time python convert_to_tfrecords.py --data_dir ./data >img.log | |
| sudo chmod 777 /var/jfsCache/ | |
| cd /var/jfsCache/ | |
| df -lh | |
| - name: Train | |
| run: | | |
| cd /jfs/SVHNClassifier/ | |
| time python train.py --data_dir ./data --train_logdir ./logs/train | |
| cd /var/jfsCache/ | |
| df -lh | |
| - name: Evaluate | |
| run: | | |
| cd /jfs/SVHNClassifier/ | |
| time python eval.py --data_dir ./data --checkpoint_dir ./logs/train --eval_logdir ./logs/eval | |
| cd /var/jfsCache/ | |
| df -lh | |
| - name: log | |
| if: always() | |
| run: | | |
| tail -300 /var/log/juicefs.log | |
| grep "<FATAL>:" /var/log/juicefs.log && exit 1 || true | |
| - name: Send Slack Notification | |
| if: failure() | |
| uses: juicedata/slack-notify-action@main | |
| with: | |
| channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}" | |
| slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}" | |