Skip to content

aitests

aitests #883

Workflow file for this run

name: "aitests"
on:
# push:
# branches:
# - 'release-*'
# paths-ignore:
# - 'docs/**'
# pull_request:
# #The branches below must be a subset of the branches above
# branches:
# - 'release-*'
# paths-ignore:
# - 'docs/**'
schedule:
- cron: '30 20 * * *'
workflow_dispatch:
jobs:
aitests:
timeout-minutes: 90
strategy:
fail-fast: false
matrix:
type: [ 'normal', 'cache']
runs-on: ubuntu-20.04
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 1
- name: Build
timeout-minutes: 10
uses: ./.github/actions/build
- name: Run Redis
run: |
sudo docker run -d --name redis -v redis-data:/data \
-p 6379:6379 redis redis-server --appendonly yes
- name: Setup minio
run: |
docker run -d -p 9000:9000 --name minio \
-e "MINIO_ACCESS_KEY=minioadmin" \
-e "MINIO_SECRET_KEY=minioadmin" \
-v /tmp/data:/data \
-v /tmp/config:/root/.minio \
minio/minio server /data
sleep 5
- name: Juicefs Format
run: |
sudo ./juicefs format --trash-days 0 --storage minio --bucket http://127.0.0.1:9000/mypics \
--access-key minioadmin \
--secret-key minioadmin \
redis://127.0.0.1:6379/1 pics
- name: Juicefs Mount
run: |
if [ "${{matrix.type}}" == "normal" ]; then
sudo ./juicefs mount -d redis://127.0.0.1:6379/1 /jfs/ --enable-xattr --no-usage-report
elif [ "${{matrix.type}}" == "cache" ]; then
sudo ./juicefs mount -d redis://127.0.0.1:6379/1 /jfs/ --enable-xattr --cache-size 3072 --no-usage-report
else
echo "matrix.type error: ${{matrix.type}} "
fi
python --version
- name: Set up Conda
uses: s-weigand/setup-conda@v1
with:
python-version: 2.7
- name: Install Dependency
run: |
conda --version
python --version
conda install tensorflow=1.12.0
sudo .github/scripts/apt_install.sh libhdf5-dev
pip install h5py
pip install pillow
- name: Prepare Data
run: |
stat /jfs/
cd /jfs/
git clone https://github.com/sanwan/SVHNClassifier
cd SVHNClassifier
git checkout 1a8c6f1
mkdir data
cd data
wget -O train.tar.gz -nv http://ufldl.stanford.edu/housenumbers/train.tar.gz
wget -O test.tar.gz -nv http://ufldl.stanford.edu/housenumbers/test.tar.gz
ls -lah
time tar -zxf train.tar.gz
time tar -zxf test.tar.gz
mkdir extra
mv test/1.png extra/
cp test/digitStruct.mat extra/
cp test/see_bboxes.m extra/
rm train.tar.gz
rm test.tar.gz
ls -lah
du -sh
cd ..
time python convert_to_tfrecords.py --data_dir ./data >img.log
sudo chmod 777 /var/jfsCache/
cd /var/jfsCache/
df -lh
- name: Train
run: |
cd /jfs/SVHNClassifier/
time python train.py --data_dir ./data --train_logdir ./logs/train
cd /var/jfsCache/
df -lh
- name: Evaluate
run: |
cd /jfs/SVHNClassifier/
time python eval.py --data_dir ./data --checkpoint_dir ./logs/train --eval_logdir ./logs/eval
cd /var/jfsCache/
df -lh
- name: log
if: always()
run: |
tail -300 /var/log/juicefs.log
grep "<FATAL>:" /var/log/juicefs.log && exit 1 || true
- name: Send Slack Notification
if: failure()
uses: juicedata/slack-notify-action@main
with:
channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}"