aitests.yml

name: "aitests"

on:
#  push:
#    branches:
#      - 'release-*'
#    paths-ignore:
#      - 'docs/**'
#  pull_request:
#    #The branches below must be a subset of the branches above
#    branches:
#      - 'release-*'
#    paths-ignore:
#      - 'docs/**'
  schedule:
    - cron:  '30 20 * * *'
  workflow_dispatch:


jobs:
  aitests:
    timeout-minutes: 90
    strategy:
      fail-fast: false
      matrix:
        type: [ 'normal', 'cache']

    runs-on: ubuntu-20.04
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          fetch-depth: 1

      - name: Build
        timeout-minutes: 10
        uses: ./.github/actions/build

      - name: Run Redis
        run: |
          sudo docker run -d --name redis -v redis-data:/data  \
          -p 6379:6379  redis redis-server --appendonly yes

      - name: Setup minio
        run: |
          docker run -d -p 9000:9000 --name minio \
                    -e "MINIO_ACCESS_KEY=minioadmin" \
                    -e "MINIO_SECRET_KEY=minioadmin" \
                    -v /tmp/data:/data \
                    -v /tmp/config:/root/.minio \
                    minio/minio server /data
          sleep 5

      - name: Juicefs Format
        run: |
          sudo ./juicefs format --trash-days 0 --storage minio --bucket http://127.0.0.1:9000/mypics \
             --access-key minioadmin \
             --secret-key minioadmin \
             redis://127.0.0.1:6379/1 pics

      - name: Juicefs Mount
        run: |
          if [ "${{matrix.type}}" == "normal" ]; then
            sudo ./juicefs mount -d redis://127.0.0.1:6379/1 /jfs/ --enable-xattr --no-usage-report
          elif [ "${{matrix.type}}" == "cache" ]; then
            sudo ./juicefs mount -d redis://127.0.0.1:6379/1 /jfs/ --enable-xattr --cache-size 3072 --no-usage-report
          else
            echo "matrix.type error: ${{matrix.type}} "
          fi
          
          python --version

      - name: Set up Conda
        uses: s-weigand/setup-conda@v1
        with:
          python-version: 2.7

      - name: Install Dependency
        run: |
          conda --version
          python --version
          conda install tensorflow=1.12.0
          sudo .github/scripts/apt_install.sh  libhdf5-dev
          pip install h5py
          pip install pillow

      - name: Prepare Data
        run: |
          stat /jfs/
          cd /jfs/
          git clone https://github.com/sanwan/SVHNClassifier
          cd SVHNClassifier
          git checkout 1a8c6f1
          mkdir data
          cd data
          wget -O train.tar.gz -nv http://ufldl.stanford.edu/housenumbers/train.tar.gz
          wget -O test.tar.gz -nv http://ufldl.stanford.edu/housenumbers/test.tar.gz
          ls -lah
          time tar -zxf train.tar.gz
          time tar -zxf test.tar.gz
          mkdir extra
          mv test/1.png extra/
          cp test/digitStruct.mat extra/
          cp test/see_bboxes.m extra/
          rm train.tar.gz
          rm test.tar.gz
          ls -lah
          du -sh
          cd ..
          time python convert_to_tfrecords.py --data_dir ./data >img.log
          sudo chmod 777 /var/jfsCache/
          cd /var/jfsCache/
          df -lh


      - name: Train
        run: |
          cd /jfs/SVHNClassifier/
          time python train.py --data_dir ./data --train_logdir ./logs/train
          cd /var/jfsCache/
          df -lh

      - name: Evaluate
        run: |
          cd /jfs/SVHNClassifier/
          time python eval.py --data_dir ./data --checkpoint_dir ./logs/train --eval_logdir ./logs/eval
          cd /var/jfsCache/
          df -lh

      - name: log
        if: always()
        run: | 
          tail -300 /var/log/juicefs.log
          grep "<FATAL>:" /var/log/juicefs.log && exit 1 || true

      - name: Send Slack Notification
        if: failure()
        uses: juicedata/slack-notify-action@main
        with:
          channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
          slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}"