examples-as-test.yml

name: Examples as Tests

on:
  schedule:
    # * is a special character in YAML so you have to quote this string
    # Format: minute hour day-of-month month day-of-week(starts on sunday)
    # Scheduled for 2 am, everyday
    - cron: '0 10 * * *'
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
  cancel-in-progress: true

jobs:
  generate_example_tests:
    name: Generate Example Tests
    runs-on: [self-hosted, linux, gpu, dataset-enabled]
    defaults:
      run:
        shell: micromamba run -n omnigibson /bin/bash -leo pipefail {0}
    steps:
      - name: Checkout source
        uses: actions/checkout@v2
        with:
          submodules: true
          path: omnigibson-src
      
      # These are temporarily disabled due to some errors they cause where some
      # unknown important dependency has its version changed and Isaac Sim will
      # no longer launch properly. The result is that if dependencies change,
      # the tests will stop working. This is a temporary fix until we can figure
      # out how to fix this issue.

      # - name: Install dev requirements
      #   working-directory: omnigibson-src
      #   run: pip install -r requirements-dev.txt
        
      # - name: Install
      #   working-directory: omnigibson-src
      #   run: pip install -e .

      - name: Generate example tests
        working-directory: omnigibson-src
        run: python tests/create_tests_of_examples.py

      - name: Get list of generated tests
        id: get-test-list
        working-directory: omnigibson-src
        run: |
          echo "example_tests=$(cat tests/example_tests.json)" >> $GITHUB_OUTPUT

    outputs:
      example_tests: ${{ steps.get-test-list.outputs.example_tests }}
  
  run_test:
    name: Run Example Tests
    needs: [generate_example_tests]
    runs-on: [self-hosted, linux, gpu, dataset-enabled]

    strategy:
      matrix:
        test_file:
          - ${{ needs.generate_example_tests.outputs.example_tests != '' && fromJson(needs.generate_example_tests.outputs.example_tests) }}
      fail-fast: true

    defaults:
      run:
        shell: micromamba run -n omnigibson /bin/bash -leo pipefail {0}

    steps:
      - name: Check for generated tests
        if: ${{ needs.generate_example_tests.outputs.example_tests == '' }}
        run: |
          echo "No tests were generated. Failing the job."
          exit 1

      - name: Fix home
        run: echo "HOME=/root" >> $GITHUB_ENV

      - name: Checkout source
        uses: actions/checkout@v2
        with:
          submodules: true
          path: omnigibson-src

      - name: Install dev requirements
        working-directory: omnigibson-src
        run: pip install -r requirements-dev.txt

      - name: Install
        working-directory: omnigibson-src
        run: pip install -e .

      - name: Run tests
        working-directory: omnigibson-src
        run: pytest -s tests/tests_of_examples/${{ matrix.test_file }}.py --junitxml=${{ matrix.test_file }}.xml && cp ${{ matrix.test_file }}.xml ${GITHUB_WORKSPACE}/

      - name: Deploy artifact
        uses: actions/upload-artifact@v3
        with:
          name: ${{ github.run_id }}-tests-${{ matrix.test_file }}
          path: ${{ matrix.test_file }}.xml

      - name: Fail on failure or error
        run: grep -Eq "<failure|error" ${{ matrix.test_file }}.xml; if [ $? -eq 0 ]; then exit 1; else exit 0; fi

  upload_report:
    name: Compile Example Test Report
    runs-on: [self-hosted, linux]
    defaults:
      run:
        shell: micromamba run -n omnigibson /bin/bash -leo pipefail {0}
    needs: [run_test]
    steps:
      - name: Checkout source
        uses: actions/checkout@v2
        with:
          submodules: true
          path: omnigibson-src
      - name: Pull reports
        uses: actions/download-artifact@v3
        with:
          path: omnigibson-src
      - name: Example Test Report0
        uses: dorny/test-reporter@v1
        with:
          name: Example Test Results
          working-directory: omnigibson-src
          path: ${{ github.run_id }}-tests-*/*_test.xml
          reporter: java-junit
          fail-on-error: 'true'
          fail-on-empty: 'true'

      # - name: Upload coverage to Codecov
      #   uses: codecov/codecov-action@v2.1.0
      #   with:
      #     token: ${{ secrets.CODECOV_TOKEN }}