crawl.yaml

name: Unit Test

on: [push]

jobs:
  build:

    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v2

    - name: Set up Python 3.8
      uses: actions/setup-python@v2
      with:
        python-version: 3.8

    - name: Install dependencies
      run: |
        python -m venv .env
        source .env/bin/activate
        pip install -r requirements.txt

    - name: Test crawler and indexer\
      env:
        SAVE_CRAWLS: 0
        SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
        SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }}
        CHUNK_SIZE: 50
        CHUNK_LIMIT: 50
      run: |
        source .env/bin/activate
        python3 -m phantom.main --num_threads 2 --urls "https://github.com/AnsahMohammad" "https://github.com/AnsahMohammad" --sleep 2

        python3 -m phantom.core.indexer

        echo "Crawling completed"
    
    - name: Unittests
      run: |
        pip install -r requirements.txt
        python3 -m unittest discover -s tests
        echo "Unit test done"