diff --git a/.github/workflows/crawling.yml b/.github/workflows/crawling.yml index fb89c54..ff5bda4 100644 --- a/.github/workflows/crawling.yml +++ b/.github/workflows/crawling.yml @@ -27,12 +27,12 @@ jobs: - name: Install run: yarn install --frozen-lockfile - + - name: Pip uses: actions/setup-python@v4 with: - python-version: '3.9' - cache: 'pip' # caching pip dependencies + python-version: "3.9" + cache: "pip" # caching pip dependencies - name: Pip Install run: pip install -r requirements.txt @@ -41,20 +41,19 @@ jobs: run: yarn start env: LOG_FORMAT: json - NUM_TERMS: 6 + NUM_TERMS: 4 ALWAYS_SCRAPE_CURRENT_TERM: 1 DETAILS_CONCURRENCY: 256 DATA_FOLDER: ./data NODE_EXTRA_CA_CERTS: ${{ github.workspace }}/intermediate.pem - + - name: Revision run: python ./src/Revise.py - - name: Upload uses: JamesIves/github-pages-deploy-action@releases/v4 with: branch: crawler-gh-pages folder: ./data clean: true - single-commit: true \ No newline at end of file + single-commit: true