Skip to content

Scraper Worker

Scraper Worker #137

Workflow file for this run

name: Scraper Worker
on:
schedule:
- cron: '30 15 * * *' # Runs every day at 11:30 AM UTC
jobs:
scrape:
runs-on: ubuntu-latest
env:
VECTOR_DB_API_KEY: ${{ secrets.VECTOR_DB_API_KEY }}
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
SERVICE_EMAIL: ${{ secrets.SERVICE_EMAIL }}
PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }}
SHEET_ID: ${{ secrets.SHEET_ID }}
MONGODB_URI : ${{secrets.MONGODB_URI}}
HUGGING_FACE_API_TOKEN : ${{secrets.HUGGING_FACE_API_TOKEN}}
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Change directory to 'packages/scraper'
run: cd packages/scraper
- name: Setup Node.js
uses: actions/setup-node@v2
with:
node-version: '20'
- name: Install dependencies
run: npm install
- name: Install playwright deps
run: npx playwright install --with-deps chromium
- name: Run Scraper
run: npm run start:scraper