Fetch Data #15
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Fetch Data | |
| on: | |
| schedule: | |
| # Normal schedule | |
| # # at 01:15 on all days in first month of each quarter | |
| # - cron: '15 1 * 1,4,7,10 *' | |
| # # at 01:15 on days 1-14 in second month of each quarter | |
| # - cron: '15 1 1-14 2,5,8,11 *' | |
| # Temp schedule | |
| # at 01:15 on all days in all months | |
| - cron: '15 1 * * *' | |
| workflow_dispatch: | |
| jobs: | |
| fetch: | |
| runs-on: ubuntu-latest | |
| steps: | |
| # CC Technology team members: | |
| # See cc-quantifying-bot GitHub entry in Bitwarden for information on | |
| # BOT_ secrets | |
| - name: Configure git | |
| run: | | |
| git config --global init.defaultBranch main | |
| git config --global user.name "${{ secrets.BOT_NAME }}" | |
| git config --global user.email "${{ secrets.BOT_EMAIL }}" | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| # Default fetch-depth is 1, however that value results in errors | |
| # when GitPython attempts to push changes: | |
| # "failed to push some refs" | |
| fetch-depth: 0 | |
| token: ${{ secrets.BOT_TOKEN }} | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install Python dependencies | |
| run: | | |
| pip install --upgrade pip pipenv | |
| - name: Sync Python modules | |
| run: | | |
| pipenv sync --system | |
| # CC Technology team members: | |
| # See cc-quantifying-bot Google Workspace entry in Bitwarden for | |
| # information on GCS_ secrets | |
| - name: Fetch from Google Custom Search (GCS) | |
| run: | | |
| ./scripts/1-fetch/gcs_fetch.py \ | |
| --limit=100 --enable-save --enable-git | |
| env: | |
| GCS_DEVELOPER_KEY: ${{ secrets.GCS_DEVELOPER_KEY }} | |
| GCS_CX: ${{ secrets.GCS_CX }} | |
| - name: Fetch from GitHub | |
| run: | | |
| ./scripts/1-fetch/github_fetch.py \ | |
| --enable-save --enable-git | |
| env: | |
| GH_TOKEN: ${{ secrets.BOT_TOKEN }} |