Skip to content

db_ingestion_by_scraping_sitemap #204

db_ingestion_by_scraping_sitemap

db_ingestion_by_scraping_sitemap #204

name: db_ingestion_by_scraping_sitemap
on:
workflow_dispatch: # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#workflow_dispatch
schedule: # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule
- cron: '00 2 * * *' # every day at 02:00 UTC
env:
PYTHON_VERSION: '3.10'
POETRY_VERSION: '1.2.1'
jobs:
db_ingestion_by_scraping_sitemamp:
runs-on: ubuntu-latest
steps:
- uses: actions/setup-python@v2
with:
python-version: ${{ env.PYTHON_VERSION }}
- uses: actions/checkout@v2
- name: Install Poetry
uses: snok/install-poetry@v1
with:
version: ${{ env.POETRY_VERSION }}
virtualenvs-create: true
virtualenvs-in-project: true
installer-parallel: true
- name: Verify poetry.lock agrees with pyproject.toml
run: poetry lock --check
- name: Install dependencies
run: poetry install --no-interaction --no-ansi
- name: RunSitemapScrappingAndIngestScript
run: |
poetry run python quotaclimat/data_ingestion/ingest_db/ingest_sitemap_in_db.py -p ${{ secrets.D4G_VM_DB_PWD }}