From 2bc78e901c17f8672e4eda175f3e5848c4000b3b Mon Sep 17 00:00:00 2001 From: jbeshir Date: Wed, 24 Apr 2024 09:07:41 +0100 Subject: [PATCH] Airtable: Take base and table IDs from environment (#200) * Airtable: Switch to using copy of base+table I have a token for. Might be a better approach here, but this works for now. * Airtable: Take base and table IDs from environment. This allows them to be swapped out for alternative copies, for different users running the script. * Airtable: Prefix airtable base+table ID env vars with the dataset. Also shuffle where we use them up to the dataset config level. Makes it easier to add additional Airtable datasets. --- .env.example | 3 +++ .github/workflows/check-articles.yml | 2 ++ .github/workflows/fetch-daily.yml | 2 ++ .github/workflows/fetch-dataset.yml | 8 ++++++++ .github/workflows/fetch-weekly.yml | 2 ++ align_data/settings.py | 2 ++ align_data/sources/agisf/__init__.py | 6 ++++-- align_data/sources/airtable.py | 4 ++++ 8 files changed, 27 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index 7964e08b..a93f5489 100644 --- a/.env.example +++ b/.env.example @@ -3,7 +3,10 @@ MIN_CONFIDENCE="0.5" CODA_TOKEN="" YOUTUBE_API_KEY="" + AIRTABLE_API_KEY="" +AGISF_AIRTABLE_BASE_ID="" +AGISF_AIRTABLE_TABLE_ID="" ARD_DB_USER="user" ARD_DB_PASSWORD="we all live in a yellow submarine" diff --git a/.github/workflows/check-articles.yml b/.github/workflows/check-articles.yml index 3d17cbbd..90d359f2 100644 --- a/.github/workflows/check-articles.yml +++ b/.github/workflows/check-articles.yml @@ -57,6 +57,8 @@ jobs: env: CODA_TOKEN: ${{ secrets.CODA_TOKEN }} AIRTABLE_API_KEY: ${{ secrets.AIRTABLE_API_KEY }} + AGISF_AIRTABLE_BASE_ID: ${{ secrets.AGISF_AIRTABLE_BASE_ID }} + AGISF_AIRTABLE_TABLE_ID: ${{ secrets.AGISF_AIRTABLE_TABLE_ID }} YOUTUBE_API_KEY: ${{ secrets.YOUTUBE_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ARD_DB_USER: ${{ secrets.ARD_DB_USER }} diff --git a/.github/workflows/fetch-daily.yml b/.github/workflows/fetch-daily.yml index f4a07050..772639b2 100644 --- a/.github/workflows/fetch-daily.yml +++ b/.github/workflows/fetch-daily.yml @@ -15,6 +15,8 @@ jobs: datasource: ${{ matrix.datasource }} coda_token: ${{ inputs.coda_token }} airtable_api_key: ${{ inputs.airtable_api_key }} + agisf_airtable_base_id: ${{ inputs.agisf_airtable_base_id }} + agisf_airtable_table_id: ${{ inputs.agisf_airtable_table_id }} youtube_api_key: ${{ inputs.youtube_api_key }} db_user: ${{ inputs.db_user }} db_password: ${{ inputs.db_password }} diff --git a/.github/workflows/fetch-dataset.yml b/.github/workflows/fetch-dataset.yml index 7731097b..f176b02f 100644 --- a/.github/workflows/fetch-dataset.yml +++ b/.github/workflows/fetch-dataset.yml @@ -12,6 +12,12 @@ on: airtable_api_key: type: string required: true + agisf_airtable_base_id: + type: string + required: true + agisf_airtable_table_id: + type: string + required: true youtube_api_key: type: string required: true @@ -72,6 +78,8 @@ jobs: env: CODA_TOKEN: ${{ secrets.CODA_TOKEN || inputs.coda_token }} AIRTABLE_API_KEY: ${{ secrets.AIRTABLE_API_KEY || inputs.airtable_api_key }} + AGISF_AIRTABLE_BASE_ID: ${{ secrets.AGISF_AIRTABLE_BASE_ID || inputs.agisf_airtable_base_id }} + AGISF_AIRTABLE_TABLE_ID: ${{ secrets.AGISF_AIRTABLE_TABLE_ID || inputs.agisf_airtable_table_id }} YOUTUBE_API_KEY: ${{ secrets.YOUTUBE_API_KEY || inputs.youtube_api_key }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY || inputs.openai_api_key }} ARD_DB_USER: ${{ secrets.ARD_DB_USER || inputs.db_user }} diff --git a/.github/workflows/fetch-weekly.yml b/.github/workflows/fetch-weekly.yml index 64d17c07..38eb3c84 100644 --- a/.github/workflows/fetch-weekly.yml +++ b/.github/workflows/fetch-weekly.yml @@ -15,6 +15,8 @@ jobs: datasource: ${{ matrix.datasource }} coda_token: ${{ inputs.coda_token }} airtable_api_key: ${{ inputs.airtable_api_key }} + agisf_airtable_base_id: ${{ inputs.agisf_airtable_base_id }} + agisf_airtable_table_id: ${{ inputs.agisf_airtable_table_id }} youtube_api_key: ${{ inputs.youtube_api_key }} db_user: ${{ inputs.db_user }} db_password: ${{ inputs.db_password }} diff --git a/align_data/settings.py b/align_data/settings.py index 383e89f3..adf57ca4 100644 --- a/align_data/settings.py +++ b/align_data/settings.py @@ -32,6 +32,8 @@ ### Airtable ### AIRTABLE_API_KEY = os.environ.get("AIRTABLE_API_KEY") +AGISF_AIRTABLE_BASE_ID = os.environ.get("AGISF_AIRTABLE_BASE_ID") +AGISF_AIRTABLE_TABLE_ID = os.environ.get("AGISF_AIRTABLE_TABLE_ID") ### MYSQL ### user = os.environ.get("ARD_DB_USER", "user") diff --git a/align_data/sources/agisf/__init__.py b/align_data/sources/agisf/__init__.py index bf484913..2fe8c9f2 100644 --- a/align_data/sources/agisf/__init__.py +++ b/align_data/sources/agisf/__init__.py @@ -2,12 +2,14 @@ from align_data.sources.airtable import AirtableDataset from align_data.sources.agisf.agisf import AGISFPodcastDataset +from align_data.settings import AGISF_AIRTABLE_BASE_ID, AGISF_AIRTABLE_TABLE_ID + datasets = [ AirtableDataset( name="agisf_governance", - base_id="app9q0E0jlDWlsR0z", - table_id="tblgTb3kszvSbo2Mb", + base_id=AGISF_AIRTABLE_BASE_ID, + table_id=AGISF_AIRTABLE_TABLE_ID, mappings={ "title": "[>] Resource", "url": "[h] [>] Link", diff --git a/align_data/sources/airtable.py b/align_data/sources/airtable.py index 20564b5c..81dfb213 100644 --- a/align_data/sources/airtable.py +++ b/align_data/sources/airtable.py @@ -21,6 +21,10 @@ class AirtableDataset(AlignmentDataset): def setup(self): if not AIRTABLE_API_KEY: raise ValueError("No AIRTABLE_API_KEY provided!") + if not self.base_id: + raise ValueError("No airtable base ID provided!") + if not self.table_id: + raise ValueError("No airtable table ID provided!") super().setup() self.at = airtable.Airtable(self.base_id, AIRTABLE_API_KEY)