Airtable: Take base and table IDs from environment (#200)

* Airtable: Switch to using copy of base+table I have a token for.

Might be a better approach here, but this works for now.

* Airtable: Take base and table IDs from environment.

This allows them to be swapped out for alternative copies,
for different users running the script.

* Airtable: Prefix airtable base+table ID env vars with the dataset.

Also shuffle where we use them up to the dataset config level.

Makes it easier to add additional Airtable datasets.

.env.example

@@ -3,7 +3,10 @@ MIN_CONFIDENCE="0.5"
 
 CODA_TOKEN=""
 YOUTUBE_API_KEY=""
+
 AIRTABLE_API_KEY=""
+AGISF_AIRTABLE_BASE_ID=""
+AGISF_AIRTABLE_TABLE_ID=""
 
 ARD_DB_USER="user"
 ARD_DB_PASSWORD="we all live in a yellow submarine"

.github/workflows/check-articles.yml

@@ -57,6 +57,8 @@ jobs:
       env:
         CODA_TOKEN: ${{ secrets.CODA_TOKEN }}
         AIRTABLE_API_KEY: ${{ secrets.AIRTABLE_API_KEY }}
+        AGISF_AIRTABLE_BASE_ID: ${{ secrets.AGISF_AIRTABLE_BASE_ID }}
+        AGISF_AIRTABLE_TABLE_ID: ${{ secrets.AGISF_AIRTABLE_TABLE_ID }}
         YOUTUBE_API_KEY: ${{ secrets.YOUTUBE_API_KEY }}
         OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         ARD_DB_USER: ${{ secrets.ARD_DB_USER }}

.github/workflows/fetch-daily.yml

@@ -15,6 +15,8 @@ jobs:
       datasource: ${{ matrix.datasource }}
       coda_token: ${{ inputs.coda_token }}
       airtable_api_key: ${{ inputs.airtable_api_key }}
+      agisf_airtable_base_id: ${{ inputs.agisf_airtable_base_id }}
+      agisf_airtable_table_id: ${{ inputs.agisf_airtable_table_id }}
       youtube_api_key: ${{ inputs.youtube_api_key }}
       db_user: ${{ inputs.db_user }}
       db_password: ${{ inputs.db_password }}

.github/workflows/fetch-dataset.yml

@@ -12,6 +12,12 @@ on:
       airtable_api_key:
         type: string
         required: true
+      agisf_airtable_base_id:
+        type: string
+        required: true
+      agisf_airtable_table_id:
+        type: string
+        required: true
       youtube_api_key:
         type: string
         required: true
@@ -72,6 +78,8 @@ jobs:
       env:
         CODA_TOKEN: ${{ secrets.CODA_TOKEN || inputs.coda_token }}
         AIRTABLE_API_KEY: ${{ secrets.AIRTABLE_API_KEY || inputs.airtable_api_key }}
+        AGISF_AIRTABLE_BASE_ID: ${{ secrets.AGISF_AIRTABLE_BASE_ID || inputs.agisf_airtable_base_id }}
+        AGISF_AIRTABLE_TABLE_ID: ${{ secrets.AGISF_AIRTABLE_TABLE_ID || inputs.agisf_airtable_table_id }}
         YOUTUBE_API_KEY: ${{ secrets.YOUTUBE_API_KEY || inputs.youtube_api_key }}
         OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY || inputs.openai_api_key }}
         ARD_DB_USER: ${{ secrets.ARD_DB_USER || inputs.db_user }}

.github/workflows/fetch-weekly.yml

@@ -15,6 +15,8 @@ jobs:
       datasource: ${{ matrix.datasource }}
       coda_token: ${{ inputs.coda_token }}
       airtable_api_key: ${{ inputs.airtable_api_key }}
+      agisf_airtable_base_id: ${{ inputs.agisf_airtable_base_id }}
+      agisf_airtable_table_id: ${{ inputs.agisf_airtable_table_id }}
       youtube_api_key: ${{ inputs.youtube_api_key }}
       db_user: ${{ inputs.db_user }}
       db_password: ${{ inputs.db_password }}

align_data/settings.py

@@ -32,6 +32,8 @@
 
 ### Airtable ###
 AIRTABLE_API_KEY = os.environ.get("AIRTABLE_API_KEY")
+AGISF_AIRTABLE_BASE_ID = os.environ.get("AGISF_AIRTABLE_BASE_ID")
+AGISF_AIRTABLE_TABLE_ID = os.environ.get("AGISF_AIRTABLE_TABLE_ID")
 
 ### MYSQL ###
 user = os.environ.get("ARD_DB_USER", "user")

align_data/sources/agisf/__init__.py

@@ -2,12 +2,14 @@
 from align_data.sources.airtable import AirtableDataset
 from align_data.sources.agisf.agisf import AGISFPodcastDataset
 
+from align_data.settings import AGISF_AIRTABLE_BASE_ID, AGISF_AIRTABLE_TABLE_ID
+
 
 datasets = [
     AirtableDataset(
         name="agisf_governance",
-        base_id="app9q0E0jlDWlsR0z",
-        table_id="tblgTb3kszvSbo2Mb",
+        base_id=AGISF_AIRTABLE_BASE_ID,
+        table_id=AGISF_AIRTABLE_TABLE_ID,
         mappings={
             "title": "[>] Resource",
             "url": "[h] [>] Link",

align_data/sources/airtable.py

@@ -21,6 +21,10 @@ class AirtableDataset(AlignmentDataset):
     def setup(self):
         if not AIRTABLE_API_KEY:
             raise ValueError("No AIRTABLE_API_KEY provided!")
+        if not self.base_id:
+            raise ValueError("No airtable base ID provided!")
+        if not self.table_id:
+            raise ValueError("No airtable table ID provided!")
         super().setup()
         self.at = airtable.Airtable(self.base_id, AIRTABLE_API_KEY)