Fetch Alignment Research Dataset #49

	name: Fetch Alignment Research Dataset

	on:
	workflow_call:
	inputs:
	datasource:
	type: string
	required: true
	coda_token:
	type: string
	required: true
	airtable_api_key:
	type: string
	required: true
	youtube_api_key:
	type: string
	required: true
	db_user:
	type: string
	required: true
	db_password:
	type: string
	required: true
	db_host:
	type: string
	required: true
	workflow_dispatch: # allow manual triggering
	inputs:
	datasource:
	description: 'The datasource to process'
	type: choice
	options:
	- agentmodels
	- agisf
	- aisafety.info
	- alignment_newsletter
	- alignmentforum
	- arbital
	- arxiv
	- blogs
	- distill
	- eaforum
	- indices
	- lesswrong
	- special_docs
	- youtube

	jobs:
	build-dataset:
	runs-on: ubuntu-latest

	steps:
	- name: Checkout repository
	uses: actions/checkout@v2

	- name: Setup Python environment
	uses: actions/setup-python@v2
	with:
	python-version: '3.x'

	- name: Install Pandoc
	run: \|
	if [ "${{ inputs.datasource }}" = "gdocs" ]; then
	sudo apt-get update
	sudo apt-get -y install pandoc
	fi

	- name: Install dependencies
	run: pip install -r requirements.txt

	- name: Process dataset
	env:
	CODA_TOKEN: ${{ secrets.CODA_TOKEN \|\| inputs.coda_token }}
	AIRTABLE_API_KEY: ${{ secrets.AIRTABLE_API_KEY \|\| inputs.airtable_api_key }}
	YOUTUBE_API_KEY: ${{ secrets.YOUTUBE_API_KEY \|\| inputs.youtube_api_key }}
	ARD_DB_USER: ${{ secrets.ARD_DB_USER \|\| inputs.db_user }}
	ARD_DB_PASSWORD: ${{ secrets.ARD_DB_PASSWORD \|\| inputs.db_password }}
	ARD_DB_HOST: ${{ secrets.ARD_DB_HOST \|\| inputs.db_host }}
	ARD_DB_NAME: alignment_research_dataset
	run: python main.py fetch ${{ inputs.datasource }}

Provide feedback