Extract data #1291

	name: Extract data

	on:
	push:
	branches:
	- main
	schedule:
	- cron: '0 /12 * *'

	jobs:
	build:
	runs-on: ubuntu-latest
	steps:
	- name: checkout repo content
	uses: actions/checkout@v2

	- name: install git lfs
	run: \|
	curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh \| sudo bash
	sudo apt-get install git-lfs
	git lfs install

	- name: setup python
	uses: actions/setup-python@v2
	with:
	python-version: '3.10'

	- name: install python packages
	run: \|
	python -m pip install --upgrade pip
	pip install .

	- name: track large files
	run: git lfs track "database/pipeline.pkl"

	- name: execute py script # run run.py to get the latest data
	env:
	TWITTER_TOKEN: ${{ secrets.TWITTER_TOKEN }}
	HACKERNEWS_PASSWORD: ${{ secrets.HACKERNEWS_PASSWORD }}
	HACKERNEWS_USERNAME: ${{ secrets.HACKERNEWS_USERNAME }}
	ZOTERO_API_KEY: ${{ secrets.ZOTERO_API_KEY }}
	ZOTERO_LIBRARY_ID: ${{ secrets.ZOTERO_LIBRARY_ID }}
	run: python run.py

	- name: commit files
	run: \|
	git config --local user.email "[email protected]"
	git config --local user.name "GitHub Action"
	git add .gitattributes
	git add -A
	git commit -m "update data" -a

	- name: push changes
	uses: ad-m/[email protected]
	with:
	github_token: ${{ secrets.GITHUB_TOKEN }}
	branch: main

Provide feedback