diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..8ad96e54 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,6 @@ +# Each line is a file pattern followed by one or more owners. +# These owners will be the default owners for everything in +# the repo. Unless a later match takes precedence, +# The users listed below are global owners and will be +# requested for review when someone opens a pull request. +* @marijaselakovic @hammerhead diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..caf8b2e8 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,17 @@ +version: 2 + +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "monthly" + + - package-ecosystem: "docker" + directory: "/" + schedule: + interval: "monthly" + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..bc4f7c8c --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.venv* +*.egg-info +.eggs +__pycache__ diff --git a/README.md b/README.md new file mode 100644 index 00000000..6a461adc --- /dev/null +++ b/README.md @@ -0,0 +1,23 @@ +# crate-airflow-tutorial +Orchestration Project - Astronomer/Airflow tutorials + + +## About + +This repository contains a few examples of Apache Airflow DAGs for automating recurrent +queries. All DAGs run on Astronomer infrastructure installed on Ubuntu 20.04.3 LTS. + +In the [dags](dags) directory you can find a subset of our DAG examples. +Each DAG is accompanied by a tutorial: + +* [data_retention_delete_dag.py](dags/data_retention_delete_dag.py) ([tutorial](https://community.crate.io/t/cratedb-and-apache-airflow-implementation-of-data-retention-policy/913)): implements a retention policy algorithm that drops expired partitions +* [data_retention_reallocate_dag.py](dags/data_retention_reallocate_dag.py) ([tutorial](https://community.crate.io/t/cratedb-and-apache-airflow-building-a-hot-cold-storage-data-retention-policy/934)): implements a retention policy algorithm that reallocates expired partitions from hot nodes to cold nodes +* [data_retention_snapshot_dag.py](dags/data_retention_snapshot_dag.py): implements a retention policy algorithm that snapshots expired partitions to a repository + + +## Testing + +```shell +pip install --editable=.[develop,test] +poe check +```