From b7df89a917fe83719cb44ef4581ea6f8ff4f417b Mon Sep 17 00:00:00 2001 From: NewAgeAirbender <34139325+NewAgeAirbender@users.noreply.github.com> Date: Thu, 25 Apr 2024 14:44:54 -0500 Subject: [PATCH 1/4] NY: run 2x day --- tasks/ny.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tasks/ny.yml b/tasks/ny.yml index b9101d9..9671e78 100644 --- a/tasks/ny.yml +++ b/tasks/ny.yml @@ -5,7 +5,8 @@ NY-scrape: environment: scrapers memory: 1024 triggers: - - cron: 0 1,19 * * ? + # - cron: 0 1,19 * * ? + - cron: 0 */12 * * ? timeout_minutes: 600 # regularly 7 hours, 10 to be safe next_tasks: - NY-text From 5e11ebdd71d5bd9fdaf182dc18a95011a65b2e79 Mon Sep 17 00:00:00 2001 From: NewAgeAirbender <34139325+NewAgeAirbender@users.noreply.github.com> Date: Tue, 30 Apr 2024 17:08:13 -0500 Subject: [PATCH 2/4] IL: bump up trigger for runtime --- tasks/il.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/il.yml b/tasks/il.yml index cfe64e4..9c88a2c 100644 --- a/tasks/il.yml +++ b/tasks/il.yml @@ -5,7 +5,7 @@ IL-scrape: environment: scrapers memory: 1024 triggers: - - cron: 0 */8 * * ? + - cron: 0 */14 * * ? # - cron: 10 6 * * ? timeout_minutes: 2160 # regularly 12 hours, 36 hours to be safe :( next_tasks: From c77800012a3bc0f4717e9fec505a84497fff1666 Mon Sep 17 00:00:00 2001 From: NewAgeAirbender <34139325+NewAgeAirbender@users.noreply.github.com> Date: Wed, 1 May 2024 13:53:18 -0500 Subject: [PATCH 3/4] ND: run via airflow instead --- tasks/nd.yml | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100644 tasks/nd.yml diff --git a/tasks/nd.yml b/tasks/nd.yml deleted file mode 100644 index 8a33882..0000000 --- a/tasks/nd.yml +++ /dev/null @@ -1,26 +0,0 @@ -ND-scrape: - image: openstates/scrapers - entrypoint: "poetry run os-update nd bills votes --realtime" - enabled: true - environment: scrapers - triggers: -# - cron: 16 */8 * * ? - - cron: 0 3 * * ? - timeout_minutes: 360 - next_tasks: - - ND-text - -ND-text: - image: openstates/core - entrypoint: "poetry run os-text-extract update nd" - enabled: true - environment: scrapers - -ND-events: - image: openstates/scrapers - entrypoint: "poetry run os-update nd events" - enabled: true - environment: scrapers - triggers: - - cron: 10 */6 * * ? - timeout_minutes: 60 From 5f1ba92aad566bf46ba348dbfe2fd637fc59230d Mon Sep 17 00:00:00 2001 From: NewAgeAirbender <34139325+NewAgeAirbender@users.noreply.github.com> Date: Wed, 8 May 2024 16:42:10 -0500 Subject: [PATCH 4/4] DE: delete duplicate task --- tasks/de.yml | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100644 tasks/de.yml diff --git a/tasks/de.yml b/tasks/de.yml deleted file mode 100644 index 79be751..0000000 --- a/tasks/de.yml +++ /dev/null @@ -1,18 +0,0 @@ -DE-scrape: - image: openstates/scrapers - entrypoint: "poetry run os-update de bills" - enabled: true - environment: scrapers - triggers: - - cron: 0 */2 * * ? -# - cron: 0 5 * * ? - memory: 1024 - timeout_minutes: 2160 # regularly 12 hours, 36 hours to be safe :( - next_tasks: - - DE-text - -DE-text: - image: openstates/core - entrypoint: "poetry run os-text-extract update de" - enabled: true - environment: scrapers