Update contents #20495
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Update contents | |
| on: | |
| schedule: | |
| - cron: '0 * * * *' | |
| - cron: '5,15,30,45 11,12,13 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| from-timestamp: | |
| type: string | |
| description: Pull data from this timestamp and onwards - requires both from and end timestamps | |
| required: false | |
| default: '' | |
| end-timestamp: | |
| type: string | |
| description: Pull data until reaching this timestamp - requires both from and end timestamps | |
| required: false | |
| default: '' | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - .github/workflows/pull-data.yaml | |
| - src/data-*.sh | |
| - src/*-data-pull.sh | |
| - configuration/zones.json | |
| concurrency: commits-pages | |
| env: | |
| publish-branch: gh-pages | |
| jobs: | |
| update-energy-price-data: | |
| runs-on: ubuntu-latest | |
| name: Update energy price data | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/checkout@v4 | |
| with: | |
| ssh-key: ${{ secrets.POWERCASTDEPLOY }} | |
| ref: ${{ env.publish-branch }} | |
| path: github-pages | |
| - shell: bash | |
| name: Setup configuration | |
| run: | | |
| mkdir -p ./github-pages/api/energy-price/ > /dev/null | |
| sh src/data-matrix.sh configuration/zones.json energy-price ./github-pages/api/energy-price/ ${{ inputs.from-timestamp }} ${{ inputs.end-timestamp }} | tee configuration.json | |
| echo "delayed=false" >> $GITHUB_ENV | |
| echo "stale=false" >> $GITHUB_ENV | |
| echo "changed=false" >> $GITHUB_ENV | |
| echo "customized=false" >> $GITHUB_ENV | |
| if [ ! -z "${{ inputs.from-timestamp }}" ]; then | |
| echo "customized=true" >> $GITHUB_ENV | |
| fi | |
| - shell: bash | |
| name: Check for stale data | |
| run: | | |
| NOW=$(date +%s) | |
| jq -rc '.[]' configuration.json | while read CONFIG; do | |
| ZONE=$(echo "$CONFIG" | jq -r '.zone') | |
| FROM=$(echo "$CONFIG" | jq -r '.latest') | |
| HOURS=$(((FROM-NOW)/3600)) | |
| echo "Checking stale data for ${ZONE}:" | |
| echo " - ${FROM} - current data" | |
| echo " - ${NOW} - now" | |
| echo " - ${HOURS} - hours" | |
| if [ "$ZONE" = "DE" ]; then | |
| echo "Ignoring..." | |
| continue | |
| fi | |
| if [ $HOURS -le 12 ]; then | |
| echo "stale=true" >> $GITHUB_ENV | |
| fi | |
| if [ $HOURS -lt 8 ] && [ "${{ env.customized}}" = "false" ]; then | |
| echo "delayed=true" >> $GITHUB_ENV | |
| sh src/incident-open.sh $ZONE $NOW delay ./github-pages/api/incidents/energy-price/ | |
| fi | |
| done | |
| - shell: bash | |
| name: Pull data | |
| if: env.stale == 'true' | |
| run: | | |
| jq -rc '.[]' configuration.json | while read CONFIG; do | |
| ZONE=$(echo "$CONFIG" | jq -r '.zone') | |
| FROM=$(echo "$CONFIG" | jq -r '.latest') | |
| END=$(echo "$CONFIG" | jq -r '.end') | |
| echo "Pull data for ${ZONE} until ${END}..." | |
| sh src/energy-price-data-pull.sh $ZONE $FROM $END > "${ZONE}.generated" | |
| echo "changed=true" >> $GITHUB_ENV | |
| done | |
| - shell: bash | |
| name: Write data | |
| if: env.changed == 'true' | |
| run: | | |
| for GENERATED in *.generated; do | |
| [ -f "$GENERATED" ] || continue | |
| ZONE=$(basename $GENERATED .generated) | |
| echo "Using generated data for ${ZONE}..." | |
| sh src/data-write.sh $GENERATED ./github-pages/api/energy-price/ $ZONE | |
| done | |
| - shell: bash | |
| name: Index data | |
| if: env.changed == 'true' | |
| run: | | |
| rm ./github-pages/api/energy-price/latest.json 2> /dev/null || true | |
| sh src/data-index.sh configuration/zones.json ./github-pages/api/energy-price/ /api/energy-price | tee ./github-pages/api/energy-price/index.json | |
| - shell: bash | |
| name: Handle open indicents | |
| if: env.changed == 'true' && env.delayed == 'true' | |
| run: | | |
| NOW=$(date +%s) | |
| sh src/data-matrix.sh configuration/zones.json energy-price ./github-pages/api/energy-price/ ${{ inputs.from-timestamp }} ${{ inputs.end-timestamp }} | tee configuration.json | |
| jq -rc '.[]' configuration.json | while read CONFIG; do | |
| ZONE=$(echo "$CONFIG" | jq -r '.zone') | |
| FROM=$(echo "$CONFIG" | jq -r '.latest') | |
| HOURS=$(((FROM-NOW)/3600)) | |
| echo "Checking stale data for ${ZONE}:" | |
| echo " - ${FROM} - current data" | |
| echo " - ${NOW} - now" | |
| echo " - ${HOURS} - hours" | |
| if [ "$ZONE" = "DE" ]; then | |
| echo "Ignoring..." | |
| continue | |
| fi | |
| if [ $HOURS -ge 12 ] && [ "${{ env.customized}}" = "false" ]; then | |
| sh src/incident-close.sh $ZONE $NOW delay ./github-pages/api/incidents/energy-price/ | |
| fi | |
| done | |
| - shell: bash | |
| name: Create patch | |
| run: | | |
| cd github-pages | |
| git add api/energy-price/ | |
| git diff --cached api/energy-price/ | tee ../energy-price.patch | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: energy-price | |
| path: energy-price.patch | |
| update-renewables-data: | |
| runs-on: ubuntu-latest | |
| name: Update renewables data | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/checkout@v4 | |
| with: | |
| ssh-key: ${{ secrets.POWERCASTDEPLOY }} | |
| ref: ${{ env.publish-branch }} | |
| path: github-pages | |
| - shell: bash | |
| name: Setup configuration | |
| run: | | |
| mkdir -p ./github-pages/api/renewables/ > /dev/null | |
| sh src/data-matrix.sh configuration/zones.json renewables-data ./github-pages/api/renewables/ ${{ inputs.from-timestamp }} ${{ inputs.end-timestamp }} | tee configuration.json | |
| echo "delayed=false" >> $GITHUB_ENV | |
| echo "stale=false" >> $GITHUB_ENV | |
| echo "changed=false" >> $GITHUB_ENV | |
| echo "customized=false" >> $GITHUB_ENV | |
| if [ ! -z "${{ inputs.from-timestamp }}" ]; then | |
| echo "customized=true" >> $GITHUB_ENV | |
| fi | |
| - shell: bash | |
| name: Check for stale data | |
| run: | | |
| jq -rc '.[]' configuration.json | while read CONFIG; do | |
| ZONE=$(echo "$CONFIG" | jq -r '.zone') | |
| FROM=$(echo "$CONFIG" | jq -r '.latest') | |
| NOW=$(date +%s) | |
| HOURS=$(((FROM-NOW)/3600)) | |
| echo "Checking stale data for ${ZONE}:" | |
| echo " - ${FROM} - current data" | |
| echo " - ${NOW} - now" | |
| echo " - ${HOURS} - hours" | |
| if [ $HOURS -le 12 ]; then | |
| echo "stale=true" >> $GITHUB_ENV | |
| fi | |
| if [ $HOURS -lt 4 ] && [ "${{ env.customized}}" = "false" ]; then | |
| echo "delayed=true" >> $GITHUB_ENV | |
| sh src/incident-open.sh $ZONE $NOW delay ./github-pages/api/incidents/renewables/ | |
| fi | |
| done | |
| - shell: bash | |
| name: Pull data | |
| if: env.stale == 'true' | |
| run: | | |
| jq -rc '.[]' configuration.json | while read CONFIG; do | |
| ZONE=$(echo "$CONFIG" | jq -r '.zone') | |
| FROM=$(echo "$CONFIG" | jq -r '.latest') | |
| END=$(echo "$CONFIG" | jq -r '.end') | |
| echo "Pull data for ${ZONE} until ${END}..." | |
| sh src/renewables-data-pull.sh $ZONE $FROM $END > "${ZONE}.generated" | |
| echo "changed=true" >> $GITHUB_ENV | |
| done | |
| - shell: bash | |
| name: Write data | |
| if: env.changed == 'true' | |
| run: | | |
| for GENERATED in *.generated; do | |
| [ -f "$GENERATED" ] || continue | |
| ZONE=$(basename $GENERATED .generated) | |
| echo "Using generated data for ${ZONE}..." | |
| sh src/data-write.sh $GENERATED ./github-pages/api/renewables/ $ZONE | |
| done | |
| - shell: bash | |
| name: Index data | |
| if: env.changed == 'true' | |
| run: | | |
| rm ./github-pages/api/renewables/latest.json 2> /dev/null || true | |
| sh src/data-index.sh configuration/zones.json ./github-pages/api/renewables/ /api/renewables | tee ./github-pages/api/renewables/index.json | |
| - shell: bash | |
| name: Handle open indicents | |
| if: env.changed == 'true' && env.delayed == 'true' | |
| run: | | |
| NOW=$(date +%s) | |
| sh src/data-matrix.sh configuration/zones.json renewables-data ./github-pages/api/renewables/ ${{ inputs.from-timestamp }} ${{ inputs.end-timestamp }} | tee configuration.json | |
| jq -rc '.[]' configuration.json | while read CONFIG; do | |
| ZONE=$(echo "$CONFIG" | jq -r '.zone') | |
| FROM=$(echo "$CONFIG" | jq -r '.latest') | |
| HOURS=$(((FROM-NOW)/3600)) | |
| echo "Checking stale data for ${ZONE}:" | |
| echo " - ${FROM} - current data" | |
| echo " - ${NOW} - now" | |
| echo " - ${HOURS} - hours" | |
| if [ "$ZONE" = "DE" ]; then | |
| echo "Ignoring..." | |
| continue | |
| fi | |
| if [ $HOURS -ge 12 ] && [ "${{ env.customized}}" = "false" ]; then | |
| sh src/incident-close.sh $ZONE $NOW delay ./github-pages/api/incidents/renewables/ | |
| fi | |
| done | |
| - shell: bash | |
| name: Create patch | |
| run: | | |
| cd github-pages | |
| git add api/renewables/ | |
| git diff --cached api/renewables/ | tee ../renewables.patch | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: renewables | |
| path: renewables.patch | |
| update-co2-emission-data: | |
| runs-on: ubuntu-latest | |
| name: Update CO2 emission data | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/checkout@v4 | |
| with: | |
| ssh-key: ${{ secrets.POWERCASTDEPLOY }} | |
| ref: ${{ env.publish-branch }} | |
| path: github-pages | |
| - shell: bash | |
| name: Setup configuration | |
| run: | | |
| mkdir -p ./github-pages/api/emission/co2/ > /dev/null | |
| sh src/data-matrix.sh configuration/zones.json co2-emission ./github-pages/api/emission/co2/ ${{ inputs.from-timestamp }} ${{ inputs.end-timestamp }} | tee configuration.json | |
| echo "delayed=false" >> $GITHUB_ENV | |
| echo "stale=false" >> $GITHUB_ENV | |
| echo "changed=false" >> $GITHUB_ENV | |
| echo "customized=false" >> $GITHUB_ENV | |
| if [ ! -z "${{ inputs.from-timestamp }}" ]; then | |
| echo "customized=true" >> $GITHUB_ENV | |
| fi | |
| - shell: bash | |
| name: Check for stale data | |
| run: | | |
| jq -rc '.[]' configuration.json | while read CONFIG; do | |
| ZONE=$(echo "$CONFIG" | jq -r '.zone') | |
| FROM=$(echo "$CONFIG" | jq -r '.latest') | |
| NOW=$(date +%s) | |
| HOURS=$(((FROM-NOW)/3600)) | |
| echo "Checking stale data for ${ZONE}:" | |
| echo " - ${FROM} - current data" | |
| echo " - ${NOW} - now" | |
| echo " - ${HOURS} - hours" | |
| if [ $HOURS -le 12 ]; then | |
| echo "stale=true" >> $GITHUB_ENV | |
| fi | |
| if [ $HOURS -lt 8 ] && [ "${{ env.customized}}" = "false" ]; then | |
| echo "delayed=true" >> $GITHUB_ENV | |
| sh src/incident-open.sh $ZONE $NOW delay ./github-pages/api/incidents/emission/co2/ | |
| fi | |
| done | |
| - shell: bash | |
| name: Pull data | |
| if: env.stale == 'true' | |
| run: | | |
| jq -rc '.[]' configuration.json | while read CONFIG; do | |
| ZONE=$(echo "$CONFIG" | jq -r '.zone') | |
| FROM=$(echo "$CONFIG" | jq -r '.latest') | |
| END=$(echo "$CONFIG" | jq -r '.end') | |
| echo "Pull data for ${ZONE} until ${END}..." | |
| sh src/co2-emission-data-pull.sh $ZONE $FROM $END > "${ZONE}.generated" | |
| echo "changed=true" >> $GITHUB_ENV | |
| done | |
| - shell: bash | |
| name: Write data | |
| if: env.changed == 'true' | |
| run: | | |
| for GENERATED in *.generated; do | |
| [ -f "$GENERATED" ] || continue | |
| ZONE=$(basename $GENERATED .generated) | |
| echo "Using generated data for ${ZONE}..." | |
| sh src/data-write.sh $GENERATED ./github-pages/api/emission/co2/ $ZONE | |
| done | |
| - shell: bash | |
| name: Index data | |
| if: env.changed == 'true' | |
| run: | | |
| rm ./github-pages/api/emission/co2/latest.json 2> /dev/null || true | |
| sh src/data-index.sh configuration/zones.json ./github-pages/api/emission/co2/ /api/emission/co2 | tee ./github-pages/api/emission/co2/index.json | |
| - shell: bash | |
| name: Handle open indicents | |
| if: env.changed == 'true' && env.delayed == 'true' | |
| run: | | |
| NOW=$(date +%s) | |
| sh src/data-matrix.sh configuration/zones.json co2-emission ./github-pages/api/emission/co2/ ${{ inputs.from-timestamp }} ${{ inputs.end-timestamp }} | tee configuration.json | |
| jq -rc '.[]' configuration.json | while read CONFIG; do | |
| ZONE=$(echo "$CONFIG" | jq -r '.zone') | |
| FROM=$(echo "$CONFIG" | jq -r '.latest') | |
| HOURS=$(((FROM-NOW)/3600)) | |
| echo "Checking stale data for ${ZONE}:" | |
| echo " - ${FROM} - current data" | |
| echo " - ${NOW} - now" | |
| echo " - ${HOURS} - hours" | |
| if [ "$ZONE" = "DE" ]; then | |
| echo "Ignoring..." | |
| continue | |
| fi | |
| if [ $HOURS -ge 12 ] && [ "${{ env.customized}}" = "false" ]; then | |
| sh src/incident-close.sh $ZONE $NOW delay ./github-pages/api/incidents/emission/co2/ | |
| fi | |
| done | |
| - shell: bash | |
| name: Create patch | |
| run: | | |
| cd github-pages | |
| git add api/emission/co2/ | |
| git diff --cached api/emission/co2/ | tee ../emission-co2.patch | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: emission-co2 | |
| path: emission-co2.patch | |
| apply-updates: | |
| runs-on: ubuntu-latest | |
| name: Apply updates | |
| needs: | |
| - update-energy-price-data | |
| - update-renewables-data | |
| - update-co2-emission-data | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| merge-multiple: true | |
| - shell: bash | |
| name: List patches | |
| run: ls -lsh *.patch | |
| - uses: actions/checkout@v4 | |
| with: | |
| ssh-key: ${{ secrets.POWERCASTDEPLOY }} | |
| ref: ${{ env.publish-branch }} | |
| path: github-pages | |
| - shell: bash | |
| name: Apply patches | |
| run: | | |
| cd github-pages | |
| git apply --allow-empty ../*.patch | |
| - uses: EndBug/add-and-commit@v9 | |
| name: Publish changes | |
| if: github.ref == 'refs/heads/main' | |
| with: | |
| cwd: github-pages | |
| push: --set-upstream origin ${{ env.publish-branch }} | |
| default_author: github_actions | |
| message: Publish via workflow |