Skip to content
Process Module Metadata

Rename pipe section header to make it easier to link to in URLs (#948) #294

Sign in to view logs

Rename pipe section header to make it easier to link to in URLs (#948)

Rename pipe section header to make it easier to link to in URLs (#948) #294

Workflow file for this run

.github/workflows/pull_metadata.yml at 86512ac
# This is a workflow to pull all of the metadata from all of the modules into a single file
# each time something is merged to main, this workflow will run and rebuild the metadata file.
name: Process Module Metadata
# Controls when the workflow will run
on:
# Triggers the workflow on push or pull request events but only for the "main" branch
push:
branches: "main" #CHANGE TO MAIN BEFORE MERGING
paths-ignore:
- 'assets/**'
- '_for_authors/**'
- '_module_templates/**'
# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
run_script:
name: Pull module metadata
runs-on: ubuntu-latest
# Map a step output to a job output
# outputs:
# changes: ${{ steps.check_if_changed.outputs.changes }}
steps:
- uses: actions/checkout@v4
# Process all the metadata using the script for that
- name: Process Metadata
run: |
bash .github/scripts/process_metadata.sh
- name: Upload json file as an artifact
uses: actions/upload-artifact@v4
with:
name: json_artifact
path: module_data.json
check_for_changes:
name: Has metadata changed?
runs-on: ubuntu-latest
needs: run_script
# Map a step output to a job output
outputs:
changes: ${{ steps.check_if_changed.outputs.changes }}
steps:
- name: Download artifact
uses: actions/download-artifact@v4
with:
name: json_artifact
# Check if processing the data has changed the python file at all
- name: Check if anything changed
id: check_if_changed
run: |
wget https://raw.githubusercontent.com/arcus/education_modules/metadata_workflow/assets/metadata/module_data.json -O current_data_file.txt
wc current_data_file.txt
changes=$(diff module_data.json current_data_file.txt | wc -l)
echo there are $changes changed lines
changes=$([ $changes -gt 0 ] && echo "true" || echo "false")
echo the changes variable is now $changes
echo "changes=$changes" >> "$GITHUB_OUTPUT"
# This message is for me to check that things ran correctly even if nothing actually updated
nothing_to_update_message:
name: No - nothing to update
runs-on: ubuntu-latest
needs: check_for_changes
if: needs.check_for_changes.outputs.changes != 'true'
steps:
- env:
OUTPUT3: ${{needs.check_for_changes.outputs.changes}}
name: Nothing was committed
run: echo "Nothing was committed because the any_change variable was $OUTPUT3"
# This message is for me to check that things ran correctly even if nothing actually updated
metadata_changed_message:
name: Yes - updates needed
runs-on: ubuntu-latest
needs: check_for_changes
if: needs.check_for_changes.outputs.changes == 'true'
steps:
- env:
OUTPUT3: ${{needs.check_for_changes.outputs.changes}}
name: Metadata has changed
run: echo "The any_change variable was $OUTPUT3 so now the changes need to be propagated"
# If changes were made to the json file, push those changes to the metadata_workflow branch
update_metadata_workflow_branch:
name: Module repo - Push to branch metadata_workflow
runs-on: ubuntu-latest
needs: metadata_changed_message
steps:
- env:
OUTPUT3: ${{needs.run_script.outputs.changes}}
run: echo "The changes variable is $OUTPUT3"
- uses: actions/checkout@v4
- name: Download json artifact
uses: actions/download-artifact@v4
with:
name: json_artifact
- name: Check that artifacts downloaded
run: |
wc module_data.json
- name: Commit newly updated files
run: |
git config --local user.name actions-user
git config --local user.email "[email protected]"
git fetch
git checkout metadata_workflow
mv -f module_data.json assets/metadata/
git add assets/metadata/module_data.json
git commit -am "update metadata records"
git push origin metadata_workflow
# The module_discovery repository automatically checks weekly to see if the module data has changed, but a message here informs users how to manually run that workflow.
update_module_discovery_repository:
name: App - Sun. cron job or manual update
runs-on: ubuntu-latest
needs: metadata_changed_message
steps:
- env:
OUTPUT3: ${{needs.run_script.outputs.changes}}
run: echo "The changes variable is $OUTPUT3"
- name: Print module_discovery instructions
run: |
echo "The module_discovery repository will automatically update on Sunday morning. If you have changed metadata that needs to be updated there immediately, please manually activate that workflow at https://github.com/arcus/module_discovery/actions/workflows/update_module_data.yml"
# If the metadata changed, generate the list of modules for arcus.github.io/education_modules
generate_webpage_markdown:
name: Website - Generate page
runs-on: ubuntu-latest
needs: metadata_changed_message
steps:
- uses: actions/checkout@v4
- name: Build webpage markdown file
run: |
bash .github/scripts/public_list_of_modules.sh
- name: Upload webpage file as an artifact
uses: actions/upload-artifact@v4
with:
name: webpage_artifact
path: new_list_of_modules.md
# It is possible for the metadata to change without any of the things displayed on the public facing website actually needing to be updated.
check_list_of_modules_for_changes:
name: Website - Did it change?
runs-on: ubuntu-latest
needs: generate_webpage_markdown
# Map a step output to a job output
outputs:
changes: ${{ steps.check_if_website_changed.outputs.changes }}
steps:
- name: Download artifact
uses: actions/download-artifact@v4
with:
name: webpage_artifact
# Check if processing the data has changed the webpage file at all
- name: Check if anything changed
id: check_if_website_changed
run: |
wget https://raw.githubusercontent.com/arcus/education_modules/gh-pages/docs/list_of_modules.md -O current_webpage.txt
wc current_webpage.txt
changes=$(diff new_list_of_modules.md current_webpage.txt | wc -l)
echo there are $changes changed lines
changes=$([ $changes -gt 0 ] && echo "true" || echo "false")
echo the changes variable is now $changes
echo "changes=$changes" >> "$GITHUB_OUTPUT"
# Maybe the public facing website doesn't need updates
no_update_to_gh-pages:
name: Website - No changes
runs-on: ubuntu-latest
needs: check_list_of_modules_for_changes
if: needs.check_list_of_modules_for_changes.outputs.changes != 'true'
steps:
- env:
OUTPUT3: ${{needs.check_list_of_modules_for_changes.outputs.changes}}
name: Nothing was committed
run: echo "Nothing was committed because the any_change variable was $OUTPUT3"
# The public facing website does need updates.
update_gh-pages:
name: Update website
runs-on: ubuntu-latest
needs: check_list_of_modules_for_changes
if: needs.check_list_of_modules_for_changes.outputs.changes == 'true'
steps:
- uses: actions/checkout@v4
- name: Download webpage artifact
uses: actions/download-artifact@v4
with:
name: webpage_artifact
- name: Check that artifacts downloaded
run: |
wc new_list_of_modules.md
- name: Commit newly updated file
run: |
git config --local user.name actions-user
git config --local user.email "[email protected]"
git fetch
git checkout gh-pages
mv -f new_list_of_modules.md docs/list_of_modules.md
git add docs/list_of_modules.md
git commit -am "update webpage records"
git push origin gh-pages