Update .github/workflows/fetch_filter_resources.yaml #4
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Weekly resource fetching and community filtering | ||
| on: | ||
| workflow_dispatch: | ||
| schedule: | ||
| #Every Sunday at 8:00 am | ||
| - cron: "0 8 * * 0" | ||
| # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. | ||
| # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. | ||
| concurrency: | ||
| group: "tools" | ||
| cancel-in-progress: false | ||
| jobs: | ||
| fetch-servers: | ||
| runs-on: ubuntu-20.04 | ||
| name: Merge tools, fetch tutorials and filter the resources for communities | ||
| steps: | ||
| - name: Checkout main | ||
| uses: actions/checkout@v4 | ||
| - uses: actions/setup-python@v5 | ||
| with: | ||
| python-version: ${{ matrix.python-version }} | ||
| - name: Install requirement | ||
| run: | | ||
| python -m pip install -r requirements.txt | ||
| sudo apt-get install jq | ||
| - name: Fetch list of all available servers | ||
| run: | | ||
| python sources/bin/get_public_galaxy_servers.py -o sources/data/available_public_servers.csv | ||
| - name: Archive available servers | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: available-servers | ||
| path: sources/data/available_public_servers.csv | ||
| fetch-tools-stepwise: | ||
| runs-on: ubuntu-20.04 | ||
| name: Fetch tool stepwise | ||
| strategy: | ||
| #max-parallel: 1 #need to run one after another, since otherwise there is a chance, that mulitple jobs want to push to the results branch at the same time (which fails due to merge) | ||
| matrix: | ||
| python-version: [3.11] | ||
| subset: | ||
| - repositories01.list | ||
| - repositories02.list | ||
| - repositories03.list | ||
| - repositories04.list | ||
| steps: | ||
| - name: Checkout main | ||
| uses: actions/checkout@v4 | ||
| - uses: actions/setup-python@v5 | ||
| with: | ||
| python-version: ${{ matrix.python-version }} | ||
| - name: Install requirement | ||
| run: python -m pip install -r requirements.txt | ||
| - name: Download a single artifact | ||
| uses: actions/download-artifact@v4 | ||
| with: | ||
| name: available-servers | ||
| path: sources/data/ | ||
| - name: Fetch all tool stepwise | ||
| run: | | ||
| bash sources/bin/extract_all_tools.sh "${{ matrix.subset }}" | ||
| env: | ||
| GITHUB_API_KEY: ${{ secrets.GH_API_TOKEN }} | ||
| - name: Archive tool sublists production artifacts | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: tools-${{ matrix.subset }} | ||
| path: communities/all/resources/repositories${{ matrix.subset }}.list_tools.tsv | ||
| merge-fetch-filter: | ||
| runs-on: ubuntu-20.04 | ||
| name: Merge tools, fetch tutorials and filter the resources for communities | ||
| steps: | ||
| - name: Checkout main | ||
| uses: actions/checkout@v4 | ||
| - uses: actions/setup-python@v5 | ||
| with: | ||
| python-version: ${{ matrix.python-version }} | ||
| - name: Install requirement | ||
| run: | | ||
| python -m pip install -r requirements.txt | ||
| sudo apt-get install jq | ||
| - name: Download All Artifacts | ||
| uses: actions/download-artifact@v4 | ||
| with: | ||
| path: tools | ||
| pattern: tools-* | ||
| merge-multiple: true | ||
| path: communities/all/resources/ | ||
| - name: Merge all tools | ||
| run: | #merge files with only one header -> https://stackoverflow.com/questions/16890582/unixmerge-multiple-csv-files-with-same-header-by-keeping-the-header-of-the-firs; map(.[]) -> https://stackoverflow.com/questions/42011086/merge-arrays-of-json (get flat array, one tool per entry) | ||
| awk 'FNR==1 && NR!=1{next;}{print}' communities/all/resources/repositories*.list_tools.tsv > communities/all/resources/tools.tsv | ||
| jq -s 'map(.[])' communities/all/resources/repositories*.list_tools.json > communities/all/resources/all_tools.json | ||
| - name: Generate wordcloud and interactive table | ||
| run: | | ||
| bash sources/bin/format_tools.sh | ||
| - name: Fetch all tutorials | ||
| run: | | ||
| bash bin/extract_all_tutorials.sh | ||
| env: | ||
| PLAUSIBLE_API_KEY: ${{ secrets.PLAUSIBLE_API_TOKEN }} | ||
| - name: Filter tutorials for communities | ||
| run: | | ||
| bash bin/get_community_tutorials.sh | ||
| - name: Update tool to keep and exclude for communities | ||
| run: | | ||
| bash bin/update_tools_to_keep_exclude.sh | ||
| - name: Filter tools for communities | ||
| run: | | ||
| bash bin/get_community_tools.sh | ||
| - name: Create Pull Request | ||
| uses: peter-evans/create-pull-request@v4 | ||
| with: | ||
| commit-message: Update resources | ||
| title: Automatic resources update | ||
| body: Automatic resource update done via GitHub Action once a week | ||
| base: main | ||
| branch: resource-update | ||
| delete-branch: true | ||
