Reorganize GitHub action workflows

- Merge weekly workflows into 1 workflow that wil open a PR with the changes
- Update other workflows to the new structure

.github/workflows/fetch_filter_resources.yaml

@@ -0,0 +1,75 @@
+name: Weekly resource fetching and community filtering
+
+on:
+  workflow_dispatch:
+  schedule:
+    #Every Sunday at 8:00 am
+    - cron: "0 8 * * 0" 
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "tools"
+  cancel-in-progress: false
+
+jobs:
+  fetch-filter:
+    runs-on: ubuntu-20.04
+    name: Fetch tool stepwise and merge, fetch tutorials and filter the resources for communities
+    strategy:
+      max-parallel: 1 #need to run one after another, since otherwise there is a chance, that mulitple jobs want to push to the results branch at the same time (which fails due to merge)
+      matrix:
+        python-version: [3.11]
+        subset:
+         - repositories01.list
+         - repositories02.list
+         - repositories03.list
+         - repositories04.list
+    steps:
+      - name: Checkout main
+        uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install requirement
+        run: |
+          python -m pip install -r requirements.txt
+          sudo apt-get install jq
+      - name: Fetch list of all available servers
+          run: | 
+            python sources/bin/get_public_galaxy_servers.py -o sources/data/available_public_servers.csv
+      - name: Fetch all tool stepwise
+        run: | 
+          bash sources/bin/extract_all_tools.sh "${{ matrix.subset }}"
+        env: 
+          GITHUB_API_KEY: ${{ secrets.GH_API_TOKEN }}
+      - name: Merge all tools
+        run: | #merge files with only one header -> https://stackoverflow.com/questions/16890582/unixmerge-multiple-csv-files-with-same-header-by-keeping-the-header-of-the-firs; map(.[]) -> https://stackoverflow.com/questions/42011086/merge-arrays-of-json (get flat array, one tool per entry)
+          awk 'FNR==1 && NR!=1{next;}{print}' communities/all/resources/repositories*.list_tools.tsv > communities/all/resources/tools.tsv
+          jq -s 'map(.[])' communities/all/resources/repositories*.list_tools.json > communities/all/resources/all_tools.json 
+      - name: Generate wordcloud and interactive table
+        run: |
+          bash sources/bin/format_tools.sh
+      - name: Fetch all tutorials
+        run: | 
+          bash bin/extract_all_tutorials.sh
+        env: 
+          PLAUSIBLE_API_KEY: ${{ secrets.PLAUSIBLE_API_TOKEN }}
+      - name: Filter tutorials for communities
+        run: | 
+          bash bin/get_community_tutorials.sh
+      - name: Update tool to keep and exclude for communities
+        run: | 
+          bash bin/update_tools_to_keep_exclude.sh
+      - name: Filter tools for communities
+        run: | 
+          bash bin/get_community_tools.sh
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v4
+        with:
+          commit-message: Update resources
+          title: Automatic resources update
+          body: Automatic resource update done via GitHub Action once a week
+          base: main
+          branch: resource-update
+          delete-branch: true

.github/workflows/filter_communities.yaml

@@ -1,18 +1,11 @@
-name: Filter community tools
+name: Filter community resources
 
 on:
   workflow_dispatch:
-
-  # the workflow it triggered when all tools are fetched
-  workflow_run:
-      workflows: ["Fetch all tools"]
-      types:
-        - completed
-
   # the workflow it also triggered when the community definitions are changed
   push:
     paths:
-    - 'data/communities**'
+    - 'data/communities/*/metadata/*'
     branches: ["main"]
 
 # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
@@ -21,104 +14,33 @@ concurrency:
   group: "filter"
   cancel-in-progress: false
 
-permissions:
-  contents: write
-
 jobs:
-  filter-all-tutorials:
-    name: Filter tutorials
+  filter-resources:
+    name: Filter resources for communities
     runs-on: ubuntu-20.04
     steps:
     - name: Checkout main
       uses: actions/checkout@v4
-      with:
-        ref: main #pull latest code produced by job 1, not the revision that started the workflow (https://github.com/actions/checkout/issues/439)
-    - name: Checkout results
-      uses: actions/checkout@v4
-      with:
-        ref: results
-        path: results
     - uses: actions/setup-python@v5
       with:
         python-version: '3.11' 
     - name: Install requirement
       run: python -m pip install -r requirements.txt
-    - name: Run script
+    - name: Filter tutorials for communities
       run: | 
         bash bin/get_community_tutorials.sh
-    - name: Commit results
-      # commit the new filtered data, only if stuff was changed
-      uses: s0/git-publish-subdir-action@develop
-      env:
-        BRANCH: results
-        FOLDER: results
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        REPO: self
-        SKIP_EMPTY_COMMITS: true
-        MESSAGE: "Update results step: filter tutorials - commmit {sha}"
-
-  update-tools-to-keep-exclude:
-    name: Update tool to keep and exclude
-    runs-on: ubuntu-20.04
-    steps:
-    - name: Checkout main
-      uses: actions/checkout@v4
-      with:
-        ref: main #pull latest code produced by job 1, not the revision that started the workflow (https://github.com/actions/checkout/issues/439)
-    - name: Checkout results
-      uses: actions/checkout@v4
-      with:
-        ref: results
-        path: results
-    - uses: actions/setup-python@v5
-      with:
-        python-version: '3.11' 
-    - name: Install requirement
-      run: python -m pip install -r requirements.txt
-    - name: Run script
+    - name: Update tool to keep and exclude for communities
       run: | 
         bash bin/update_tools_to_keep_exclude.sh
-    - name: Commit results
-      # commit the new filtered data, only if stuff was changed
-      uses: s0/git-publish-subdir-action@develop
-      env:
-        BRANCH: results
-        FOLDER: results
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        REPO: self
-        SKIP_EMPTY_COMMITS: true
-        MESSAGE: "Update results step: update tool to keep and exclude - commmit {sha}"
-
-  filter-all-tools:
-    name: Filter all tools
-    runs-on: ubuntu-20.04
-    steps:
-    - name: Checkout main
-      uses: actions/checkout@v4
-      with:
-        ref: main #pull latest code produced by job 1, not the revision that started the workflow (https://github.com/actions/checkout/issues/439)
-    - name: Checkout results
-      uses: actions/checkout@v4
-      with:
-        ref: results
-        path: results
-    - uses: actions/setup-python@v5
-      with:
-        python-version: '3.11' 
-    - name: Install requirement
-      run: python -m pip install -r requirements.txt
-    - name: Run script
+    - name: Filter tools for communities
       run: | 
         bash bin/get_community_tools.sh
-    - name: Commit results
-      # commit the new filtered data, only if stuff was changed
-      uses: s0/git-publish-subdir-action@develop
-      env:
-        BRANCH: results
-        FOLDER: results
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        REPO: self
-        SKIP_EMPTY_COMMITS: true
-        MESSAGE: "Update results step: filter all tools - commmit {sha}"
-
-
+    - name: Create Pull Request
+      uses: peter-evans/create-pull-request@v4
+      with:
+        commit-message: Update resources
+        title: Resource filtering
+        body: Resource filering done via GitHub Action when the community definitions are changed 
+        base: main
+        branch: resource-filtering
+        delete-branch: true