Add scripts to extract workflows from WorkflowHub and UseGalaxy serve…

…rs (#158)

* Add workflow extraction

* Add functional tests for workflows

* Add mkdir for community result folder

* Fix linting

* Fix microgalaxy wf tags

* Try to fix CI linting

* Remove linting for Python 3.8

* Add microgalaxy wf tag and fix scripts

* Fix linting on Python 3.8 - @nsoranzo commit

* Remove Python 3.8 from the tests

* Fix GitHub workflow

- Add matrix strategy for test tutorial and test workflows
- Replace pull_request_target by pull_request

* Add tests

.github/workflows/fetch_filter_resources.yaml

@@ -101,6 +101,12 @@ jobs:
       - name: Generate wordcloud and interactive table
         run: |
           bash sources/bin/format_tools.sh
+      - name: Fetch all workflows
+        run: | 
+          bash sources/bin/extract_all_workflows.sh 
+      - name: Filter workflows for communities
+        run: | 
+          bash sources/bin/get_community_workflows.sh
       - name: Fetch all tutorials
         run: | 
           bash sources/bin/extract_all_tutorials.sh

.github/workflows/lint.yaml

@@ -6,14 +6,11 @@ concurrency:
 jobs:
   lint:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ['3.8', '3.11']
     steps:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5
       with:
-        python-version: ${{ matrix.python-version }}
+          python-version: '3.11'
     - name: Install tox
       run: python -m pip install 'tox>=1.8.0'
     - name: Lint

.github/workflows/run_tests.yaml

@@ -1,16 +1,19 @@
 name: Run tests
-on: [push, pull_request_target]
+on: [push, pull_request]
 
 jobs:
   test-tools:
     runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        python-version: ['3.11']
     environment: fetch-tools
     steps:
-      - name: Checkout main
+      - name: Checkout
         uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.11'
+          python-version: ${{ matrix.python-version }}
       - name: Install requirement
         run: python -m pip install -r requirements.txt
       - name: Tool extraction
@@ -26,12 +29,15 @@ jobs:
           bash sources/bin/format_tools.sh
   test-tutorials:
     runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        python-version: ['3.11']
     steps:
-      - name: Checkout main
+      - name: Checkout
         uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.11'
+          python-version: ${{ matrix.python-version }}
       - name: Install requirement
         run: python -m pip install -r requirements.txt
       - name: Tutorial extraction
@@ -41,4 +47,23 @@ jobs:
           PLAUSIBLE_API_KEY: ${{ secrets.PLAUSIBLE_API_TOKEN }}
       - name: Tutorial filtering
         run: | 
-          bash sources/bin/get_community_tutorials.sh test
+          bash sources/bin/get_community_tutorials.sh test
+  test-workflows:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        python-version: ['3.11']
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install requirement
+        run: python -m pip install -r requirements.txt
+      - name: Workflow extraction
+        run: | 
+          bash sources/bin/extract_all_workflows.sh test
+      - name: Workflow filtering
+        run: | 
+          bash sources/bin/get_community_workflows.sh test

README.md

@@ -6,4 +6,4 @@ Galaxy Communities Dock aka Galaxy Codex is a catalog of Galaxy resources (tools
 This repository stores the sources to build this catalog. The catalog is automatically updated every week.
 
 
-Any Galaxy community can be added to this project and benefit from the dedicated resources, including interactive tables that can be embedded into subdomains and website via an iframe. **Learn [how to add your community](https://training.galaxyproject.org/training-material//topics/dev/tutorials/community-tool-table/tutorial.html) in the dedicated GTN toturial**. 
+Any Galaxy community can be added to this project and benefit from the dedicated resources, including interactive tables that can be embedded into subdomains and website via an iframe. **Learn [how to add your community](https://training.galaxyproject.org/training-material//topics/dev/tutorials/community-tool-table/tutorial.html) in the dedicated GTN tutorial**.