Fix community filtering scripts

galaxyproject · Oct 2, 2024 · 5efb070 · 5efb070
1 parent d34a91a
commit 5efb070
Show file tree

Hide file tree

Showing 3 changed files with 47 additions and 39 deletions.
diff --git a/.github/workflows/fetch_filter_resources.yaml b/.github/workflows/fetch_filter_resources.yaml
@@ -95,7 +95,9 @@ jobs:
       - name: Merge all tools
         run: | #merge files with only one header -> https://stackoverflow.com/questions/16890582/unixmerge-multiple-csv-files-with-same-header-by-keeping-the-header-of-the-firs; map(.[]) -> https://stackoverflow.com/questions/42011086/merge-arrays-of-json (get flat array, one tool per entry)
           awk 'FNR==1 && NR!=1{next;}{print}' communities/all/resources/repositories*.list_tools.tsv > communities/all/resources/tools.tsv
-          jq -s 'map(.[])' communities/all/resources/repositories*.list_tools.json > communities/all/resources/all_tools.json 
+          jq -s 'map(.[])' communities/all/resources/repositories*.list_tools.json > communities/all/resources/tools.json 
+          rm communities/all/resources/repositories*.list_tools.json
+          rm communities/all/resources/repositories*.list_tools.tsv
       - name: Generate wordcloud and interactive table
         run: |
           bash sources/bin/format_tools.sh
@@ -107,9 +109,9 @@ jobs:
       - name: Filter tutorials for communities
         run: | 
           bash sources/bin/get_community_tutorials.sh
-      - name: Update tool to keep and exclude for communities
-        run: | 
-          bash sources/bin/update_tools_to_keep_exclude.sh
+      #- name: Update tool to keep and exclude for communities
+      #  run: | 
+      #    bash sources/bin/update_tools_to_keep_exclude.sh
       - name: Filter tools for communities
         run: | 
           bash sources/bin/get_community_tools.sh

diff --git a/sources/bin/get_community_tools.sh b/sources/bin/get_community_tools.sh
@@ -32,33 +32,35 @@ else
                 if [[ -d "$com_data_fp" && ! -L "$com_data_fp" ]]; then
                         community=`basename "$com_data_fp"`
 
-                        echo "$community";
+                        if [ "$community" != "all" ]; then
 
-                        mkdir -p "communities/$community/resources"
+                                echo "$community";
 
-                        python sources/bin/extract_galaxy_tools.py \
-                                filter \
-                                --all "communities/all/resources/tools.json" \
-                                --ts-filtered "communities/$community/resources/tools_filtered_by_ts_categories.tsv" \
-                                --filtered "communities/$community/resources/tools.tsv" \
-                                --categories "communities/$community/metadata/categories" \
-                                --status "communities/$community/metadata/tool_status.tsv"
+                                mkdir -p "communities/$community/resources"
 
-                        python sources/bin/create_interactive_table.py \
-                                --input "communities/$community/resources/tools.tsv" \
-                                --remove-col "Reviewed" \
-                                --remove-col "To keep" \
-                                --filter-col "To keep" \
-                                --template "sources/data/interactive_table_template.html" \
-                                --output "communities/$community/resources/tools.html"
+                                python sources/bin/extract_galaxy_tools.py \
+                                        filter \
+                                        --all "communities/all/resources/tools.json" \
+                                        --ts-filtered "communities/$community/resources/tools_filtered_by_ts_categories.tsv" \
+                                        --filtered "communities/$community/resources/tools.tsv" \
+                                        --categories "communities/$community/metadata/categories" \
+                                        --status "communities/$community/metadata/tool_status.tsv"
 
-                        python sources/bin/create_wordcloud.py \
-                                --input  "communities/$community/resources/tools.tsv" \
-                                --name-col "Galaxy wrapper id" \
-                                --stat-col "No. of tool users (2022-2023) (usegalaxy.eu)" \
-                                --wordcloud_mask "sources/data/usage_stats/wordcloud_mask.png" \
-                                --output "communities/$community/resources/tools_wordcloud.png"
+                                python sources/bin/create_interactive_table.py \
+                                        --input "communities/$community/resources/tools.tsv" \
+                                        --remove-col "Reviewed" \
+                                        --remove-col "To keep" \
+                                        --filter-col "To keep" \
+                                        --template "sources/data/interactive_table_template.html" \
+                                        --output "communities/$community/resources/tools.html"
 
+                                python sources/bin/create_wordcloud.py \
+                                        --input  "communities/$community/resources/tools.tsv" \
+                                        --name-col "Galaxy wrapper id" \
+                                        --stat-col "No. of tool users (2022-2023) (usegalaxy.eu)" \
+                                        --wordcloud_mask "sources/data/usage_stats/wordcloud_mask.png" \
+                                        --output "communities/$community/resources/tools_wordcloud.png"
+                        fi;
                 fi;
         done
 fi

diff --git a/sources/bin/get_community_tutorials.sh b/sources/bin/get_community_tutorials.sh
@@ -14,24 +14,28 @@ then
                 --output "communities/microgalaxy/resources/tutorials.html"
 
 else
-        for com_data_fp in data/communities/* ; do
+        for com_data_fp in communities/* ; do
                 if [[ -d "$com_data_fp" && ! -L "$com_data_fp" ]]; then
                         community=`basename "$com_data_fp"`
+                        if [ "$community" != "all" ]; then
+                                echo "$community"
 
-                        echo "$community";
+                                if [[ -f "communities/$community/metadata/tutorial_tags" && -f "communities/$community/resources/tutorials.tsv" ]]; then
+                                        echo "Filter tutorials"
 
-                        if [[ -f "data/communities/$community/tutorial_tags" && -f "results/$community/tutorials.tsv" ]]; then
+                                        python sources/bin/extract_gtn_tutorials.py \
+                                                filter \
+                                                --all "communities/all/resources/tutorials.json" \
+                                                --filtered "communities/$community/resources/tutorials.tsv" \
+                                                --tags "communities/$community/metadata/tutorial_tags"
 
-                                python sources/bin/extract_gtn_tutorials.py \
-                                        filter \
-                                        --all "communities/all/resources/tutorials.json" \
-                                        --filtered "communities/$community/resources/tutorials.tsv" \
-                                        --tags "communities/$community/metadata/tutorial_tags"
-
-                                python sources/bin/create_interactive_table.py \
-                                        --input "communities/$community/resources/tutorials.tsv" \
-                                        --template "sources/data/interactive_table_template.html" \
-                                        --output "communities/$community/resources/tutorials.html"
+                                        python sources/bin/create_interactive_table.py \
+                                                --input "communities/$community/resources/tutorials.tsv" \
+                                                --template "sources/data/interactive_table_template.html" \
+                                                --output "communities/$community/resources/tutorials.html"
+                                fi;
+
+                                echo ""
                         fi;
                 fi;
         done