run CI on result change, move community data to dedicated subfolder (#42

)

.github/workflows/filter_communities.yaml

@@ -7,7 +7,6 @@ on:
   push:
     paths:
     - 'results/all_tools_tsv'
-    branches: ["main"]
 
 # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
 # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
@@ -34,5 +33,8 @@ jobs:
       run: |
         git config user.name github-actions
         git config user.email [email protected]
-        git diff --quiet || (git add results && git commit -m "filter communities bot") 
+        git pull --no-rebase -s recursive -X ours
+        git add results
+        git status
+        git diff --quiet && git diff --staged --quiet || (git commit -m "fetch all tools bot  - step filter") 
         git push

.github/workflows/static.yml

@@ -2,11 +2,10 @@
 name: Deploy static content to Pages
 
 on:
-  # the workflow is triggered only when results are changed
+  # the workflow is triggered when any of the results are changed
   push:
     paths:
-    - 'results'
-    branches: ["main"]
+    - 'results/**'
 
 
   # Allows you to run this workflow manually from the Actions tab

bin/get_community_tools.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 
-for com_data_fp in data/* ; do
+for com_data_fp in data/communities/* ; do
         if [[ -d "$com_data_fp" && ! -L "$com_data_fp" ]]; then
                 community=`basename "$com_data_fp"`
 
@@ -11,12 +11,12 @@ for com_data_fp in data/* ; do
                         curl \
                                 -L \
                                 "https://docs.google.com/spreadsheets/d/1Nq_g-CPc8t_eC4M1NAS9XFJDflA7yE3b9hfSg3zu9L4/export?format=tsv&gid=1533244711" \
-                                -o "data/$community/tools_to_keep"
+                                -o "data/communities/$community/tools_to_keep"
 
                         curl \
                                 -L \
                                 "https://docs.google.com/spreadsheets/d/1Nq_g-CPc8t_eC4M1NAS9XFJDflA7yE3b9hfSg3zu9L4/export?format=tsv&gid=672552331" \
-                                -o "data/$community/tools_to_exclude"
+                                -o "data/communities/$community/tools_to_exclude"
                 fi;
 
 
@@ -26,9 +26,9 @@ for com_data_fp in data/* ; do
                         filtertools \
                         --tools "results/all_tools.tsv" \
                         --filtered_tools "results/$community/tools.tsv" \
-                        --categories "data/$community/categories" \
-                        --exclude "data/$community/tools_to_exclude" \
-                        --keep "data/$community/tools_to_keep"
+                        --categories "data/communities/$community/categories" \
+                        --exclude "data/communities/$community/tools_to_exclude" \
+                        --keep "data/communities/$community/tools_to_keep"
 
                 python bin/create_interactive_table.py \
                         --table "results/$community/tools.tsv" \