update scripts / add proxy test script

scripts/rewards/getlogs.sh

@@ -1,10 +1,13 @@
 #!/usr/bin/bash
-timeout 30.0m unbuffer skywire-cli log --minv v1.3.19 -s $(tail -n1 survey-wl.conf) | tee skywire-cli-log.txt
+timeout 30.0m unbuffer skywire-cli log --minv v1.3.21 -s $(tail -n1 survey-wl.conf) | tee skywire-cli-log.txt
 #echo -e "skywire survey and transport log collection $(date)\n\n$(cat skywire-cli-log.txt)\n"
  echo -e "skywire survey and transport log collection $(date)\n\n$(cat skywire-cli-log.txt)\n" | tee skywire-cli-log0.txt >> /dev/null
 echo "finished "$(date) | tee -a skywire-cli-log0.txt
 mv skywire-cli-log0.txt skywire-cli-log.txt
 
+#Delete json files more than 1 week old
+find log_backups/*/*.json -type f -mmin +$((168 * 60)) -delete
+find log_collecting/*/*.json -type f -mmin +$((168 * 60)) -delete
 #remove empty files and dirs
 find log_collecting/*/ -empty -type f -delete && printf "removed empty files... \n"  || true
 find log_collecting/*/ -type f -size 19c -delete && printf "removed files with http 404 errors... \n" || true
@@ -22,20 +25,8 @@ find log_collecting/*/$(date +'%Y-%m-%d').csv -type f -print | while read _file
 find log_collecting/*/$(date --date="yesterday" +'%Y-%m-%d').csv -type f -print | while read _file ; do [[ $(head -n 1 $_file) == *"tp_id,recv,sent,time_stamp"* ]] && sed -i '1d' $_file ; done || true
 find log_backups/*/$(date +'%Y-%m-%d').csv -type f -print | while read _file ; do [[ $(head -n 1 $_file) == *"tp_id,recv,sent,time_stamp"* ]] && sed -i '1d' $_file ; done || true
 find log_backups/*/$(date --date="yesterday" +'%Y-%m-%d').csv -type f -print | while read _file ; do [[ $(head -n 1 $_file) == *"tp_id,recv,sent,time_stamp"* ]] && sed -i '1d' $_file ; done || true
-grep -l "404 page not found" log_collecting/*/$(date +'%Y-%m-%d').csv | xargs rm -f || true
-grep -l "404 page not found" log_collecting/*/$(date --date="yesterday" +'%Y-%m-%d').csv | xargs rm -f || true
-grep -l "404 page not found" log_backups/*/$(date +'%Y-%m-%d').csv | xargs rm -f || true
-grep -l "404 page not found" log_backups/*/$(date --date="yesterday" +'%Y-%m-%d').csv | xargs rm -f || true
-
-
-#check / decrypt surveys
-printf "checking surveys... \n"
-find log_collecting/*/node-info.json -type f -print | xargs grep -l "404 page not found" | parallel rm || true
-find log_collecting/*/node-info.json -type f -print | xargs grep -l "Not Found" | parallel rm || true
-find log_collecting/*/node-info.json -type f -print | xargs grep -l "PGP MESSAGE" | parallel rm  || true #xargs -I {} ./decrypt.sh {} #./decrypt.sh
-find log_backups/*/node-info.json -type f -print | xargs grep -l "404 page not found" | parallel rm || true
-find log_backups/*/node-info.json -type f -print | xargs grep -l "Not Found" | parallel rm || true
-find log_backups/*/node-info.json -type f -print | xargs grep -l "PGP MESSAGE" | parallel rm || true #xargs -I {} ./decrypt.sh {} #./decrypt.sh
+find log_collecting/*/*.json -type f -print | while read _file; do if ! jq '.' "$_file" >/dev/null 2>&1; then echo "invalid json $_file" ; rm $_file; fi; done
+find log_backups/*/*.json -type f -print | while read _file; do if ! jq '.' "$_file" >/dev/null 2>&1; then echo "invalid json $_file" ; rm $_file; fi; done
 
 printf "checking tp logs... \n"
 [[ -f log_collecting/*/$(date +'%Y-%m-%d').csv ]] && find log_collecting/*/$(date +'%Y-%m-%d').csv -type f -print | xargs grep -l "404 page not found" | parallel rm || true
@@ -47,6 +38,7 @@ printf "checking tp logs... \n"
 [[ -f log_collecting/*/$(date +'%Y-%m-%d').csv ]] && find log_backups/*/$(date +'%Y-%m-%d').csv -type f -print | xargs grep -l "Not Found" | parallel rm || true
 [[ -f log_collecting/*/log_collecting/*/$(date --date="yesterday" +'%Y-%m-%d').csv ]] && find log_backups/*/$(date --date="yesterday" +'%Y-%m-%d').csv -type f -print | xargs grep -l "Not Found" | parallel rm || true
 
+
 #back up the collected files
 rsync -r log_collecting/ log_backups || true
 [[ -f log_backups/*/*~ ]] && rm log_backups/*/*~ || true

scripts/rewards/gettps.sh

@@ -1,10 +1,24 @@
 #!/usr/bin/bash
-[[ ! -d tp_setup ]] && mkdir tp_setup
+#refresh the cached uptime tracker data at /tmp/ut.json
 skywire cli ut -s
-#jq '.[] | select(.version=="v1.3.21" and .on==true ) | .pk' /tmp/ut.json | tr -d '"' | while read _pk ; do
-printf "%s\n%s\n" $(jq '.[] | select(.version=="v1.3.21" and .on==true ) | .pk' /tmp/ut.json | tr -d '"') $(find ./log_backups/ -type d -print0 | xargs -0 -n1 basename) | sort | uniq -c | sort -nr | grep "2 " | sed 's/      2 //g' | tac | while read _pk ; do
-  mkdir -p tp_setup/${_pk}
-skywire svc tps list -1 ${_pk} 2>&1 | tee tp_setup/${_pk}/tp.json
+#online status alone is too broad of a qualifier for surveying. But there are sometimes errors with the dmsghttp logserver that wouldn't otherwise cause the transport setup node to fail. Some nodes may be offline though UT shows online and vice versa.
+find ./log_backups/ -type d -print0 | xargs -0 -n1 basename | while read _pk ; do
+mkdir -p tp_setup/${_pk}
 done
+printf "%s\n%s\n%s\n" $(jq '.[] | select(.version=="v1.3.21" and .on==true ) | .pk' /tmp/ut.json | tr -d '"') $(find ./log_backups/ -type d -print0 | xargs -0 -n1 basename) $(find log_backups/ -type f -name "health.json" -mmin -60 | cut -d '/' -f2) | sort | uniq -c | sort -nr | grep -v "1 " | sed 's/      [23] //g' | tac | while read _pk ; do
+timeout 3.0m parallel -j 25 'skywire svc tps list -1 {} 2>&1 | tee tp_setup/{}/tp.json'
+done
+skywire cli ut -s
+find ./log_backups/ -type d -print0 | xargs -0 -n1 basename | while read _pk ; do
+mkdir -p tp_setup/${_pk}
+done
+printf "%s\n%s\n%s\n" $(jq '.[] | select(.version=="v1.3.21" and .on==true ) | .pk' /tmp/ut.json | tr -d '"') $(find ./log_backups/ -type d -print0 | xargs -0 -n1 basename) $(find log_backups/ -type f -name "health.json" -mmin -60 | cut -d '/' -f2) | sort | uniq -c | sort -nr | grep -v "1 " | sed 's/      [23] //g' | while read _pk ; do
+timeout 3.0m parallel -j 25 'skywire svc tps list -1 {} 2>&1 | tee tp_setup/{}/tp.json'
+done
+skywire cli ut -s
+printf "%s\n%s\n" $(jq '.[] | select(.version=="v1.3.21" and .on==true ) | .pk' /tmp/ut.json | tr -d '"') $(find tp_setup/ -type f -name "tp.json" -mmin -60 | cut -d '/' -f2) | sort | uniq -c | sort -nr | sed 's/      [12] //g' | while read _pk ; do
+timeout 3.0m parallel -j 25 'jq -e . tp_setup/{}/tp.json  > /dev/null 2>&1 || skywire svc tps list -1 {} 2>&1 | tee tp_setup/{}/tp.json'
+done
+echo "cleaning up"
 find ./tp_setup/ -type f -mtime +2 -delete
 find ./tp_setup/ -type d -empty -delete