-
Notifications
You must be signed in to change notification settings - Fork 12
/
decoding.sh
executable file
·2370 lines (2129 loc) · 155 KB
/
decoding.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/bin/bash
############## Decoding ################################################
### For each real receiver/band there is one decode daemon and one recording daemon
### Waits for a new wav file then decodes and posts it to all of the posting client
declare -r DECODING_CLIENTS_SUBDIR="decoding_clients.d" ### Each decoding daemon will create its own subdir where it will copy YYMMDD_HHMM_wspr_spots.txt
declare MAX_ALL_WSPR_SIZE=200000 ### Delete the ALL_WSPR.TXT file once it reaches this size.. Stops wsprdaemon from filling ${WSPRDAEMON_TMP_DIR}/..
declare FFT_WINDOW_CMD=${WSPRDAEMON_ROOT_DIR}/wav_window.py
declare C2_FFT_ENABLED="yes" ### If "yes", then use the c2 file produced by wsprd to calculate FFT noise levels
declare C2_FFT_CMD=${WSPRDAEMON_ROOT_DIR}/c2_noise.py
function get_decode_mode_list() {
local modes_variable_to_return=$1
local receiver_modes_arg=$2
local receiver_band=$3
local temp_receiver_modes
temp_receiver_modes=${receiver_modes_arg}
if [[ ${receiver_modes_arg} == "DEFAULT" ]]; then
### Translate DEFAULT mode to a list of modes for this band
local default_modes=""
get_default_modes_for_band default_modes ${receiver_band}
local ret_code=$?
if [[ ${ret_code} -ne 0 ]]; then
wd_logger 1 "ERROR: 'get_default_modes_for_band default_modes ${receiver_band}' => ${ret_code}"
sleep 1
return ${ret_code}
fi
wd_logger 1 "Translated decode mode '${receiver_modes_arg}' to '${default_modes}'"
temp_receiver_modes=${default_modes}
fi
### Validate the mode list
is_valid_mode_list ${temp_receiver_modes}
local ret_code=$?
if [[ ${ret_code} -ne 0 ]] ; then
wd_logger 1 "ERROR: 'is_valid_mode_list ${temp_receiver_modes}' => ${ret_code}"
return 1
fi
wd_logger 2 "Returning modes ${temp_receiver_modes}"
eval ${modes_variable_to_return}=${temp_receiver_modes}
return 0
}
##########
function get_af_db() {
local return_variable_name=$1
local local real_receiver_name=$2 ### 'real' as opposed to 'merged' receiver
local real_receiver_rx_band=$3
local default_value
local af_info_field="$(get_receiver_af_list_from_name ${real_receiver_name})"
if [[ -z "${af_info_field}" ]]; then
wd_logger 2 "Found no AF field for receiver ${real_receiver_name}, so return AF=0"
eval ${return_variable_name}=0
return 0
fi
local af_info_list=(${af_info_field//,/ })
wd_logger 1 "af_info_list= '${af_info_list[*]}'"
for element in ${af_info_list[@]}; do
local fields=(${element//:/ })
if [[ ${fields[0]} == "DEFAULT" ]]; then
default_value=${fields[1]}
wd_logger 1 "Found default value ${default_value}"
elif [[ ${fields[0]} == ${real_receiver_rx_band} ]]; then
wd_logger 1 "Found AF value ${fields[1]} for receiver ${real_receiver_name}, band ${real_receiver_rx_band}"
eval ${return_variable_name}=${fields[1]}
return 0
fi
done
if [[ -z "${default_value-}" ]]; then
wd_logger 1 "ERROR: can't find af value for receiver ${real_receiver_name}, band ${real_receiver_rx_band}, AND there is no DEFAULT. So return 0"
default_value=0
else
wd_logger 1 "Returning default value ${default_value} for receiver ${real_receiver_name}, band ${real_receiver_rx_band}"
fi
eval ${return_variable_name}=${default_value}
return 0
}
function calculate_nl_adjustments() {
local return_rms_corrections_variable_name=$1
local return_fft_corrections_variable_name=$2
local receiver_band=$3
local wspr_band_freq_khz=$(get_wspr_band_freq_khz ${receiver_band})
local wspr_band_freq_mhz=$( printf "%2.4f\n" $(bc <<< "scale = 5; ${wspr_band_freq_khz}/1000.0" ) )
local wspr_band_freq_hz=$( bc <<< "scale = 0; ${wspr_band_freq_khz}*1000.0/1" )
if [[ -f ${WSPRDAEMON_ROOT_DIR}/noise_plot/noise_ca_vals.csv ]]; then
local cal_vals=($(sed -n '/^[0-9]/s/,/ /gp' ${WSPRDAEMON_ROOT_DIR}/noise_plot/noise_ca_vals.csv))
fi
### In each of these assignments, if cal_vals[] was not defined above from the file 'noise_ca_vals.csv', then use the default value. e.g. cal_c2_correction will get the default value '-187.7
local cal_nom_bw=${cal_vals[0]-320} ### In this code I assume this is 320 hertz
local cal_ne_bw=${cal_vals[1]-246}
local cal_rms_offset=${cal_vals[2]--50.4}
local cal_fft_offset=${cal_vals[3]--41.0}
local cal_fft_band=${cal_vals[4]--13.9}
local cal_threshold=${cal_vals[5]-13.1}
local cal_c2_correction=${cal_vals[6]--187.7}
local kiwi_amplitude_versus_frequency_correction="$(bc <<< "scale = 10; -1 * ( (2.2474 * (10 ^ -7) * (${wspr_band_freq_mhz} ^ 6)) - (2.1079 * (10 ^ -5) * (${wspr_band_freq_mhz} ^ 5)) + \
(7.1058 * (10 ^ -4) * (${wspr_band_freq_mhz} ^ 4)) - (1.1324 * (10 ^ -2) * (${wspr_band_freq_mhz} ^ 3)) + \
(1.0013 * (10 ^ -1) * (${wspr_band_freq_mhz} ^ 2)) - (3.7796 * (10 ^ -1) * ${wspr_band_freq_mhz} ) - (9.1509 * (10 ^ -1)))" )"
if [[ $(bc <<< "${wspr_band_freq_mhz} > 30") -eq 1 ]]; then
### Don't adjust Kiwi's af when fed by transverter
kiwi_amplitude_versus_frequency_correction=0
fi
local antenna_factor_adjust
get_af_db antenna_factor_adjust ${receiver_name} ${receiver_band}
local ret_code=$?
if [[ ${ret_code} -ne 0 ]]; then
wd_logger 1 "ERROR: can't find AF for ${receiver_name} ${receiver_band}"
exit 1
fi
wd_logger 1 "Got AF = ${antenna_factor_adjust} for ${receiver_name} ${receiver_band}"
local rx_khz_offset=$(get_receiver_khz_offset_list_from_name ${receiver_name})
local total_correction_db=$(bc <<< "scale = 10; ${kiwi_amplitude_versus_frequency_correction} + ${antenna_factor_adjust}")
local calculated_rms_nl_adjust=$(bc -l <<< "var=(${cal_rms_offset} + (10 * (l( 1 / ${cal_ne_bw}) / l(10) ) ) + ${total_correction_db}); scale=2; var/1.0" ) ## bc -l invokes the math extension, l(x)/l(10) == log10(x)
wd_logger 1 "calculated_rms_nl_adjust=\$(bc -l <<< \"var=(${cal_rms_offset} + (10 * (l( 1 / ${cal_ne_bw}) / l(10) ) ) + ${total_correction_db}); scale=2; var/1.0\" )"
eval ${return_rms_corrections_variable_name}=${calculated_rms_nl_adjust}
## G3ZIL implementation of algorithm using the c2 file by Christoph Mayer
local calculated_fft_nl_adjust=$(bc <<< "scale = 2;var=${cal_c2_correction};var+=${total_correction_db}; (var * 100)/100")
wd_logger 1 "calculated_fft_nl_adjust = ${calculated_fft_nl_adjust} from calculated_fft_nl_adjust=\$(bc <<< \"scale = 2;var=${cal_c2_correction};var+=${total_correction_db}; (var * 100)/100\")"
eval ${return_fft_corrections_variable_name}="'${calculated_fft_nl_adjust}'"
}
declare WAV_SAMPLES_LIST=(
"${SIGNAL_LEVEL_PRE_TX_SEC} ${SIGNAL_LEVEL_PRE_TX_LEN}"
"${SIGNAL_LEVEL_TX_SEC} ${SIGNAL_LEVEL_TX_LEN}"
"${SIGNAL_LEVEL_POST_TX_SEC} ${SIGNAL_LEVEL_POST_TX_LEN}"
)
### Record an error line to the log file if the wav file contains audio samples which exceed these levels
declare WAV_MIN_LEVEL=${WAV_MIN_LEVEL--1.0}
declare WAV_MAX_LEVEL=${WAV_MAX_LEVEL-1.0}
function get_wav_levels()
{
local __return_levels_var=$1
local wav_filename=$2
local sample_start_sec=$3
local sample_length_secs=$4
local rms_adjust=$5
if [[ ${sample_start_sec} == ${SIGNAL_LEVEL_PRE_TX_SEC} ]]; then
### This function is called three times for each wav file. We only need to check the whole wav file once to determine the min/max values
### So execute this check only the first time
### To see if the AGC might need to change from its default 60, check to see if any samples in the whole wav file closely approach the MAX or MIN sample values
### 'sox -n stats' output this information on seperate line:
### DC offset Min level Max level Pk lev dB RMS lev dB RMS Pk dB RMS Tr dB Crest factor Flat factor Pk count Bit-depth Num samples Length s Scale max Window s
### Field #: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14
### Run 'man sox' and search for 'stats' to find a description of those statistic fields
local full_wav_stats=$(sox ${wav_filename} -n stats 2>&1) ### sox -n stats prints those to stderr
local full_wav_stats_list=( $(echo "${full_wav_stats}" | awk '{printf "%s\t", $NF }') ) ### store them in an array
if [[ ${#full_wav_stats_list[@]} -ne ${EXPECTED_SOX_STATS_FIELDS_COUNT-15} ]]; then
wd_logger 1 "ERROR: Got ${#full_wav_stats_list[@]} stats from 'sox -n stats', not the expected ${EXPECTED_SOX_STATS_FIELDS_COUNT-15} fields:\n${full_wav_stats}"
else
local full_wav_min_level=${full_wav_stats_list[1]}
local full_wav_max_level=${full_wav_stats_list[2]}
local full_wav_peak_level_count=${full_wav_stats_list[9]}
local full_wav_bit_depth=${full_wav_stats_list[10]}
local full_wav_len_secs=${full_wav_stats_list[12]}
### Min and Max level are floating point numbers and their absolute values are less than or equal to 1.0000
if [[ $( echo "${full_wav_min_level} <= ${WAV_MIN_LEVEL}" | bc ) == "1" || $( echo "${full_wav_max_level} >= ${WAV_MAX_LEVEL}" | bc ) == "1" ]] ; then
wd_logger 1 "ERROR: ${full_wav_peak_level_count} full level (+/-1.0) samples detected in file ${wav_filename} of length=${full_wav_len_secs} seconds and with Bit-depth=${full_wav_bit_depth}: the min/max levels are: min=${full_wav_min_level}, max=${full_wav_max_level}"
else
wd_logger 2 "In file ${wav_filename} of length=${full_wav_len_secs} seconds and with Bit-depth=${full_wav_bit_depth}: the min/max levels are: min=${full_wav_min_level}, max=${full_wav_max_level}"
fi
### Create a status file associated with this indsividual wav file from which the decoding daemon will extract wav overload information for the spots decoded from this wav file
echo "WAV_stats: ${full_wav_min_level} ${full_wav_max_level} ${full_wav_peak_level_count}" > ${wav_filename}.stats
### Append these stats to a log file which can be searched by a yet-to-be-implemented 'wd-...' command
local wav_status_file="${WAV_STATUS_LOG_FILE-wav_status.log}"
touch ${wav_status_file} ### In case it doesn't yet exist
if grep -q "${wav_filename}" ${wav_status_file} ; then
wd_logger 1 "ERROR: unexpectly found log line for wav file ${wav_filename} in ${wav_status_file}"
else
wd_logger 1 "Appending '${wav_filename}: ${full_wav_min_level} ${full_wav_max_level} ${full_wav_peak_level_count}' to the log file '${wav_status_file}'"
echo "${wav_filename}: ${full_wav_min_level} ${full_wav_max_level} ${full_wav_peak_level_count}" >> ${wav_status_file}
truncate_file ${wav_status_file} 100000 ### Limit the size of this log file to 100 Kb
fi
fi
fi
local wav_levels_list=( $(sox ${wav_filename} -t wav - trim ${sample_start_sec} ${sample_length_secs} 2>/dev/null | sox - -n stats 2>&1 | awk '/dB/{print $(NF)}'))
if [[ ${#wav_levels_list[@]} -ne 4 ]]; then
wd_logger 1 "ERROR: found only ${#wav_levels_list[@]} dB lines, not the four expected dB lines from 'sox ${wav_filename} -t wav - trim ${sample_start_sec} ${sample_length_secs}'"
return 1
fi
wd_logger 2 "Got sox dB values: '${wav_levels_list[*]}'"
local return_line=""
for db_val in ${wav_levels_list[@]}; do
local adjusted_val=$(bc <<< "scale = 2; (${db_val} + ${rms_adjust})/1") ### '/1' forces bc to use the scale = 2 setting
return_line="${return_line} ${adjusted_val}"
done
wd_logger 2 "Returning adjusted dB values: '${return_line}'"
eval ${__return_levels_var}=\"${return_line}\"
return 0
}
declare WAV_SECOND_RANGE=${WAV_SECOND_RANGE-10} ### wav files of +/- this number of seconds are deemed OK for wsprd to decode
declare TARGET_RAW_WAV_SECONDS=60
declare MIN_VALID_RAW_WAV_SECONDS=${MIN_VALID_RAW_WAV_SECONDS-$(( ${TARGET_RAW_WAV_SECONDS} - ${WAV_SECOND_RANGE} )) }
declare MAX_VALID_RAW_WAV_SECONDS=${MAX_VALID_RAW_WAV_SECONDS-$(( ${TARGET_RAW_WAV_SECONDS} + ${WAV_SECOND_RANGE} )) }
declare TARGET_WSPR_WAV_SECONDS=120
declare MIN_VALID_WSPR_WAV_SECONDS=${MIN_VALID_WSPR_WAV_SECONDS-$(( ${TARGET_WSPR_WAV_SECONDS} - ${WAV_SECOND_RANGE} )) }
declare MAX_VALID_WSPR_WAV_SECONDS=${MAX_VALID_WSPR_WAV_SECONDS-$(( ${TARGET_WSPR_WAV_SECONDS} + ${WAV_SECOND_RANGE} )) }
function is_valid_wav_file()
{
local wav_filename=$1
local min_valid_secs=$2
local max_valid_secs=$3
if [[ ! -f ${wav_filename} ]]; then
wd_logger 1 "ERROR: no wav file ${wav_filename}"
return 1
fi
if [[ ! -s ${wav_filename} ]]; then
wd_logger 1 "ERROR: zero length wav file ${wav_filename}"
return 1
fi
local wav_stats=$(sox ${wav_filename} -n stats 2>&1 ) ### Don't add ' --keep-foreign-metadata"
local ret_code=$?
if [[ ${ret_code} -ne 0 ]]; then
wd_logger 1 "ERROR: 'sox ${wav_filename} -n stats' => ${ret_code}"
return 1
fi
wd_logger 2 "'sox ${wav_filename} -n stats 2>&1' =>\n${wav_stats}"
local wav_length_line_list=( $(grep '^Length' <<< "${wav_stats}") )
if [[ ${#wav_length_line_list[@]} -eq 0 ]]; then
wd_logger 1 "ERROR: can't find wav file 'Length' line in output of 'sox ${wav_filename} -n stats'"
return 1
fi
if [[ ${#wav_length_line_list[@]} -ne 3 ]]; then
wd_logger 1 "ERROR: 'sox ${wav_filename} -n stats' ouput 'Length' line has ${#wav_length_line_list[@]} fields in it instead of the expected 3 fields"
return 1
fi
local wav_length_secs=${wav_length_line_list[2]/.*}
if [[ -z "${wav_length_secs}" ]]; then
wd_logger 1 "ERROR: 'sox ${wav_filename} -n stats' reports invalid wav file length '${wav_length_line_list[2]}'"
return 1
fi
if [[ ! ${wav_length_secs} =~ ^[0-9]+$ ]]; then
wd_logger 1 "ERROR: 'sox ${wav_filename} -n stats' reports wav file length ${wav_length_line_list[2]} which doesn't contain an integer number"
return 1
fi
if [[ ${wav_length_secs} -lt ${min_valid_secs} || ${wav_length_secs} -gt ${max_valid_secs} ]]; then
wd_logger 1 "ERROR: 'sox ${wav_filename} -n stats' reports invalid wav file length of ${wav_length_secs} seconds. valid min=${min_valid_secs}, valid max=${max_valid_secs}"
return 1
fi
return 0
}
function get_rms_levels()
{
local __return_var_name=$1
local __return_string_name=$2
local wav_filename=$3
local rms_adjust=$4
if ! is_valid_wav_file ${wav_filename} ${MIN_VALID_WSPR_WAV_SECONDS} ${MAX_VALID_WSPR_WAV_SECONDS} ; then
local rc=$?
wd_logger 1 "ERROR: 'valid_wav_file ${wav_filename}' => ${rc}"
return 1
fi
local output_line=""
local sample_info
for sample_info in "${WAV_SAMPLES_LIST[@]}"; do
local sample_line_list=( ${sample_info} )
local sample_start_sec=${sample_line_list[0]}
local sample_length_secs=${sample_line_list[1]}
local sample_vals
get_wav_levels sample_vals ${wav_filename} ${sample_start_sec} ${sample_length_secs} ${rms_adjust}
local ret_code=$?
if [[ ${ret_code} -ne 0 ]]; then
wd_logger 1 "ERROR: 'get_wav_levels sample_vals ${wav_filename} ${sample_start_sec} ${sample_length_secs}' => {ret_code}"
return 1
fi
output_line="${output_line} ${sample_vals}"
done
local output_line_list=( ${output_line} )
if [[ ${#output_line_list[@]} -ne 12 ]]; then
wd_logger 1 "ERROR: expected 12 fields of dB info, but got only ${#output_line_list[@]} fields from calls to get_wav_levels()"
return 1
fi
local return_rms_value
local pre_rms_value=${output_line_list[3]} # RMS level is the minimum of the Pre and Post 'RMS Tr dB'
local post_rms_value=${output_line_list[11]} # RMS level is the minimum of the Pre and Post 'RMS Tr dB'
if [[ $(bc --mathlib <<< "${pre_rms_value} < ${post_rms_value}") -eq "1" ]]; then
return_rms_value=${pre_rms_value}
wd_logger 2 "So returning rms_level ${return_rms_value} which is from pre_tx"
else
return_rms_value=${post_rms_value}
wd_logger 2 "So returning rms_level ${return_rms_value} which is from post_tx"
fi
local signal_level_line=" ${output_line} ${return_rms_value}"
eval ${__return_var_name}=${return_rms_value}
eval ${__return_string_name}=\"${signal_level_line}\"
wd_logger 2 "Returning rms_value=${return_rms_value} and signal_level_line='${signal_level_line}'"
return 0
}
### Runs wsprd and outputs new spots to ALL_WSPR.TXT.new
function decode_wspr_wav_file() {
local wav_file_name=$1
local wspr_decode_capture_freq_hz=$2
local rx_khz_offset=$3
local stdout_file=$4
local wsprd_cmd_flags="$5" ### ${WSPRD_CMD_FLAGS}
local wsprd_spreading_cmd_flags="$6" ### ${WSPRD_CMD_FLAGS}
wd_logger 2 "Decode file ${wav_file_name} for frequency ${wspr_decode_capture_freq_hz} and send stdout to ${stdout_file}. rx_khz_offset=${rx_khz_offset}, wsprd_cmd_flags='${wsprd_cmd_flags}'"
local wspr_decode_capture_freq_hzx=${wav_file_name#*_} ### Remove the year/date/time
wspr_decode_capture_freq_hzx=${wspr_decode_capture_freq_hz%_*} ### Remove the _usb.wav
local wspr_decode_capture_freq_hzx=$( bc <<< "${wspr_decode_capture_freq_hz} + (${rx_khz_offset} * 1000)" )
local wspr_decode_capture_freq_mhz=$( printf "%2.4f\n" $(bc <<< "scale = 5; ${wspr_decode_capture_freq_hz}/1000000.0" ) )
if ! [[ -f ALL_WSPR.TXT ]]; then
touch ALL_WSPR.TXT
fi
sort -k 1,2 -k 5,5 ALL_WSPR.TXT > ALL_WSPR.TXT.save
cp -p ALL_WSPR.TXT.save ALL_WSPR.TXT
timeout ${WSPRD_TIMEOUT_SECS-110} nice -n ${WSPR_CMD_NICE_LEVEL} ${WSPRD_CMD} -c ${wsprd_cmd_flags} -f ${wspr_decode_capture_freq_mhz} ${wav_file_name} > ${stdout_file}
local ret_code=$?
if [[ ${ret_code} -ne 0 ]]; then
wd_logger 1 "ERROR: Command 'timeout ${WSPRD_TIMEOUT_SECS-110} nice -n ${WSPR_CMD_NICE_LEVEL} ${WSPRD_CMD} -c ${wsprd_cmd_flags} -f ${wspr_decode_capture_freq_mhz} ${wav_file_name} > ${stdout_file}' returned error ${ret_code}"
return ${ret_code}
fi
sort -k 1,2 -k 5,5 ALL_WSPR.TXT > sort.tmp
mv sort.tmp ALL_WSPR.TXT
comm --nocheck-order -13 ALL_WSPR.TXT.save ALL_WSPR.TXT | sort -k 1,2 -k 5,5 > ALL_WSPR.TXT.new.tmp
wd_logger 1 "wsprd added $(wc -l < ALL_WSPR.TXT.new.tmp) spots to ALL_WSPR.txt and we saved those new spots in ALL_WSPR.TXT.new.tmp:\n$(< ALL_WSPR.TXT.new.tmp)"
### Start with the original ALL_WSPR.TXT and see what spots are reported by wsprd.spreading
wd_logger 2 "Decoding WSPR a second time to obtain spreading information"
cp -p ALL_WSPR.TXT.save ALL_WSPR.TXT
local n_arg="-n"
if [[ ${OS_RELEASE} =~ 20.04 ]]; then
n_arg="" ## until we get a wsprd.spreading for U 20.04
fi
if [[ ${WSPRD_TWO_PASS-no} == "no" ]]; then
wd_logger 2 "Skipping wsprd second pass because WSPRD_TWO_PASS == 'no'"
> ${stdout_file}.spreading
else
timeout ${WSPRD_TIMEOUT_SECS-110} nice -n ${WSPR_CMD_NICE_LEVEL} ${WSPRD_SPREADING_CMD} ${n_arg} -c ${wsprd_spreading_cmd_flags} -f ${wspr_decode_capture_freq_mhz} ${wav_file_name} > ${stdout_file}.spreading
local rc=$?
if [[ ${rc} -ne 0 ]]; then
wd_logger 1 "ERROR: Command 'timeout ${WSPRD_TIMEOUT_SECS-110} nice -n ${WSPR_CMD_NICE_LEVEL} ${WSPRD_SPREADING_CMD} -n -c ${wsprd_spreading_cmd_flags} -f ${wspr_decode_capture_freq_mhz} ${wav_file_name} > ${stdout_file}.spreading' returned error ${rc}"
# return ${ret_code}
fi
fi
sort -k 1,2 -k 5,5 ALL_WSPR.TXT > sort.tmp
mv sort.tmp ALL_WSPR.TXT
comm --nocheck-order -13 ALL_WSPR.TXT.save ALL_WSPR.TXT | sort -k 1,2 -k 5,5 > ALL_WSPR.TXT.new.tmp.spreading
wd_logger 1 "wsprd.spreading added $(wc -l < ALL_WSPR.TXT.new.tmp.spreading) spots to ALL_WSPR.txt and added those new spots in ALL_WSPR.TXT.nspreading_ew.tmp:\n$(< ALL_WSPR.TXT.new.tmp.spreading)"
cat ALL_WSPR.TXT.new.tmp.spreading >> ALL_WSPR.TXT.new.tmp
### Restore ALL_WSPR.TXT to its state before either of the decodes added spots
mv ALL_WSPR.TXT.save ALL_WSPR.TXT
### Find the best set of spots from the two passes, giving preference to spots with WSPR-2 spreading information, and append them to ALL_WSPR.TXT so it can use them in the next decoding
awk -f ${AWK_FIND_BEST_SPOT_LINES} ALL_WSPR.TXT.new.tmp | sort -k 1,2 -k 5,5 > ALL_WSPR.TXT.new
cat ALL_WSPR.TXT.new >> ALL_WSPR.TXT
wd_logger 1 "Added the $(wc -l < ALL_WSPR.TXT.new) spots which are the union of the standard and spreading decodes:\n$(< ALL_WSPR.TXT.new)"
truncate_file ALL_WSPR.TXT ${MAX_ALL_WSPR_SIZE}
return ${ret_code}
}
declare WSPRD_BIN_DIR=${WSPRDAEMON_ROOT_DIR}/bin
declare WSPRD_X86_SPREADING_CMD=${WSPRD_BIN_DIR}/wsprd.spread_nodrift.x86
declare WSPRD_ARM_SPREADING_CMD=${WSPRD_BIN_DIR}/wsprd.spread_nodrift.arm
declare AWK_FIND_BEST_SPOT_LINES=${WSPRDAEMON_ROOT_DIR}/best_spots.awk
declare WSPR_CMD_NICE_LEVEL="${WSPR_CMD_NICE_LEVEL-19}"
declare JT9_CMD_NICE_LEVEL="${JT9_CMD_NICE_LEVEL-19}"
declare WSPRD_STDOUT_FILE=wsprd_stdout.txt ### wsprd stdout goes into this file, but we use wspr_spots.txt
declare MAX_ALL_WSPR_SIZE=200000 ### Truncate the ALL_WSPR.TXT file once it reaches this size.. Stops wsprdaemon from filling ${WSPRDAEMON_TMP_DIR}/..
declare RAW_FILE_FULL_SIZE=1440000 ### Approximate number of bytes in a full size one minute long raw or wav file
### We use 'soxi' to check the length of the 1 minute long wav files created by kiwirecorder.py in a field with the form HOURS:MINUTES:SECONDS.MILLISECONDS
### Because bash can only do integer comparisons, we strip the ':'s and '.' from that field
### As a result, valid wav files will bein the ranges from 6000 - (${MIN_VALID_RAW_WAV_SECONDS} * 100) to 5999
### or in the range from 10000 to (10000 + ${MIN_VALID_RAW_WAV_SECONDS})
### So this code gets the time duration of the wave file into an integer which has the form HHMMSSUU and thus can be compared by a bash expression
### Because the field rolls over from second 59 to minute 1, There can be no fields which have the values 6000 through 9999
declare WAV_FILE_MIN_HHMMSSUU=$(( ${MIN_VALID_RAW_WAV_SECONDS} * 100 )) ### by default this = 55 seconds == 5500
declare WAV_FILE_MAX_HHMMSSUU=$(( 10000 + ( ${WAV_SECOND_RANGE} * 100) )) ### by default this = 65 seconds == 10500
### If the wav recording daemon is running, we can calculate how many seconds until it starts to fill the raw file (if 0 length first file) or fills the 2nd raw file. Sleep until then
function flush_wav_files_older_than()
{
local reference_file=$1
if [[ ! -f ${reference_file} ]]; then
wd_logger 1 "ERROR: can't find expected reference file '${reference_file}"
return 1
fi
wd_logger 1 "Delete any files older than ${reference_file}"
local olders=0
local newers=0
local wav_file
for wav_file in $(find -name '*wav'); do
if [[ ${wav_file} -ot ${reference_file} ]]; then
(( ++olders ))
wd_logger 1 "Deleting older wav file '${wav_file}'"
local rc
wd_rm ${wav_file}
rc=$?
if [[ ${rc} -ne 0 ]]; then
wd_logger 1 "ERROR: Deleting older wav file '${wav_file}', 'wd_rm ${wav_file}' => ${rc}"
fi
elif [[ ${wav_file} -nt ${reference_file} ]]; then
(( ++newers ))
wd_logger 2 "Found wav file '${wav_file}' is newer than ${reference_file}"
else
### 'find' prepends './' to the filenames it returns, so we can't compare flenames. But if two wav file timestamps in the same directory match each other, then they must be the same wav file
wd_logger 1 "Found expected reference file ${reference_file}"
fi
done
if [[ ${olders} -gt 0 || ${newers} -gt 0 ]]; then
wd_logger 1 "Deleted ${olders} older wav files and/or found ${newers} new wav files"
fi
return 0
}
declare WD_RECORD_HDR_SIZE_BYTES=44 ## wd-record writes a wav file header and then waits until the first sample of the next minute before starting to write samples to the file
declare WAV_FILE_SIZE_POLL_SECS=${WAV_FILE_SIZE_POLL_SECS-2} ## Check that the wav file is growing every NN seconds, 2 seconds by default
function sleep_until_raw_file_is_full() {
local filename=$1
if [[ ! -f ${filename} ]]; then
wd_logger 1 "ERROR: ${filename} doesn't exist"
return 1
fi
local old_file_size=$( ${GET_FILE_SIZE_CMD} ${filename} )
local new_file_size
local start_seconds=${SECONDS}
sleep ${WAV_FILE_SIZE_POLL_SECS}
while [[ -f ${filename} ]] && new_file_size=$( ${GET_FILE_SIZE_CMD} ${filename}) && [[ ${new_file_size} -eq ${WD_RECORD_HDR_SIZE_BYTES} || ${new_file_size} -gt ${old_file_size} ]]; do
wd_logger 3 "Waiting for file ${filename} to stop growing in size. old_file_size=${old_file_size}, new_file_size=${new_file_size}"
old_file_size=${new_file_size}
sleep ${WAV_FILE_SIZE_POLL_SECS}
done
local loop_seconds=$(( SECONDS - start_seconds ))
if [[ ! -f ${filename} ]]; then
wd_logger 1 "ERROR: file ${filename} disappeared after ${loop_seconds} seconds"
return 1
fi
wd_logger 2 "'${filename}' stopped growing after ${loop_seconds} seconds"
local file_start_minute=${filename:11:2}
local file_start_second=${filename:13:2}
if [[ ${file_start_second} != "00" ]]; then
wd_logger 2 "'${filename} starts at second ${file_start_second}, not at the required second '00', so delete this file which should be the first file created after startup AND any older wav files"
local rc
flush_wav_files_older_than ${filename}
rc=$?
if [[ ${rc} -ne 0 ]]; then
wd_logger 1 "ERROR: Deleting non 00 second wav file'${filename}', 'flush_wav_files_older_than ${filename}' => ${rc}"
fi
wd_rm ${filename}
rc=$?
if [[ ${rc} -ne 0 ]]; then
wd_logger 1 "ERROR: Deleting non 00 second wav file'${filename}', 'wd_rm ${filename}' => ${rc}"
fi
return 2
fi
### Previously, I had just checked the size of the wav file to validate the duration of the recording
### My guesess of the min and max valid wav file size in bytes were too narrow and useful wav files were being thrown away
local wav_file_duration_hh_mm_sec_msec=$(soxi ${filename} | awk '/Duration/{print $3}')
local wav_file_duration_integer=$(sed 's/[\.:]//g' <<< "${wav_file_duration_hh_mm_sec_msec}")
wd_logger 1 "Got wav file ${filename} header which reports duration = ${wav_file_duration_hh_mm_sec_msec} => wav_file_duration_integer = ${wav_file_duration_integer}. WAV_FILE_MIN_HHMMSSUU=${WAV_FILE_MIN_HHMMSSUU}, WAV_FILE_MAX_HHMMSSUU=${WAV_FILE_MAX_HHMMSSUU}"
if [[ 10#${wav_file_duration_integer} -lt ${WAV_FILE_MIN_HHMMSSUU} ]]; then ### The 10#... forces bash to treat wav_file_duration_integer as a decimal, since its leading zeros would otherwise identify it at an octal number
wd_logger 2 "The wav file stabilized at invalid too short duration ${wav_file_duration_hh_mm_sec_msec} which almost always occurs at startup. Flush this file since it can't be used as part of a WSPR wav file"
local rc
flush_wav_files_older_than ${filename}
rc=$?
if [[ ${rc} -ne 0 ]]; then
wd_logger 1 "ERROR: While flushing too short wav file'${filename}', 'flush_wav_files_older_than ${filename}' => ${rc}"
fi
wd_rm ${filename}
rc=$?
if [[ ${rc} -ne 0 ]]; then
wd_logger 1 "ERROR: While flushing too shortwav file'${filename}', 'wd_rm ${filename}' => ${rc}"
fi
return 2
fi
if [[ 10#${wav_file_duration_integer} -gt ${WAV_FILE_MAX_HHMMSSUU} ]]; then
### If the wav file has grown to longer than one minute, then it is likely there are two kiwirecorder jobs running
### We really need to know the IP address of the Kiwi recording this band, since this freq may be recorded by other other Kiwis in a Merged group
local this_dir_path_list=( ${PWD//\// } )
local kiwi_name=${this_dir_path_list[-2]}
local kiwi_freq=${filename#*_}
kiwi_freq=${kiwi_freq::3}
local ps_output=$(ps aux | grep "${KIWI_RECORD_COMMAND}.*${kiwi_freq}.*${receiver_ip_address/:*}" | grep -v grep)
local kiwirecorder_pids=( $(awk '{print $2}' <<< "${ps_output}" ) )
if [[ ${#kiwirecorder_pids[@]} -eq 0 ]]; then
wd_logger 1 "ERROR: wav file stabilized at invalid too long duration ${wav_file_duration_hh_mm_sec_msec}, but can't find any kiwirecorder processes which would be creating it;\n$(soxi ${filename})"
else
wd_kill ${kiwirecorder_pids[@]}
local rc=$?
if [[ ${rc} -ne 0 ]]; then
wd_logger 1 "ERROR: 'wd_kill ${kiwirecorder_pids[*]}' => ${rc}"
fi
wd_logger 1 "ERROR: wav file stabilized at invalid too long duration ${wav_file_duration_hh_mm_sec_msec}, so there appear to be more than one instance of the KWR running. 'ps' output was:\n${ps_output}\nSo executed 'wd_kill ${kiwirecorder_pids[*]}'"
fi
local rc
flush_wav_files_older_than ${filename}
rc=$?
if [[ ${rc} -ne 0 ]]; then
wd_logger 1 "ERROR: Deleting non 00 second wav file'${filename}', 'flush_wav_files_older_than ${filename}' => ${rc}"
fi
wd_rm ${filename}
rc=$?
if [[ ${rc} -ne 0 ]]; then
wd_logger 1 "ERROR: Deleting non 00 second wav file'${filename}', 'wd_rm ${filename}' => ${rc}"
fi
return 3
fi
wd_logger 2 "File ${filename} for minute ${filename:11:2} stabilized at size ${new_file_size} after ${loop_seconds} seconds"
return 0
}
### Returns the minute and epoch of the first sample in 'filename'. Variations in CPU and OS make using the file's timestamp a poor choice for the time source.
### So use the time in the file's name
function get_file_start_time_info()
{
local __epoch_return_variable_name=$1
local __minute_return_variable_name=$2
local file_name=$3
local epoch_from_file_stat=$( ${GET_FILE_MOD_TIME_CMD} ${file_name})
local ret_code=$?
if [[ ${ret_code} -ne 0 ]]; then
wd_logger 1 "ERROR: '${GET_FILE_MOD_TIME_CMD} ${file_name}' => ${ret_code}"
return 1
fi
local minute_from_file_epoch=$( printf "%(%M)T" ${epoch_from_file_stat} )
local year_from_file_name="${file_name:0:4}"
local month_from_file_name=${file_name:4:2}
local day_from_file_name=${file_name:6:2}
local hour_from_file_name=${file_name:9:2}
local minute_from_file_name=${file_name:11:2}
local file_spec_for_date_cmd="${month_from_file_name}/${day_from_file_name}/${year_from_file_name} ${hour_from_file_name}:${minute_from_file_name}:00"
local epoch_from_file_name=$( date --date="${file_spec_for_date_cmd}" +%s )
if [[ ${minute_from_file_epoch} != ${minute_from_file_name} ]]; then
wd_logger 1 "INFO: minute_from_file_epoch=${minute_from_file_epoch} != minute_from_file_name=${minute_from_file_name}, but always use file_name times"
fi
wd_logger 1 "File '${file_name}' => epoch_from_file_stat=${epoch_from_file_stat}, epoch_from_file_name=${epoch_from_file_name}, minute_from_file_epoch=${minute_from_file_epoch}, minute_from_file_name=${minute_from_file_name}"
eval ${__epoch_return_variable_name}=${epoch_from_file_name}
eval ${__minute_return_variable_name}=${minute_from_file_name}
return 0
}
###
### Get the epoch from the wav filename
function epoch_from_filename()
{
local file_name=$1
local file_date_format="${file_name:0:8} ${file_name:9:2}:${file_name:11:2}:${file_name:13:2}"
local file_epoch=$(date -d "${file_date_format}" +%s)
echo "${file_epoch}"
return 0
}
### Given a list of filenames, start from the newest file, the one at the end of the list (i.e. [-1]), and work towards the front of the list
### Make sure that each earlier filename is 1 minute earlier. If not, then flush all the older files from the list
function cleanup_wav_file_list()
{
local __return_clean_files_string_name=$1
local check_file_list=( $2 )
if [[ ${#check_file_list[@]} -eq 0 ]]; then
wd_logger 1 "Was given an empty file list"
eval ${__return_clean_files_string_name}=\"\"
return 0
fi
wd_logger 2 "Testing list of ${#check_file_list[@]} raw files: '${check_file_list[*]}'"
if [[ ${#check_file_list[@]} -lt 1 ]]; then
wd_logger 1 "ERROR: check_file_list[] is empty"
return 1
fi
local epoch_of_newest_file=$( epoch_from_filename "${check_file_list[-1]}" )
wd_logger 2 "Checking for valid list of wav_raw files which end with file ${check_file_list[-1]} = epoch ${epoch_of_newest_file} = minute $(( ( ${epoch_of_newest_file} % 3600 ) / 60 ))"
local flush_files="no"
### Walk back from the end of the file list verifying that each preceeding file starts one minute earlier and is full sized.
### If a invalid file is found, flush it and all earlier files
local raw_file_index=$(( ${#check_file_list[@]} - 2 )) ### Start testing the second to last file in the list
local epoch_of_last_file=${epoch_of_newest_file} ### So the epoch of the last file is the last
local return_clean_files_string="${check_file_list[-1]}" ### The last file is clean
### Now walk backwards through the check_file_list[] verifying that each file is full length and 60 seconds earlier than than its successor file
while [[ ${raw_file_index} -ge 0 ]]; do
local test_file_name
test_file_name=${check_file_list[${raw_file_index}]}
wd_logger 2 "Testing file ${test_file_name}"
if [[ ${flush_files} == "yes" ]]; then
wd_logger 1 "flush_files == 'yes', so flushing file ${test_file_name}"
wd_rm ${test_file_name}
rc=$?
if [[ ${rc} -ne 0 ]]; then
wd_logger 1 "ERROR: for flush_files == 'yes' ${test_file_name}', 'wd_rm ${test_file_name}' => ${rc}"
fi
(( --raw_file_index ))
continue
fi
local ret_code
is_valid_wav_file ${test_file_name} ${MIN_VALID_RAW_WAV_SECONDS} ${MAX_VALID_RAW_WAV_SECONDS}
ret_code=$?
if [[ ${ret_code} -ne 0 ]]; then
### Found a wav file with invalid size
wd_logger 1 "ERROR: found wav file '${test_file_name}' has invalid size. Flush it and all earlier wav files"
wd_rm ${test_file_name}
rc=$?
if [[ ${rc} -ne 0 ]]; then
wd_logger 1 "ERROR: Failed to flush the first invalid file we found, ${test_file_name}', 'wd_rm ${test_file_name}' => ${rc}"
fi
flush_files="yes"
(( --raw_file_index ))
continue
fi
### wav file size is valid
local epoch_of_test_file=$( epoch_from_filename ${test_file_name} )
wd_logger 2 "test_file_name=${test_file_name} = ${epoch_of_test_file} = minute $(( ( ${epoch_of_test_file} % 3600 ) / 60 ))"
### see if it is one minute (60 second) earlier than the previous file
local file_epoch_gap=$(( ${epoch_of_last_file} - ${epoch_of_test_file} ))
if [[ ${file_epoch_gap} -ne 60 ]]; then
wd_logger 1 "ERROR: test_file_name=${test_file_name} is file_epoch_gap=${file_epoch_gap} seocnds, not 1 minute (60 seconds), earlier than the next file in the list. So delete it and all earlier files in the list"
local rc
wd_rm ${test_file_name}
rc=$?
if [[ ${rc} -ne 0 ]]; then
wd_logger 1 "ERROR: Failed to flush ${test_file_name}' which is not one minute earlier than the next wav file in the list: 'wd_rm ${test_file_name}' => ${rc}"
fi
flush_files="yes"
wd_logger 1 "test_file_name=${test_file_name} is 1 minute (60 seconds) earlier than the next file in the list"
epoch_of_last_file=${epoch_of_test_file}
(( --raw_file_index ))
continue
fi
wd_logger 2 "test_file_name='${test_file_name}' from index ${raw_file_index} is clean and 60 seconds earlier the the next file in the list. Proceed to check previous file on the list"
epoch_of_last_file=${epoch_of_test_file}
return_clean_files_string="${return_clean_files_string} ${test_file_name}"
(( --raw_file_index ))
done
local clean_files_list=( ${return_clean_files_string} )
wd_logger 2 "Given check_file_list[${#check_file_list[@]}] ='${check_file_list[*]}'"
wd_logger 2 "Returning clean_file_list[${#clean_files_list[*]}] ='${clean_files_list[*]}'"
if [[ ${#check_file_list[@]} -ne ${#clean_files_list[*]} ]]; then
wd_logger 1 "ERROR: Found errors in wav file list, so cleaned list check_file_list[${#check_file_list[@]}]='${check_file_list[*]}' => clean_file_list[${#clean_files_list[*]}]='${clean_files_list[*]}'"
fi
eval ${__return_clean_files_string_name}=\"${return_clean_files_string}\"
return 0
}
### Waits for wav files needed to decode one or more of the WSPR packet length wav file have been fully recorded
### Then returns zero or more space-seperated strings each of which has the form 'WSPR_PKT_SECONDS:ONE_MINUTE_WAV_FILENAME_0,ONE_MINUTE_WAV_FILENAME_1[,ONE_MINUTE_WAV_FILENAME_2...]'
function get_wav_file_list() {
local return_variable_name=$1 ### returns a string with a space-separated list each element of which is of the form MODE:first.wav[,second.wav,...]
local receiver_name=$2 ### Used when we need to start or restart the wav recording daemon
local receiver_band=$3
local receiver_modes=$4
local wav_archive_dir=$5
local target_modes_list=( ${receiver_modes//:/ } ) ### Argument has form MODE1[:MODE2...] put it in local array
local -ia target_minutes_list=( $( IFS=$'\n' ; echo "${target_modes_list[*]/?/}" | sort -nu ) ) ### Chop the "W" or "F" from each mode element to get the minutes for each mode NOTE THE "s which are requried if arithmatic is being done on each element!!!!
if [[ " ${target_minutes_list[*]} " =~ " 0 " ]] ; then
### The configuration validtor verified that jobs which have mode 'W0' specified will have no other modes
### In mode W0 we are only goign to run the wsprd decoder in order to get the RMS can C2 noise levels
wd_logger 1 "Found that mode 'W0' has been specified"
target_minutes_list=( 2 )
fi
local -ia target_seconds_list=( "${target_minutes_list[@]/%/*60}" ) ### Multiply the minutes of each mode by 60 to get the number of seconds of wav files needed to decode that mode NOTE that both ' and " are needed for this to work
local oldest_file_needed=${target_seconds_list[-1]}
wd_logger 2 "Start with args '${return_variable_name} ${receiver_name} ${receiver_band} ${receiver_modes}', then receiver_modes => ${target_modes_list[*]} => target_minutes=( ${target_minutes_list[*]} ) => target_seconds=( ${target_seconds_list[*]} )"
### This code requires that the list of wav files to be generated is in ascending seconds order, i.e "120 300 900 1800)
if ! spawn_wav_recording_daemon ${receiver_name} ${receiver_band} ; then
local ret_code=$?
wd_logger 1 "ERROR: 'spawn_wav_recording_daemon ${receiver_name} ${receiver_band}' => ${ret_code}"
return ${ret_code}
fi
local raw_file_list=( $( find -maxdepth 1 \( -name \*.wav -o -name \*.raw \) | sed 's/\.\///' | sort ) ) ### minute-*.raw *_usb.wav) ### Get list of the one minute long 'raw' wav files being created by the Kiwi (.wav) or SDR ((.raw)
wd_logger 2 "Found ${#raw_file_list[@]} raw/wav files: '${raw_file_list[*]}'"
case ${#raw_file_list[@]} in
0 )
wd_logger 2 "There are no raw files. Wait up to 10 seconds for the first file to appear"
declare WAIT_FOR_FIRST_WAV_SECS=10
local timeout=0
while raw_file_list=( $( find -maxdepth 1 \( -name \*.wav -o -name \*.raw \) | sed 's/\.\///' | sort ) ) \
&& [[ ${#raw_file_list[@]} -eq 0 ]] \
&& [[ ${timeout} -lt ${WAIT_FOR_FIRST_WAV_SECS} ]]; do
sleep 1
(( ++timeout ))
done
if [[ ${#raw_file_list[@]} -eq 0 ]]; then
wd_logger 1 "Timeout after ${timeout} seconds while waiting for the first wav file to appear"
else
wd_logger 2 "First file appeared after waiting ${timeout} seconds"
fi
return 1 ### Signal to calling function to try again
;;
1 )
wd_logger 2 "There is only 1 raw file ${raw_file_list[0]} which is for minute ${raw_file_list[0]:11:2}, but all modes need at least 2 one minute wav files. So wait for this file to be filled"
local ret_code
sleep_until_raw_file_is_full ${raw_file_list[0]}
ret_code=$?
if [[ ${ret_code} -ne 0 ]]; then
wd_logger 1 "Error: while waiting for the first wav file to fill, 'sleep_until_raw_file_is_full ${raw_file_list[0]}' => ${ret_code} "
else
if [[ -f ${raw_file_list[0]} ]]; then
wd_logger 2 "First file '${raw_file_list[0]}' which is for minute ${raw_file_list[0]:11:2} is filled and good, but since there is only one good file return error 2"
else
wd_logger 1 "First file '${raw_file_list[0]}' was filled, but it was flushed"
fi
fi
return 2
;;
* )
wd_logger 2 "Found ${#raw_file_list[@]} files, so once this file is full we *may* have enough 1 minute wav files to make up a WSPR pkt. Wait until the last file is full, then proceed to process the list."
sleep_until_raw_file_is_full ${raw_file_list[-1]}
local ret_code=$?
if [[ ${ret_code} -ne 0 ]]; then
wd_logger 1 "ERROR: while waiting for the last of ${#raw_file_list[@]} wav files to fill, 'sleep_until_raw_file_is_full ${raw_file_list[-1]}' => ${ret_code}. Sleep 5 before resuming search"
wd_sleep 5
return 4
fi
wd_logger 2 "Check the ${#raw_file_list[@]} files starting with file '${raw_file_list[0]}' which is for minute ${raw_file_list[0]:11:2}"
;;
esac
wd_logger 2 "Found ${#raw_file_list[@]} full raw files, enough that we *may* have a set which can create a new pkt wav file. First clean the list of raw files"
local clean_files_string
cleanup_wav_file_list clean_files_string "${raw_file_list[*]}"
local clean_file_list=( ${clean_files_string} )
if [[ ${#clean_file_list[@]} -lt 2 ]]; then
wd_logger 1 "ERROR: (not really) after cleanup of raw_file_list[], clean_file_list[] has only ${#clean_file_list[@]} files, not enough to create even a 2 minute wav file"
return 5
fi
if [[ ${#clean_file_list[@]} -eq ${#raw_file_list[@]} ]]; then
wd_logger 2 "raw_file_list[@] is clean"
else
if [[ ${#clean_file_list[@]} -lt 2 ]]; then
wd_logger 1 "ERROR: (not really) After cleaning, clean_file_list[]='${clean_file_list[*]}' has less than the minimum 2 packets needed for the smallest WSPR packet. So return error and try again to find a good list"
return 6
fi
raw_file_list=( ${clean_file_list[@]} )
wd_logger 1 "ERROR: (not really) Cleanup trimmed $(( ${#raw_file_list[@]} - ${#clean_file_list[@]})) files from raw_file_list[], but there are still enough for a minimm sized WSPR packet"
wd_logger 1 "ERROR: raw_file_list[]= '${raw_file_list[*]}'"
wd_logger 1 "ERROR: clean_file_list[]='${clean_file_list[*]}'"
fi
### We now have a clean list of two or more full size raw files
## Search for newly completed pkt wav files lists
local epoch_of_first_raw_file=$(epoch_from_filename ${raw_file_list[0]})
local minute_of_first_raw_file=$(( ( ${epoch_of_first_raw_file} % 3600 ) / 60 ))
local epoch_of_last_raw_file=$(epoch_from_filename ${raw_file_list[-1]})
local minute_of_last_raw_file=$(( ( ${epoch_of_last_raw_file} % 3600 ) / 60 ))
wd_logger 2 "============== Starting to search for newly completed wspr wav files in the ${#raw_file_list[@]} raw files which start at time ${epoch_of_first_raw_file} = minute ${minute_of_first_raw_file} and ends at epoch ${epoch_of_last_raw_file} = minute ${minute_of_last_raw_file} ==================="
local return_list=() ### contaains zero or more WAV_SECONDS:WAV_FILE_0,WAV_FILE_1[,WAV_FILE_2...] entries
local index_of_last_file_which_should_be_flushed=$(( ${#raw_file_list[@]} - 1 )) ### By default flush all the raw_wav files, After all searches we will flush the raw_file_list[@] files with indexes up to this, since those wav files are not candidates for future pkt wav files
wd_logger 2 "Start by planning to flush all the ${#raw_file_list[@]} raw_file_list[] files up to and including index ${index_of_last_file_which_should_be_flushed}"
### For each 2/5/15/30 minute wav file we have been asked to return, serach for earliest run of one minute wav files which satisfy the needed run of needed minute wav files
local seconds_in_wspr_pkt
for seconds_in_wspr_pkt in ${target_seconds_list[@]} ; do
local minutes_in_wspr_pkt=$(( ${seconds_in_wspr_pkt} / 60 ))
local seconds_into_wspr_pkt_of_first_raw_file=$(( ${epoch_of_first_raw_file} % ${seconds_in_wspr_pkt} ))
local modulo_of_first_raw_file=$(( ${seconds_into_wspr_pkt_of_first_raw_file} / 60 ))
wd_logger 2 "============== Checking for ${seconds_in_wspr_pkt} second = ${minutes_in_wspr_pkt} minute wspr packet which is modulo_of_first_raw_file=${modulo_of_first_raw_file} in the raw wav list which start at minute ${minute_of_first_raw_file} and ends at ${minute_of_last_raw_file} =============="
### Find where to start searching for a start file in the raw_file[]. Check to see if we have returned some of these files in a previous call to this function
### The '-secs' files contain the name of the first file of a complete ${seconds_in_wspr_pkt} wav file which was previously reporeted
shopt -s nullglob
local wav_raw_pkt_sec_list=( *.wav.${seconds_in_wspr_pkt}-secs )
shopt -u nullglob
local epoch_of_first_unreported_wspr_packet
if [[ ${#wav_raw_pkt_sec_list[@]} -eq 0 ]]; then
### No previosuly reported wspr files in the raw list, so search from index 0
if [[ ${modulo_of_first_raw_file} -eq 0 ]]; then
epoch_of_first_unreported_wspr_packet=${epoch_of_first_raw_file}
wd_logger 2 "Found no previously reported ${minutes_in_wspr_pkt} minute wav_secs files and raw_file_list[0] is the first file of this wspr pkt, so first unreported wspr pkt starts in raw_file_list[0]"
else
epoch_of_first_unreported_wspr_packet=$(( ${epoch_of_first_raw_file} + ${seconds_in_wspr_pkt} - ( ${modulo_of_first_raw_file} * 60 ) ))
wd_logger 2 "Found no previously reported ${minutes_in_wspr_pkt} minute wav_secs files and raw_file_list[0] is minute ${modulo_of_first_raw_file} of this wspr pkt, so epoch_of_first_unreported_wspr_packet=${epoch_of_first_unreported_wspr_packet}"
fi
else
### We have previously reported a wspr file for this wspr pkt length
local wav_raw_pkt_sec_list_count=${#wav_raw_pkt_sec_list[@]}
if [[ ${wav_raw_pkt_sec_list_count} -gt 1 ]]; then
local wav_raw_pkt_sec_list_flush_count=$(( ${wav_raw_pkt_sec_list_count} - 1 ))
local wav_raw_pkt_sec_flush_list=( ${wav_raw_pkt_sec_list[@]:0:${wav_raw_pkt_sec_list_flush_count}} )
wd_logger 1 "For ${minutes_in_wspr_pkt} minute wspr packet search, found wav_raw_pkt_sec_list_count=${wav_raw_pkt_sec_list_count} files '${wav_raw_pkt_sec_list[*]}', not the one file needed. So flush the first ${wav_raw_pkt_sec_list_flush_count} files: ${wav_raw_pkt_sec_flush_list[*]}"
local rc
wd_rm ${wav_raw_pkt_sec_flush_list[@]}
rc=$?
if [[ ${rc} -ne 0 ]]; then
wd_logger 1 "ERROR: for ${minutes_in_wspr_pkt} minute wspr packet search, Failed flushing or archiving extra wav_raw_pkt_sec_list[]: 'flush_or_archive_raw_wav_files ${wav_archive_dir} ${wav_raw_pkt_sec_flush_list[*]}' => ${rc}"
fi
fi
local epoch_of_previously_reported_wspr_pkt=$( epoch_from_filename ${wav_raw_pkt_sec_list[-1]} )
local minute_of_previously_reported_wspr_pkt=$(( (${epoch_of_previously_reported_wspr_pkt} % 3600 ) / 60 ))
epoch_of_first_unreported_wspr_packet=$(( ${epoch_of_previously_reported_wspr_pkt} + ${seconds_in_wspr_pkt} ))
local minute_of_first_unreported_wspr_packet=$(( ( ${epoch_of_first_unreported_wspr_packet} % 3600 ) / 60 ))
wd_logger 2 "We previously reported a wspr packet that started at epoch ${epoch_of_previously_reported_wspr_pkt} = minute ${minute_of_previously_reported_wspr_pkt}, so we are now looking for a raw wav list file which starts at epoch ${epoch_of_first_unreported_wspr_packet} = minute ${minute_of_first_unreported_wspr_packet}"
fi
if [[ ${epoch_of_first_unreported_wspr_packet} -gt ${epoch_of_last_raw_file} ]]; then
### The start of an unreported wspr pkt can't be found in the current raw file list
wd_logger 2 "Next unreported wspr pkt will start at epoch ${epoch_of_first_unreported_wspr_packet}, but epoch_of_last_raw_file=${epoch_of_last_raw_file} so start of new wspr pkt isn't in raw file list"
continue
fi
### Start of unreported wav file is present in the raw wav file list
local index_of_start_wspr_packet=$(( (${epoch_of_first_unreported_wspr_packet} - ${epoch_of_first_raw_file}) / 60 ))
if [[ ${index_of_start_wspr_packet} -lt 0 ]]; then
wd_logger 1 "ERROR: index_of_start_wspr_packet=${index_of_start_wspr_packet} is less than zero"
exit 1
fi
wd_logger 2 "For ${minutes_in_wspr_pkt} minute wspr packet, first raw wav file is at index_of_start_wspr_packet=${index_of_start_wspr_packet}"
if [[ ${index_of_start_wspr_packet} -le ${index_of_last_file_which_should_be_flushed} ]]; then
### Don't flush this wspr start file from raw wav file list
local new_index_of_last_file_which_should_be_flushed=$(( ${index_of_start_wspr_packet} - 1 ))
wd_logger 2 "Found start of an unreported wspr pkt is found at raw wav list index ${index_of_start_wspr_packet} is <= to index_of_last_file_which_should_be_flushed=${index_of_last_file_which_should_be_flushed}, so change it to index ${new_index_of_last_file_which_should_be_flushed}"
index_of_last_file_which_should_be_flushed=${new_index_of_last_file_which_should_be_flushed}
fi
local index_of_end_packet=$(( ${index_of_start_wspr_packet} + ${minutes_in_wspr_pkt} - 1 ))
if [[ ${index_of_end_packet} -ge ${#raw_file_list[@]} ]]; then
### Can't find full length wspr packet, so ensure that first unreported raw wav file is preserved
wd_logger 2 "First wav file of wspr packet starts at index ${index_of_start_wspr_packet} which is for time ${epoch_of_first_unreported_wspr_packet}, but the last wav file is not yet recorded, so can't yet create wspr pkt"
continue
fi
local epoch_of_last_file_of_unreported_wspr_pkt=$( epoch_from_filename ${raw_file_list[${index_of_end_packet}]} )
local epoch_expected_of_last_file_of_unreported_wspr_pkt=$(( ${epoch_of_first_unreported_wspr_packet} + ${seconds_in_wspr_pkt} - 60 ))
if [[ ${epoch_of_last_file_of_unreported_wspr_pkt} -ne ${epoch_expected_of_last_file_of_unreported_wspr_pkt} ]]; then
wd_logger 1 "ERROR: epoch_of_last_file_of_unreported_wspr_pkt=${epoch_of_last_file_of_unreported_wspr_pkt} != epoch_expected_of_last_file_of_unreported_wspr_pkt=${epoch_expected_of_last_file_of_unreported_wspr_pkt} "
exit 1
fi
wd_logger 2 "Found a complete ${minutes_in_wspr_pkt} minute wspr packet which starts at index_of_start_wspr_packet=${index_of_start_wspr_packet} for time ${epoch_of_first_unreported_wspr_packet} and ends at index_of_end_packet=${index_of_end_packet} for epoch ${epoch_of_last_file_of_unreported_wspr_pkt}"
local comma_seperated_file_list_of_minute_raw_files=$( IFS=, ; echo -n "${raw_file_list[*]:${index_of_start_wspr_packet}:${minutes_in_wspr_pkt}}" )
local add_to_return_list="${seconds_in_wspr_pkt}:${comma_seperated_file_list_of_minute_raw_files}"
wd_logger 2 "The raw_file_list[] file ${raw_file_list[${index_of_start_wspr_packet}]} for minute ${epoch_of_first_unreported_wspr_packet} at index ${index_of_start_wspr_packet} is the start of a full ${minutes_in_wspr_pkt} minute WSPR pkt, so add '${add_to_return_list}' to the return list"
return_list+=( ${add_to_return_list} )
local wav_list_returned_file=${raw_file_list[${index_of_start_wspr_packet}]}.${seconds_in_wspr_pkt}-secs
touch -r ${raw_file_list[${index_of_start_wspr_packet}]} ${wav_list_returned_file}
wd_logger 2 "Created '${wav_list_returned_file}' so we won't return agan this list raw wav files which make up the wspr pkt files"
done ### with search for all the different wspr wav file lengths
wd_logger 2 "=========== Finished search for all wspr pkts in raw wav list ============"
if [[ ${index_of_last_file_which_should_be_flushed} -lt 0 ]] ; then
wd_logger 1 "INFO: No raw files should be flushed"
else
local files_to_flush_count=$(( ${index_of_last_file_which_should_be_flushed} + 1 ))
local rc
wd_logger 1 "INFO: We can flush or archive raw wav files up to index ${index_of_last_file_which_should_be_flushed}, so flushing raw_wav_file[] entries: '${raw_file_list[@]:0:${files_to_flush_count}}'"
flush_or_archive_raw_wav_files ${wav_archive_dir} ${raw_file_list[@]:0:${files_to_flush_count}}
rc=$?
if [[ ${rc} -ne 0 ]]; then
wd_logger 1 "ERROR: Failed flushing or archiving old raw_file_list[]: 'flush_or_archive_raw_wav_files ${wav_archive_dir} ${raw_file_list[@]:0:${files_to_flush_count}}' => ${rc}"
fi
fi
if [[ ${#return_list[@]} -ne 0 ]]; then
wd_logger 1 "INFO: Returning ${#return_list[@]} wspr pkt lists: '${return_list[*]}'"
else
wd_logger 1 "INFO: Returning no wav file lists"
fi
eval ${return_variable_name}=\"${return_list[*]}\"
return 0
}
function flush_or_archive_raw_wav_files() {
local wav_archive_dir=$1
local wav_file_list=( ${@:2} ) ### Get list of variable number of arguments from $2 to last argument
local config_archive_raw_wav_files
local rc
wd_logger 1 "wav_archive_dir=${wav_archive_dir}, rm or archive: ${wav_file_list[*]}"
get_config_file_variable config_archive_raw_wav_files "ARCHIVE_RAW_WAV_FILES"
if [[ "${config_archive_raw_wav_files}" != "yes" && -f "archive-raw-wav-files" ]]; then
wd_logger 1 "Found file 'archive-raw-wav-file', so save this raw file"
config_archive_raw_wav_files="yes"
fi
if [[ "${config_archive_raw_wav_files}" != "yes" ]]; then
wd_rm ${wav_file_list[@]}
rc=$?
if [[ ${rc} -ne 0 ]]; then
wd_logger 1 "ERROR: Failed flushing old raw_file_list[]: 'wd_rum ${wav_file_list[*]}' => ${rc}"
fi
return ${rc}
else
local rrc=0
local raw_wav_file
for raw_wav_file in ${wav_file_list[@]} ; do
queue_wav_file ${raw_wav_file} ${wav_archive_dir}
rc=$?
if [[ ${rc} -eq 0 ]]; then
wd_logger 1 "INFO: Archived wav file ${raw_wav_file}"
else
wd_logger 1 "ERROR: 'queue_wav_file ${raw_wav_file}' => $?"
rrc=${rc}
fi
done
if [[ ${rrc} -eq 0 ]]; then
wd_logger 1 "INFO: Archived all wav files"
else
wd_logger 1 "ERROR: 'queue_wav_file ${raw_wav_file}' failed one or more times=> $${rrc}"
fi
return ${rrc}
fi
}
### Called by the decoding_daemon() to create an enhanced_spot file from the output of ALL_WSPR.TXT
### That enhanced_spot file is then posted to the subdirectory where the posting_daemon will process it (and other enhanced_spot files if this receiver is part of a MERGEd group)
### For future reference, here is the output lines in ALL_WSPR.TXT taken from the wsjt-x 2.1-2 source code:
# In WSJT-x v 2.2+, the wsprd decoder was enhanced. That new wsprd can be detected because it outputs 17 fields to each line of ALL_WSPR.TXT
# fprintf(fall_wspr, "%6s %4s %3.0f %5.2f %11.7f %-22s %2d %5.2f %2d %2d %4d %2d %3d %5u %5d \n",
# date, time, snr, dt, freq, message, (int)drift, sync, ipass+1, blocksize, jitter, decodetype, nhardmin, cycles/81, metric);
declare FIELD_COUNT_DECODE_LINE_WITH_GRID=19 ### wsprd v2.2 adds two fields and we have added the 'upload to wsprnet.org' field, so lines with a GRID will have 17 + 1 + 2 noise level fields. V3.x added spot_mode to the end of each line
declare FIELD_COUNT_DECODE_LINE_WITHOUT_GRID=$((FIELD_COUNT_DECODE_LINE_WITH_GRID - 1)) ### Lines without a GRID will have one fewer field
function create_enhanced_spots_file_and_queue_to_posting_daemon () {
local real_receiver_wspr_spots_file=$1 ### file with the new spot lines found in ALL_WSPR.TXT
local spot_file_date=$2 ### These are prepended to the output file name
local spot_file_time=$3
local wspr_cycle_rms_noise=$4 ### The following fields are the same for every spot in the wspr cycle
local wspr_cycle_fft_noise=$5
local wspr_cycle_kiwi_overloads_count=$6
local real_receiver_call_sign=$7 ### For real receivers, these are taken from the conf file line
local real_receiver_grid=$8 ### But for MERGEd receivers, the posting daemon will change them to the call+grid of the MERGEd receiver
local freq_adj_mhz=$9
local proxy_upload_this_spot=0 ### This is the last field of the enhanced_spot line. If ${SIGNAL_LEVEL_UPLOAD} == "proxy" AND this is the only spot (or best spot among a MERGEd group),
### then the posting daemon will modify this last field to '1' to signal to the upload_server to forward this spot to wsprnet.org
local cached_spots_file_name="${spot_file_date}_${spot_file_time}_spots.txt"
if grep -q "<...>" ${real_receiver_wspr_spots_file} ; then
grep -v "<...>" ${real_receiver_wspr_spots_file} > no_unknown_type3_spots.txt
wd_logger 1 "Posting 'no_unknown_type3_spots.txt' since found '<...>' calls in ${real_receiver_wspr_spots_file}"
real_receiver_wspr_spots_file=no_unknown_type3_spots.txt
fi
if [[ ! ${REMOVE_WD_DUP_SPOTS-yes} =~ [Yy][Ee][Ss] ]]; then
wd_logger 1 "WD is configured to record duplicate spots, so skip duplicate removal"
else
local spot_count=$(wc -l < ${real_receiver_wspr_spots_file} )
local tx_calls=$( awk '{print $6}' ${real_receiver_wspr_spots_file} | sort -u )
local tx_calls_list=( ${tx_calls} )
if [[ ${#tx_calls_list[@]} -eq ${spot_count} ]]; then
wd_logger 1 "Found no dup spots among the ${#tx_calls_list[@]} spots in ${real_receiver_wspr_spots_file}, so record all the spots"
else