-
Notifications
You must be signed in to change notification settings - Fork 13
/
common.sh
3316 lines (2803 loc) · 86.9 KB
/
common.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/bin/bash
__argv0=$0
if [ "$__argv0" == "-bash" -o "$__argv0" == "/bin/bash" ]; then
__argv0='.'
fi
TESTNAME=`basename $__argv0`
TESTDIR=$(cd `dirname $__argv0` ; pwd)
DIR=$(cd "$(dirname ${BASH_SOURCE[0]})" &>/dev/null && pwd)
SET_MACS="$DIR/set-macs.sh"
SOS_REPORT_COLLECTOR="/.autodirect/net_linux_verification/release/doca/scripts/collect_sos_report.sh"
COREDUMP_PATH="/.autodirect/net_linux_verification/release/doca/coredumps"
OVS_MEMORY="$DIR/ovs-memory.sh"
: "${OVS_MEMORY_CSV_OUTPUT:="/workspace/ovs-memory.csv"}"
OVS_VSCTL_TIMEOUT=10
CONNECT_TIMEOUT=3
COLOR0="\033["
NOCOLOR="\033[0;0m"
RED="\033[0;31m"
GREEN="\033[0;32m"
YELLOW="\033[0;33m"
CYAN="\033[0;36m"
GRAY="\033[0;37m"
BLUE="\033[0;34m"
BLUE_BOLD="\033[0;94m"
# global var to set if test fails. should change to error but never back to
# success.
TEST_FAILED=0
# global var to use for last error msg. like errno and %m.
ERRMSG=""
VENDOR_MELLANOX="0x15b3"
<<EOT
#define PCI_DEVICE_ID_MELLANOX_CONNECTX3 0x1003
#define PCI_DEVICE_ID_MELLANOX_CONNECTX3_PRO 0x1007
#define PCI_DEVICE_ID_MELLANOX_CONNECTIB 0x1011
#define PCI_DEVICE_ID_MELLANOX_CONNECTX4 0x1013
#define PCI_DEVICE_ID_MELLANOX_CONNECTX4_LX 0x1015
{ PCI_VDEVICE(MELLANOX, 0x1011) }, /* Connect-IB */
{ PCI_VDEVICE(MELLANOX, 0x1012), MLX5_PCI_DEV_IS_VF}, /* Connect-IB VF */
{ PCI_VDEVICE(MELLANOX, 0x1013) }, /* ConnectX-4 */
{ PCI_VDEVICE(MELLANOX, 0x1014), MLX5_PCI_DEV_IS_VF}, /* ConnectX-4 VF */
{ PCI_VDEVICE(MELLANOX, 0x1015) }, /* ConnectX-4LX */
{ PCI_VDEVICE(MELLANOX, 0x1016), MLX5_PCI_DEV_IS_VF}, /* ConnectX-4LX VF */
{ PCI_VDEVICE(MELLANOX, 0x1017) }, /* ConnectX-5, PCIe 3.0 */
{ PCI_VDEVICE(MELLANOX, 0x1018), MLX5_PCI_DEV_IS_VF}, /* ConnectX-5 VF */
{ PCI_VDEVICE(MELLANOX, 0x1019) }, /* ConnectX-5 Ex */
{ PCI_VDEVICE(MELLANOX, 0x101a), MLX5_PCI_DEV_IS_VF}, /* ConnectX-5 Ex VF */
{ PCI_VDEVICE(MELLANOX, 0x101b) }, /* ConnectX-6 */
{ PCI_VDEVICE(MELLANOX, 0x101c), MLX5_PCI_DEV_IS_VF}, /* ConnectX-6 VF */
{ PCI_VDEVICE(MELLANOX, 0x101d) }, /* ConnectX-6 Dx */
{ PCI_VDEVICE(MELLANOX, 0x101e), MLX5_PCI_DEV_IS_VF}, /* ConnectX Family mlx5Gen Virtual Function */
{ PCI_VDEVICE(MELLANOX, 0x101f) }, /* ConnectX-6 LX */
{ PCI_VDEVICE(MELLANOX, 0x1021) }, /* ConnectX-7 */
EOT
declare -A PCI_MAP
declare -A TRAFFIC_INFO
# set in __test_for_devlink_compat()
devlink_compat=0
# Special variables
__ignore_errors=0
PF_IN_NS=""
function is_dpdk() {
[ "$DPDK" == 1 ] && return 0
return 1
}
function get_mlx_iface() {
local i
for i in /sys/class/net/* ; do
if [ ! -r $i/device/vendor ]; then
continue
fi
t=`cat $i/device/vendor`
if [ "$t" == "$VENDOR_MELLANOX" ]; then
. $i/uevent
NIC=$INTERFACE
echo "Found Mellanox iface $NIC"
return
fi
done
}
function __test_for_devlink_compat() {
local pci=$PCI
if is_bf_host; then
pci=$BF_PCI
__devlink_compat_dir="/sys/class/net/\$nic/compat/devlink"
elif [ -e /sys/kernel/debug/mlx5/$PCI/compat ]; then
__devlink_compat_dir="/sys/kernel/debug/mlx5/\$pci/compat"
elif [ -e /sys/class/net/$NIC/compat/devlink ]; then
__devlink_compat_dir="/sys/class/net/\$nic/compat/devlink"
fi
if [ -n "$DEVLINK_COMPAT" ]; then
devlink_compat=$DEVLINK_COMPAT
elif __devlink dev param show pci/$pci name flow_steering_mode &>/dev/null ; then
return
else
devlink_compat=${DEVLINK_COMPAT:-1}
fi
log "Using devlink compat $devlink_compat"
}
function get_nic_fw() {
ethtool -i $1 | grep firmware-version | awk {'print $2'}
}
function get_rx_bytes() {
ethtool -S $1 | grep -E 'rx_bytes_phy|vport_rx_bytes' | awk {'print $2'} | tail -1
}
function get_tx_bytes() {
ethtool -S $1 | grep -E 'tx_bytes_phy|vport_tx_bytes' | awk {'print $2'} | tail -1
}
function get_rx_pkts() {
ethtool -S $1 | grep -E 'rx_packets_phy|vport_rx_packets' | awk {'print $2'} | tail -1
}
function get_tx_pkts() {
ethtool -S $1 | grep -E 'tx_packets_phy|vport_tx_packets' | awk {'print $2'} | tail -1
}
function get_tx_pkts_ns() {
local ns=$1
local dev=$2
ip netns exec $ns cat /sys/class/net/$dev/statistics/tx_packets
}
function get_rx_pkts_ns() {
local ns=$1
local dev=$2
ip netns exec $ns cat /sys/class/net/$dev/statistics/rx_packets
}
function require_cmd() {
local i
for i in $@ ; do
if ! `which $i &>/dev/null` ; then
fail "Missing required command $i"
fi
done
}
function print_key_val() {
local m=$@
local c=$CYAN
awk "{for (i=1; i<=NF; i+=2) print \"$c\"\$i\"$NOCOLOR\", \$(i+1)}" <<< $m | xargs echo
}
function __get_device_name() {
device_name="NA"
short_device_name="NA"
local tmp=`lspci -s $PCI | cut -d\[ -f2 | tr -d ]`
if [[ "$tmp" == *"BlueField-2"* ]]; then
device_name="BlueField-2"
short_device_name="bf2"
elif [[ "$tmp" == *"BlueField-3"* ]]; then
device_name="BlueField-3"
short_device_name="bf3"
elif [ -n "$tmp" ]; then
device_name=$tmp
short_device_name=`echo $device_name | tr "[:upper:]" "[:lower:]" | sed -e 's/connectx-/cx/' -e 's/ //g'`
if [ "$short_device_name" == "cx5ex" ]; then
short_device_name="cx5"
fi
fi
}
function __get_pci_device_name() {
pci_device_name="NA"
if [ "$short_device_name" == "cx5" ]; then
pci_device_name="galil"
elif [ "$short_device_name" == "cx6" ]; then
pci_device_name="negev"
elif [ "$short_device_name" == "cx6lx" ]; then
pci_device_name="tamar"
elif [ "$short_device_name" == "cx6dx" ]; then
pci_device_name="arava"
elif [ "$short_device_name" == "cx7" ]; then
pci_device_name="carmel"
fi
}
function check_ovs_asan() {
local count=$((bf_wrap "nm /usr/sbin/ovs-vswitchd 2>/dev/null && nm -D /usr/sbin/ovs-vswitchd 2>/dev/null") | grep -E "(__sanitizer_syscall_|__asan_init)" | wc -l)
if [ $count -gt 0 ]; then
log "OVS ASAN"
IS_ASAN=1
fi
}
function is_asan() {
[ "$IS_ASAN" == 1 ] && return 0
return 1
}
function check_simx() {
if lspci -s $PCI -vvv | grep -q SimX ; then
log "SimX"
IS_SIMX=1
if [ "$ENABLE_SOS_COLLECTOR" == "1" ]; then
# [SimX - NIC] Bug SW #3991818: [OVS-DOCA, SIMX] sosreport use registers which are not supported by SimX | mlx5_access_registers: assertion failed: D0:P1:F0:V0 Dispatcher: invalid access register id
redmine_info 3991818
if redmine_bug_is_open_or_err ; then
warn "Disable sos collector on simx"
ENABLE_SOS_COLLECTOR=0
fi
fi
fi
}
function is_simx() {
[ "$IS_SIMX" == 1 ] && return 0
return 1
}
function is_not_simx() {
is_simx && return 1
return 0
}
function ofed_ver_lte() {
local ver=$1
if ! is_ofed ; then
return 1
fi
local major=`modinfo --field version mlx5_core | tr ".-" " " | awk {'print $1'}`
local minor=`modinfo --field version mlx5_core | tr ".-" " " | awk {'print $2'}`
local cur="${major}.$minor"
local o=`bc <<< "$cur <= $ver"`
if [ "$o" == 1 ]; then
return 0
fi
return 1
}
function print_mlnx_ofed_version() {
if is_ofed || is_bf_host ; then
# first try version field and fallback to ofed_info script.
local ofed_version=`bf_wrap modinfo --field version mlx5_core`
if [ -z "$ofed_version" ]; then
ofed_version=`bf_wrap ofed_info -s 2>/dev/null | tr -d :`
fi
log "MLNX_OFED $ofed_version"
fi
}
function is_lockdep_enabled() {
[ -f /proc/config.gz ] || return 1
if zcat /proc/config.gz | grep -iq 'CONFIG_LOCKDEP=y'; then
return 0
fi
return 1
}
function __setup_common() {
[ -f /etc/os-release ] && . /etc/os-release
ANSI_COLOR0="$COLOR0${ANSI_COLOR}m"
if [ -n "$PRETTY_NAME" ]; then
kmsg $PRETTY_NAME
echo -e "${ANSI_COLOR0}$PRETTY_NAME$NOCOLOR"
fi
date
log `uname -nsrp`
require_interfaces NIC NIC2
require_cmd lspci ethtool tc bc jq
fail_if_err
bf_wrap "ulimit -c unlimited ; sysctl -w fs.suid_dumpable=1"
sysfs_pci_device=`readlink -f /sys/class/net/$NIC/../../`
SRIOV_NUMVFS_NIC=$sysfs_pci_device/sriov_numvfs
sysfs_pci_device2=`readlink -f /sys/class/net/$NIC2/../../`
SRIOV_NUMVFS_NIC2=$sysfs_pci_device2/sriov_numvfs
PCI=$(basename `readlink /sys/class/net/$NIC/device`)
PCI2=$(basename `readlink /sys/class/net/$NIC2/device`)
PCI_MAP=( [$NIC]=$PCI [$NIC2]=$PCI2 )
DEVICE=`cat /sys/class/net/$NIC/device/device`
PCI_DEVICE_NUM=`printf '%d' $DEVICE`
FW=`get_nic_fw $NIC`
__get_device_name
__get_pci_device_name
status="NIC $NIC FW $FW PCI $PCI DEVICE $DEVICE $device_name"
log $status
reset_is_bf_host
is_bf_host && . $DIR/common-bf.sh
is_bf && . $DIR/common-bf.sh
print_mlnx_ofed_version
__test_for_devlink_compat
setup_expected_steering_mode
setup_iptables_legacy
clear_warn_once
kmemleak_scan_per_test && kmemleak_clear
__set_testpmd
set_ovs_debug_logs
check_ovs_asan
check_simx
simx_append_log "# TEST $TESTNAME #"
is_simx && OVS_VSCTL_TIMEOUT=100
is_lockdep_enabled && CONNECT_TIMEOUT=30
add_expected_error_for_issue 3944219 "mlx5_hwmon_dev_register failed with error code -22|\ .*syndrome .*0x7d79ae.*"
add_expected_error_for_issue 4035785 "doca_offload_entry_process.*del aux"
add_expected_error_msg "dump_create: failed to get ifindex for .*: No such device"
if is_upstream ; then
add_expected_error_for_issue 3437831 "CREATE_EQ(0x301) recovered after timeout"
add_expected_error_for_issue 3437831 "MANAGE_PAGES(0x108) recovered after timeout"
fi
add_expected_bf_error_for_issue 3883402 "eal_memalloc_alloc_seg_bulk"
check_for_local_ethaddr $NIC
check_for_local_ethaddr $NIC2
}
function check_for_local_ethaddr() {
local nic=$1
local addr=$(cat /sys/class/net/$nic/address)
local ea0=0x$(cat /sys/class/net/$nic/address | cut -d: -f1)
local is_local=$(($ea0 & 2))
[ $is_local != 0 ] && warn "Mac address of $nic $addr is a local address."
}
ovs_log_path="/var/log/openvswitch/ovs-vswitchd.log"
function set_ovs_debug_logs() {
if [ "$CLEAR_OVS_LOG" == 1 ]; then
bf_wrap "test -f $ovs_log_path && echo > $ovs_log_path"
fi
local lvl
if [ "$DPDK" != 1 ]; then
lvl+=" tc:syslog:warn"
fi
if [ "$ENABLE_OVS_DEBUG" == "1" ]; then
lvl+=" netdev_offload:file:DBG netdev_offload_tc:file:DBG tc:file:DBG"
fi
ovs_set_log_levels $lvl
}
function __set_testpmd() {
if [ -x /opt/mellanox/dpdk/bin/dpdk-testpmd ]; then
testpmd="/opt/mellanox/dpdk/bin/dpdk-testpmd"
else
testpmd="$DIR/testpmd/testpmd"
fi
}
function kmemleak_scan_per_test() {
[ "$KMEMLEAK_SCAN" == 1 ] && return 0
return 1
}
kmemleak_sysfs="/sys/kernel/debug/kmemleak"
function kmemleak_clear() {
[ -w $kmemleak_sysfs ] && echo clear > $kmemleak_sysfs
}
function kmemleak_scan() {
[ ! -w $kmemleak_sysfs ] && return
log "Initiate kmemleak scan"
# looks like we don't get a report on first scan but doing double scan works.
echo scan > $kmemleak_sysfs && echo scan > $kmemleak_sysfs
}
function clear_warn_once() {
local fs="/sys/kernel/debug/clear_warn_once"
[ -w $fs ] && echo 1 > $fs
}
function setup_iptables_legacy() {
if [ -f /usr/sbin/iptables-legacy ]; then
if update-alternatives --list 2>/dev/null | grep -w iptables | grep -q legacy ; then
return
fi
update-alternatives --set iptables /usr/sbin/iptables-legacy
fi
}
function set_trusted_vf_mode() {
local nic=$1
local pci=$(basename `readlink /sys/class/net/$nic/device`)
mlxreg -d $pci --reg_id 0xc007 --reg_len 0x40 --indexes "0x0.31:1=1" --yes --set "0x4.0:32=0x1" || fail "Failed to set trusted vf mode"
}
function is_mlxdump_supported() {
is_simx && return 1
local mode=`get_flow_steering_mode $PCI`
if [ "$mode" == "dmfs" ]; then
return 0
fi
return 1
}
function get_flow_steering_mode() {
local pci=$1
if [ "$devlink_compat" -lt 1 ]; then
__devlink dev param show pci/$pci name flow_steering_mode | grep "runtime value" | awk {'print $NF'}
else
local nic=`get_pf_nic $pci`
bf_wrap cat `devlink_compat_dir $nic`/steering_mode 2>/dev/null
fi
}
function set_flow_steering_mode() {
local pci=$1
local mode=$2
if is_bf_host; then
warn "Changing steering mode is not supported over bf host"
fi
log "Set $mode flow steering mode on $pci"
if [ "$devlink_compat" -lt 1 ]; then
__devlink dev param set pci/$pci name flow_steering_mode value $mode cmode runtime || fail "Failed to set $mode flow steering mode"
else
local nic=`get_pf_nic $pci`
echo $mode > `devlink_compat_dir $nic`/steering_mode || fail "Failed to set $mode flow steering mode"
fi
}
function show_current_steering_mode() {
local pci=`get_pf_pci`
local pci2=`get_pf_pci2`
local mode1=`get_flow_steering_mode $pci`
local mode2=`get_flow_steering_mode $pci2`
# if mode is empty assume old ofed 4.6 which doesn't support steering mode.
if [ -n "$mode1" ]; then
log "Flow steering mode for $pci is $mode1"
log "Flow steering mode for $pci2 is $mode2"
fi
}
function setup_expected_steering_mode() {
if [ -z "$STEERING_MODE" ]; then
show_current_steering_mode
return
fi
local pci=`get_pf_pci`
local pci2=`get_pf_pci2`
local mode1=`get_flow_steering_mode $pci`
local mode2=`get_flow_steering_mode $pci2`
if [ -n "$mode1" ] && [ "$mode1" != $STEERING_MODE ]; then
config_sriov 2
enable_legacy $NIC
set_flow_steering_mode $pci $STEERING_MODE
fi
if [ -n "$mode2" ] && [ "$mode2" != $STEERING_MODE ]; then
config_sriov 2 $NIC2
enable_legacy $NIC2
set_flow_steering_mode $pci2 $STEERING_MODE
fi
show_current_steering_mode
}
function enable_esw_multiport() {
local pci=`get_pf_pci`
__devlink dev param show pci/$pci name esw_multiport &>/dev/null
if [ $? -ne 0 ]; then
return
fi
log "Enable multiport eswitch"
__devlink dev param set pci/$pci name esw_multiport value 1 cmode runtime || fail "Failed to enable multiport eswitch"
}
function disable_esw_multiport() {
local pci=`get_pf_pci`
__devlink dev param show pci/$pci name esw_multiport &>/dev/null
if [ $? -ne 0 ]; then
return
fi
log "Disable multiport eswitch"
__devlink dev param set pci/$pci name esw_multiport value 0 cmode runtime
}
function config_mpesw() {
enable_lag_resource_allocation_mode
set_lag_port_select_mode "multiport_esw"
config_sriov 2
config_sriov 2 $NIC2
enable_switchdev
enable_switchdev $NIC2
enable_esw_multiport
bind_vfs $NIC
bind_vfs $NIC2
ip link set $NIC up
ip link set $NIC2 up
}
function cleanup_mpesw() {
ovs_clear_bridges
reset_tc $NIC $NIC2 $REP
clear_remote_bonding
ip netns del ns0 &> /dev/null
set_port_state_up &> /dev/null
disable_esw_multiport
restore_lag_port_select_mode
restore_lag_resource_allocation_mode
reload_modules
config_devices
ip link set $NIC up
ip link set $NIC2 up
}
function get_dev_mac() {
local device=${1:-$NIC}
cat /sys/class/net/$device/address
}
function devlink_port_eswitch_enable() {
local port=$1
if is_ofed ; then
mlxdevm port function cap set $port eswitch true || err "Failed to set port $port eswitch"
else
devlink port function set $port eswitch enable || err "Failed to set port $port eswitch"
fi
}
function devlink_port_show() {
local port=$1
if is_ofed ; then
mlxdevm port show $port
else
devlink port show $port
fi
}
function is_vf_lag_activated() {
local rc
for _ in `seq 10`; do
sleep 1
# noticed up to 10 sec on nic mode and debug kernel.
# look for "lag map" and not "modify lag map".
# "lag map" print is from create lag.
# "modify lag map" print is from modify lag.
# In later kernel only printing shared_fdb and mode.
dmesg | grep -v -E -i "(disagrees about version of symbol|Unknown symbol)" | tail -n10 | grep "shared_fdb" | grep -v "modify lag map"
rc=$?
if [ $rc -eq 0 ]; then
# wait for driver to actually create the lag and check for error.
sleep 1
dmesg | tail -n10 | grep -q "Failed to create LAG" && err "Failed to create lag"
break
fi
done
return $rc
}
function is_rh72_kernel() {
local k=`uname -r`
if [ "$k" == "3.10.0-327.el7.x86_64" ]; then
return 0 # true
fi
return 1 # false
}
function __config_bonding() {
local nic1=${1:-$NIC}
local nic2=${2:-$NIC2}
local mode=${3:-active-backup}
local xmit_hash_policy=$4
local cmd
log "Config bonding $nic1 $nic2 mode $mode"
unbind_vfs $nic1 $nic2
if is_rh72_kernel ; then
ip link add name bond0 type bond
echo 100 > /sys/class/net/bond0/bonding/miimon
echo $mode > /sys/class/net/bond0/bonding/mode
if [ -n "$xmit_hash_policy" ]; then
err "xmit_hash_policy not supported"
fi
else
cmd="ip link add name bond0 type bond mode $mode miimon 100"
[ -n "$xmit_hash_policy" ] && cmd+=" xmit_hash_policy $xmit_hash_policy"
eval $cmd || fail "Failed to create bond interface"
fi
ip link set dev $nic1 down
ip link set dev $nic2 down
ip link set dev $nic1 master bond0
local rc1=$?
ip link set dev $nic2 master bond0
local rc2=$?
if [ $rc1 -ne 0 ] || [ $rc2 -ne 0 ]; then
fail "Failed to attach devices to bond0"
return $rc1
fi
ip link set dev bond0 up
ip link set dev $nic1 up
ip link set dev $nic2 up
return 0
}
function config_bonding() {
__config_bonding $@
if ! is_vf_lag_activated ; then
err "VF LAG not activated in the driver?"
return 1
fi
reset_tc bond0
}
function clear_bonding() {
local nic1=${1:-$NIC}
local nic2=${2:-$NIC2}
log "Clear bonding"
unbind_vfs $nic1 $nic2
ip link del bond0 &>/dev/null
ip link set dev $nic1 nomaster &>/dev/null
ip link set dev $nic2 nomaster &>/dev/null
}
function change_slaves() {
title "Change active lag slave from $slave1 to $slave2"
if [[ -z "$slave1" || -z "$slave2" || -z "$remote_active" ]]; then
err "change_slaves: Missing variables"
return
fi
local tmpslave=$slave1
slave1=$slave2
slave2=$tmpslave
ifconfig $tmpslave down
if [ "$B2B" == 1 ]; then
if [ "$remote_active" == $REMOTE_NIC ]; then
remote_active=$REMOTE_NIC2
else
remote_active=$REMOTE_NIC
fi
on_remote "echo $remote_active > /sys/class/net/bond0/bonding/active_slave"
fi
sleep 2
ifconfig $tmpslave up
}
function remote_disable_sriov() {
local nic1=$REMOTE_NIC
local nic2=$REMOTE_NIC2
echo "Disabling sriov in remote server"
on_remote_exec disable_sriov
}
function config_remote_bonding() {
local nic1=$REMOTE_NIC
local nic2=$REMOTE_NIC2
local mode=${3:-active-backup}
log "Config remote bonding $nic1 $nic2 mode $mode"
on_remote modprobe -q bonding || fail "Remote missing module bonding"
clear_remote_bonding
on_remote ip link add name bond0 type bond || fail "Failed to create remote bond interface"
on_remote "echo 100 > /sys/class/net/bond0/bonding/miimon
echo $mode > /sys/class/net/bond0/bonding/mode
ip link set dev $nic1 down
ip link set dev $nic2 down
ip link set dev $nic1 master bond0
ip link set dev $nic2 master bond0
ip link set dev bond0 up
ip link set dev $nic1 up
ip link set dev $nic2 up"
}
function clear_remote_bonding() {
on_remote "ip link set dev $REMOTE_NIC nomaster &>/dev/null
ip link set dev $REMOTE_NIC2 nomaster &>/dev/null
ip link del bond0 &>/dev/null"
}
function require_mlxreg() {
require_cmd mlxreg
}
function require_mlxdump() {
require_cmd mlxdump
}
function require_mlxconfig() {
require_cmd mlxconfig
}
function require_module() {
local module
for module in $@ ; do
modprobe -q $module || fail "Missing module $module"
done
}
function require_min_kernel_5() {
local v=`uname -r | cut -d. -f1`
if [ $v -lt 5 ]; then
fail "Require minimum kernel 5"
fi
}
function simx_log() {
local ip=`ip route get 1 | grep -o "src [0-9.]*" | awk {'print $2'}`
echo "/workspace/logs/${ip}_simx.log"
}
function sleep_if_simx() {
is_simx && debug "${FUNCNAME[1]} : sleep simx" && sleep 5
}
function cloud_fw_reset() {
local ip=`ip route get 1 | grep -o "src [0-9.]*" | awk {'print $2'}`
disable_sriov
unload_modules
/workspace/cloud_tools/cloud_firmware_reset.sh -ips $ip || err "cloud_firmware_reset failed"
load_modules
}
function remote_bf_wrap() {
# overridden in common-bf.sh
on_remote "$@"
}
function remote_bf_wrap_exec() {
# overridden in common-bf.sh
on_remote_exec "$@"
}
function bf_wrap() {
# overridden in common-bf.sh
eval "$@"
}
function bf_wrap_exec() {
# overridden in common-bf.sh
eval "$@"
}
function is_bf() {
lspci -s 00:00.0 2>/dev/null | grep -wq "PCI bridge: Mellanox Technologies"
}
IS_BF_HOST=""
function is_bf_host() {
[ -n "$IS_BF_HOST" ] && return $IS_BF_HOST
# BF NIC mode should not have BF vars.
if [ -z "$BF_NIC" ] || [ -z "$BF_IP" ]; then
IS_BF_HOST=1
return 1
fi
# Same $PCI could appear in arm so return if arm.
if lspci -s 00:00.0 2>/dev/null | grep -wq "PCI bridge: Mellanox Technologies"; then
IS_BF_HOST=1
return 1
fi
lspci -s $PCI 2>/dev/null | grep -wq "Mellanox .* BlueField.* integrated"
IS_BF_HOST=$?
return $IS_BF_HOST
}
function reset_is_bf_host() {
IS_BF_HOST=""
}
function is_ofed() {
modprobe -q mlx_compat && return 0
return 1
}
function is_upstream() {
if [[ `uname -r` == *"upstream"* ]] || [[ `uname -r` == *"linust"* ]] || [[ `ofed_info -s 2>/dev/null` == *"upstream"* ]]; then
return 0 # true
fi
return 1 # false
}
function is_cloud() {
if [ -e /workspace/cloud_tools/ ]; then
return 0 # true
fi
return 1 # false
}
function fw_reset() {
log "fw reset"
if is_cloud ; then
cloud_fw_reset
else
mlxfwreset -y -d $PCI reset || err "mlxfwreset failed"
fi
wait_for_ifaces
setup_expected_steering_mode
}
function fw_config() {
log "fw config $@"
mlxconfig -y -d $PCI set $@ || err "mlxconfig failed to set $@"
}
function fw_query_val() {
mlxconfig -d $PCI q | grep $1 | awk {'print $2'}
}
function set_port_state() {
local state=${1:-UP}
title "Set $NIC port state $state"
mlxlink -d $PCI --port_state $state &>/tmp/mlxlink.log || err "Failed to set port state\n`cat /tmp/mlxlink.log`"
}
function set_port_state_up() {
set_port_state UP
}
function set_port_state_down() {
set_port_state DN
}
function scp2() {
scp -q -o ConnectTimeout=$CONNECT_TIMEOUT "$@"
}
function ssh2() {
__USING_REMOTE_SERVER=1
ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o BatchMode=yes -o ConnectTimeout=$CONNECT_TIMEOUT "$@"
local rc=$?
if [ $rc -eq 255 ]; then
debug "SSH connection $1 rc 255"
fi
return $rc
}
# Run given code on remote server which provide all function and env vars
function on_remote_exec() {
[ -z "$REMOTE_SERVER" ] && fail "No remote server configured"
__on_remote_exec $REMOTE_SERVER "$@"
}
__foo_copied=()
# use a static file to avoid creating endless temp files.
__FOO="/tmp/foo.sh"
function __foo_copy() {
if [[ "${__foo_copied[@]}" =~ $remote ]]; then
return
fi
local i
for i in _longopt _parse_help _parse_usage; do unset -f $i ; done
declare -p PCI_MAP > $__FOO
declare -p TRAFFIC_INFO >> $__FOO
set | grep -Ev "^(BASH|SHELLOPTS|UID|EUID|PPID|_OVS_VSCTL|IS_BF_HOST|COVFILE|COVERR)" >> $__FOO
echo ". /etc/os-release" >> $__FOO
scp2 $__FOO $remote:/tmp/
__foo_copied+=($remote)
}
function __on_remote_exec() {
local remote=$1
__foo_copy
ssh2 $remote ". $__FOO; ${@:2}"
}
function on_vm1() {
[ -z "$NESTED_VM_IP1" ] && fail "No vm ip1 configured"
__on_remote_clean $NESTED_VM_IP1 "$@"
}
function on_vm2() {
[ -z "$NESTED_VM_IP2" ] && fail "No vm ip2 configured"
__on_remote_clean $NESTED_VM_IP2 "$@"
}
function on_remote() {
[ -z "$REMOTE_SERVER" ] && fail "No remote server configured"
__on_remote $REMOTE_SERVER "$@"
}
function __on_remote_clean() {
local remote=$1
ssh2 $remote "${@:2}"
}
function __on_remote() {
local remote=$1
ssh2 $remote "[ -n $CONFIG ] && . $CONFIG ; ${@:2}"
}
function print_remote_test_separator() {
local remote=$1
local tmp="## TEST $TESTNAME REMOTE ##"
local count=${#tmp}
local sep=$(printf '%*s' $count | tr ' ' '#')
__on_remote $remote "echo -e \"$sep\n$tmp\n$sep\" >> /dev/kmsg"
}
__USING_REMOTE_SERVER=0
function require_remote_server() {
if [ -z "$REMOTE_SERVER" ]; then
fail "Remote server is not configured"
fi
if [ -z "$REMOTE_NIC" ]; then
fail "Remote nic is not configured"
fi
log "Remote server $REMOTE_SERVER"
on_remote true || fail "Remote command failed"
print_remote_test_separator $REMOTE_SERVER
__USING_REMOTE_SERVER=1
}
function kmsg() {
local m=$@
if [ -w /dev/kmsg ]; then
echo -e ":test: $m" >>/dev/kmsg
fi
}
function title_test_name() {
title_box "TEST $1" $YELLOW
}
function title_box() {
local title=${1:-$TESTNAME}
local color=${2:-$CYAN}
local tmp="## $title ##"
local count=${#tmp}
local sep=$(printf '%*s' $count | tr ' ' '#')
echo -e "Start test
${color}${sep}${NOCOLOR}
${color}${tmp}${NOCOLOR}
${color}${sep}${NOCOLOR}"
kmsg "Start test
$sep
$tmp
$sep"
}
function max() {
echo $(($1>$2?$1:$2))
}
function min() {
echo $(($1<$2?$1:$2))
}
function ethtool_hw_tc_offload() {
local nic="$1"
ethtool -K $nic1 hw-tc-offload on &>/dev/null
}
function reset_tc() {
local nic1
for nic1 in $@ ; do
ethtool_hw_tc_offload $nic1
tc qdisc del dev $nic1 ingress >/dev/null 2>&1 || true
tc qdisc add dev $nic1 $BLOCK_INDEX ingress $TC_ARG || err "Failed to add ingress qdisc to $nic1"
done
}
function reset_tc_cacheable() {
TC_ARG="cacheable"
reset_tc $@
unset TC_ARG
}
function reset_tc_block_index() {