-
Notifications
You must be signed in to change notification settings - Fork 11
/
main
executable file
·1802 lines (1578 loc) · 67.9 KB
/
main
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/bin/bash
DRY=1 # Assume dry, '-run' flag will flip the bit later.
echo "" # Professional padding
# If the script is not outputting to a terminal, don't add colors (good for piping into 'less', among other commands)
# Otherwise, color away.
if [ -t 1 ]
then
# Some colors
declare -A colors
colors[none]="\e[0m"
colors[red]="\e[31m"
colors[green]="\e[32m"
colors[yellow]="\e[33m"
colors[blue]="\e[34m"
colors[magenta]="\e[35m"
colors[cyan]="\e[36m"
colors[lightgray]="\e[37m"
colors[white]="\e[97m"
fi
# _____ _ _
#| ___| _ _ __ ___| |_(_) ___ _ __ ___
#| |_ | | | | '_ \ / __| __| |/ _ \| '_ \/ __|
#| _|| |_| | | | | (__| |_| | (_) | | | \__ \
#|_| \__,_|_| |_|\___|\__|_|\___/|_| |_|___/
#
function printer {
if [ -z "$quiet" ] || [ "$2" == "fault" ] ; then echo -e "${1} ${colors[none]}" ; fi
}
function printHelp {
printer "This script exists to make vfio-related work less of a pain. Among other use-cases such as"
printer "for LiveCD or kernel/initramfs testing without having to setup an entire libvirt vm, quick throwaway environments for quick testing, checking if a physical drive will boot, PXE testing and imaging to physically attached drives and anything else with optional PCI/USB passthrough."
printer "Hopefully somebody finds it useful.\n"
printer "Valid example arguments: [ ${colors[blue]}There's more in the README.md! ${colors[none]}]\n"
printer " ${colors[cyan]}-avoidVirtio / -novirtio"
printer " Intentionally avoid virtio qemu hardware."
printer " Useful for a fresh Win install where you don't have the virtio iso available yet or wish to give the guest physical networking hardware"
printer " Also good for general kernel hackery where the virtio drivers aren't in the kernel or initramfs causing a panic.\n"
printer " ${colors[cyan]}-bios /usr/share/ovmf/x64/OVMF_CODE.secboot.4m.fd"
printer " Manually specify a bios for qemu to use. Required if the script cannot automatically find one."
printer " ${colors[cyan]}-noBios / -legacy / -legacyboot"
printer " Skip adding a pflash device for UEFI booting. This will boot in qemu's default BIOS mode instead of UEFI."
printer " ${colors[cyan]}-biosvars /tmp/OVMF_VARS.4m.fd"
printer " Manually specify a bios variables file for qemu to use. Needed for Secureboot / Win11."
printer " You must copy the default OVMF_VARS.fd on your system to a new unique path for your VM OS."
printer " ${colors[cyan]}-ignorevtcon / -ignoreframebuffer / -leavefb / -leaveframebuffer / -leavevtcon"
printer " Intentionally leave the vtcon and efi-framebuffer bindings as is."
printer " Added to work around kernel bug 216475 temporarily (https://bugzilla.kernel.org/show_bug.cgi?id=216475)\n"
printer " ${colors[cyan]}-image /dev/zvol/zpool/windows"
printer " A file or blockdevice to be made available to the guest. Can be specified multiple times."
printer " ${colors[cyan]}-imageformat/${colors[cyan]}-format [raw / qcow / vdi / etc]"
printer " A QEMU-recognised disk format. Must be specified after each invocation of -image."
printer " Can be omitted for virtual disk files as QEMU will guess the format. But it won't guess 'raw'"
printer " ${colors[cyan]}-nvme"
printer " Present storage to the guest virtualizing NVMe rather than using virtio-blk-pci or ide-hd (with -avoidVirtio)."
printer " This can prove useful when booting a physical installation on NVMe which lacks other storage drivers in their bootloader"
printer " And general permanent P2V scenarios. For long term usage I strongly recommend installing the virtio driver on the guest instead.\n"
printer " ${colors[cyan]}-cmdline console=ttyS0 root=/dev/vda1"
printer " ${colors[cyan]}-initrd /boot/initramfs.img"
printer " ${colors[cyan]}-kernel /boot/vmlinuz-linux"
printer " Optonally boot directly from a kernel file with a cmdline and optional initramfs if needed\n"
printer " ${colors[cyan]}-iso /path/to/a/diskimage.iso"
printer " If you're installing the OS for drivers you may wish to include this\n"
printer " ${colors[cyan]}-iommu / -iommugroups / -iommugrouping"
printer " (Try to) Print IOMMU Groups then exit. Useful for the -PCI arg.\n"
printer " ${colors[cyan]}-cputhreads / -showpairs / -showthreads / -showcpu"
printer " (Try to) Print thread parings for each core.\n"
printer " Useful if you plan to use -pinvcpus or plan to update your kernel arguments for isolation"
printer " ${colors[cyan]}-bridge br0\t\t${colors[none]}(Attach vm to a bridge with a new tap interface. Also creates the bridge if missing.)"
printer " ${colors[cyan]}-tap/-interface tap1\t${colors[none]}(Attach the VM's network card to a new tap interface tap1 - or attaches to it if existing.)"
printer " ${colors[cyan]}-hostint/-ethernet enp0s1\t${colors[none]}(Also attaches the specified host interface to the bridge when specified; allowing the VM true Layer 2 networking on the host LAN.)"
printer " If no network options are specified the default qemu NAT adapter will be used which is OK for most use-cases.\n"
printer " ${colors[cyan]}-nonet"
printer " If specified, the VM is not given a network adapter. Useful if you're passing a USB or PCI network card.\n"
printer " ${colors[cyan]}-memory 8192M / ${colors[cyan]}-memory 8G / ${colors[cyan]}-mem xxxxM / -m xT"
printer " Set VM memory (Default is half of system total memory)\n"
printer " ${colors[cyan]}-hugepages / ${colors[cyan]}-huge /optional/hugepage/mount"
printer " Try to allocate hugepages for qemu with the -mem value specified."
printer " If no hugepage mountpoint given as argument, tries /dev/hugepages which is often present."
printer " Supports hugepage preallocation from boot time and will use existing pages if free."
printer " (May want to specify -mem for this. Half of host ram may be too much.)\n"
printer " ${colors[cyan]}-hyperv"
printer " Enable hyper-v enlightenments for nested virtualization."
printer " Can help invasive anticheats play along however HyperV will need to be enabled in the guest's Features.\n"
printer " ${colors[cyan]}-hostaudio"
printer " Enable an audiodev for VM audio to go through the host's pulseaudio server."
printer " Will fail if pulseaudio is not installed on the host.\n"
printer " ${colors[cyan]}-usb 'SteelSeries|Keyboard|Xbox|1234:5678'"
printer " A regex for lsusb parsing. Matches against device lines from lsusb output.\n"
printer " ${colors[cyan]}-pci 'Realtek|NVIDIA|10ec:8168'"
printer " A regex for lspci parsing. Matches against device lines from lspci output.\n"
printer " ${colors[cyan]}-looking-glass / -lookingglass / -lg "
printer " Adds a 64MB shared memory module for the Looking Glass project and a spice server onto the guest for input from host during Looking Glass usage.\n"
printer " ${colors[cyan]}-extras '-device abc123,b,c,d=e' / -extras '-device AC97' "
printer " Accepts a string for arbitrary extra qemu arguments. Highly useful when I want to use extra qemu features without having to implement them into the script right away.\n"
printer " ${colors[cyan]}-romfile/-vbios path-to-vbios.bin"
printer " Accepts a file path to a rom file to use on any detected GPU during -pci argument processing. You should check the vbios dump you're about to use is safe before using it on a GPU, rom-parser is a good project for this.\n"
printer " ${colors[cyan]}-pinvcpus 0,1,2,3,4,5,6${colors[cyan]} / ${colors[cyan]}-pinvcpus 0,2,4,8"
printer " A comma delimited list of host threads the Guest's vcpus are allowed to execute on."
printer " (Setting this also shapes the guest CPU topology to match.)"
printer " If you've configured core isolation on the host, this is the argument for you!\n"
printer " ${colors[cyan]}-portforward tcp:2222:22 -portforward tcp:33389:3389"
printer " A colon:separated argument with a protocol, host port and destination guest port for guest portforwarding."
printer " Can be specified multiple times."
printer " Only applicable when not using a bridge (The default User-mode networking).\n"
printer " ${colors[cyan]}-quiet/-q/-silence/-s"
printer " Try to be quiet, only print if we have an error.\n"
printer " ${colors[cyan]}-secureboot"
printer " Try to search for the secboot version of your system's OVMF_CODE for VM usage.\n"
printer " You must also provide -biosvars for your secureboot-enabled VM to use.\n"
printer " ${colors[cyan]}-tpm"
printer " Start swtpm and have qemu use that for the guest's TPM needs (Added primarily for Win11 support)\n"
printer " ${colors[cyan]}-win11"
printer " Automatically enables -tpm and -secureboot.\n"
printer "Please see the README.md file for more information about the flags."
exit 1
}
function permissionsManager { # chown's things before qemu starts, but also tries to return them in the end.
isDry && return # Noop on dry runs
if ! declare -p Permissions >/dev/null 2>&1 ; then declare -g -A Permissions ; fi # Check array is ready for use.
if [ "$1" == "takeown" ] # Takes ownership of a file or dir with sudo
then
for path in "${@:2}"
do
originalOwner="$(stat -c %U ${path} 2>/dev/null)"
if [ "${originalOwner}" == "${USER}" ]
then
continue # If we already own this, do nothing.
else
# Take ownership but keep track
Permissions[${path}]=${originalOwner} ; sudo chown ${USER} ${path} 2>/dev/null
fi
done
elif [ "$1" == "return" ]
then
# Returns all stored in the array to original owner
for entry in ${!Permissions[@]}
do
originalOwner="${Permissions[$entry]}" ; sudo chown ${originalOwner} ${entry} 2>/dev/null
done
fi
}
# Check if script is in dry mode.
function isDry { if [ $DRY -eq 1 ] ; then return 0 ; else return 1 ; fi ;}
function hostIsHyperthreaded { grep -qs ' ht ' /proc/cpuinfo ;}
# Read numeric value from meminfo
function readMeminfo { if [ -n "$1" ]; then grep -m1 "$1" /proc/meminfo | grep -Eo '[0-9]+'; fi ; }
function readHugeInfo { if [ -n "$1" ] && [ -n "$2" ]; then cat /sys/kernel/mm/hugepages/hugepages-${1}kB/${2}; fi ; }
function writeHugeInfo { if [ -n "$1" ] && [ -n "$2" ]; then sudo tee /sys/kernel/mm/hugepages/hugepages-${1}kB/${2} > /dev/null; fi ; }
function isGpu { if grep -o '\[VGA controller\]' <<<$(lspci -vnnn -s "$1") > /dev/null 2>&1 ; then return 0 ; else return 1 ; fi }
function errorBindingTimeout {
if [ -n "$1" ]; then attemptedTask=$1 ; else attemptedTask="bind or unbind" ; fi
printer "${colors[red]} The device $fullBuspath ${colors[red]}// $vendorClass ${colors[red]} Was unable to $attemptedTask after 5 seconds, is something else using it?" "fault"
printer "${colors[red]} (E.g This will happen to a GPU in use by X)" "fault"
printer "${colors[red]} Giving up." "fault"
exit 1
}
# Try to undo any mess we've caused at the end of a run
function do_cleanup {
printer "\n${colors[yellow]}Cleaning up.."
permissionsManager return # Restore original permissions if any
if ! isDry
then
# Undo any custom networking
if [[ -n "${networking}" ]]; then networking stop ; fi
# Rebind the console and PCI devices if they were rebound
if [ -n "$pciREGEX" ]
then
enumeratePCIs restorebind
if [ -n "$vtconUnboundThisSession" ]; then vtconBindings bind; fi
if [ "$dm" == "seen" ]; then echo "Attempting to restore display-manager..." ; sudo systemctl start display-manager ; fi # Start DM if seen before run.
fi
if [ -n "$HUGEPAGES" ] && [ -n "$hugepagesSelfAllocated" ] # Clean up hugepages
then
echo 0 | writeHugeInfo ${hugepageSizeKB} nr_hugepages # pack up
fi
fi
[ -f '/dev/shm/looking-glass' ] && sudo rm -fv /dev/shm/looking-glass
[ -n "$swtpmUUID" ] && { kill $swtpmPid >/dev/null 2>&1 ; find /tmp/${swtpmUUID}/ -type f -delete; rmdir /tmp/${swtpmUUID} ;}
printer "${colors[green]}Cleanup complete."
exit
}
function regexCleanup {
if [ -z "$@" ] ; then echo "No argument given to regexCleanup" ; return ; fi
REGEX="$@"
if grep -qE '^\|' <<< $1; then REGEX=${REGEX#?} ; fi
if grep -qE '\|$' <<< $1; then REGEX=${REGEX%?} ; fi
echo $REGEX
}
function etherGrep {
if [ -n "${1}" ]
then
ip link show "${1}" | grep -Po '(?<=ether\ )([a-f]|[0-9]|\:)+'
else
return
fi
}
function buildGuestMac {
if [ -z "$1" ]
then
# Generate a mac address for the guest if no host int or bridge to influence from
echo "52:54:00:$(openssl rand -hex 3 | sed 's/.\{2\}/&:/g;s/:$//')"
else
echo "52:54:00:$(cut -d':' -f4,5,6 <<<${1})"
fi
}
function genTap { # Think of a free tap name.
inc=0
# Look for a suitable numbered tap interface name.
until [ ! -L /sys/class/net/tap${inc} ] || [ $inc -ge 100 ]
do
((inc++))
done
if [ $inc -ge 100 ]
then
return 1
else
echo "tap${inc}"
fi
}
function networking {
# A function for handling guest tap interfaces, bridge interfaces, host ethernet interfaces
# and tying them all together, pre-existing or not.
# Do not act if there are run issues.
[ -n "$expectLaunchIssue" ] && ! isDry && return 0
if [ "$1" == "start" ] && ! isDry
then
printer "\t\tDetermining network situation..."
# Determine how we will approach the network situation
# If we've been invoked without a tap interface name - generate a free one.
if [ -z "${tap}" ]
then
if tap=$(genTap)
then
tapAutoDiscovered=1
printer "${colors[blue]}\t\tUsing tap name ${tap} for VM"
else
printer "\t\tFailed to discover a free tap interface name to use." "fault"
return 1
fi
# Otherwise check if the user-provided tap exists already
else
if [ -L /sys/class/net/${tap} ]
then
printer "\t\tSpecified tap ${tap} already exists, will attempt to attach VM to that."
tapPreexists=1
else
printer "\t\tSpecified tap ${tap} does not exist, will create and attach VM to that."
fi
fi
# If the user hasn't specified a bridge or host interface,
# assume the VM's networking will be solely this tap interface
if [ -z "${bridge}${hostInt}" ]
then
printer "${colors[green]}\t\tNo bridge or host interface specified. The VM will be attached to tap ${tap} only."
printer "${colors[yellow]}\t\tThe ${tap} interface must be configured on the host for the VM to communicate with it."
tapOnly=1
# Otherwise if there's a host interface check it exists and is not already in use.
elif [ -n "${hostInt}" ]
then
if [ -d /sys/class/net/${hostInt} ]
then
intExists=1
else
printer "${colors[red]}\t\tinterface ${hostInt} specified but not found. Refusing to continue." "fault"
brStartFault=1
return 1
fi
if [ -L /sys/class/net/${hostInt}/master ]
then
printer "${colors[red]}\t\tHost interface ${hostInt} already belongs to a bridge. Refusing to modify its existing configuration." "fault"
brStartFault=1
return 1
fi
else # At this point the only other possibility is that a bridge interface has been specified.
printer "\t\tHost int not specified, will attach VM tap to existing bridge"
fi
# If a bridge has been defined and exists we'll attach the tap interface to it.
if [ -n "${bridge}" ]
then
if [ -d "/sys/class/net/${bridge}" ]
then
printer "${colors[green]}\t\tBridge ${bridge} exists, will attach ${tap} to that."
bridgePreexists=1
# If it doesn't exist but a hostInt has been set we will create the bridge and attach the tap and host interface to it.
elif [ -n "${hostInt}" ]
then
printer "${colors[green]}\t\tBridge ${bridge} doesn't exist but host interface ${int} has been specified."
printer "${colors[green]}\t\tWe'll create it and attach both ${tap} and ${hostInt} to it and run dhclient on it for the host's own networking."
bridgePreexists=0
else
printer "${colors[red]}\t\tbridge ${bridge} does not exist, but no host interface has been specified." "fault"
printer "${colors[red]}\t\tRefusing to create a bridge interface for no reason." "fault"
printer "${colors[red]}\t\tEither include a host interface with -hostint or drop the -bridge argument to use only a tap adapter." "fault"
printer "${colors[red]}\t\tOr drop all networking arguments to use user-mode NAT networking." "fault"
brStartFault=1
return 1
fi
fi
# Create a MAC Address for the VM based of either the host int or bridge bridge for easy L2 identification.
# Otherwise generate one at random.
tapMAC="$(buildGuestMac $(etherGrep ${hostInt:-${bridge}}))"
# Start creating things
# Create our bridge if required and stop NetworkManager if it were running.
if [ -n "${bridge}" ] && [ ${bridgePreexists} -eq 0 ]
then
# Handle NM
if systemctl is-active NetworkManager >/dev/null
then
sudo systemctl stop NetworkManager
export nm="seen"
fi
# Create the bridge and bring it up
sudo ip link add ${bridge} type bridge
sudo ip link set ${bridge} up
if [ -n "${intExists}" ]
then
# Remove IPs from int and attach it to the bridge. Copy the MAC and bring it up.
sudo ip addr flush dev ${hostInt} && \
sudo ip link set dev ${hostInt} master ${bridge} && \
sudo ip link set dev ${bridge} address $(etherGrep ${hostInt}) && \
sudo ip link set ${hostInt} up
fi
fi
# Create the tap if required
if [ -z "${tapPreexists}" ]
then
sudo ip tuntap add ${tap} mode tap
fi
# Bring the tap device up if it isn't already.
if [[ "$(</sys/class/net/${tap}/operstate)" == "down" ]]
then
sudo ip link set ${tap} up
fi
# Bring the bridge device up if it isn't already.
if [ -n "${bridge}" ]
then
if [[ "$(</sys/class/net/${bridge}/operstate)" == "down" ]]
then
sudo ip link set ${bridge} up
fi
fi
# Take ownership of the tun special device for running qemu without root.
permissionsManager takeown /dev/net/tun
# Attach our new tap interface to the bridge
if [ -n "${bridge}" ] # This check may not be required with the logic early in this function.
then
sudo ip link set ${tap} master ${bridge} && sudo ip link set ${tap} up
printer '\t\t------------------'
printer "\t\tBridge details:"
printer "\t\t\t${bridge}:"
for interface in $(ls -1 /sys/class/net/)
do
if [ -e "/sys/class/net/${interface}/master" ]
then
master=$(basename $(readlink /sys/class/net/${interface}/master))
if [ "$master" == "${bridge}" ]
then
printer "\t\t\t ${interface}"
fi
fi
done
if [ ${bridgePreexists} -eq 0 ] && [ -n "${hostInt}" ]
then
printer "\t\tRunning dhclient on ${bridge}..."
sudo dhclient -v ${bridge} 2>&1|grep '^DHCPACK'
else
printer "${colors[green]}\t\tBridge already existed, not running dhclient -r on it."
fi
fi
echo '------------------'
elif [ "$1" == "stop" ] && ! isDry && [ -z "$brStartFault" ]
then
if [ -n "${bridge}" ] && [ ${bridgePreexists} -eq 0 ]
then
printer "${colors[yellow]}Undoing our bridge..."
dhclient -r ${bridge} >/dev/null 2>&1 && printer "${colors[green]}dhcp lease released and dhclient process ended..."
sudo ip link set ${bridge} down
sudo ip link del ${bridge}
if ! ip link show ${bridge} >/dev/null 2>&1
then
printer "${colors[green]}Bridge ${bridge} removed."
fi
fi
if [ -z "${tapPreexists}" ]
then
sudo ip link set ${tap} down
sudo ip link del ${tap}
printer "${colors[green]}Tap ${tap} removed."
fi
if [ "$nm" == "seen" ]
then
sudo systemctl restart NetworkManager
printer "${colors[green]}NetworkManager restarted."
fi
fi
}
function vtconBindings { # Toggles rendering to Ctrl+Alt+F[0-9]. Needs to be unhooked for GPU passthrough.
if ! isDry
then
[ -n "$ignoreVtconn" ] && return 0
if [[ "$1" -eq 'bind' ]]; then bindBoolean="1"; bindState="$1" ; else bindBoolean="0" ; bindState="unbind" ; fi
echo efi-framebuffer.0 | sudo tee /sys/bus/platform/drivers/efi-framebuffer/$bindState >/dev/null
for vtconsole in /sys/class/vtconsole/*; do echo $bindBoolean | sudo tee $vtconsole/bind >/dev/null; done
vtconUnboundThisSession=1
fi
}
function enumerateUSBs { # Generate usb related arguments for the guest.
[ -n "$expectLaunchIssue" ] && ! isDry && return 0 # No action if issue present
echo "USB:"
lsusbOutput="$(lsusb)"
usbArgs="-device qemu-xhci" # Add a backwards compatable xhci USB3 controller to the guest by default
USBDevices="$(grep -E "$usbREGEX" <<<"$lsusbOutput")"
while read USBDevice
do
vendorProduct="$( grep -Eo '(([0-9]|[a-f]){4}|:){3}' <<<$USBDevice)"
vendor="$( cut -d':' -f1 <<<"$vendorProduct")"
product="$( cut -d':' -f2 <<<"$vendorProduct")"
Name="$(cut -d' ' -f7- <<<"$USBDevice")"
busPath="$(cut -d' ' -f2 <<<"$USBDevice")"
devicePath="$(cut -d' ' -f4 <<<"$USBDevice" | grep -Eo '[0-9]+')"
if [[ -n "$vendor" ]] && [[ -n "$product" ]]
then
newUsbArg="-device usb-host,vendorid=0x$vendor,productid=0x$product"
usbArgs+=" $newUsbArg"
printer " Matched: ${colors[blue]}$vendor:$product '$Name'"
permissionsManager takeown /dev/bus/usb/${busPath}/${devicePath} # Take ownership, required for USB storage if not root.
printer "${colors[green]} Added to USB Args as:\t$newUsbArg\n"
else
printer "${colors[red]}Skipping: '$USBDevice' as there was an issue finding it.\n"
fi
done <<<"$USBDevices"
}
function enumeratePCIs { # This makes arguments and also unbinds from drivers. Optional vfio-pci rebinding.
[ -n "$expectLaunchIssue" ] && ! isDry && ! [[ "$@" == *"restorebind"* ]] && return 0 # No action if issue present
function getPciCurrentDriver { lspci -nn -k -s "$1" | grep -oP '(?<=driver in use:\ ).*' ;}
function gpuLockHandler {
processesUsingGpu="$(sudo fuser /dev/dri/by-path/pci-${fullBuspath}* 2>/dev/null)"
if [ -n "$processesUsingGpu" ]
then
processNamesUsingGpu="$(ps $processesUsingGpu)"
if [[ "$processNamesUsingGpu" = *"Xorg"* ]]
then
printer "${colors[yellow]} It appears Xorg has latched onto this GPU, cannot unbind from driver and give to guest without killing Xorg." "fault"
if [ -z "$KILLX" ]
then
printer "${colors[yellow]} Specify -killX to allow killing your graphical session for the guest to take a card." "fault"
printer "${colors[yellow]} If you're on a dual GPU setup, consider configuring Xorg to ignore your second GPU. (See README.md)" "fault"
((++expectLaunchIssue))
return 1
else
if systemctl is-active display-manager >/dev/null && ! isDry
then
printer "${colors[yellow]} Stopping display-manager and unbinding console drivers..." "fault"
sudo systemctl stop display-manager ; export dm="seen"
else
printer "${colors[yellow]} Could not find a service for display-manager. Confused but trying to continue anyway." "fault"
fi
vtconBindings unbind
fi
else
printer "${colors[yellow]} Some processes are latched onto this card, but NOT Xorg. They must be killed to unbind this card." "fault"
printer "${colors[yellow]} Xorg can be left alive and these processes relaunched:" "fault"
for pid in ${processesUsingGpu} # Keep track of what we killed to try and restore after stealing a GPU.
do
if grep "${pid}" <<< $(ps -u $USER)
then
killedCmdlines+=("$(tr -d '\0' </proc/${pid}/cmdline)") 2>/dev/null
fi
done
kill ${processesUsingGpu}
fi
printer "$processNamesUsingGpu" # Share the problematic process list with the user
else
printer "${colors[green]} This GPU is free." "fault"
fi
}
function unbindWithTimeout { echo "$fullBuspath" | sudo timeout --signal 9 5 tee /sys/bus/pci/devices/$fullBuspath/driver/unbind >/dev/null 2>&1 ;}
function tryGracefulGpuUnbind {
printer "${colors[green]} Unbinding GPU from:\t${colors[none]}${currentDriver}..."
gpuLockHandler || return $?
unbindWithTimeout
if [ $? -eq 137 ]
then
if [ -n "$KILLX" ]
then
printer "${colors[green]} ${colors[red]}Failed... Trying again with X killed..."
gpuLockHandler killx
unbindWithTimeout
if [ $? -eq 137 ]; then errorBindingTimeout "Gpu unbind after killing X" ; return 1; fi
else
errorBindingTimeout "Gpu unbind without killing X"
fi
fi
}
printer "PCI:"
declare -A -g deviceDrivers # Array for tracking devices and their driver.
if ! lsmod|grep -q vfio_pci
then
printer "${colors[yellow]} vfio-pci isn't loaded. Loading it now."
result="$(sudo modprobe vfio-pci 2>&1)"
if [ "$?" -ne "0" ]
then
printer "$result" "fault"
((++expectLaunchIssue))
printer "${colors[red]} Could not modprobe vfio-pci. Please fix this." "fault"
if grep -qs 'Module vfio-pci not found in directory' <<< "$result"
then
printer " ${colors[red]}This error is common after a kernel upgrade pre reboot. Either reboot into your new kernel or downgrade back to the above version to avoid rebooting." "fault"
fi
return 1
fi
fi
lspciOutput="$(lspci -nn)" ; PCIDevices="$(grep -E "$pciREGEX" <<<"$lspciOutput")"
if [ -z "$PCIDevices" ]; then echo "Couldn't find any PCI devices with regex '$pciREGEX', please try another one. Panicking."; exit 1; fi
inc=0
while read PCIDevice
do
shortBuspath="$(cut -d' ' -f1<<<$PCIDevice)"
fullBuspath="0000:$shortBuspath"
iommuGroup="$(basename $(readlink /sys/bus/pci/devices/$fullBuspath/iommu_group))"
printer " Matched:\t${colors[blue]}$PCIDevice"
printer " IOMMU Group:\t${colors[blue]}${iommuGroup}"
vendorClass="$( grep -Eo '(([0-9]|[a-f]){4}|:){3}' <<<$PCIDevice )"
vendor="$( cut -d':' -f1 <<<"$vendorClass")"
class="$( cut -d':' -f2 <<<"$vendorClass")"
currentDriver="$(getPciCurrentDriver "$fullBuspath")" # Always read the current driver for script usage.
if [ -n "${deviceDrivers["$vendorClass"]}" ]
then
originalDriver="${deviceDrivers["$vendorClass"]}"
fi
if ! [[ "$@" == *"restorebind"* ]] # Do our initial driver checks and changes unless we're here to restore an original driver right now.
then
if [ -n "$currentDriver" ]
then
deviceDrivers["$vendorClass"]="$currentDriver" # Add the PCI card's driver to the driver array for later.
originalDriver="${deviceDrivers["$vendorClass"]}"
if [ "$currentDriver" == "vfio-pci" ]
then
printer " [INFO] ${colors[green]}Already bound to vfio-pci, leaving as is."
else
printer " [INFO] ${colors[green]}Detected driver ${originalDriver}${colors[green]} is using this device. It will be re-bound on VM exit."
fi
else
printer " [INFO] ${colors[green]}Has no driver, it will be bound to vfio-pci and will be left that way."
fi
fi
driver="${deviceDrivers["$vendorClass"]}"
if [[ "$@" == *"unbind"* ]] && isDry
then
printer " [DRY] ${colors[green]}Leaving device as is."
elif [[ "$@" == *"unbind"* ]] && ! isDry
then
if isGpu "$fullBuspath"
then
if [ -n "${currentDriver}" ] && [ ! "${currentDriver}" == "vfio-pci" ]
then
tryGracefulGpuUnbind || return $?
fi
else
unbindWithTimeout
fi
fi
if [[ "$@" == *"vfiobind"* ]] && ! isDry
then
if [ "$currentDriver" != "vfio-pci" ]
then
printer "${colors[green]} Adding ID and binding to:\t${colors[none]}vfio-pci"
echo "0x$vendor 0x$class" | sudo timeout --signal 9 5 tee /sys/bus/pci/drivers/vfio-pci/new_id >/dev/null 2>&1
if [ $? -eq 137 ]; then errorBindingTimeout "bind via new_id" ; exit 1 ; fi
echo "$fullBuspath" | sudo timeout --signal 9 5 tee /sys/bus/pci/drivers/vfio-pci/bind >/dev/null 2>&1 # Try and bind the PCI address for this device just in case ID already added and not automatically bound
if [ $? -eq 137 ]; then errorBindingTimeout bind ; exit 1 ; fi
fi
while ! [ -c /dev/vfio/$iommuGroup ] # Wait a moment for the device to initialize under vfio-pci.
do
sleep 1 && ((++iommuTimeout))
if [ $iommuTimeout -gt 5 ]
then
printer "${colors[red]} Timed out waiting for bound device to appear under /dev/vfio" "fault"
((++expectLaunchIssue))
return 1
fi
done
permissionsManager takeown /dev/vfio/${iommuGroup}
echo 1 | sudo tee /sys/bus/pci/rescan >/dev/null
elif [[ "$@" == *"vfiobind"* ]] && isDry
then
:
echo
elif [[ "$@" == *"restorebind"* ]] && ! isDry
then
if [ "${currentDriver}" == "${originalDriver}" ]
then
: # Nothing to do here
elif [ -n "${originalDriver}" ] && [ "${originalDriver}" != "vfio-pci" ]
then
printer "${colors[green]} Rebinding $vendorClass back to driver:\t${colors[none]}${originalDriver}"
echo "0x$vendor 0x$class" | sudo tee /sys/bus/pci/drivers/vfio-pci/remove_id >/dev/null 2>&1 # Removing the ID from vfio-pci is not strictly necessary.
echo "$fullBuspath" | sudo tee /sys/bus/pci/drivers/vfio-pci/unbind >/dev/null 2>&1 # Above remove_id skips this step but run it "just in case" for certain setups.
echo "$fullBuspath" | sudo tee /sys/bus/pci/drivers/${originalDriver}/bind >/dev/null 2>&1 # Try binding back to the real host driver to finish up.
if [ "$?" -eq "0" ]; then printer "${colors[green]} Successfully rebound." ; else printer "${colors[red]} Was unable to rebind it to ${originalDriver}." "fault"; fi
else
printer "${colors[green]} Device $vendorClass had no driver so it has been left bound to vfio-pci."
fi
fi
if isGpu "${fullBuspath}" && [[ "$@" != *"restorebind"* ]] # Special condition for GPUs
then
guestGetsGPU=1
pciArgs+=" -device vfio-pci,x-vga=on,multifunction=on,host=$fullBuspath,id=hostdev$inc"
if [ -n "$romfilePath" ] && [ -f "$romfilePath" ]
then
printer "${colors[green]} Romfile will be passed to the GPU on VM start.\t"
pciArgs+=",romfile=$romfilePath" #Tack romfile on the end if specified
fi
else
pciArgs+=" -device vfio-pci,host=$fullBuspath,id=hostdev$inc"
fi
((++inc))
done <<<"$PCIDevices"
if [[ $var == *"vfiobind"* ]] ; then echo -ne "Here are the completed pciArgs:\n $pciArgs"; fi
}
function setPidPrio {
# Sets a given pid argument to 'First-in First-out' scheduling policy and priority 20.
sudo chrt -f -p 20 $1 > /dev/null
}
function getQemuThreads {
if [ -z "$1" ]
then
printer "${FUNCNAME[0]} Needs a qemu pid as argument." >&2
printer "\tAlso accepts second argument as thread filter (e.g. 'CPU' or 'IO')" >&2
return 1
fi
for child in /proc/$1/task/*
do
threadName="$(cat $child/comm)"
if [ -n "$2" ] # Omit thread if match desired.
then
if ! [[ "$threadName" =~ "$2" ]]
then
continue
fi
fi
threadPid="$(basename $child)"
echo "$threadPid $threadName"
done
}
function pinVcpus {
if [ -d /proc/$1 ]
then
# First, this function will wait for all threads to be accounted for.
timeout=15 ; timer=0
while [ $(getQemuThreads $1 CPU | wc -l) -lt ${#vcpuArray[@]} ] && [ $timer -lt $timeout ]
do
((++timer))
printer "${colors[yellow]}Waiting for vcpus to wake up... $(getQemuThreads $1 CPU | wc -l) found." >&2
sleep 1
done
if [ $timer -eq $timeout ] ; then printer "${colors[red]}Was unable to see all vCPU's appear under the qemu task in time. This can happen on large launches or slower systems." "fault" >&2 ; fi
printer "Successfully found ${#vcpuArray[@]} vCPUs on the guest." >&2
vcpuPosition=0
while read vcpuPid
do
printer "${colors[green]}Pinning vCPU $vcpuPosition to host core ${vcpuArray[$vcpuPosition]}" >&2
sudo taskset -cp ${vcpuArray[$vcpuPosition]} $vcpuPid >/dev/null # Pin this vcpu to a host thread.
setPidPrio $vcpuPid
#sudo chrt -f -p 20 $vcpuPid >/dev/null # Use FIFO scheduling on qemu vcpu threads.
((++vcpuPosition))
done <<< "$(getQemuThreads $1 CPU | cut -d' ' -f1)"
fi
}
function checkFileExists {
# Check for valid Symlink or real file
if [ -e "$1" ]
then
return 0
else
printer "\t\t ${colors[red]}:$1: Doesn't appear to exist?" "fault"
((++expectLaunchIssue))
return 1
fi
}
function printCpuThreads {
unset processorArray
declare -A processorArray
eval $(awk '/^processor/ {thread=$NF;next} /^core/ { core=$NF ; printf "processorArray[" ; printf $NF ; printf "]+=" ; printf thread ; print "," }' /proc/cpuinfo)
printer "${colors[green]}Host has ${#processorArray[@]} cores. Below is the cpu thread pairs per core. Useful to know as a -pinvcpus visual aid:"
coreIndex=0
for core in $(tr ' ' '\n' <<< "${!processorArray[@]}" | sort -h | tr '\n' ' ')
do
printer "core ${coreIndex}'s threads:" ; (((coreIndex++)))
printer "\t${colors[cyan]}${processorArray[$core]}${colors[none]}"
done
exit $?
}
function printIOMMU {
printer "${colors[green]}Ok, printing IOMMU Groups then exiting..."
iommuDir="/sys/kernel/iommu_groups"
if [ -d $iommuDir ]; then for g in `ls -1v $iommuDir`; do printer "IOMMU Group $g"; for d in $iommuDir/$g/devices/* ; do printer "${colors[cyan]}\t$(lspci -nns ${d##*/})"; done; done ; else printer ${colors[red]}"Couldn't find $iommuDir" "fault" ; fi
exit $?
}
function checkIOMMU {
if [ -z "$(find /sys/class/iommu/ -type l)" ]
then
printer "${colors[red]}IOMMU directory does not appear to be functional."
printer "\tFor an Intel machine you must add 'intel_iommu=on' to your kernel boot options in your bootloader." "fault"
printer "\tIf you're on an AMD machine instead add 'iommu=1'\n" "fault"
printer "\tYou should also add 'iommu=pt' as per the Archwiki to skip PCI devices which don't support passthrough" "fault"
printer "\tIf you've already done this and rebooted your host may require extra configuration or may not be compatible with IOMMU if not a bios option to change." "fault"
printer "\tFeel free to ask about this in case it's a script fault." "fault"
((++expectLaunchIssue))
fi
}
function startVirtualTPM {
swtpmUUID=$(uuidgen)
if ! isDry
then
printer "${colors[blue]}\t\tStarting a virtual TPM for this guest"
mkdir -p /tmp/${swtpmUUID}
swtpm socket --tpmstate dir=/tmp/${swtpmUUID},mode=0600 \
--ctrl type=unixio,path=/tmp/${swtpmUUID}.sock,mode=0600 \
--log file=/tmp/${swtpmUUID}/tpm.log --terminate --tpm2 &
swtpmPid=$!
fi
TPMArgs="-chardev socket,id=chrtpm,path=/tmp/${swtpmUUID}.sock -tpmdev emulator,id=tpm0,chardev=chrtpm -device tpm-tis,tpmdev=tpm0"
}
function checkDependency {
for dependency in $@
do
which $dependency >/dev/null 2>&1 || missing="$missing $dependency "
done
if [[ -n "$missing" ]]
then
echo -ne "Couldn't find these programs:$missing\nPlease install the relevant packages or check your \$PATH."
exit 1
fi
}
function checkKvmHypervisorEligibility {
# Check for kvm module presence and host support. Try to self resolve any issues which pop up.
function cpuSupportsVirtualization { grep -Eqs "vmx|svm|0xc0f" /proc/cpuinfo ; return $? ;}
function kernelModuleExists { modinfo $1 > /dev/null 2>&1 ; return $? ;}
function kvmVendorLoaded { # Boolean return on whether vendor kvm modules are loaded
lsmod|grep -qs '^kvm_' ; return $?
}
function checkDmesgKvm { # Try to throw a descriptive error if everything else has gone wrong.
sudo dmesg -t |grep 'kvm' | sort -u | while read dmesgLine
do
case "$dmesgLine" in
*"no hardware support for"*)
if [[ ${hostCpuCheck} == "unknown" ]]
then
printer "${colors[red]}Host CPU doesn't seem to be AMD or Intel and the 'kvm' module claims your CPU or Motherboard doesn't have hardware support for virtualization."
else
printer "${colors[red]}Host detected as ${hostCpuCheck} but the ${KvmVendorModule} kernel module doesn't believe there's hardware support."
fi
printer "${colors[red]}If this isn't expected for your hardware please check your bios for any virtualization-relevant flags to enable."
printer "${colors[red]}If the issue persists or the script has gotten something wrong feel free to make an issue on github to look into this."
return 1
;;
*"disabled by bios"*)
printer "${colors[red]}Host CPU detected as ${hostCpuCheck} but the ${KvmVendorModule} kernel module believes you haven't enabled all relevant support features in your bios"
printer "${colors[red]}Visit your bios and enable the relevant ${hostVirtualizationBiosFlag} flags before trying again."
return 1
;;
esac
done
}
function tryLoadKvmVendor { # Try to load the appropriate vendor module, unloading it first if present to potentially produce useful dmesg output.
case $(grep -m1 'model name' /proc/cpuinfo | tr '[:upper:]' '[:lower:]') in
*amd*)
hostCpuCheck=amd
KvmVendorModule=kvm_amd
hostVirtualizationBiosFlag="AMD-Vi (Sometimes labeled: 'SVM Mode / IOMMU Mode / IOMMU')"
;;
*intel*)
hostCpuCheck=intel
KvmVendorModule=kvm_intel
hostVirtualizationBiosFlag="VT-d (Intel Virtualization Technology)"
;;
*)
echo "Not sure what CPU vendor you're with. Will try reprobing the 'kvm' module."
hostCpuCheck=unknown
KvmVendorModule=kvm
;;
esac
if kernelModuleExists ${KvmVendorModule}
then
printer "Attempting modprobe of ${KvmVendorModule}..."
sudo modprobe ${KvmVendorModule}
if [ ! -e /dev/kvm ] # If still missing: check diagnostics and give up.
then
((++expectLaunchIssue))
if checkDmesgKvm
then
printer "${colors[red]}Something else went wrong trying to get the kvm-relevant drivers active."
printer "${colors[red]}Please check your dmesg output for more information on what specifically went wrong for kvm. (e.g. 'dmesg -t|grep kvm | sort -u')"
((++expectLaunchIssue))
return 1
else
: #((++expectLaunchIssue)) # Bad
fi
fi
else
printer "${colors[red]}The ${KvmVendorModule} module isn't present. This host can't use KVM without the relevant modules available."
((++expectLaunchIssue))
return 1
fi
}
# Basic virtualization check
if ! cpuSupportsVirtualization
then
printer "${colors[red]}CPU doesn't claim to support virtualization at all [No feature flag for 'vmx', 'svm' or '0xc0f']."
printer "${colors[red]}This may be a result of bios settings or lack of hardware support."
fi
# Continued poking around for issues
if [ -e /dev/kvm ] && [ ! -c /dev/kvm ] # Theoretical invalid device file catch
then
printer "${colors[red]}Path /dev/kvm exists but is not in its expected character-device file type."
printer "${colors[red]}This must be manually resolved."
((++expectLaunchIssue))
elif [ ! -e /dev/kvm ] # If kvm devicefile missing or loaded with troubled kvm_amd / kvm_intel module, try reloading.
then
printer "${colors[yellow]}kvm module doesn't appear to be active. Finding cause..."
tryLoadKvmVendor
[ -e /dev/kvm ] && printer "${colors[blue]}KVM ready."
fi
}
function checkLibraryExists {
while [ $# -gt 0 ]
do
case "$(tr '[:upper:]' '[:lower:]'<<<$1)" in
-library)
library="$2"
shift
;;
-64)
archPath="lib64"
shift
;;
esac
shift
done
[ -z "$archPath" ] && archPath="lib"
# May slow things down on a well established system. Should add a common lookup table for common distro locations.
[ -n "$(find /usr/${archPath} -type f -name "${library}" 2>/dev/null)" ]
return $?
}
# _
# / \ _ __ __ _ ___
# / _ \ | '__/ _` / __|
# / ___ \| | | (_| \__ \
#/_/ \_\_| \__, |___/
# |___/
if [[ -z "$@" ]] ; then echo "No args seen. Printing help" ; printHelp ; fi
while [ $# -gt 0 ]
do
case "$(tr '[:upper:]' '[:lower:]'<<<$1)" in
-append|-cmdline)
append="$2"
shift
;;
-avoidvirtio|-novirtio)
avoidVirtio="1"