diff --git a/.github/scripts/install_and_run_vm.sh b/.github/scripts/install_and_run_vm.sh index 7f12835..b0115fb 100755 --- a/.github/scripts/install_and_run_vm.sh +++ b/.github/scripts/install_and_run_vm.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash sudo apt-get update -sudo apt-get install -y cloud-image-utils qemu qemu-kvm +sudo apt-get install cloud-image-utils qemu -y cloud-localds seed.iso user-data meta-data @@ -25,8 +25,8 @@ sudo umount /mnt/ubuntu-iso sudo rm -rf /mnt/ubuntu-iso sudo qemu-system-x86_64 --enable-kvm -smp 2 \ - -m 4G -object memory-backend-file,id=mem,size=4G,mem-path=/dev/shm,share=on -numa node,memdev=mem \ - -chardev socket,id=char0,path=/tmp/vfsd.sock -device vhost-user-fs-pci,queue-size=1024,chardev=char0,tag=myfs \ - -drive file=image.img,format=raw,cache=none,if=virtio \ - -net user,hostfwd=tcp::2222-:22 -net nic \ - -nographic -boot c + -m 4G -object memory-backend-file,id=mem,size=4G,mem-path=/dev/shm,share=on -numa node,memdev=mem \ + -chardev socket,id=char0,path=/tmp/vfsd.sock -device vhost-user-fs-pci,queue-size=1024,chardev=char0,tag=myfs \ + -drive file=image.img,format=raw,cache=none,if=virtio \ + -net user,hostfwd=tcp::2222-:22 -net nic \ + -nographic -boot c diff --git a/README.md b/README.md index 3538cf0..246c182 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ The following components are required: ### Install QEMU and VMs ```shell -$ sudo apt-get -y qemu # debian/ubuntu +$ sudo apt-get install qemu -y # debian/ubuntu ``` Download and install the VM, taking Ubuntu as an example: @@ -25,7 +25,7 @@ $ truncate -s 10G image.img $ sudo qemu-system-x86_64 -enable-kvm -smp 2 -m 4G \ -cdrom ubuntu-20.04.6-live-server-amd64.iso \ -drive file=image.img,format=raw,cache=none,if=virtio \ - -boot d + -no-reboot -boot d ``` ### Mount shared directory on VMs @@ -40,7 +40,7 @@ host# cargo run --release ```markdown - fs://?root= -- s3://?root=&bucket=&endpoint=®ion=&access_key_id=&secret_access_key= +- s3://?bucket=&endpoint=&access_key_id=&secret_access_key=®ion= ``` Run the VM through QEMU and create a VirtioFS device: @@ -59,12 +59,11 @@ Mount a shared directory in the VM: guest# sudo mount -t virtiofs ``` -For more examples, please refer to the test scripts in github actions of this repository. +> Notes: For more examples or some useful Ubuntu unattended autoinstall scripts, see the scripts [here](./scripts/). ## Periodic Reports During GSoC 2024 And Acknowledgements The following are reports from the implementation phase, all synchronized in the [OpenDAL dev mailing list](https://lists.apache.org/list.html?dev@opendal.apache.org). - - [05.12-05.19](./docs/reports/05.12-05.19.md) - [05.20-06.02](./docs/reports/05.20-06.02.md) - [06.02-06.27](./docs/reports/06.02-06.27.md) diff --git a/scripts/example.txt b/scripts/example.txt new file mode 100644 index 0000000..08b17bb --- /dev/null +++ b/scripts/example.txt @@ -0,0 +1,3 @@ +OpenDAL: access data freely. +OpenDAL: access data freely. +OpenDAL: access data freely. diff --git a/scripts/install_and_run_vm.sh b/scripts/install_and_run_vm.sh index c6db43d..1774771 100755 --- a/scripts/install_and_run_vm.sh +++ b/scripts/install_and_run_vm.sh @@ -1,8 +1,7 @@ #!/usr/bin/env bash sudo apt-get update -sudo apt-get install cloud-image-utils -sudo apt-get install qemu qemu-kvm +sudo apt-get install cloud-image-utils qemu -y cloud-localds seed.iso user-data meta-data @@ -11,9 +10,10 @@ wget https://releases.ubuntu.com/20.04/ubuntu-20.04.6-live-server-amd64.iso sudo mkdir /mnt/ubuntu-iso sudo mount -o loop ubuntu-20.04.6-live-server-amd64.iso /mnt/ubuntu-iso -truncate -s 20G image.img +truncate -s 10G image.img -qemu-system-x86_64 -enable-kvm -smp 8 -m 16G \ +# Notes: The user and password of the VM created unattended here are both ubuntu. +sudo qemu-system-x86_64 -enable-kvm -smp 2 -m 4G \ -drive file=image.img,format=raw,cache=none,if=virtio \ -drive file=seed.iso,format=raw,cache=none,if=virtio \ -cdrom ubuntu-20.04.6-live-server-amd64.iso \ @@ -25,7 +25,6 @@ qemu-system-x86_64 -enable-kvm -smp 8 -m 16G \ sudo umount /mnt/ubuntu-iso sudo rm -rf /mnt/ubuntu-iso -qemu-system-x86_64 -enable-kvm -smp 8 -m 16G \ +sudo qemu-system-x86_64 -enable-kvm -smp 2 -m 4G \ -drive file=image.img,format=raw,cache=none,if=virtio \ - -net user,hostfwd=tcp::2222-:22 -net nic \ -nographic -boot c diff --git a/scripts/install_and_run_vm_with_virtiofs.sh b/scripts/install_and_run_vm_with_virtiofs.sh new file mode 100755 index 0000000..38da0f3 --- /dev/null +++ b/scripts/install_and_run_vm_with_virtiofs.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +sudo apt-get update +sudo apt-get install cloud-image-utils qemu -y + +cloud-localds seed.iso user-data meta-data + +wget https://releases.ubuntu.com/20.04/ubuntu-20.04.6-live-server-amd64.iso + +sudo mkdir /mnt/ubuntu-iso +sudo mount -o loop ubuntu-20.04.6-live-server-amd64.iso /mnt/ubuntu-iso + +truncate -s 10G image.img + +# Notes: The user and password of the VM created unattended here are both ubuntu. +sudo qemu-system-x86_64 -enable-kvm -smp 2 -m 4G \ + -drive file=image.img,format=raw,cache=none,if=virtio \ + -drive file=seed.iso,format=raw,cache=none,if=virtio \ + -cdrom ubuntu-20.04.6-live-server-amd64.iso \ + -kernel /mnt/ubuntu-iso/casper/vmlinuz \ + -initrd /mnt/ubuntu-iso/casper/initrd \ + -append "console=ttyS0 autoinstall" \ + -nographic -no-reboot -boot d + +sudo umount /mnt/ubuntu-iso +sudo rm -rf /mnt/ubuntu-iso + +sudo qemu-system-x86_64 --enable-kvm -smp 2 \ + -m 4G -object memory-backend-file,id=mem,size=4G,mem-path=/dev/shm,share=on -numa node,memdev=mem \ + -chardev socket,id=char0,path=/tmp/vfsd.sock -device vhost-user-fs-pci,queue-size=1024,chardev=char0,tag=myfs \ + -drive file=image.img,format=raw,cache=none,if=virtio \ + -nographic -boot c diff --git a/scripts/run_ovfs_with_fs.sh b/scripts/run_ovfs_with_fs.sh new file mode 100755 index 0000000..e11f06c --- /dev/null +++ b/scripts/run_ovfs_with_fs.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +RUST_LOG=debug cargo run --manifest-path ../Cargo.toml --release \ + /tmp/vfsd.sock \ + fs://?root=$PWD diff --git a/scripts/run_ovfs_with_s3.sh b/scripts/run_ovfs_with_s3.sh new file mode 100755 index 0000000..11ba10b --- /dev/null +++ b/scripts/run_ovfs_with_s3.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +RUST_LOG=debug cargo run --manifest-path ../Cargo.toml --release \ + /tmp/vfsd.sock \ + "s3://?bucket=test&endpoint=http://127.0.0.1:9000&access_key_id=minioadmin&secret_access_key=minioadmin®ion=us-east-1" diff --git a/scripts/run_s3_with_minio.sh b/scripts/run_s3_with_minio.sh index 1c236fb..7b4ae8b 100755 --- a/scripts/run_s3_with_minio.sh +++ b/scripts/run_s3_with_minio.sh @@ -1,6 +1,15 @@ +#!/usr/bin/env bash + +sudo apt-get update +sudo apt-get install awscli -y + docker compose -f docker-compose-minio.yml up -d --wait export AWS_ACCESS_KEY_ID="minioadmin" export AWS_SECRET_ACCESS_KEY="minioadmin" export AWS_EC2_METADATA_DISABLED="true" + aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test +aws --endpoint-url http://127.0.0.1:9000/ s3 cp example.txt s3://test +aws --endpoint-url http://127.0.0.1:9000/ s3 ls s3://test +aws --endpoint-url http://127.0.0.1:9000/ s3 cp s3://test/example.txt - diff --git a/scripts/s3_scripts.sh b/scripts/s3_scripts.sh deleted file mode 100755 index 32585bd..0000000 --- a/scripts/s3_scripts.sh +++ /dev/null @@ -1,5 +0,0 @@ -export AWS_ACCESS_KEY_ID="minioadmin" -export AWS_SECRET_ACCESS_KEY="minioadmin" -export AWS_EC2_METADATA_DISABLED="true" - -aws --endpoint-url http://127.0.0.1:9000/ s3 ls s3://test