From 023e1da4398cdc066c97707dbd1f581c5e88c65a Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Sat, 17 Jun 2023 02:28:00 +0000 Subject: [PATCH] Add ability to parse OCI images Signed-off-by: James Sturtevant --- .github/workflows/ci.yml | 12 +- .gitignore | 1 + Cargo.lock | 503 +++++++++++++++++- Makefile | 31 +- README.md | 4 + crates/containerd-shim-wasm/Cargo.toml | 2 + .../src/sandbox/containerd.rs | 69 +++ .../containerd-shim-wasm/src/sandbox/mod.rs | 1 + .../containerd-shim-wasm/src/sandbox/oci.rs | 109 ++++ .../containerd-shim-wasmedge/src/executor.rs | 68 ++- .../containerd-shim-wasmtime/src/instance.rs | 144 +++-- crates/oci-tar-builder/src/bin.rs | 5 +- test/k8s/Dockerfile | 2 +- 13 files changed, 851 insertions(+), 100 deletions(-) create mode 100644 crates/containerd-shim-wasm/src/sandbox/containerd.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 160ba2b52..9f48fb15b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: - name: Setup OCI runtime build env run: | sudo apt -y update - sudo apt install -y pkg-config libsystemd-dev libdbus-glib-1-dev build-essential libelf-dev libseccomp-dev libclang-dev + sudo apt install -y pkg-config libsystemd-dev libdbus-glib-1-dev build-essential libelf-dev libseccomp-dev libclang-dev protobuf-compiler - name: Setup WasmEdge build env run: | make bin/wasmedge @@ -48,7 +48,7 @@ jobs: - name: Setup OCI runtime build env run: | sudo apt -y update - sudo apt install -y pkg-config libsystemd-dev libdbus-glib-1-dev build-essential libelf-dev libseccomp-dev libclang-dev + sudo apt install -y pkg-config libsystemd-dev libdbus-glib-1-dev build-essential libelf-dev libseccomp-dev libclang-dev protobuf-compiler - name: Setup WasmEdge build env run: | make bin/wasmedge @@ -100,7 +100,7 @@ jobs: - name: Setup OCI runtime build env run: | sudo apt -y update - sudo apt install -y pkg-config libsystemd-dev libdbus-glib-1-dev build-essential libelf-dev libseccomp-dev libclang-dev + sudo apt install -y pkg-config libsystemd-dev libdbus-glib-1-dev build-essential libelf-dev libseccomp-dev libclang-dev protobuf-compiler - name: run run: | echo "LD_LIBRARY_PATH=$PWD/bin/wasmedge/lib" >> $GITHUB_ENV @@ -109,4 +109,8 @@ jobs: make test/k3s - name: cleanup if: always() - run: make test/k3s/clean + run: | + # run log collection after running tests, so that we can see the logs in case pod doesn't start properly + timeout 5 bash -c -- 'sudo bin/k3s kubectl logs deployments/wasi-demo' + timeout 5 bash -c -- 'sudo bin/k3s kubectl get pods -o wide' + make test/k3s/clean diff --git a/.gitignore b/.gitignore index c99b23042..99f49351d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ test/out/img.tar !crates/wasmtime/src/bin/ test/k8s/_out release/ +.vscode/ diff --git a/Cargo.lock b/Cargo.lock index 6788875c0..3e47a21e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -125,6 +125,28 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2d098ff73c1ca148721f37baad5ea6a465a13f9573aba8641fbbbae8164a54e" +[[package]] +name = "async-stream" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.18", +] + [[package]] name = "async-trait" version = "0.1.68" @@ -142,6 +164,57 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "axum" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8175979259124331c1d7bf6586ee7e0da434155e4b2d48ec2c8386281d8df39" +dependencies = [ + "async-trait", + "axum-core", + "bitflags 1.3.2", + "bytes", + "futures-util", + "http", + "http-body", + "hyper", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.21.2" @@ -453,6 +526,21 @@ dependencies = [ "thiserror", ] +[[package]] +name = "containerd-client" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea3bda9d79e851f456e86638109e556ee70bd2fcff661cde0f214f16e094590f" +dependencies = [ + "axum-core", + "prost 0.11.9", + "prost-types 0.11.9", + "tokio", + "tonic", + "tonic-build", + "tower", +] + [[package]] name = "containerd-shim" version = "0.3.0" @@ -513,6 +601,7 @@ dependencies = [ "chrono", "clone3", "command-fds", + "containerd-client", "containerd-shim", "env_logger", "libc", @@ -528,6 +617,7 @@ dependencies = [ "signal-hook", "tempfile", "thiserror", + "tokio", "ttrpc", "ttrpc-codegen", ] @@ -1330,6 +1420,25 @@ dependencies = [ "cfg-if 0.1.10", ] +[[package]] +name = "h2" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d357c7ae988e7d2182f7d7871d0b963962420b0678b0997ce7de72001aeab782" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "half" version = "1.8.2" @@ -1387,12 +1496,82 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "http" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + [[package]] name = "humantime" version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "hyper" +version = "0.14.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + [[package]] name = "iana-time-zone" version = "0.1.57" @@ -1685,6 +1864,12 @@ dependencies = [ "libc", ] +[[package]] +name = "matchit" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" + [[package]] name = "maybe-owned" version = "0.3.4" @@ -1742,6 +1927,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1757,6 +1948,17 @@ dependencies = [ "adler", ] +[[package]] +name = "mio" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.48.0", +] + [[package]] name = "multimap" version = "0.8.3" @@ -1977,6 +2179,36 @@ dependencies = [ "indexmap", ] +[[package]] +name = "petgraph" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" +dependencies = [ + "fixedbitset 0.4.2", + "indexmap", +] + +[[package]] +name = "pin-project" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c95a7476719eab1e366eaf73d0260af3021184f18177925b07f54b30089ceead" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.18", +] + [[package]] name = "pin-project-lite" version = "0.2.9" @@ -2051,6 +2283,16 @@ dependencies = [ "yansi", ] +[[package]] +name = "prettyplease" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" +dependencies = [ + "proc-macro2", + "syn 1.0.109", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -2115,7 +2357,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de5e2533f59d08fcf364fd374ebda0692a70bd6d7e66ef97f306f45c6c5d8020" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.8.0", +] + +[[package]] +name = "prost" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +dependencies = [ + "bytes", + "prost-derive 0.11.9", ] [[package]] @@ -2129,9 +2381,31 @@ dependencies = [ "itertools", "log", "multimap", - "petgraph", - "prost", - "prost-types", + "petgraph 0.5.1", + "prost 0.8.0", + "prost-types 0.8.0", + "tempfile", + "which", +] + +[[package]] +name = "prost-build" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" +dependencies = [ + "bytes", + "heck 0.4.1", + "itertools", + "lazy_static", + "log", + "multimap", + "petgraph 0.6.3", + "prettyplease", + "prost 0.11.9", + "prost-types 0.11.9", + "regex", + "syn 1.0.109", "tempfile", "which", ] @@ -2149,6 +2423,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "prost-derive" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "prost-types" version = "0.8.0" @@ -2156,7 +2443,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "603bbd6394701d13f3f25aada59c7de9d35a6a5887cfc156181234a44002771b" dependencies = [ "bytes", - "prost", + "prost 0.8.0", +] + +[[package]] +name = "prost-types" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +dependencies = [ + "prost 0.11.9", ] [[package]] @@ -2437,6 +2733,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "rustversion" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" + [[package]] name = "ryu" version = "1.0.13" @@ -2599,6 +2901,16 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +[[package]] +name = "socket2" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -2639,6 +2951,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + [[package]] name = "system-interface" version = "0.25.7" @@ -2766,6 +3084,71 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tokio" +version = "1.28.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" +dependencies = [ + "autocfg", + "bytes", + "libc", + "mio", + "num_cpus", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.48.0", +] + +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-macros" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.18", +] + +[[package]] +name = "tokio-stream" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + [[package]] name = "toml" version = "0.5.11" @@ -2775,6 +3158,83 @@ dependencies = [ "serde", ] +[[package]] +name = "tonic" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f219fad3b929bef19b1f86fbc0358d35daed8f2cac972037ac0dc10bbb8d5fb" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64 0.13.1", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost 0.11.9", + "prost-derive 0.11.9", + "tokio", + "tokio-stream", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "tracing", + "tracing-futures", +] + +[[package]] +name = "tonic-build" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bf5e9b9c0f7e0a7c027dcfaba7b2c60816c7049171f679d99ee2ff65d0de8c4" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build 0.11.9", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap", + "pin-project", + "pin-project-lite", + "rand", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + [[package]] name = "tracing" version = "0.1.37" @@ -2808,6 +3268,22 @@ dependencies = [ "once_cell", ] +[[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "pin-project", + "tracing", +] + +[[package]] +name = "try-lock" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" + [[package]] name = "ttrpc" version = "0.6.1" @@ -2842,9 +3318,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b73a5e485d4c3b5e4a5e27123886fb36ed47bfa41b7022c76714ab20e9ec224f" dependencies = [ "derive-new", - "prost", - "prost-build", - "prost-types", + "prost 0.8.0", + "prost-build 0.8.0", + "prost-types 0.8.0", "protobuf 2.28.0", "protobuf-codegen 2.28.0", "tempfile", @@ -2963,6 +3439,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -3203,7 +3688,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfd5ef82889a6a35b3790025b3fd33e6ee4902a5408c09f716c771a38c47cd10" dependencies = [ "anyhow", - "base64", + "base64 0.21.2", "bincode", "directories-next", "file-per-thread-logger", diff --git a/Makefile b/Makefile index c78c10c08..35e9bf4f0 100644 --- a/Makefile +++ b/Makefile @@ -33,21 +33,34 @@ test: cargo test --all --verbose .PHONY: install -install: +install: mkdir -p $(PREFIX)/bin $(foreach runtime,$(RUNTIMES), \ - $(INSTALL) target/$(TARGET)/containerd-shim-$(runtime)-v1 $(PREFIX)/bin/; \ - $(INSTALL) target/$(TARGET)/containerd-shim-$(runtime)d-v1 $(PREFIX)/bin/; \ - $(INSTALL) target/$(TARGET)/containerd-$(runtime)d $(PREFIX)/bin/; \ + sudo $(INSTALL) target/$(TARGET)/containerd-shim-$(runtime)-v1 $(PREFIX)/bin/; \ + sudo $(INSTALL) target/$(TARGET)/containerd-shim-$(runtime)d-v1 $(PREFIX)/bin/; \ + sudo $(INSTALL) target/$(TARGET)/containerd-$(runtime)d $(PREFIX)/bin/; \ ) +.PHONY: install/all +install/all: test-image/clean build install test-image load + +.PHONY: instal/oci/all +install/oci/all: test-image/oci/clean build install test-image/oci load/oci + .PHONY: test-image test-image: target/wasm32-wasi/$(TARGET)/img.tar -.PHONY: test-image +.PHONY: test-image/oci +test-image/oci: bin/$(TARGET)/wasi-demo-oci.tar + +.PHONY: test-image/clean test-image/clean: rm -rf target/wasm32-wasi/$(TARGET)/ +.PHONY: test-image/oci/clean +test-image/oci/clean: + rm -rf bin/$(TARGET) + .PHONY: target/wasm32-wasi/$(TARGET)/wasi-demo-app.wasm target/wasm32-wasi/$(TARGET)/wasi-demo-app.wasm: rustup target add wasm32-wasi @@ -60,6 +73,14 @@ target/wasm32-wasi/$(TARGET)/img.tar: target/wasm32-wasi/$(TARGET)/wasi-demo-app load: target/wasm32-wasi/$(TARGET)/img.tar sudo ctr -n $(CONTAINERD_NAMESPACE) image import --all-platforms $< +load/oci: bin/$(TARGET)/wasi-demo-oci.tar + sudo ctr -n $(CONTAINERD_NAMESPACE) image import --all-platforms $< + +.PHONY: +bin/$(TARGET)/wasi-demo-oci.tar: target/wasm32-wasi/$(TARGET)/wasi-demo-app.wasm + mkdir -p ${CURDIR}/bin/$(TARGET)/ + cargo run --bin oci-tar-builder -- --name wasi-demo-oci --repo localhost:5000 --module ./target/wasm32-wasi/$(TARGET)/wasi-demo-app.wasm -o ./bin/$(TARGET) + bin/kind: test/k8s/Dockerfile $(DOCKER_BUILD) --output=bin/ -f test/k8s/Dockerfile --target=kind . diff --git a/README.md b/README.md index 06f6a5b2c..3890ea093 100644 --- a/README.md +++ b/README.md @@ -261,3 +261,7 @@ So they'll continue singing it forever just because... To kill the process from demo 2, you can run in other session: `sudo ctr task kill -s SIGKILL testwasm`. The test binary supports commands for different type of functionality, check [crates/wasi-demo-app/src/main.rs](crates/wasi-demo-app/src/main.rs) to try it out. + +## Demo 3 using OCI artifacts + +todo diff --git a/crates/containerd-shim-wasm/Cargo.toml b/crates/containerd-shim-wasm/Cargo.toml index 594f2d741..10a3fb349 100644 --- a/crates/containerd-shim-wasm/Cargo.toml +++ b/crates/containerd-shim-wasm/Cargo.toml @@ -13,6 +13,7 @@ doctest = false [dependencies] containerd-shim = { workspace = true } +containerd-client = "0.3.0" anyhow = { workspace = true } serde_json = { workspace = true } oci-spec = { workspace = true } @@ -28,6 +29,7 @@ clone3 = "0.2" libc = { workspace = true } caps = "0.5" proc-mounts = "0.3" +tokio = { version = "1.28.2", features = [ "full" ] } [build-dependencies] ttrpc-codegen = { version = "0.3", optional = true } diff --git a/crates/containerd-shim-wasm/src/sandbox/containerd.rs b/crates/containerd-shim-wasm/src/sandbox/containerd.rs new file mode 100644 index 000000000..9ada6f4a4 --- /dev/null +++ b/crates/containerd-shim-wasm/src/sandbox/containerd.rs @@ -0,0 +1,69 @@ +use tokio::runtime::Runtime; + +use crate::sandbox::error::Error as ShimError; +use crate::sandbox::error::Result; +use client::tonic::{self, Streaming}; +use client::{ + services::v1::{content_client::ContentClient, ReadContentRequest, ReadContentResponse}, + tonic::transport::Channel, + with_namespace, +}; +use containerd_client as client; +use tonic::Request; + +pub struct SyncContentClient { + inner: ContentClient, + rt: Runtime, +} + +// implementation from https://tokio.rs/tokio/topics/bridging +impl SyncContentClient { + // wrapper around connection that will establish a connection and create a content client + pub fn connect(address: String) -> Result { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + + let inner = rt.block_on(async { + client::connect(address) + .await + .map_err(|err| ShimError::Others(err.to_string())) + })?; + + Ok(SyncContentClient { + inner: ContentClient::new(inner), + rt, + }) + } + + // wrapper around read that will read the entire content file + pub fn read_content(&mut self, digest: String, namespace: &str) -> Result> { + let resp: Result> = self.rt.block_on(async { + let req = ReadContentRequest { + digest, + offset: 0, + size: 0, + }; + let req = with_namespace!(req, namespace); + let response: tonic::Response> = self + .inner + .read(req) + .await + .map_err(|err| ShimError::Others(err.to_string()))?; + let mut resp_stream = response.into_inner(); + + let mut data = vec![]; + while let Some(mut next_message) = resp_stream + .message() + .await + .map_err(|err| ShimError::Others(err.to_string()))? + { + data.append(&mut next_message.data); + } + + Ok(data) + }); + + resp + } +} diff --git a/crates/containerd-shim-wasm/src/sandbox/mod.rs b/crates/containerd-shim-wasm/src/sandbox/mod.rs index 72d7ac2f3..660fef3c2 100644 --- a/crates/containerd-shim-wasm/src/sandbox/mod.rs +++ b/crates/containerd-shim-wasm/src/sandbox/mod.rs @@ -14,6 +14,7 @@ pub use instance::{EngineGetter, Instance, InstanceConfig}; pub use manager::{Sandbox as SandboxService, Service as ManagerService}; pub use shim::{Cli as ShimCli, Local}; +pub mod containerd; pub mod oci; pub mod testutil; diff --git a/crates/containerd-shim-wasm/src/sandbox/oci.rs b/crates/containerd-shim-wasm/src/sandbox/oci.rs index 909e9afc8..9865322bd 100644 --- a/crates/containerd-shim-wasm/src/sandbox/oci.rs +++ b/crates/containerd-shim-wasm/src/sandbox/oci.rs @@ -36,6 +36,39 @@ pub fn get_args(spec: &Spec) -> &[String] { } } +pub fn get_module(spec: &Spec) -> (Option, String) { + let args = get_args(spec); + + if !args.is_empty() { + let start = args[0].clone(); + let mut iterator = start.split('#'); + let mut cmd = iterator.next().unwrap().to_string(); + + let stripped = cmd.strip_prefix(std::path::MAIN_SEPARATOR); + if let Some(strpd) = stripped { + cmd = strpd.to_string(); + } + let method = iterator.next().unwrap_or("_start"); + return (Some(cmd), method.to_string()); + } + + (None, "_start".to_string()) +} + +pub fn get_oci_artifact(spec: &Spec) -> Option { + match spec.annotations().clone() { + Some(annotations) + if annotations.contains_key("application/vnd.w3c.wasm.module.v1+wasm") => + { + let containerd_module = annotations + .get("application/vnd.w3c.wasm.module.v1+wasm") + .unwrap(); + Some(containerd_module.to_string()) + } + _ => None, + } +} + pub fn spec_from_file>(path: P) -> Result { let file = File::open(path)?; let cfg: Spec = json::from_reader(file)?; @@ -160,3 +193,79 @@ pub fn setup_prestart_hooks(hooks: &Option) -> Result< } Ok(()) } + +#[cfg(test)] +mod oci_tests { + use super::*; + use oci_spec::runtime::{ProcessBuilder, RootBuilder, SpecBuilder}; + + #[test] + fn test_get_args() -> Result<()> { + let spec = SpecBuilder::default() + .root(RootBuilder::default().path("rootfs").build()?) + .process( + ProcessBuilder::default() + .cwd("/") + .args(vec!["hello.wat".to_string()]) + .build()?, + ) + .build()?; + + let args = get_args(&spec); + assert_eq!(args.len(), 1); + assert_eq!(args[0], "hello.wat"); + + Ok(()) + } + + #[test] + fn test_get_args_return_empty() -> Result<()> { + let spec = SpecBuilder::default() + .root(RootBuilder::default().path("rootfs").build()?) + .process(ProcessBuilder::default().cwd("/").args(vec![]).build()?) + .build()?; + + let args = get_args(&spec); + assert_eq!(args.len(), 0); + + Ok(()) + } + + #[test] + fn test_get_args_returns_all() -> Result<()> { + let spec = SpecBuilder::default() + .root(RootBuilder::default().path("rootfs").build()?) + .process( + ProcessBuilder::default() + .cwd("/") + .args(vec![ + "hello.wat".to_string(), + "echo".to_string(), + "hello".to_string(), + ]) + .build()?, + ) + .build()?; + + let args = get_args(&spec); + assert_eq!(args.len(), 3); + assert_eq!(args[0], "hello.wat"); + assert_eq!(args[1], "echo"); + assert_eq!(args[2], "hello"); + + Ok(()) + } + + #[test] + fn test_get_module_returns_none_when_not_present() -> Result<()> { + let spec = SpecBuilder::default() + .root(RootBuilder::default().path("rootfs").build()?) + .process(ProcessBuilder::default().cwd("/").args(vec![]).build()?) + .build()?; + + let (module, _) = get_module(&spec); + assert_eq!(module, None); + + Ok(()) + } +} diff --git a/crates/containerd-shim-wasmedge/src/executor.rs b/crates/containerd-shim-wasmedge/src/executor.rs index 1158271bc..5c4c261ad 100644 --- a/crates/containerd-shim-wasmedge/src/executor.rs +++ b/crates/containerd-shim-wasmedge/src/executor.rs @@ -1,11 +1,12 @@ use anyhow::Result; +use containerd_shim_wasm::sandbox::{containerd, oci}; use nix::unistd::{dup, dup2}; use oci_spec::runtime::Spec; use libc::{STDERR_FILENO, STDIN_FILENO, STDOUT_FILENO}; use libcontainer::workload::{Executor, ExecutorError}; +use log::debug; use std::os::unix::io::RawFd; - use wasmedge_sdk::{ config::{CommonConfigOptions, ConfigBuilder, HostRegistrationConfigOptions}, params, VmBuilder, @@ -21,16 +22,6 @@ pub struct WasmEdgeExecutor { impl Executor for WasmEdgeExecutor { fn exec(&self, spec: &Spec) -> Result<(), ExecutorError> { - // parse wasi parameters - let args = get_args(spec); - if args.is_empty() { - return Err(ExecutorError::InvalidArg); - } - - let mut cmd = args[0].clone(); - if let Some(stripped) = args[0].strip_prefix(std::path::MAIN_SEPARATOR) { - cmd = stripped.to_string(); - } let envs = env_to_wasi(spec); // create configuration with `wasi` option enabled @@ -51,15 +42,47 @@ impl Executor for WasmEdgeExecutor { .ok_or_else(|| anyhow::Error::msg("Not found wasi module")) .map_err(|err| ExecutorError::Execution(err.into()))?; + let args = oci::get_args(spec); wasi_module.initialize( Some(args.iter().map(|s| s as &str).collect()), Some(envs.iter().map(|s| s as &str).collect()), None, ); - let vm = vm - .register_module_from_file("main", cmd) - .map_err(|err| ExecutorError::Execution(err))?; + let (module_name, method) = oci::get_module(spec); + let module_name = match module_name { + Some(m) => m, + None => { + return Err(ExecutorError::Execution( + anyhow::Error::msg( + "no module provided, cannot load module from file within container", + ) + .into(), + )) + } + }; + + let vm = match oci::get_oci_artifact(spec) { + Some(oci_module) => { + debug!("loading module from annotations"); + let mut ctrd_client = containerd::SyncContentClient::connect( + "/run/containerd/containerd.sock".to_string(), + ) + .map_err(|err| ExecutorError::Execution(err.into()))?; + let module = ctrd_client + .read_content(oci_module, "default") + .map_err(|err| ExecutorError::Execution(err.into()))?; + + vm.register_module_from_bytes("main", module) + .map_err(|err| ExecutorError::Execution(err))? + } + None => { + debug!("loading module from file"); + + vm.register_module_from_file("main", module_name.clone()) + .map_err(|err| ExecutorError::Execution(err))? + } + }; if let Some(stdin) = self.stdin { let _ = dup(STDIN_FILENO); @@ -74,9 +97,8 @@ impl Executor for WasmEdgeExecutor { let _ = dup2(stderr, STDERR_FILENO); } - // TODO: How to get exit code? - // This was relatively straight forward in go, but wasi and wasmtime are totally separate things in rust - match vm.run_func(Some("main"), "_start", params!()) { + debug!("running {:?} with method {}", module_name, method); + match vm.run_func(Some("main"), method, params!()) { Ok(_) => std::process::exit(0), Err(_) => std::process::exit(137), }; @@ -91,18 +113,6 @@ impl Executor for WasmEdgeExecutor { } } -fn get_args(spec: &Spec) -> &[String] { - let p = match spec.process() { - None => return &[], - Some(p) => p, - }; - - match p.args() { - None => &[], - Some(args) => args.as_slice(), - } -} - fn env_to_wasi(spec: &Spec) -> Vec { let default = vec![]; let env = spec diff --git a/crates/containerd-shim-wasmtime/src/instance.rs b/crates/containerd-shim-wasmtime/src/instance.rs index 4fbaa7c9a..1819ca715 100644 --- a/crates/containerd-shim-wasmtime/src/instance.rs +++ b/crates/containerd-shim-wasmtime/src/instance.rs @@ -6,9 +6,9 @@ use std::thread; use anyhow::Context; use chrono::{DateTime, Utc}; use containerd_shim_wasm::sandbox::error::Error; -use containerd_shim_wasm::sandbox::exec; use containerd_shim_wasm::sandbox::instance::Wait; use containerd_shim_wasm::sandbox::oci; +use containerd_shim_wasm::sandbox::{containerd, exec}; use containerd_shim_wasm::sandbox::{EngineGetter, Instance, InstanceConfig}; use log::{debug, error}; use nix::sys::signal::SIGKILL; @@ -121,22 +121,36 @@ pub fn prepare_module( let wctx = wasi_builder.build(); debug!("wasi context ready"); - let start = args[0].clone(); - let mut iterator = start.split('#'); - let mut cmd = iterator.next().unwrap().to_string(); + let (module_name, method) = oci::get_module(spec); + let module_name = match module_name { + Some(m) => m, + None => { + return Err(WasmtimeError::Other(anyhow::format_err!( + "no module provided, cannot load module from file within container" + ))) + } + }; + + if let Some(oci_module) = oci::get_oci_artifact(spec) { + println!("loading module from annotations"); + let mut ctrd_client = + containerd::SyncContentClient::connect("/run/containerd/containerd.sock".to_string())?; + let module = ctrd_client.read_content(oci_module, "default")?; + let module = Module::from_binary(&engine, &module) + .map_err(|err| Error::Others(format!("could not load module from file: {}", err)))?; - let stripped = cmd.strip_prefix(std::path::MAIN_SEPARATOR); - if let Some(strpd) = stripped { - cmd = strpd.to_string(); + return Ok((wctx, module, method)); } - let method = iterator.next().unwrap_or("_start"); - let mod_path = oci::get_root(spec).join(cmd); - debug!("loading module from file"); + debug!( + "loading module from file in container {} with method {}", + module_name, method + ); + let mod_path = oci::get_root(spec).join(module_name); let module = Module::from_file(&engine, mod_path) .map_err(|err| Error::Others(format!("could not load module from file: {}", err)))?; - Ok((wctx, module, method.to_string())) + Ok((wctx, module, method)) } impl Instance for Wasi { @@ -279,34 +293,36 @@ mod wasitest { use super::*; // This is taken from https://github.com/bytecodealliance/wasmtime/blob/6a60e8363f50b936e4c4fc958cb9742314ff09f3/docs/WASI-tutorial.md?plain=1#L270-L298 - const WASI_HELLO_WAT: &[u8]= r#"(module - ;; Import the required fd_write WASI function which will write the given io vectors to stdout - ;; The function signature for fd_write is: - ;; (File Descriptor, *iovs, iovs_len, nwritten) -> Returns number of bytes written - (import "wasi_unstable" "fd_write" (func $fd_write (param i32 i32 i32 i32) (result i32))) - - (memory 1) - (export "memory" (memory 0)) - - ;; Write 'hello world\n' to memory at an offset of 8 bytes - ;; Note the trailing newline which is required for the text to appear - (data (i32.const 8) "hello world\n") - - (func $main (export "_start") - ;; Creating a new io vector within linear memory - (i32.store (i32.const 0) (i32.const 8)) ;; iov.iov_base - This is a pointer to the start of the 'hello world\n' string - (i32.store (i32.const 4) (i32.const 12)) ;; iov.iov_len - The length of the 'hello world\n' string - - (call $fd_write - (i32.const 1) ;; file_descriptor - 1 for stdout - (i32.const 0) ;; *iovs - The pointer to the iov array, which is stored at memory location 0 - (i32.const 1) ;; iovs_len - We're printing 1 string stored in an iov - so one. - (i32.const 20) ;; nwritten - A place in memory to store the number of bytes written + fn hello_world_module(startfn: String) -> Vec { + format!(r#"(module + ;; Import the required fd_write WASI function which will write the given io vectors to stdout + ;; The function signature for fd_write is: + ;; (File Descriptor, *iovs, iovs_len, nwritten) -> Returns number of bytes written + (import "wasi_unstable" "fd_write" (func $fd_write (param i32 i32 i32 i32) (result i32))) + + (memory 1) + (export "memory" (memory 0)) + + ;; Write 'hello world\n' to memory at an offset of 8 bytes + ;; Note the trailing newline which is required for the text to appear + (data (i32.const 8) "hello world\n") + + (func $main (export "{startfn}") + ;; Creating a new io vector within linear memory + (i32.store (i32.const 0) (i32.const 8)) ;; iov.iov_base - This is a pointer to the start of the 'hello world\n' string + (i32.store (i32.const 4) (i32.const 12)) ;; iov.iov_len - The length of the 'hello world\n' string + + (call $fd_write + (i32.const 1) ;; file_descriptor - 1 for stdout + (i32.const 0) ;; *iovs - The pointer to the iov array, which is stored at memory location 0 + (i32.const 1) ;; iovs_len - We're printing 1 string stored in an iov - so one. + (i32.const 20) ;; nwritten - A place in memory to store the number of bytes written + ) + drop ;; Discard the number of bytes written from the top of the stack ) - drop ;; Discard the number of bytes written from the top of the stack ) - ) - "#.as_bytes(); + "#).as_bytes().to_vec() + } #[test] fn test_delete_after_create() { @@ -322,40 +338,58 @@ mod wasitest { #[test] fn test_wasi() -> Result<(), Error> { - let dir = tempdir()?; - create_dir(dir.path().join("rootfs"))?; + let module = hello_world_module("_start".to_string()); + let dir = create_module_dir(module)?; - let mut f = File::create(dir.path().join("rootfs/hello.wat"))?; - f.write_all(WASI_HELLO_WAT)?; + let spec = SpecBuilder::default() + .root(RootBuilder::default().path("rootfs").build()?) + .process( + ProcessBuilder::default() + .cwd("/") + .args(vec!["hello.wat".to_string()]) + .build()?, + ) + .build()?; - let stdout = File::create(dir.path().join("stdout"))?; - drop(stdout); + spec.save(dir.path().join("config.json"))?; + + run_module(dir)?; + + Ok(()) + } + + #[test] + fn test_wasi_entrypoint() -> Result<(), Error> { + let module = hello_world_module("foo".to_string()); + let dir = create_module_dir(module)?; let spec = SpecBuilder::default() .root(RootBuilder::default().path("rootfs").build()?) .process( ProcessBuilder::default() .cwd("/") - .args(vec!["hello.wat".to_string()]) + .args(vec!["hello.wat#foo".to_string()]) .build()?, ) .build()?; spec.save(dir.path().join("config.json"))?; + run_module(dir)?; + + Ok(()) + } + + fn run_module(dir: tempfile::TempDir) -> Result<(), Error> { let mut cfg = InstanceConfig::new(Engine::default(), "test_namespace".into()); let cfg = cfg .set_bundle(dir.path().to_str().unwrap().to_string()) .set_stdout(dir.path().join("stdout").to_str().unwrap().to_string()); - let wasi = Wasi::new("test".to_string(), Some(cfg)); - wasi.start()?; - let (tx, rx) = channel(); let waiter = Wait::new(tx); wasi.wait(&waiter).unwrap(); - let res = match rx.recv_timeout(Duration::from_secs(10)) { Ok(res) => res, Err(e) => { @@ -367,14 +401,22 @@ mod wasitest { } }; assert_eq!(res.0, 0); - let output = read_to_string(dir.path().join("stdout"))?; assert_eq!(output, "hello world\n"); - wasi.delete()?; - Ok(()) } + + fn create_module_dir(module: Vec) -> Result { + let dir = tempdir()?; + println!("{}", dir.path().to_str().unwrap()); + create_dir(dir.path().join("rootfs"))?; + let mut f = File::create(dir.path().join("rootfs/hello.wat"))?; + f.write_all(&module)?; + let stdout = File::create(dir.path().join("stdout"))?; + drop(stdout); + Ok(dir) + } } impl EngineGetter for Wasi { diff --git a/crates/oci-tar-builder/src/bin.rs b/crates/oci-tar-builder/src/bin.rs index d1f82da85..967ad8122 100644 --- a/crates/oci-tar-builder/src/bin.rs +++ b/crates/oci-tar-builder/src/bin.rs @@ -53,7 +53,10 @@ pub fn main() { } } - let config = spec::ConfigBuilder::default().build().unwrap(); + let config = spec::ConfigBuilder::default() + .entrypoint(vec![args.name.clone() + ".wasm"]) + .build() + .unwrap(); let img = spec::ImageConfigurationBuilder::default() .config(config) diff --git a/test/k8s/Dockerfile b/test/k8s/Dockerfile index e95149eab..da415bdf2 100644 --- a/test/k8s/Dockerfile +++ b/test/k8s/Dockerfile @@ -12,7 +12,7 @@ ENV PATH="/root/.cargo/bin:${PATH}" RUN rustup install stable WORKDIR /shim COPY . . -RUN apt-get update && apt-get install --no-install-recommends -y build-essential git clang pkg-config libsystemd-dev libdbus-glib-1-dev build-essential libelf-dev libseccomp-dev libclang-dev +RUN apt-get update && apt-get install --no-install-recommends -y build-essential git clang pkg-config libsystemd-dev libdbus-glib-1-dev build-essential libelf-dev libseccomp-dev libclang-dev protobuf-compiler RUN curl -sSf https://raw.githubusercontent.com/WasmEdge/WasmEdge/master/utils/install.sh | bash -s -- --version=0.12.1 RUN \ --mount=type=cache,target=/usr/local/cargo/registry \