From 27e04c20fe40558900f4ba4e9cf945f1aaa91a03 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Tue, 1 Aug 2023 20:14:01 +0000 Subject: [PATCH 1/2] Add ability to parse OCI images Signed-off-by: James Sturtevant --- .github/workflows/action-test-image.yml | 12 +- .github/workflows/action-test-k3s.yml | 2 +- .github/workflows/action-test-kind.yml | 14 +- .github/workflows/action-test-smoke.yml | 2 +- .github/workflows/ci.yml | 28 +- Cargo.lock | 262 +++++++++++++++++- Makefile | 44 ++- README.md | 23 ++ crates/containerd-shim-wasm/Cargo.toml | 3 + .../src/container/context.rs | 66 ++++- .../containerd-shim-wasm/src/container/mod.rs | 1 + .../src/sandbox/containerd.rs | 136 +++++++++ .../src/sandbox/instance.rs | 2 +- .../containerd-shim-wasm/src/sandbox/mod.rs | 1 + .../containerd-shim-wasm/src/sandbox/oci.rs | 10 + .../src/services/sandbox_ttrpc.rs | 8 +- .../src/sys/unix/container/executor.rs | 28 +- .../src/sys/unix/container/instance.rs | 12 +- crates/containerd-shim-wasm/src/testing.rs | 2 +- .../containerd-shim-wasmedge/src/instance.rs | 38 ++- crates/containerd-shim-wasmer/src/instance.rs | 30 +- .../containerd-shim-wasmtime/src/instance.rs | 25 +- crates/oci-tar-builder/Cargo.toml | 2 +- crates/oci-tar-builder/src/bin.rs | 67 +++-- crates/oci-tar-builder/src/lib.rs | 25 +- crates/wasi-demo-app/Cargo.toml | 2 +- scripts/setup-linux.sh | 2 +- scripts/setup-windows.sh | 1 + test/k8s/Dockerfile.oci | 35 +++ test/k8s/deploy.yaml | 2 +- 30 files changed, 775 insertions(+), 110 deletions(-) create mode 100644 crates/containerd-shim-wasm/src/sandbox/containerd.rs create mode 100644 test/k8s/Dockerfile.oci diff --git a/.github/workflows/action-test-image.yml b/.github/workflows/action-test-image.yml index e62dfc08b..b40605194 100644 --- a/.github/workflows/action-test-image.yml +++ b/.github/workflows/action-test-image.yml @@ -2,17 +2,21 @@ name: Run end to end tests on kind on: workflow_call: + inputs: + image: + type: string + default: img jobs: test-image: - name: build test image + name: build test ${{ inputs.image }} runs-on: "ubuntu-latest" steps: - uses: actions/checkout@v3 - name: build - run: make dist/img.tar + run: make dist/${{ inputs.image }}.tar - name: Upload artifacts uses: actions/upload-artifact@master with: - name: test-image - path: dist/img.tar + name: test-${{ inputs.image }} + path: dist/${{ inputs.image }}.tar diff --git a/.github/workflows/action-test-k3s.yml b/.github/workflows/action-test-k3s.yml index fecf56867..bc2f256d9 100644 --- a/.github/workflows/action-test-k3s.yml +++ b/.github/workflows/action-test-k3s.yml @@ -32,7 +32,7 @@ jobs: - name: Download test image uses: actions/download-artifact@master with: - name: test-image + name: test-img path: dist - name: run timeout-minutes: 5 diff --git a/.github/workflows/action-test-kind.yml b/.github/workflows/action-test-kind.yml index 530045c54..4de788d6a 100644 --- a/.github/workflows/action-test-kind.yml +++ b/.github/workflows/action-test-kind.yml @@ -9,10 +9,16 @@ on: runtime: required: true type: string + image: + type: string + default: img + test-command: + type: string + required: true jobs: e2e-kind: - name: e2e kind test on ${{ inputs.os }} + name: e2e kind test on ${{ inputs.os }} with ${{ inputs.image }} runs-on: ${{ inputs.os }} steps: - uses: actions/checkout@v3 @@ -32,11 +38,11 @@ jobs: - name: Download test image uses: actions/download-artifact@master with: - name: test-image + name: test-${{ inputs.image }} path: dist - name: run - timeout-minutes: 5 - run: make test/k8s-${{ inputs.runtime }} + timeout-minutes: 7 + run: ${{ inputs.test-command }} # only runs when the previous step fails - name: inspect failed pods if: failure() diff --git a/.github/workflows/action-test-smoke.yml b/.github/workflows/action-test-smoke.yml index 46da903aa..532943f5a 100644 --- a/.github/workflows/action-test-smoke.yml +++ b/.github/workflows/action-test-smoke.yml @@ -32,7 +32,7 @@ jobs: - name: Download test image uses: actions/download-artifact@master with: - name: test-image + name: test-img path: dist - name: run timeout-minutes: 5 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 81413fab5..222b5b980 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,6 +28,15 @@ jobs: runtime: ["common"] # not required, but groups jobs uses: ./.github/workflows/action-test-image.yml + test-image-oci: + name: ${{ matrix.runtime }} + strategy: + matrix: + runtime: ["common"] # not required, but groups jobs + uses: ./.github/workflows/action-test-image.yml + with: + image: img-oci + build-ubuntu: name: ${{ matrix.runtime }} strategy: @@ -67,7 +76,7 @@ jobs: os: ${{ matrix.os }} runtime: ${{ matrix.runtime }} - e2e-wasmtime: + e2e-kind: name: ${{ matrix.runtime }} needs: [build-ubuntu, test-image] strategy: @@ -79,7 +88,22 @@ jobs: with: os: ${{ matrix.os }} runtime: ${{ matrix.runtime }} - + test-command: make test/k8s-${{ matrix.runtime }} + + e2e-kind-oci: + name: ${{ matrix.runtime }} + needs: [build-ubuntu, test-image-oci] + strategy: + matrix: + os: ["ubuntu-22.04"] + runtime: ["wasmtime", "wasmedge", "wasmer"] + uses: ./.github/workflows/action-test-kind.yml + with: + os: ${{ matrix.os }} + runtime: ${{ matrix.runtime }} + image: img-oci + test-command: make test/k8s-oci-${{ matrix.runtime }} + e2e-k3s: name: ${{ matrix.runtime }} needs: [build-ubuntu, test-image] diff --git a/Cargo.lock b/Cargo.lock index 312387c1f..650d7d7de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -176,6 +176,51 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "axum" +version = "0.6.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6a1de45611fdb535bfde7b7de4fd54f4fd2b17b1737c0a59b69bf9b92074b8c" +dependencies = [ + "async-trait", + "axum-core", + "bitflags 1.3.2", + "bytes", + "futures-util", + "http", + "http-body", + "hyper", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + [[package]] name = "backtrace" version = "0.3.69" @@ -218,7 +263,7 @@ dependencies = [ "lazy_static", "lazycell", "peeking_take_while", - "prettyplease", + "prettyplease 0.2.15", "proc-macro2", "quote", "regex", @@ -508,6 +553,20 @@ dependencies = [ "thiserror", ] +[[package]] +name = "containerd-client" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcbd55a5b186b60273ed7361d18d566ede8d66db962bafd702dd4db7fd30f23f" +dependencies = [ + "prost 0.11.9", + "prost-types 0.11.9", + "tokio", + "tonic", + "tonic-build", + "tower", +] + [[package]] name = "containerd-shim" version = "0.5.0" @@ -555,11 +614,13 @@ dependencies = [ "anyhow", "caps", "chrono", + "containerd-client", "containerd-shim", "containerd-shim-wasm-test-modules", "crossbeam", "dbus", "env_logger", + "futures", "git-version", "libc", "libcontainer", @@ -571,6 +632,7 @@ dependencies = [ "serde_json", "tempfile", "thiserror", + "tokio", "ttrpc", "ttrpc-codegen", "wat", @@ -1794,6 +1856,18 @@ dependencies = [ "tokio-rustls", ] +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -2143,6 +2217,12 @@ dependencies = [ "libc", ] +[[package]] +name = "matchit" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" + [[package]] name = "maybe-owned" version = "0.3.4" @@ -2672,6 +2752,16 @@ dependencies = [ "nix 0.27.1", ] +[[package]] +name = "prettyplease" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" +dependencies = [ + "proc-macro2", + "syn 1.0.109", +] + [[package]] name = "prettyplease" version = "0.2.15" @@ -2753,7 +2843,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de5e2533f59d08fcf364fd374ebda0692a70bd6d7e66ef97f306f45c6c5d8020" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.8.0", +] + +[[package]] +name = "prost" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +dependencies = [ + "bytes", + "prost-derive 0.11.9", ] [[package]] @@ -2768,8 +2868,30 @@ dependencies = [ "log", "multimap", "petgraph 0.5.1", - "prost", - "prost-types", + "prost 0.8.0", + "prost-types 0.8.0", + "tempfile", + "which", +] + +[[package]] +name = "prost-build" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" +dependencies = [ + "bytes", + "heck 0.4.1", + "itertools", + "lazy_static", + "log", + "multimap", + "petgraph 0.6.4", + "prettyplease 0.1.25", + "prost 0.11.9", + "prost-types 0.11.9", + "regex", + "syn 1.0.109", "tempfile", "which", ] @@ -2787,6 +2909,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "prost-derive" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "prost-types" version = "0.8.0" @@ -2794,7 +2929,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "603bbd6394701d13f3f25aada59c7de9d35a6a5887cfc156181234a44002771b" dependencies = [ "bytes", - "prost", + "prost 0.8.0", +] + +[[package]] +name = "prost-types" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +dependencies = [ + "prost 0.11.9", ] [[package]] @@ -3272,6 +3416,12 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustversion" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" + [[package]] name = "ryu" version = "1.0.15" @@ -3673,6 +3823,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + [[package]] name = "system-configuration" version = "0.5.1" @@ -3848,12 +4004,24 @@ dependencies = [ "libc", "mio", "num_cpus", + "parking_lot", "pin-project-lite", + "signal-hook-registry", "socket2 0.5.4", "tokio-macros", "windows-sys 0.48.0", ] +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-macros" version = "2.1.0" @@ -3885,6 +4053,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.9" @@ -3967,6 +4146,73 @@ dependencies = [ "winnow", ] +[[package]] +name = "tonic" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" +dependencies = [ + "async-trait", + "axum", + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost 0.11.9", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-build" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6fdaae4c2c638bb70fe42803a26fbd6fc6ac8c72f5c59f67ecc2a2dcabf4b07" +dependencies = [ + "prettyplease 0.1.25", + "proc-macro2", + "prost-build 0.11.9", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + [[package]] name = "tower-service" version = "0.3.2" @@ -4048,9 +4294,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0672eb06e5663ad190c7b93b2973f5d730259859b62e4e3381301a12a7441107" dependencies = [ "derive-new", - "prost", - "prost-build", - "prost-types", + "prost 0.8.0", + "prost-build 0.8.0", + "prost-types 0.8.0", "protobuf 2.28.0", "protobuf-codegen 2.28.0", "tempfile", diff --git a/Makefile b/Makefile index 89152c174..10899c4c4 100644 --- a/Makefile +++ b/Makefile @@ -140,13 +140,33 @@ dist: $(RUNTIMES:%=dist-%); dist-%: [ -f $(PWD)/dist/bin/containerd-shim-$*-v1 ] || $(MAKE) install-$* CARGO=$(CARGO) PREFIX="$(PWD)/dist" OPT_PROFILE="$(OPT_PROFILE)" +.PHONY: dist/clean +dist/clean: + rm -rf dist + +.PHONY: install/all +install/all: test-image/clean build install test-image load + +.PHONY: install/oci/all +install/oci/all: test-image/oci/clean build install test-image/oci load/oci + .PHONY: test-image test-image: dist/img.tar -.PHONY: test-image +.PHONY: test-image/oci +test-image/oci: dist/img-oci.tar + +.PHONY: test-image/clean test-image/clean: rm -rf target/wasm32-wasi/$(OPT_PROFILE)/ +.PHONY: test-image/oci/clean +test-image/oci/clean: + rm -rf target/wasm32-wasi/$(OPT_PROFILE)/img-oci.tar + +.PHONY: demo-app +demo-app: target/wasm32-wasi/$(OPT_PROFILE)/wasi-demo-app.wasm + .PHONY: target/wasm32-wasi/$(OPT_PROFILE)/wasi-demo-app.wasm target/wasm32-wasi/$(OPT_PROFILE)/wasi-demo-app.wasm: rustup target add wasm32-wasi @@ -161,9 +181,21 @@ dist/img.tar: [ -f $(PWD)/dist/img.tar ] || $(MAKE) target/wasm32-wasi/$(OPT_PROFILE)/img.tar [ -f $(PWD)/dist/img.tar ] || cp target/wasm32-wasi/$(OPT_PROFILE)/img.tar "$@" +dist/img-oci.tar: target/wasm32-wasi/$(OPT_PROFILE)/img-oci.tar + @mkdir -p "dist/" + cp "$<" "$@" + load: dist/img.tar sudo ctr -n $(CONTAINERD_NAMESPACE) image import --all-platforms $< +load/oci: dist/img-oci.tar + sudo ../containerd/bin/ctr -n $(CONTAINERD_NAMESPACE) image import --all-platforms $< + +.PHONY: +target/wasm32-wasi/$(OPT_PROFILE)/img-oci.tar: target/wasm32-wasi/$(OPT_PROFILE)/wasi-demo-app.wasm + mkdir -p ${CURDIR}/bin/$(OPT_PROFILE)/ + cargo run --bin oci-tar-builder -- --name wasi-demo-app --repo ghcr.io/containerd/runwasi --tag latest --module ./target/wasm32-wasi/$(OPT_PROFILE)/wasi-demo-app.wasm -o target/wasm32-wasi/$(OPT_PROFILE)/img-oci.tar + bin/kind: test/k8s/Dockerfile $(DOCKER_BUILD) --output=bin/ -f test/k8s/Dockerfile --target=kind . @@ -173,6 +205,9 @@ test/k8s/_out/img-%: override CARGO=cross TARGET= TARGET_DIR= test/k8s/_out/img-%: test/k8s/Dockerfile dist-% mkdir -p $(@D) && $(DOCKER_BUILD) -f test/k8s/Dockerfile --build-arg="RUNTIME=$*" --iidfile=$(@) --load . +test/k8s/_out/img-oci-%: test/k8s/Dockerfile.oci dist-% + mkdir -p $(@D) && $(DOCKER_BUILD) -f test/k8s/Dockerfile.oci --build-arg="RUNTIME=$*" --iidfile=$(@) --load . + .PHONY: test/nginx test/nginx: docker pull docker.io/nginx:latest @@ -183,8 +218,13 @@ test/k8s/cluster-%: dist/img.tar bin/kind test/k8s/_out/img-% bin/kind create cluster --name $(KIND_CLUSTER_NAME) --image="$(shell cat test/k8s/_out/img-$*)" && \ bin/kind load image-archive --name $(KIND_CLUSTER_NAME) $(<) +.PHONY: test/k8s/cluster-oci-% +test/k8s/cluster-oci-%: dist/img-oci.tar bin/kind test/k8s/_out/img-oci-% + bin/kind create cluster --name $(KIND_CLUSTER_NAME) --image="$(shell cat test/k8s/_out/img-oci-$*)" && \ + bin/kind load image-archive --name $(KIND_CLUSTER_NAME) $(<) + .PHONY: test/k8s-% -test/k8s-%: test/k8s/cluster-% +test/k8s-%: test/k8s/clean test/k8s/cluster-% kubectl --context=kind-$(KIND_CLUSTER_NAME) apply -f test/k8s/deploy.yaml kubectl --context=kind-$(KIND_CLUSTER_NAME) wait deployment wasi-demo --for condition=Available=True --timeout=90s diff --git a/README.md b/README.md index ff3395d11..9dc15a9c7 100644 --- a/README.md +++ b/README.md @@ -322,3 +322,26 @@ So they'll continue singing it forever just because... To kill the process from demo 2, you can run in other session: `sudo ctr task kill -s SIGKILL testwasm`. The test binary supports commands for different type of functionality, check [crates/wasi-demo-app/src/main.rs](crates/wasi-demo-app/src/main.rs) to try it out. + +## Demo 3 using WASM OCI artifacts + +The previous demos run with an OCI Container image containing the wasm module in the file system. Another option is to provide a cross-platform OCI artifact that that will not have the wasm module or components in the file system of the container that wraps the wasmtime/wasmedge process. This OCI artifact can be run across any platform and provides for de-duplication in the Containerd content store among other benefits. + +To learn more about this approach checkout the [design document](https://docs.google.com/document/d/11shgC3l6gplBjWF1VJCWvN_9do51otscAm0hBDGSSAc/edit). + +> **Note**: This requires containerd components based on https://github.com/containerd/containerd/pull/8699. Both CTR and containerd need to be build with that patch. If you do not have this patch for both `containerd` and `ctr` you will end up with an error message such as `mismatched image rootfs and manifest layers` at the import and run steps + +Build and import the OCI artifact image: + +``` +make test-image/oci +make load/oci +``` + +Run the image with `sudo ctr run --rm --runtime=io.containerd.[ wasmedge | wasmtime | wasmer ].v1 --label application/vnd.bytecodealliance.wasm.module=oci.wasm ghcr.io/containerd/runwasi/wasi-demo-oci:latest testwasmoci` + +``` +sudo ctr run --rm --runtime=io.containerd.wasmtime.v1 ghcr.io/containerd/runwasi/wasi-demo-oci:latest testwasmoci wasi-demo-oci.wasm echo 'hello' +hello +exiting +``` diff --git a/crates/containerd-shim-wasm/Cargo.toml b/crates/containerd-shim-wasm/Cargo.toml index 1f1e4f327..f5cf50f08 100644 --- a/crates/containerd-shim-wasm/Cargo.toml +++ b/crates/containerd-shim-wasm/Cargo.toml @@ -29,12 +29,15 @@ tempfile = { workspace = true, optional = true } thiserror = { workspace = true } ttrpc = { workspace = true } wat = { workspace = true } +tokio = { version = "1.28.2", features = [ "full" ] } +futures = { version = "0.3.28" } [target.'cfg(unix)'.dependencies] caps = "0.5" dbus = { version = "*", features = ["vendored"] } libcontainer = { workspace = true, features = ["libseccomp", "systemd", "v1", "v2"]} nix = { workspace = true, features = ["sched", "mount"] } +containerd-client = "0.4.0" [target.'cfg(windows)'.dependencies] windows-sys = { workspace = true, features = ["Win32_Foundation", "Win32_Storage_FileSystem"] } diff --git a/crates/containerd-shim-wasm/src/container/context.rs b/crates/containerd-shim-wasm/src/container/context.rs index c0efc12e4..e9fb982e7 100644 --- a/crates/containerd-shim-wasm/src/container/context.rs +++ b/crates/containerd-shim-wasm/src/container/context.rs @@ -2,6 +2,8 @@ use std::path::{Path, PathBuf}; use oci_spec::runtime::Spec; +use crate::sandbox::oci::OciArtifact; + pub trait RuntimeContext { // ctx.args() returns arguments from the runtime spec process field, including the // path to the entrypoint executable. @@ -20,6 +22,8 @@ pub trait RuntimeContext { // "my_module.wat" -> { path: "my_module.wat", func: "_start" } // "#init" -> { path: "", func: "init" } fn wasi_entrypoint(&self) -> WasiEntrypoint; + + fn oci_artifacts(&self) -> &[OciArtifact]; } pub struct WasiEntrypoint { @@ -27,9 +31,15 @@ pub struct WasiEntrypoint { pub func: String, } -impl RuntimeContext for Spec { +pub(crate) struct WasiContext<'a> { + pub spec: &'a Spec, + pub oci_artifacts: &'a [OciArtifact], +} + +impl RuntimeContext for WasiContext<'_> { fn args(&self) -> &[String] { - self.process() + self.spec + .process() .as_ref() .and_then(|p| p.args().as_ref()) .map(|a| a.as_slice()) @@ -48,6 +58,10 @@ impl RuntimeContext for Spec { func: func.to_string(), } } + + fn oci_artifacts(&self) -> &[OciArtifact] { + self.oci_artifacts + } } #[cfg(test)] @@ -68,9 +82,13 @@ mod tests { .build()?, ) .build()?; - let spec = &spec; - let args = spec.args(); + let ctx = WasiContext { + spec: &spec, + oci_artifacts: &[], + }; + + let args = ctx.args(); assert_eq!(args.len(), 1); assert_eq!(args[0], "hello.wat"); @@ -83,9 +101,13 @@ mod tests { .root(RootBuilder::default().path("rootfs").build()?) .process(ProcessBuilder::default().cwd("/").args(vec![]).build()?) .build()?; - let spec = &spec; - let args = spec.args(); + let ctx = WasiContext { + spec: &spec, + oci_artifacts: &[], + }; + + let args = ctx.args(); assert_eq!(args.len(), 0); Ok(()) @@ -106,9 +128,13 @@ mod tests { .build()?, ) .build()?; - let spec = &spec; - let args = spec.args(); + let ctx = WasiContext { + spec: &spec, + oci_artifacts: &[], + }; + + let args = ctx.args(); assert_eq!(args.len(), 3); assert_eq!(args[0], "hello.wat"); assert_eq!(args[1], "echo"); @@ -123,9 +149,13 @@ mod tests { .root(RootBuilder::default().path("rootfs").build()?) .process(ProcessBuilder::default().cwd("/").args(vec![]).build()?) .build()?; - let spec = &spec; - let path = spec.wasi_entrypoint().path; + let ctx = WasiContext { + spec: &spec, + oci_artifacts: &[], + }; + + let path = ctx.wasi_entrypoint().path; assert!(path.as_os_str().is_empty()); Ok(()) @@ -146,9 +176,13 @@ mod tests { .build()?, ) .build()?; - let spec = &spec; - let WasiEntrypoint { path, func } = spec.wasi_entrypoint(); + let ctx = WasiContext { + spec: &spec, + oci_artifacts: &[], + }; + + let WasiEntrypoint { path, func } = ctx.wasi_entrypoint(); assert_eq!(path, Path::new("hello.wat")); assert_eq!(func, "foo"); @@ -170,9 +204,13 @@ mod tests { .build()?, ) .build()?; - let spec = &spec; - let WasiEntrypoint { path, func } = spec.wasi_entrypoint(); + let ctx = WasiContext { + spec: &spec, + oci_artifacts: &[], + }; + + let WasiEntrypoint { path, func } = ctx.wasi_entrypoint(); assert_eq!(path, Path::new("/root/hello.wat")); assert_eq!(func, "_start"); diff --git a/crates/containerd-shim-wasm/src/container/mod.rs b/crates/containerd-shim-wasm/src/container/mod.rs index 51bd6d093..ff2aca7f5 100644 --- a/crates/containerd-shim-wasm/src/container/mod.rs +++ b/crates/containerd-shim-wasm/src/container/mod.rs @@ -14,6 +14,7 @@ mod context; mod engine; mod path; +pub(crate) use context::WasiContext; pub use context::{RuntimeContext, WasiEntrypoint}; pub use engine::Engine; pub use instance::Instance; diff --git a/crates/containerd-shim-wasm/src/sandbox/containerd.rs b/crates/containerd-shim-wasm/src/sandbox/containerd.rs new file mode 100644 index 000000000..8c9569957 --- /dev/null +++ b/crates/containerd-shim-wasm/src/sandbox/containerd.rs @@ -0,0 +1,136 @@ +#![cfg(unix)] + +use std::path::Path; + +use containerd_client; +use containerd_client::services::v1::containers_client::ContainersClient; +use containerd_client::services::v1::content_client::ContentClient; +use containerd_client::services::v1::images_client::ImagesClient; +use containerd_client::services::v1::{GetContainerRequest, GetImageRequest, ReadContentRequest}; +use containerd_client::tonic::transport::Channel; +use containerd_client::{tonic, with_namespace}; +use futures::TryStreamExt; +use oci_spec::image::{ImageManifest, MediaType}; +use tokio::runtime::Runtime; +use tonic::Request; + +use crate::sandbox::error::{Error as ShimError, Result}; +use crate::sandbox::oci::{self, OciArtifact, COMPONENT_ARTIFACT_TYPE, MODULE_ARTIFACT_TYPE}; + +pub struct Client { + inner: Channel, + rt: Runtime, + namespace: String, +} + +// sync wrapper implementation from https://tokio.rs/tokio/topics/bridging +impl Client { + // wrapper around connection that will establish a connection and create a client + pub fn connect(address: impl AsRef, namespace: impl ToString) -> Result { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + + let inner = rt + .block_on(containerd_client::connect(address)) + .map_err(|err| ShimError::Others(err.to_string()))?; + + Ok(Client { + inner, + rt, + namespace: namespace.to_string(), + }) + } + + // wrapper around read that will read the entire content file + pub fn read_content(&self, digest: impl ToString) -> Result> { + self.rt.block_on(async { + let req = ReadContentRequest { + digest: digest.to_string(), + ..Default::default() + }; + let req = with_namespace!(req, self.namespace); + ContentClient::new(self.inner.clone()) + .read(req) + .await + .map_err(|err| ShimError::Others(err.to_string()))? + .into_inner() + .map_ok(|msg| msg.data) + .try_concat() + .await + .map_err(|err| ShimError::Others(err.to_string())) + }) + } + + pub fn get_image_content_sha(&self, image_name: impl ToString) -> Result { + self.rt.block_on(async { + let name = image_name.to_string(); + let req = GetImageRequest { name }; + let req = with_namespace!(req, self.namespace); + let digest = ImagesClient::new(self.inner.clone()) + .get(req) + .await + .map_err(|err| ShimError::Others(err.to_string()))? + .into_inner() + .image.ok_or(ShimError::Others(format!("failed to get image content sha for image {}", image_name.to_string())))? + .target.ok_or(ShimError::Others(format!("failed to get image content sha for image {}", image_name.to_string())))? + .digest; + Ok(digest) + }) + } + + pub fn get_image(&self, container_name: impl ToString) -> Result { + self.rt.block_on(async { + let id = container_name.to_string(); + let req = GetContainerRequest { id }; + let req = with_namespace!(req, self.namespace); + let image = ContainersClient::new(self.inner.clone()) + .get(req) + .await + .map_err(|err| ShimError::Others(err.to_string()))? + .into_inner() + .container + .ok_or(ShimError::Others(format!("failed to get image for container {}", container_name.to_string())))? + .image; + Ok(image) + }) + } + + // load module will query the containerd store to find an image that has an ArtifactType of WASM OCI Artifact + // If found it continues to parse the manifest and return the layers that contains the WASM modules + // and possibly other configuration artifacts + pub fn load_modules(&self, containerd_id: impl ToString) -> Result> { + let image_name = self.get_image(containerd_id.to_string())?; + let digest = self.get_image_content_sha(image_name)?; + let manifest = self.read_content(digest)?; + let manifest = manifest.as_slice(); + let manifest = ImageManifest::from_reader(manifest)?; + + let artifact_type = manifest + .artifact_type() + .as_ref() + .ok_or(ShimError::Others("manifest is not an OCI Artifact".to_string()))?; + + match artifact_type { + MediaType::Other(s) if s == COMPONENT_ARTIFACT_TYPE || s == MODULE_ARTIFACT_TYPE => { + log::info!("manifest with OCI Artifact of type {s}"); + } + _ => { + log::info!("manifest is not a known OCI Artifact: {artifact_type}"); + return Ok([].to_vec()); + } + } + + Ok(manifest + .layers() + .iter() + .map(|config| { + self.read_content(config.digest()) + .map(|module| OciArtifact { + config: config.clone(), + layer: module, + }) + }) + .collect::>>()?) + } +} diff --git a/crates/containerd-shim-wasm/src/sandbox/instance.rs b/crates/containerd-shim-wasm/src/sandbox/instance.rs index b5c853768..2674e304c 100644 --- a/crates/containerd-shim-wasm/src/sandbox/instance.rs +++ b/crates/containerd-shim-wasm/src/sandbox/instance.rs @@ -34,11 +34,11 @@ impl InstanceConfig { Self { engine, namespace, + containerd_address, stdin: None, stdout: None, stderr: None, bundle: None, - containerd_address, } } diff --git a/crates/containerd-shim-wasm/src/sandbox/mod.rs b/crates/containerd-shim-wasm/src/sandbox/mod.rs index 5099437ef..20f87fc86 100644 --- a/crates/containerd-shim-wasm/src/sandbox/mod.rs +++ b/crates/containerd-shim-wasm/src/sandbox/mod.rs @@ -17,4 +17,5 @@ pub use manager::{Sandbox as SandboxService, Service as ManagerService}; pub use shim::{Cli as ShimCli, Local}; pub use stdio::Stdio; +pub(crate) mod containerd; pub(crate) mod oci; diff --git a/crates/containerd-shim-wasm/src/sandbox/oci.rs b/crates/containerd-shim-wasm/src/sandbox/oci.rs index d14be5d8f..bc03d9e36 100644 --- a/crates/containerd-shim-wasm/src/sandbox/oci.rs +++ b/crates/containerd-shim-wasm/src/sandbox/oci.rs @@ -7,10 +7,20 @@ use std::os::unix::process::CommandExt; use std::process; use anyhow::Context; +use oci_spec::image::Descriptor; pub use oci_spec::runtime::Spec; use super::error::Result; +pub const COMPONENT_ARTIFACT_TYPE: &str = "application/vnd.bytecodealliance.component.v1+wasm"; +pub const MODULE_ARTIFACT_TYPE: &str = "application/vnd.bytecodealliance.module.v1+wasm"; + +#[derive(Clone)] +pub struct OciArtifact { + pub config: Descriptor, + pub layer: Vec, +} + fn parse_env(envs: &[String]) -> HashMap { // make NAME=VALUE to HashMap. envs.iter() diff --git a/crates/containerd-shim-wasm/src/services/sandbox_ttrpc.rs b/crates/containerd-shim-wasm/src/services/sandbox_ttrpc.rs index ab3161923..8249c7058 100644 --- a/crates/containerd-shim-wasm/src/services/sandbox_ttrpc.rs +++ b/crates/containerd-shim-wasm/src/services/sandbox_ttrpc.rs @@ -1,12 +1,9 @@ -// This file is generated by ttrpc-compiler 0.6.1. Do not edit +// This file is generated by ttrpc-compiler 0.6.2. Do not edit // @generated -// https://github.com/Manishearth/rust-clippy/issues/702 +#![cfg_attr(rustfmt, rustfmt_skip)] #![allow(unknown_lints)] #![allow(clipto_camel_casepy)] - -#![cfg_attr(rustfmt, rustfmt_skip)] - #![allow(box_pointers)] #![allow(dead_code)] #![allow(missing_docs)] @@ -17,6 +14,7 @@ #![allow(unsafe_code)] #![allow(unused_imports)] #![allow(unused_results)] +#![allow(clippy::all)] use protobuf::{CodedInputStream, CodedOutputStream, Message}; use std::collections::HashMap; use std::sync::Arc; diff --git a/crates/containerd-shim-wasm/src/sys/unix/container/executor.rs b/crates/containerd-shim-wasm/src/sys/unix/container/executor.rs index 87c0c939c..99598cd5e 100644 --- a/crates/containerd-shim-wasm/src/sys/unix/container/executor.rs +++ b/crates/containerd-shim-wasm/src/sys/unix/container/executor.rs @@ -12,7 +12,8 @@ use libcontainer::workload::{ }; use oci_spec::runtime::Spec; -use crate::container::{Engine, PathResolve, RuntimeContext, Stdio}; +use crate::container::{Engine, PathResolve, RuntimeContext, Stdio, WasiContext}; +use crate::sandbox::oci::OciArtifact; #[derive(Clone)] enum InnerExecutor { @@ -26,6 +27,7 @@ pub(crate) struct Executor { engine: E, stdio: Stdio, inner: OnceCell, + oci_artifacts: Vec, } impl LibcontainerExecutor for Executor { @@ -49,7 +51,7 @@ impl LibcontainerExecutor for Executor { } InnerExecutor::Wasm => { log::info!("calling start function"); - match self.engine.run_wasi(spec, self.stdio.take()) { + match self.engine.run_wasi(&self.ctx(spec), self.stdio.take()) { Ok(code) => std::process::exit(code), Err(err) => { log::info!("error running start function: {err}"); @@ -62,19 +64,31 @@ impl LibcontainerExecutor for Executor { } impl Executor { - pub fn new(engine: E, stdio: Stdio) -> Self { + pub fn new(engine: E, stdio: Stdio, oci_artifacts: Vec) -> Self { Self { engine, stdio, inner: Default::default(), + oci_artifacts, + } + } + + fn ctx<'a>(&'a self, spec: &'a Spec) -> WasiContext<'a> { + let oci_artifacts = &self.oci_artifacts; + WasiContext { + spec, + oci_artifacts, } } fn inner(&self, spec: &Spec) -> &InnerExecutor { self.inner.get_or_init(|| { - if is_linux_container(spec).is_ok() { + // if the spec has oci annotations we know it is wasm so short circuit checks + if !self.oci_artifacts.is_empty() { + InnerExecutor::Wasm + } else if is_linux_container(&self.ctx(spec)).is_ok() { InnerExecutor::Linux - } else if self.engine.can_handle(spec).is_ok() { + } else if self.engine.can_handle(&self.ctx(spec)).is_ok() { InnerExecutor::Wasm } else { InnerExecutor::CantHandle @@ -83,8 +97,8 @@ impl Executor { } } -fn is_linux_container(spec: &Spec) -> Result<()> { - let executable = spec +fn is_linux_container(ctx: &impl RuntimeContext) -> Result<()> { + let executable = ctx .entrypoint() .context("no entrypoint provided")? .resolve_in_path() diff --git a/crates/containerd-shim-wasm/src/sys/unix/container/instance.rs b/crates/containerd-shim-wasm/src/sys/unix/container/instance.rs index 198cf1d4a..dcc624efc 100644 --- a/crates/containerd-shim-wasm/src/sys/unix/container/instance.rs +++ b/crates/containerd-shim-wasm/src/sys/unix/container/instance.rs @@ -16,7 +16,7 @@ use nix::unistd::Pid; use crate::container::Engine; use crate::sandbox::instance_utils::{determine_rootdir, get_instance_root, instance_exists}; use crate::sandbox::sync::WaitableCell; -use crate::sandbox::{Error as SandboxError, Instance as SandboxInstance, InstanceConfig, Stdio}; +use crate::sandbox::{containerd, Error as SandboxError, Instance as SandboxInstance, InstanceConfig, Stdio}; use crate::sys::container::executor::Executor; static DEFAULT_CONTAINER_ROOT_DIR: &str = "/run/containerd"; @@ -40,8 +40,16 @@ impl SandboxInstance for Instance { let rootdir = determine_rootdir(&bundle, &namespace, rootdir)?; let stdio = Stdio::init_from_cfg(cfg)?; + // check if container is OCI artifact and attempt to read the module + let modules = containerd::Client::connect(cfg.get_containerd_address(), &namespace)? + .load_modules(&id) + .unwrap_or_else(|e| { + log::warn!("Error obtaining wasm layers for container {}. Will attempt to use files inside containerd image. Error: {}", id.clone(), e); + vec![] + }); + ContainerBuilder::new(id.clone(), SyscallType::Linux) - .with_executor(Executor::new(engine, stdio)) + .with_executor(Executor::new(engine, stdio, modules)) .with_root_path(rootdir.clone())? .as_init(&bundle) .with_systemd(false) diff --git a/crates/containerd-shim-wasm/src/testing.rs b/crates/containerd-shim-wasm/src/testing.rs index a56e680b5..4ca1df28b 100644 --- a/crates/containerd-shim-wasm/src/testing.rs +++ b/crates/containerd-shim-wasm/src/testing.rs @@ -123,7 +123,7 @@ where let mut cfg = InstanceConfig::new( WasiInstance::Engine::default(), "test_namespace".into(), - "/containerd/address".into(), + "/run/containerd/containerd.sock".into(), ); cfg.set_bundle(dir.to_string_lossy().to_string()) .set_stdout(dir.join("stdout").to_string_lossy().to_string()) diff --git a/crates/containerd-shim-wasmedge/src/instance.rs b/crates/containerd-shim-wasmedge/src/instance.rs index 9ab7636d5..a42560123 100644 --- a/crates/containerd-shim-wasmedge/src/instance.rs +++ b/crates/containerd-shim-wasmedge/src/instance.rs @@ -1,7 +1,8 @@ -use anyhow::{Context, Result}; +use anyhow::{bail, Context, Result}; use containerd_shim_wasm::container::{ Engine, Instance, PathResolve, RuntimeContext, Stdio, WasiEntrypoint, }; +use log::debug; use wasmedge_sdk::config::{ConfigBuilder, HostRegistrationConfigOptions}; use wasmedge_sdk::plugin::PluginManager; use wasmedge_sdk::VmBuilder; @@ -34,11 +35,10 @@ impl Engine for WasmEdgeEngine { fn run_wasi(&self, ctx: &impl RuntimeContext, stdio: Stdio) -> Result { let args = ctx.args(); let envs: Vec<_> = std::env::vars().map(|(k, v)| format!("{k}={v}")).collect(); - let WasiEntrypoint { path, func } = ctx.wasi_entrypoint(); - let path = path - .resolve_in_path_or_cwd() - .next() - .context("module not found")?; + let WasiEntrypoint { + path: entrypoint_path, + func, + } = ctx.wasi_entrypoint(); let mut vm = self.vm.clone(); vm.wasi_module_mut() @@ -49,20 +49,36 @@ impl Engine for WasmEdgeEngine { Some(vec!["/:/"]), ); - let mod_name = match path.file_stem() { + let mod_name = match entrypoint_path.file_stem() { Some(name) => name.to_string_lossy().to_string(), None => "main".to_string(), }; PluginManager::load(None)?; let vm = vm.auto_detect_plugins()?; - let vm = vm - .register_module_from_file(&mod_name, &path) - .context("registering module")?; + + let vm = match ctx.oci_artifacts() { + [] => { + debug!("loading module from file"); + let path = entrypoint_path + .resolve_in_path_or_cwd() + .next() + .context("module not found")?; + + vm.register_module_from_file(&mod_name, path) + .context("registering module")? + } + [module] => { + log::info!("loading module from OCI Artifact"); + vm.register_module_from_bytes(&mod_name, &module.layer) + .context("registering module")? + } + [..] => bail!("only a single module is supported when using OCI Artifact"), + }; stdio.redirect()?; - log::debug!("running {path:?} with method {func:?}"); + log::debug!("running {entrypoint_path:?} with method {func:?}"); vm.run_func(Some(&mod_name), func, vec![])?; let status = vm diff --git a/crates/containerd-shim-wasmer/src/instance.rs b/crates/containerd-shim-wasmer/src/instance.rs index 106387440..360d7ed68 100644 --- a/crates/containerd-shim-wasmer/src/instance.rs +++ b/crates/containerd-shim-wasmer/src/instance.rs @@ -1,4 +1,4 @@ -use anyhow::{Context, Result}; +use anyhow::{bail, Context, Result}; use containerd_shim_wasm::container::{ Engine, Instance, PathResolve, RuntimeContext, Stdio, WasiEntrypoint, }; @@ -22,24 +22,31 @@ impl Engine for WasmerEngine { let args = ctx.args(); let envs = std::env::vars(); let WasiEntrypoint { path, func } = ctx.wasi_entrypoint(); - let path = path - .resolve_in_path_or_cwd() - .next() - .context("module not found")?; let mod_name = match path.file_stem() { Some(name) => name.to_string_lossy().to_string(), None => "main".to_string(), }; - log::info!("redirect stdio"); - stdio.redirect()?; - log::info!("Create a Store"); let mut store = Store::new(self.engine.clone()); - log::info!("loading module from file {path:?}"); - let module = Module::from_file(&store, path)?; + let module = match ctx.oci_artifacts() { + [] => { + log::info!("loading module from file {path:?}"); + let path = path + .resolve_in_path_or_cwd() + .next() + .context("module not found")?; + + Module::from_file(&store, path)? + } + [module] => { + log::info!("loading module from OCI Artifact"); + Module::from_binary(&store, &module.layer)? + } + [..] => bail!("only a single module is supported when using OCI Artifact"), + }; let runtime = tokio::runtime::Builder::new_multi_thread() .enable_all() @@ -54,6 +61,9 @@ impl Engine for WasmerEngine { .preopen_dir("/")? .instantiate(module, &mut store)?; + log::info!("redirect stdio"); + stdio.redirect()?; + log::info!("Running {func:?}"); let start = instance.exports.get_function(&func)?; wasi_env.data(&store).thread.set_status_running(); diff --git a/crates/containerd-shim-wasmtime/src/instance.rs b/crates/containerd-shim-wasmtime/src/instance.rs index e627ddfb7..057c50fb1 100644 --- a/crates/containerd-shim-wasmtime/src/instance.rs +++ b/crates/containerd-shim-wasmtime/src/instance.rs @@ -1,4 +1,4 @@ -use anyhow::{Context, Result}; +use anyhow::{bail, Context, Result}; use containerd_shim_wasm::container::{ Engine, Instance, PathResolve, RuntimeContext, Stdio, WasiEntrypoint, }; @@ -36,13 +36,24 @@ impl Engine for WasmtimeEngine { log::info!("wasi context ready"); let WasiEntrypoint { path, func } = ctx.wasi_entrypoint(); - let path = path - .resolve_in_path_or_cwd() - .next() - .context("module not found")?; - log::info!("loading module from file {path:?}"); - let module = Module::from_file(&self.engine, &path)?; + let module = match ctx.oci_artifacts() { + [] => { + log::info!("loading module from file"); + let path = path + .resolve_in_path_or_cwd() + .next() + .context("module not found")?; + + Module::from_file(&self.engine, path)? + } + [module] => { + log::info!("loading module from OCI Artifact"); + Module::from_binary(&self.engine, &module.layer)? + } + [..] => bail!("only a single module is supported when using OCI Artifact"), + }; + let mut linker = Linker::new(&self.engine); wasmtime_wasi::add_to_linker(&mut linker, |s| s)?; diff --git a/crates/oci-tar-builder/Cargo.toml b/crates/oci-tar-builder/Cargo.toml index 963d431cd..dc59e4011 100644 --- a/crates/oci-tar-builder/Cargo.toml +++ b/crates/oci-tar-builder/Cargo.toml @@ -7,7 +7,7 @@ edition.workspace = true tar = { workspace = true } sha256 = { workspace = true } log = { workspace = true } -oci-spec = { workspace = true } +oci-spec = { workspace = true, features = ["runtime"] } anyhow = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/oci-tar-builder/src/bin.rs b/crates/oci-tar-builder/src/bin.rs index b619b6994..06575ffdb 100644 --- a/crates/oci-tar-builder/src/bin.rs +++ b/crates/oci-tar-builder/src/bin.rs @@ -7,28 +7,43 @@ use clap::Parser; use oci_spec::image as spec; use oci_tar_builder::Builder; +// Following labels and media types must match in ../containerd-shim-wasm/src/sandbox/oci.rs +// not using it here to avoid needing to have all the deps for containerd-shim-wasm +pub const WASM_LAYER_MEDIA_TYPE: &str = + "application/vnd.bytecodealliance.wasm.component.layer.v0+wasm"; + pub fn main() { let args = Args::parse(); let out_dir; if let Some(out_path) = args.out_path.as_deref() { out_dir = PathBuf::from(out_path); - fs::create_dir_all(&out_dir).unwrap(); + fs::create_dir_all(out_dir.parent().unwrap()).unwrap(); } else { out_dir = env::current_dir().unwrap(); } + let entry_point = args.name.clone() + ".wasm"; + let mut builder = Builder::default(); + for module_path in args.module.iter() { + builder.as_module_artifact(); - if let Some(module_path) = args.module.as_deref() { let module_path = PathBuf::from(module_path); - builder.add_layer_with_media_type( - &module_path, - "application/vnd.w3c.wasm.module.v1+wasm".to_string(), - ); + builder.add_layer_with_media_type(&module_path, WASM_LAYER_MEDIA_TYPE.to_string()); + } + + for layer_config in args.layer.iter() { + //split string on equals sign + let layer_options: Vec<&str> = layer_config.split('=').collect(); + + let layer_type = layer_options.first().unwrap(); + let layer_path = PathBuf::from(layer_options.last().unwrap()); + builder.add_layer_with_media_type(&layer_path, layer_type.to_string()); } if let Some(components_path) = args.components.as_deref() { + builder.as_component_artifact(); let paths = fs::read_dir(components_path).unwrap(); for path in paths { @@ -36,16 +51,7 @@ pub fn main() { let ext = path.extension().unwrap().to_str().unwrap(); match ext { "wasm" => { - builder.add_layer_with_media_type( - &path, - "application/vnd.wasm.component.v1+wasm".to_string(), - ); - } - "yml" | "yaml" => { - builder.add_layer_with_media_type( - &path, - "application/vnd.wasm.component.config.v1+yaml".to_string(), - ); + builder.add_layer_with_media_type(&path, WASM_LAYER_MEDIA_TYPE.to_string()); } _ => println!( "Skipping Unknown file type: {:?} with extension {:?}", @@ -56,7 +62,10 @@ pub fn main() { } } - let config = spec::ConfigBuilder::default().build().unwrap(); + let config = spec::ConfigBuilder::default() + .entrypoint(vec![entry_point]) + .build() + .unwrap(); let img = spec::ImageConfigurationBuilder::default() .config(config) @@ -72,15 +81,19 @@ pub fn main() { .context("failed to build image configuration") .unwrap(); - builder.add_config(img, args.repo + "/" + &args.name); + builder.add_config(img, args.repo + "/" + &args.name + ":" + &args.tag); - let p = out_dir.join(args.name + ".tar"); - let f = File::create(p.clone()).unwrap(); + println!("Creating oci tar file {}", out_dir.clone().display()); + let f = File::create(out_dir.clone()).unwrap(); match builder.build(f) { - Ok(_) => println!("Successfully created oci tar file {}", p.display()), + Ok(_) => println!("Successfully created oci tar file {}", out_dir.display()), Err(e) => { - print!("Building oci tar file {} failed: {:?}", p.display(), e); - fs::remove_file(p).unwrap_or(print!("Failed to remove temporary file")); + print!( + "Building oci tar file {} failed: {:?}", + out_dir.display(), + e + ); + fs::remove_file(out_dir).unwrap_or(print!("Failed to remove temporary file")); } } } @@ -94,11 +107,17 @@ struct Args { #[arg(short, long)] name: String, + #[arg(short, long)] + tag: String, + #[arg(short, long)] repo: String, #[arg(short, long)] - module: Option, + module: Vec, + + #[arg(short, long)] + layer: Vec, #[arg(short, long)] components: Option, diff --git a/crates/oci-tar-builder/src/lib.rs b/crates/oci-tar-builder/src/lib.rs index 40f293382..3defdacd5 100644 --- a/crates/oci-tar-builder/src/lib.rs +++ b/crates/oci-tar-builder/src/lib.rs @@ -12,10 +12,17 @@ use oci_spec::image::{ use serde::Serialize; use sha256::{digest, try_digest}; +// duplicated here to avoid pulling in all of containerd-shim-wasm +// Module is preview1 +// Component is preview2 +const COMPONENT_ARTIFACT_TYPE: &str = "application/vnd.bytecodealliance.component.v1+wasm"; +const MODULE_ARTIFACT_TYPE: &str = "application/vnd.bytecodealliance.module.v1+wasm"; + #[derive(Debug, Default)] pub struct Builder { configs: Vec<(ImageConfiguration, String)>, layers: Vec<(PathBuf, String)>, + artifact_type: Option, } #[derive(Serialize, Debug)] @@ -148,9 +155,15 @@ impl Builder { mfst.repo_tags.push(config.1.clone()); annotations.insert("io.containerd.image.name".to_string(), config.1.clone()); - let manifest = ImageManifestBuilder::default() + let mut manifest = ImageManifestBuilder::default() .schema_version(SCHEMA_VERSION) - .media_type(MediaType::ImageManifest) + .media_type(MediaType::ImageManifest); + + if self.artifact_type.is_some() { + manifest = manifest.artifact_type(self.artifact_type.clone().unwrap().as_str()); + } + + let manifest = manifest .layers(layers) .config(desc) .annotations(annotations.clone()) @@ -231,4 +244,12 @@ impl Builder { Ok(()) } + + pub fn as_module_artifact(&mut self) { + self.artifact_type = Some(MODULE_ARTIFACT_TYPE.to_string()); + } + + pub fn as_component_artifact(&mut self) { + self.artifact_type = Some(COMPONENT_ARTIFACT_TYPE.to_string()); + } } diff --git a/crates/wasi-demo-app/Cargo.toml b/crates/wasi-demo-app/Cargo.toml index b80633139..e8fcba389 100644 --- a/crates/wasi-demo-app/Cargo.toml +++ b/crates/wasi-demo-app/Cargo.toml @@ -8,7 +8,7 @@ tar = { workspace = true, optional = true } sha256 = { workspace = true, optional = true } log = { workspace = true, optional = true } env_logger = { workspace = true, optional = true } -oci-spec = { workspace = true, optional = true } +oci-spec = { workspace = true, optional=true } oci-tar-builder = { optional = true, path = "../oci-tar-builder" } anyhow = { workspace = true, optional = true } diff --git a/scripts/setup-linux.sh b/scripts/setup-linux.sh index 0081f5fb8..bec843b71 100755 --- a/scripts/setup-linux.sh +++ b/scripts/setup-linux.sh @@ -1,6 +1,6 @@ #!/bin/bash sudo apt -y update -sudo apt install -y pkg-config libsystemd-dev libdbus-glib-1-dev build-essential libelf-dev libseccomp-dev libclang-dev +sudo apt install -y pkg-config libsystemd-dev libdbus-glib-1-dev build-essential libelf-dev libseccomp-dev libclang-dev protobuf-compiler if [ ! -z "$CI" ] && ! mount | grep cgroup; then echo "cgroup is not mounted" 1>&2 diff --git a/scripts/setup-windows.sh b/scripts/setup-windows.sh index e5eef9024..5f912da03 100644 --- a/scripts/setup-windows.sh +++ b/scripts/setup-windows.sh @@ -2,6 +2,7 @@ choco install -y wasmedge --version 0.13.1 # require clang for wasmedge for bindgen, which is used in the build script to generate the rust bindings to the c codebase choco install -y llvm --version 16.0.6 +choco install -y protoc if [ ! -z "$CI" ]; then echo "WASMEDGE_LIB_DIR=C:\Program Files\WasmEdge\lib" >> ${GITHUB_ENV} diff --git a/test/k8s/Dockerfile.oci b/test/k8s/Dockerfile.oci new file mode 100644 index 000000000..4473470a0 --- /dev/null +++ b/test/k8s/Dockerfile.oci @@ -0,0 +1,35 @@ +# syntax=docker/dockerfile:1.4 + +ARG KIND_NODE_VERSION=v1.23.13 +ARG RUNTIME=wasmtime +ARG GO_VERSION="1.21.1" + +# modified from https://github.com/kubernetes-sigs/kind/blob/main/images/base/Dockerfile +# stage for building containerd +FROM golang:${GO_VERSION} as build-containerd +ARG CONTAINERD_VERSION="main" +ARG CONTAINERD_CLONE_URL="https://github.com/containerd/containerd" +# we don't build with optional snapshotters, we never select any of these +# they're not ideal inside kind anyhow, and we save some disk space +ARG BUILDTAGS="no_aufs no_zfs no_btrfs no_devmapper" +RUN git clone --filter=tree:0 "${CONTAINERD_CLONE_URL}" /containerd \ + && cd /containerd \ + && git checkout "${CONTAINERD_VERSION}" \ + && export CGO_ENABLED=1 \ + && make bin/ctr bin/containerd + +FROM kindest/node:${KIND_NODE_VERSION} +COPY --from=build-containerd /containerd/bin/containerd /usr/local/bin/ +COPY --from=build-containerd /containerd/bin/ctr /usr/local/bin/ + +RUN apt-get update -y && \ + apt-get install --no-install-recommends -y libdbus-1-3 + +ADD dist/bin/* /usr/local/bin/ + +ARG RUNTIME +RUN sed -i 's,SystemdCgroup = true,,' /etc/containerd/config.toml && \ + cat <> /etc/containerd/config.toml +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.wasm] +runtime_type = "io.containerd.${RUNTIME}.v1" +EOF \ No newline at end of file diff --git a/test/k8s/deploy.yaml b/test/k8s/deploy.yaml index 64d0de157..6c833ce5f 100644 --- a/test/k8s/deploy.yaml +++ b/test/k8s/deploy.yaml @@ -28,4 +28,4 @@ spec: - name: nginx image: docker.io/nginx:latest ports: - - containerPort: 80 \ No newline at end of file + - containerPort: 80 From 94a7fdb0919c37908e4c0656d93c6f98e2f3c536 Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Thu, 19 Oct 2023 21:09:51 +0000 Subject: [PATCH 2/2] Move away from oci artifact naming Signed-off-by: James Sturtevant --- Makefile | 5 +- README.md | 10 +- .../src/container/context.rs | 36 ++++-- .../src/sandbox/containerd.rs | 104 ++++++++++++------ .../containerd-shim-wasm/src/sandbox/error.rs | 2 + .../containerd-shim-wasm/src/sandbox/oci.rs | 5 +- .../src/sys/unix/container/executor.rs | 19 ++-- .../src/sys/unix/container/instance.rs | 15 ++- .../containerd-shim-wasmedge/src/instance.rs | 6 +- crates/containerd-shim-wasmer/src/instance.rs | 6 +- .../containerd-shim-wasmtime/src/instance.rs | 6 +- crates/oci-tar-builder/README.md | 2 +- crates/oci-tar-builder/src/bin.rs | 11 +- crates/oci-tar-builder/src/lib.rs | 24 +--- crates/wasi-demo-app/build.rs | 13 ++- cross/Dockerfile.gnu | 2 +- cross/Dockerfile.musl | 2 +- test/k8s/Dockerfile.oci | 9 +- 18 files changed, 158 insertions(+), 119 deletions(-) diff --git a/Makefile b/Makefile index 10899c4c4..90bf9b5e4 100644 --- a/Makefile +++ b/Makefile @@ -188,8 +188,11 @@ dist/img-oci.tar: target/wasm32-wasi/$(OPT_PROFILE)/img-oci.tar load: dist/img.tar sudo ctr -n $(CONTAINERD_NAMESPACE) image import --all-platforms $< +CTR_VERSION := $(shell sudo ctr version | sed -n -e '/Version/ {s/.*: *//p;q;}') load/oci: dist/img-oci.tar - sudo ../containerd/bin/ctr -n $(CONTAINERD_NAMESPACE) image import --all-platforms $< + @echo $(CTR_VERSION)\\nv1.7.7 | sort -crV || (echo "containerd version must be 1.7.7+ was $(CTR_VERSION)" && exit 1) + @echo using containerd $(CTR_VERSION) + sudo ctr -n $(CONTAINERD_NAMESPACE) image import --all-platforms $< .PHONY: target/wasm32-wasi/$(OPT_PROFILE)/img-oci.tar: target/wasm32-wasi/$(OPT_PROFILE)/wasi-demo-app.wasm diff --git a/README.md b/README.md index 9dc15a9c7..2a2c01986 100644 --- a/README.md +++ b/README.md @@ -323,22 +323,22 @@ To kill the process from demo 2, you can run in other session: `sudo ctr task ki The test binary supports commands for different type of functionality, check [crates/wasi-demo-app/src/main.rs](crates/wasi-demo-app/src/main.rs) to try it out. -## Demo 3 using WASM OCI artifacts +## Demo 3 using OCI Images with custom WASM layers -The previous demos run with an OCI Container image containing the wasm module in the file system. Another option is to provide a cross-platform OCI artifact that that will not have the wasm module or components in the file system of the container that wraps the wasmtime/wasmedge process. This OCI artifact can be run across any platform and provides for de-duplication in the Containerd content store among other benefits. +The previous demos run with an OCI Container image containing the wasm module in the file system. Another option is to provide a cross-platform OCI Image that that will not have the wasm module or components in the file system of the container that wraps the wasmtime/wasmedge process. This OCI Image with custom WASM layers can be run across any platform and provides for de-duplication in the Containerd content store among other benefits. To learn more about this approach checkout the [design document](https://docs.google.com/document/d/11shgC3l6gplBjWF1VJCWvN_9do51otscAm0hBDGSSAc/edit). -> **Note**: This requires containerd components based on https://github.com/containerd/containerd/pull/8699. Both CTR and containerd need to be build with that patch. If you do not have this patch for both `containerd` and `ctr` you will end up with an error message such as `mismatched image rootfs and manifest layers` at the import and run steps +> **Note**: This requires containerd 1.7.7+ and 1.6.25+ (not yet released). If you do not have these patches for both `containerd` and `ctr` you will end up with an error message such as `mismatched image rootfs and manifest layers` at the import and run steps. -Build and import the OCI artifact image: +Build and import the OCI image with WASM layers image: ``` make test-image/oci make load/oci ``` -Run the image with `sudo ctr run --rm --runtime=io.containerd.[ wasmedge | wasmtime | wasmer ].v1 --label application/vnd.bytecodealliance.wasm.module=oci.wasm ghcr.io/containerd/runwasi/wasi-demo-oci:latest testwasmoci` +Run the image with `sudo ctr run --rm --runtime=io.containerd.[ wasmedge | wasmtime | wasmer ].v1 ghcr.io/containerd/runwasi/wasi-demo-oci:latest testwasmoci` ``` sudo ctr run --rm --runtime=io.containerd.wasmtime.v1 ghcr.io/containerd/runwasi/wasi-demo-oci:latest testwasmoci wasi-demo-oci.wasm echo 'hello' diff --git a/crates/containerd-shim-wasm/src/container/context.rs b/crates/containerd-shim-wasm/src/container/context.rs index e9fb982e7..97fea259a 100644 --- a/crates/containerd-shim-wasm/src/container/context.rs +++ b/crates/containerd-shim-wasm/src/container/context.rs @@ -1,8 +1,9 @@ use std::path::{Path, PathBuf}; +use oci_spec::image::Platform; use oci_spec::runtime::Spec; -use crate::sandbox::oci::OciArtifact; +use crate::sandbox::oci::WasmLayer; pub trait RuntimeContext { // ctx.args() returns arguments from the runtime spec process field, including the @@ -23,7 +24,9 @@ pub trait RuntimeContext { // "#init" -> { path: "", func: "init" } fn wasi_entrypoint(&self) -> WasiEntrypoint; - fn oci_artifacts(&self) -> &[OciArtifact]; + fn wasm_layers(&self) -> &[WasmLayer]; + + fn platform(&self) -> &Platform; } pub struct WasiEntrypoint { @@ -33,7 +36,8 @@ pub struct WasiEntrypoint { pub(crate) struct WasiContext<'a> { pub spec: &'a Spec, - pub oci_artifacts: &'a [OciArtifact], + pub wasm_layers: &'a [WasmLayer], + pub platform: &'a Platform, } impl RuntimeContext for WasiContext<'_> { @@ -59,8 +63,12 @@ impl RuntimeContext for WasiContext<'_> { } } - fn oci_artifacts(&self) -> &[OciArtifact] { - self.oci_artifacts + fn wasm_layers(&self) -> &[WasmLayer] { + self.wasm_layers + } + + fn platform(&self) -> &Platform { + self.platform } } @@ -85,7 +93,8 @@ mod tests { let ctx = WasiContext { spec: &spec, - oci_artifacts: &[], + wasm_layers: &[], + platform: &Platform::default(), }; let args = ctx.args(); @@ -104,7 +113,8 @@ mod tests { let ctx = WasiContext { spec: &spec, - oci_artifacts: &[], + wasm_layers: &[], + platform: &Platform::default(), }; let args = ctx.args(); @@ -131,7 +141,8 @@ mod tests { let ctx = WasiContext { spec: &spec, - oci_artifacts: &[], + wasm_layers: &[], + platform: &Platform::default(), }; let args = ctx.args(); @@ -152,7 +163,8 @@ mod tests { let ctx = WasiContext { spec: &spec, - oci_artifacts: &[], + wasm_layers: &[], + platform: &Platform::default(), }; let path = ctx.wasi_entrypoint().path; @@ -179,7 +191,8 @@ mod tests { let ctx = WasiContext { spec: &spec, - oci_artifacts: &[], + wasm_layers: &[], + platform: &Platform::default(), }; let WasiEntrypoint { path, func } = ctx.wasi_entrypoint(); @@ -207,7 +220,8 @@ mod tests { let ctx = WasiContext { spec: &spec, - oci_artifacts: &[], + wasm_layers: &[], + platform: &Platform::default(), }; let WasiEntrypoint { path, func } = ctx.wasi_entrypoint(); diff --git a/crates/containerd-shim-wasm/src/sandbox/containerd.rs b/crates/containerd-shim-wasm/src/sandbox/containerd.rs index 8c9569957..3168c1dd7 100644 --- a/crates/containerd-shim-wasm/src/sandbox/containerd.rs +++ b/crates/containerd-shim-wasm/src/sandbox/containerd.rs @@ -10,14 +10,14 @@ use containerd_client::services::v1::{GetContainerRequest, GetImageRequest, Read use containerd_client::tonic::transport::Channel; use containerd_client::{tonic, with_namespace}; use futures::TryStreamExt; -use oci_spec::image::{ImageManifest, MediaType}; +use oci_spec::image::{Arch, ImageManifest, MediaType, Platform}; use tokio::runtime::Runtime; use tonic::Request; use crate::sandbox::error::{Error as ShimError, Result}; -use crate::sandbox::oci::{self, OciArtifact, COMPONENT_ARTIFACT_TYPE, MODULE_ARTIFACT_TYPE}; +use crate::sandbox::oci::{self, WasmLayer}; -pub struct Client { +pub(crate) struct Client { inner: Channel, rt: Runtime, namespace: String, @@ -33,7 +33,7 @@ impl Client { let inner = rt .block_on(containerd_client::connect(address)) - .map_err(|err| ShimError::Others(err.to_string()))?; + .map_err(|err| ShimError::Containerd(err.to_string()))?; Ok(Client { inner, @@ -53,12 +53,12 @@ impl Client { ContentClient::new(self.inner.clone()) .read(req) .await - .map_err(|err| ShimError::Others(err.to_string()))? + .map_err(|err| ShimError::Containerd(err.to_string()))? .into_inner() .map_ok(|msg| msg.data) .try_concat() .await - .map_err(|err| ShimError::Others(err.to_string())) + .map_err(|err| ShimError::Containerd(err.to_string())) }) } @@ -70,10 +70,22 @@ impl Client { let digest = ImagesClient::new(self.inner.clone()) .get(req) .await - .map_err(|err| ShimError::Others(err.to_string()))? + .map_err(|err| ShimError::Containerd(err.to_string()))? .into_inner() - .image.ok_or(ShimError::Others(format!("failed to get image content sha for image {}", image_name.to_string())))? - .target.ok_or(ShimError::Others(format!("failed to get image content sha for image {}", image_name.to_string())))? + .image + .ok_or_else(|| { + ShimError::Containerd(format!( + "failed to get image content sha for image {}", + image_name.to_string() + )) + })? + .target + .ok_or_else(|| { + ShimError::Containerd(format!( + "failed to get image content sha for image {}", + image_name.to_string() + )) + })? .digest; Ok(digest) }) @@ -87,50 +99,74 @@ impl Client { let image = ContainersClient::new(self.inner.clone()) .get(req) .await - .map_err(|err| ShimError::Others(err.to_string()))? + .map_err(|err| ShimError::Containerd(err.to_string()))? .into_inner() .container - .ok_or(ShimError::Others(format!("failed to get image for container {}", container_name.to_string())))? + .ok_or_else(|| { + ShimError::Containerd(format!( + "failed to get image for container {}", + container_name.to_string() + )) + })? .image; Ok(image) }) } - // load module will query the containerd store to find an image that has an ArtifactType of WASM OCI Artifact + // load module will query the containerd store to find an image that has an OS of type 'wasm' // If found it continues to parse the manifest and return the layers that contains the WASM modules - // and possibly other configuration artifacts - pub fn load_modules(&self, containerd_id: impl ToString) -> Result> { + // and possibly other configuration layers. + pub fn load_modules( + &self, + containerd_id: impl ToString, + ) -> Result<(Vec, Platform)> { let image_name = self.get_image(containerd_id.to_string())?; let digest = self.get_image_content_sha(image_name)?; let manifest = self.read_content(digest)?; let manifest = manifest.as_slice(); let manifest = ImageManifest::from_reader(manifest)?; - let artifact_type = manifest - .artifact_type() - .as_ref() - .ok_or(ShimError::Others("manifest is not an OCI Artifact".to_string()))?; + let image_config_descriptor = manifest.config(); + let image_config = self.read_content(image_config_descriptor.digest())?; + let image_config = image_config.as_slice(); - match artifact_type { - MediaType::Other(s) if s == COMPONENT_ARTIFACT_TYPE || s == MODULE_ARTIFACT_TYPE => { - log::info!("manifest with OCI Artifact of type {s}"); - } - _ => { - log::info!("manifest is not a known OCI Artifact: {artifact_type}"); - return Ok([].to_vec()); - } - } + // the only part we care about here is the platform values + let platform: Platform = serde_json::from_slice(image_config)?; + let Arch::Wasm = platform.architecture() else { + log::info!("manifest is not in WASM OCI image format"); + return Ok((vec![], platform)); + }; + log::info!("found manifest with WASM OCI image format."); - Ok(manifest + let layers = manifest .layers() .iter() + .filter(|x| !is_image_layer_type(x.media_type())) .map(|config| { - self.read_content(config.digest()) - .map(|module| OciArtifact { - config: config.clone(), - layer: module, - }) + self.read_content(config.digest()).map(|module| WasmLayer { + config: config.clone(), + layer: module, + }) }) - .collect::>>()?) + .collect::>>()?; + Ok((layers, platform)) + } +} + +fn is_image_layer_type(media_type: &MediaType) -> bool { + match media_type { + MediaType::ImageLayer + | MediaType::ImageLayerGzip + | MediaType::ImageLayerNonDistributable + | MediaType::ImageLayerNonDistributableGzip + | MediaType::ImageLayerNonDistributableZstd + | MediaType::ImageLayerZstd => true, + MediaType::Other(s) + if s.as_str() + .starts_with("application/vnd.docker.image.rootfs.") => + { + true + } + _ => false, } } diff --git a/crates/containerd-shim-wasm/src/sandbox/error.rs b/crates/containerd-shim-wasm/src/sandbox/error.rs index 6d072077a..a9a87f0c0 100644 --- a/crates/containerd-shim-wasm/src/sandbox/error.rs +++ b/crates/containerd-shim-wasm/src/sandbox/error.rs @@ -46,6 +46,8 @@ pub enum Error { #[cfg(unix)] #[error("{0}")] Libcontainer(#[from] libcontainer::error::LibcontainerError), + #[error("{0}")] + Containerd(String), } pub type Result = ::std::result::Result; diff --git a/crates/containerd-shim-wasm/src/sandbox/oci.rs b/crates/containerd-shim-wasm/src/sandbox/oci.rs index bc03d9e36..e06f50dec 100644 --- a/crates/containerd-shim-wasm/src/sandbox/oci.rs +++ b/crates/containerd-shim-wasm/src/sandbox/oci.rs @@ -12,11 +12,8 @@ pub use oci_spec::runtime::Spec; use super::error::Result; -pub const COMPONENT_ARTIFACT_TYPE: &str = "application/vnd.bytecodealliance.component.v1+wasm"; -pub const MODULE_ARTIFACT_TYPE: &str = "application/vnd.bytecodealliance.module.v1+wasm"; - #[derive(Clone)] -pub struct OciArtifact { +pub struct WasmLayer { pub config: Descriptor, pub layer: Vec, } diff --git a/crates/containerd-shim-wasm/src/sys/unix/container/executor.rs b/crates/containerd-shim-wasm/src/sys/unix/container/executor.rs index 99598cd5e..287f74d3c 100644 --- a/crates/containerd-shim-wasm/src/sys/unix/container/executor.rs +++ b/crates/containerd-shim-wasm/src/sys/unix/container/executor.rs @@ -10,10 +10,11 @@ use libcontainer::workload::{ Executor as LibcontainerExecutor, ExecutorError as LibcontainerExecutorError, ExecutorValidationError, }; +use oci_spec::image::Platform; use oci_spec::runtime::Spec; use crate::container::{Engine, PathResolve, RuntimeContext, Stdio, WasiContext}; -use crate::sandbox::oci::OciArtifact; +use crate::sandbox::oci::WasmLayer; #[derive(Clone)] enum InnerExecutor { @@ -27,7 +28,8 @@ pub(crate) struct Executor { engine: E, stdio: Stdio, inner: OnceCell, - oci_artifacts: Vec, + wasm_layers: Vec, + platform: Platform, } impl LibcontainerExecutor for Executor { @@ -64,27 +66,30 @@ impl LibcontainerExecutor for Executor { } impl Executor { - pub fn new(engine: E, stdio: Stdio, oci_artifacts: Vec) -> Self { + pub fn new(engine: E, stdio: Stdio, wasm_layers: Vec, platform: Platform) -> Self { Self { engine, stdio, inner: Default::default(), - oci_artifacts, + wasm_layers, + platform, } } fn ctx<'a>(&'a self, spec: &'a Spec) -> WasiContext<'a> { - let oci_artifacts = &self.oci_artifacts; + let wasm_layers = &self.wasm_layers; + let platform = &self.platform; WasiContext { spec, - oci_artifacts, + wasm_layers, + platform, } } fn inner(&self, spec: &Spec) -> &InnerExecutor { self.inner.get_or_init(|| { // if the spec has oci annotations we know it is wasm so short circuit checks - if !self.oci_artifacts.is_empty() { + if !self.wasm_layers.is_empty() { InnerExecutor::Wasm } else if is_linux_container(&self.ctx(spec)).is_ok() { InnerExecutor::Linux diff --git a/crates/containerd-shim-wasm/src/sys/unix/container/instance.rs b/crates/containerd-shim-wasm/src/sys/unix/container/instance.rs index dcc624efc..247bb39b5 100644 --- a/crates/containerd-shim-wasm/src/sys/unix/container/instance.rs +++ b/crates/containerd-shim-wasm/src/sys/unix/container/instance.rs @@ -12,11 +12,14 @@ use libcontainer::syscall::syscall::SyscallType; use nix::errno::Errno; use nix::sys::wait::{waitid, Id as WaitID, WaitPidFlag, WaitStatus}; use nix::unistd::Pid; +use oci_spec::image::Platform; use crate::container::Engine; use crate::sandbox::instance_utils::{determine_rootdir, get_instance_root, instance_exists}; use crate::sandbox::sync::WaitableCell; -use crate::sandbox::{containerd, Error as SandboxError, Instance as SandboxInstance, InstanceConfig, Stdio}; +use crate::sandbox::{ + containerd, Error as SandboxError, Instance as SandboxInstance, InstanceConfig, Stdio, +}; use crate::sys::container::executor::Executor; static DEFAULT_CONTAINER_ROOT_DIR: &str = "/run/containerd"; @@ -40,16 +43,16 @@ impl SandboxInstance for Instance { let rootdir = determine_rootdir(&bundle, &namespace, rootdir)?; let stdio = Stdio::init_from_cfg(cfg)?; - // check if container is OCI artifact and attempt to read the module - let modules = containerd::Client::connect(cfg.get_containerd_address(), &namespace)? + // check if container is OCI image with wasm layers and attempt to read the module + let (modules, platform) = containerd::Client::connect(cfg.get_containerd_address(), &namespace)? .load_modules(&id) .unwrap_or_else(|e| { - log::warn!("Error obtaining wasm layers for container {}. Will attempt to use files inside containerd image. Error: {}", id.clone(), e); - vec![] + log::warn!("Error obtaining wasm layers for container {id}. Will attempt to use files inside container image. Error: {e}"); + (vec![], Platform::default()) }); ContainerBuilder::new(id.clone(), SyscallType::Linux) - .with_executor(Executor::new(engine, stdio, modules)) + .with_executor(Executor::new(engine, stdio, modules, platform)) .with_root_path(rootdir.clone())? .as_init(&bundle) .with_systemd(false) diff --git a/crates/containerd-shim-wasmedge/src/instance.rs b/crates/containerd-shim-wasmedge/src/instance.rs index a42560123..ce723d439 100644 --- a/crates/containerd-shim-wasmedge/src/instance.rs +++ b/crates/containerd-shim-wasmedge/src/instance.rs @@ -57,7 +57,7 @@ impl Engine for WasmEdgeEngine { PluginManager::load(None)?; let vm = vm.auto_detect_plugins()?; - let vm = match ctx.oci_artifacts() { + let vm = match ctx.wasm_layers() { [] => { debug!("loading module from file"); let path = entrypoint_path @@ -69,11 +69,11 @@ impl Engine for WasmEdgeEngine { .context("registering module")? } [module] => { - log::info!("loading module from OCI Artifact"); + log::info!("loading module from wasm OCI layers"); vm.register_module_from_bytes(&mod_name, &module.layer) .context("registering module")? } - [..] => bail!("only a single module is supported when using OCI Artifact"), + [..] => bail!("only a single module is supported when using images with OCI layers"), }; stdio.redirect()?; diff --git a/crates/containerd-shim-wasmer/src/instance.rs b/crates/containerd-shim-wasmer/src/instance.rs index 360d7ed68..7ca35a2ec 100644 --- a/crates/containerd-shim-wasmer/src/instance.rs +++ b/crates/containerd-shim-wasmer/src/instance.rs @@ -31,7 +31,7 @@ impl Engine for WasmerEngine { log::info!("Create a Store"); let mut store = Store::new(self.engine.clone()); - let module = match ctx.oci_artifacts() { + let module = match ctx.wasm_layers() { [] => { log::info!("loading module from file {path:?}"); let path = path @@ -42,10 +42,10 @@ impl Engine for WasmerEngine { Module::from_file(&store, path)? } [module] => { - log::info!("loading module from OCI Artifact"); + log::info!("loading module wasm OCI layers"); Module::from_binary(&store, &module.layer)? } - [..] => bail!("only a single module is supported when using OCI Artifact"), + [..] => bail!("only a single module is supported when using images with OCI layers"), }; let runtime = tokio::runtime::Builder::new_multi_thread() diff --git a/crates/containerd-shim-wasmtime/src/instance.rs b/crates/containerd-shim-wasmtime/src/instance.rs index 057c50fb1..3655a2773 100644 --- a/crates/containerd-shim-wasmtime/src/instance.rs +++ b/crates/containerd-shim-wasmtime/src/instance.rs @@ -37,7 +37,7 @@ impl Engine for WasmtimeEngine { log::info!("wasi context ready"); let WasiEntrypoint { path, func } = ctx.wasi_entrypoint(); - let module = match ctx.oci_artifacts() { + let module = match ctx.wasm_layers() { [] => { log::info!("loading module from file"); let path = path @@ -48,10 +48,10 @@ impl Engine for WasmtimeEngine { Module::from_file(&self.engine, path)? } [module] => { - log::info!("loading module from OCI Artifact"); + log::info!("loading module wasm OCI layers"); Module::from_binary(&self.engine, &module.layer)? } - [..] => bail!("only a single module is supported when using OCI Artifact"), + [..] => bail!("only a single module is supported when using images with OCI layers"), }; let mut linker = Linker::new(&self.engine); diff --git a/crates/oci-tar-builder/README.md b/crates/oci-tar-builder/README.md index fd0911eae..f775f9641 100644 --- a/crates/oci-tar-builder/README.md +++ b/crates/oci-tar-builder/README.md @@ -20,7 +20,7 @@ See [wasi-demo-app build script](../wasi-demo-app/build.rs) for an example. ### Executable usage -There is an experimental executable that uses the library and can package a wasm module as an OCI artifact. This isn't currently usable with containerd/runwasi today. See the [OCI artifacts in containerd](https://docs.google.com/document/d/11shgC3l6gplBjWF1VJCWvN_9do51otscAm0hBDGSSAc) for more information. +There is an experimental executable that uses the library and can package a wasm module as an OCI image with wasm layers. See the [OCI WASM in containerd](https://docs.google.com/document/d/11shgC3l6gplBjWF1VJCWvN_9do51otscAm0hBDGSSAc) for more information. To generate the package and import to a registry using a tool such as [regctl](https://github.com/regclient/regclient/blob/main/docs/regctl.md#image-commands): diff --git a/crates/oci-tar-builder/src/bin.rs b/crates/oci-tar-builder/src/bin.rs index 06575ffdb..0d1ba80b8 100644 --- a/crates/oci-tar-builder/src/bin.rs +++ b/crates/oci-tar-builder/src/bin.rs @@ -4,11 +4,9 @@ use std::{env, fs}; use anyhow::Context; use clap::Parser; -use oci_spec::image as spec; +use oci_spec::image::{self as spec, Arch}; use oci_tar_builder::Builder; -// Following labels and media types must match in ../containerd-shim-wasm/src/sandbox/oci.rs -// not using it here to avoid needing to have all the deps for containerd-shim-wasm pub const WASM_LAYER_MEDIA_TYPE: &str = "application/vnd.bytecodealliance.wasm.component.layer.v0+wasm"; @@ -27,8 +25,6 @@ pub fn main() { let mut builder = Builder::default(); for module_path in args.module.iter() { - builder.as_module_artifact(); - let module_path = PathBuf::from(module_path); builder.add_layer_with_media_type(&module_path, WASM_LAYER_MEDIA_TYPE.to_string()); } @@ -43,7 +39,6 @@ pub fn main() { } if let Some(components_path) = args.components.as_deref() { - builder.as_component_artifact(); let paths = fs::read_dir(components_path).unwrap(); for path in paths { @@ -69,8 +64,8 @@ pub fn main() { let img = spec::ImageConfigurationBuilder::default() .config(config) - .os("wasi") - .architecture("wasm") + .os("wasip1") + .architecture(Arch::Wasm) .rootfs( spec::RootFsBuilder::default() .diff_ids(vec![]) diff --git a/crates/oci-tar-builder/src/lib.rs b/crates/oci-tar-builder/src/lib.rs index 3defdacd5..cabf785a1 100644 --- a/crates/oci-tar-builder/src/lib.rs +++ b/crates/oci-tar-builder/src/lib.rs @@ -11,18 +11,10 @@ use oci_spec::image::{ }; use serde::Serialize; use sha256::{digest, try_digest}; - -// duplicated here to avoid pulling in all of containerd-shim-wasm -// Module is preview1 -// Component is preview2 -const COMPONENT_ARTIFACT_TYPE: &str = "application/vnd.bytecodealliance.component.v1+wasm"; -const MODULE_ARTIFACT_TYPE: &str = "application/vnd.bytecodealliance.module.v1+wasm"; - #[derive(Debug, Default)] pub struct Builder { configs: Vec<(ImageConfiguration, String)>, layers: Vec<(PathBuf, String)>, - artifact_type: Option, } #[derive(Serialize, Debug)] @@ -131,7 +123,7 @@ impl Builder { .build() .context("failed to build descriptor")?; - // add all layer_digests including any OCI artifacts types that are may not be in the rootfs + // add all layer_digests including any OCI WASM types that are may not be in the rootfs let mut layers = Vec::new(); for (_k, v) in layer_digests.iter() { layers.push(v.clone()); @@ -155,14 +147,10 @@ impl Builder { mfst.repo_tags.push(config.1.clone()); annotations.insert("io.containerd.image.name".to_string(), config.1.clone()); - let mut manifest = ImageManifestBuilder::default() + let manifest = ImageManifestBuilder::default() .schema_version(SCHEMA_VERSION) .media_type(MediaType::ImageManifest); - if self.artifact_type.is_some() { - manifest = manifest.artifact_type(self.artifact_type.clone().unwrap().as_str()); - } - let manifest = manifest .layers(layers) .config(desc) @@ -244,12 +232,4 @@ impl Builder { Ok(()) } - - pub fn as_module_artifact(&mut self) { - self.artifact_type = Some(MODULE_ARTIFACT_TYPE.to_string()); - } - - pub fn as_component_artifact(&mut self) { - self.artifact_type = Some(COMPONENT_ARTIFACT_TYPE.to_string()); - } } diff --git a/crates/wasi-demo-app/build.rs b/crates/wasi-demo-app/build.rs index f5b43ed11..70a336691 100644 --- a/crates/wasi-demo-app/build.rs +++ b/crates/wasi-demo-app/build.rs @@ -1,7 +1,12 @@ #[cfg(feature = "oci-v1-tar")] use { - anyhow::Context, oci_spec::image as spec, oci_tar_builder::Builder, sha256::try_digest, - std::env, std::fs::File, std::path::PathBuf, + anyhow::Context, + oci_spec::image::{self as spec, Arch}, + oci_tar_builder::Builder, + sha256::try_digest, + std::env, + std::fs::File, + std::path::PathBuf, }; #[cfg(not(feature = "oci-v1-tar"))] @@ -39,8 +44,8 @@ fn main() { let layer_digest = try_digest(layer_path.as_path()).unwrap(); let img = spec::ImageConfigurationBuilder::default() .config(config) - .os("wasi") - .architecture("wasm") + .os("wasip1") + .architecture(Arch::Wasm) .rootfs( spec::RootFsBuilder::default() .diff_ids(vec!["sha256:".to_owned() + &layer_digest]) diff --git a/cross/Dockerfile.gnu b/cross/Dockerfile.gnu index 54ca2996e..9ba7fb199 100644 --- a/cross/Dockerfile.gnu +++ b/cross/Dockerfile.gnu @@ -5,4 +5,4 @@ FROM $CROSS_BASE_IMAGE ARG CROSS_DEB_ARCH RUN dpkg --add-architecture ${CROSS_DEB_ARCH} && \ apt-get -y update && \ - apt-get install -y pkg-config libseccomp-dev:${CROSS_DEB_ARCH} + apt-get install -y pkg-config protobuf-compiler libseccomp-dev:${CROSS_DEB_ARCH} diff --git a/cross/Dockerfile.musl b/cross/Dockerfile.musl index e359b520c..48dbd6c60 100644 --- a/cross/Dockerfile.musl +++ b/cross/Dockerfile.musl @@ -22,4 +22,4 @@ rustflags = ["-Clink-arg=-lgcc"] EOF RUN apt-get -y update && \ - apt-get install -y pkg-config + apt-get install -y pkg-config protobuf-compiler diff --git a/test/k8s/Dockerfile.oci b/test/k8s/Dockerfile.oci index 4473470a0..2dcbccc9e 100644 --- a/test/k8s/Dockerfile.oci +++ b/test/k8s/Dockerfile.oci @@ -1,13 +1,13 @@ # syntax=docker/dockerfile:1.4 -ARG KIND_NODE_VERSION=v1.23.13 +ARG KIND_NODE_VERSION=v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72 ARG RUNTIME=wasmtime -ARG GO_VERSION="1.21.1" +ARG GO_VERSION="1.21.3-bullseye" # modified from https://github.com/kubernetes-sigs/kind/blob/main/images/base/Dockerfile # stage for building containerd FROM golang:${GO_VERSION} as build-containerd -ARG CONTAINERD_VERSION="main" +ARG CONTAINERD_VERSION="v1.7.7" ARG CONTAINERD_CLONE_URL="https://github.com/containerd/containerd" # we don't build with optional snapshotters, we never select any of these # they're not ideal inside kind anyhow, and we save some disk space @@ -28,8 +28,7 @@ RUN apt-get update -y && \ ADD dist/bin/* /usr/local/bin/ ARG RUNTIME -RUN sed -i 's,SystemdCgroup = true,,' /etc/containerd/config.toml && \ - cat <> /etc/containerd/config.toml +RUN cat <> /etc/containerd/config.toml [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.wasm] runtime_type = "io.containerd.${RUNTIME}.v1" EOF \ No newline at end of file