ci: add userspace convertor E2E

Signed-off-by: Bowei Zhuang <[email protected]>

.github/workflows/ci-userspace-convertor.yml

@@ -28,6 +28,8 @@ jobs:
         - /sys/kernel/config:/sys/kernel/config
         - /mnt:/var/lib/containerd
       options: --privileged
+      env:
+        DOCKER_HOST: "unix:///app/dockerd/docker.sock"
 
     steps:
       - name: Checkout
@@ -41,12 +43,29 @@ jobs:
         shell: bash
         run: |
           bash start_services.sh
-          sleep 3s
-          docker run -d -p 5000:5000 --restart always --name registry registry:2
-          sleep 3s
+          sleep 5s
+
+      - name: Prepare Local Registry
+        working-directory: ci/scripts
+        shell: bash
+        run: |
+          bash new_registry.sh
+          bash prepare_image.sh registry.hub.docker.com/overlaybd/centos:centos7.9.2009 localhost:5000/centos:centos7.9.2009 && \
+          bash prepare_image.sh registry.hub.docker.com/overlaybd/ubuntu:22.04          localhost:5000/ubuntu:22.04 && \
+          bash prepare_image.sh registry.hub.docker.com/overlaybd/redis:7.2.3           localhost:5000/redis:7.2.3 && \
+          bash prepare_image.sh registry.hub.docker.com/overlaybd/wordpress:6.4.2       localhost:5000/wordpress:6.4.2 && \
+          bash prepare_image.sh registry.hub.docker.com/overlaybd/nginx:1.25.3          localhost:5000/nginx:1.25.3
 
       - name: CI - uconv reproduce
         working-directory: ci/uconv_reproduce
         shell: bash
         run: |
           bash ci-uconv-reproduce.sh
+
+      - name: CI - uconv E2E
+        working-directory: ci/scripts
+        shell: bash
+        run: |
+          /opt/overlaybd/snapshotter/convertor -r localhost:5000/redis -i 7.2.3 --overlaybd 7.2.3_overlaybd --turboOCI 7.2.3_turbo
+          bash run_container.sh localhost:5000/redis:7.2.3_overlaybd
+          bash run_container.sh localhost:5000/redis:7.2.3_turbo

ci/build_image/Dockerfile

@@ -48,10 +48,8 @@ RUN apt-get update && apt-get install -y apt-transport-https ca-certificates cur
       "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \
       $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null && \
     apt-get update && apt-get install -y docker-ce docker-ce-cli containerd.io && \
-    apt-get clean && rm -rf /var/lib/apt/lists/*
-
-RUN \
-    apt update && apt install -y libnl-3-200 libnl-genl-3-200 libcurl4-openssl-dev libaio-dev wget less kmod && \
+    apt-get install -y libnl-3-200 libnl-genl-3-200 libcurl4-openssl-dev libaio-dev wget less kmod && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* && \
     wget https://github.com/containerd/overlaybd/releases/download/v${OBD_VERSION}/overlaybd-${OBD_VERSION}-0ubuntu1.22.04.x86_64.deb && \
     dpkg -i overlaybd-${OBD_VERSION}-0ubuntu1.22.04.x86_64.deb && \
     dpkg -i overlaybd-snapshotter_${RELEASE_VERSION}_amd64.deb && \

ci/build_image/start_services.sh

@@ -20,5 +20,3 @@ fi
 /opt/overlaybd/snapshotter/overlaybd-snapshotter &>/var/log/overlaybd-snapshotter.log &
 
 /sbin/modprobe overlay && /usr/bin/containerd &>/var/log/containerd.log &
-
-dockerd &>/var/log/dockerd.log &

ci/scripts/new_registry.sh

@@ -0,0 +1,85 @@
+#!/bin/bash
+#
+# run a HTTPS registry
+
+set -x
+
+mkdir -p /app/dockerd/
+echo '{ "hosts": ["unix:///app/dockerd/docker.sock"] }' > /etc/docker/daemon.json
+dockerd &>/var/log/dockerd.log &
+
+sleep 3
+
+rm -rf /etc/registry/
+mkdir -p /etc/registry/certs/
+mkdir -p /etc/registry/config/
+
+# generate server certifications
+cat << EOF > /etc/registry/openssl.cnf
+[req]
+distinguished_name = req_distinguished_name
+x509_extensions = v3_req
+prompt = no
+
+[req_distinguished_name]
+C = CN
+ST = Beijing
+L = Beijing City
+O = Alibaba
+CN = localhost
+
+[v3_req]
+basicConstraints = CA:FALSE
+keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+subjectAltName = @alt_names
+
+[alt_names]
+DNS.1 = localhost
+IP.1 = 127.0.0.1
+EOF
+
+openssl req -new -x509 -newkey rsa:2048 -sha256 -nodes -config /etc/registry/openssl.cnf \
+  -days 365 -out /etc/registry/certs/server.crt -keyout /etc/registry/certs/server.key
+
+ls /etc/registry/certs/
+cp /etc/registry/certs/server.crt /usr/local/share/ca-certificates/registry.crt
+update-ca-certificates
+
+# start registry
+cat << EOF > /etc/registry/config/config.yml
+version: 0.1
+log:
+  fields:
+    service: registry
+storage:
+  cache:
+    blobdescriptor: inmemory
+  filesystem:
+    rootdirectory: /var/lib/registry
+http:
+  addr: :5000
+  headers:
+    X-Content-Type-Options: [nosniff]
+  tls:
+    certificate: /certs/server.crt
+    key: /certs/server.key
+health:
+  storagedriver:
+    enabled: true
+    interval: 10s
+    threshold: 3
+EOF
+
+docker run -d --restart=always --name registry -p 5000:5000 \
+  -v /etc/registry/certs:/certs \
+  -v /etc/registry/config:/etc/docker/registry/ \
+  registry:2
+
+sleep 5s
+
+docker ps -a
+apt-get update && apt-get install -y lsof
+lsof -i :5000
+curl http://localhost:5000/v2/_catalog
+lsof -i :5000
+curl https://localhost:5000/v2/_catalog

ci/scripts/prepare_image.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+from=${1:?}
+to=${2:?}
+
+set -x
+
+ctr i pull "${from}"
+ctr i tag "${from}" "${to}"
+ctr i push "${to}"
+ctr i rm "${from}" "${to}"

ci/scripts/run_container.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# rpull and run on-demand
+
+image=$1
+container_name=${2:-test}
+
+exit_code=0
+
+/opt/overlaybd/snapshotter/ctr rpull "${image}"
+if ! ctr run -d --net-host --snapshotter=overlaybd "${image}" "${container_name}"; then
+  exit_code=1
+fi
+if ! ctr t ls | grep "${container_name}"; then
+  exit_code=1
+fi
+ctr t kill -s 9 "${container_name}" && sleep 5s && ctr t ls
+ctr c rm "${container_name}" && ctr c ls
+ctr i rm "${image}"
+
+if [[ ${exit_code} -ne 0 ]]; then
+  cat /var/log/overlaybd.log
+fi
+
+exit ${exit_code}

ci/uconv_reproduce/ci-uconv-reproduce.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-apt install -y python3 jq
+apt update && apt install -y python3 jq
 
 convertor="/opt/overlaybd/snapshotter/convertor"
 images=("centos:centos7.9.2009" "ubuntu:22.04" "redis:7.2.3" "wordpress:6.4.2" "nginx:1.25.3")
@@ -11,11 +11,6 @@ result=0
 
 for image in "${images[@]}"
 do
-    from_img="registry.hub.docker.com/overlaybd/${image}"
-    ctr i pull "${from_img}" &> /dev/null
-    ctr i tag "${from_img}" "${registry}/${image}" &> /dev/null
-    ctr i push "${registry}/${image}" &> /dev/null
-
     img=${image%%":"*}
     tag=${image##*":"}
     echo "${img} ${tag}"
@@ -35,16 +30,16 @@ do
     output_turbo="${workspace}/convert.turbo.out"
 
     ${convertor} -r "${registry}/${img}" -i "${tag}" --overlaybd "${tag_obd}" -d "${workspace}/overlaybd_tmp_conv" &> "${output_obd}"
-    curl -H "Accept: application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.manifest.v1+json" -o "${manifest_obd}" "${registry}/v2/${img}/manifests/${tag_obd}" &> /dev/null
+    curl -H "Accept: application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.manifest.v1+json" -o "${manifest_obd}" "https://${registry}/v2/${img}/manifests/${tag_obd}" &> /dev/null
     configDigest=$(jq '.config.digest' "${manifest_obd}")
     configDigest=${configDigest//\"/}
-    curl -o "${config_obd}" "${registry}/v2/${img}/blobs/${configDigest}" &> /dev/null
+    curl -o "${config_obd}" "https://${registry}/v2/${img}/blobs/${configDigest}" &> /dev/null
 
     ${convertor} -r "${registry}/${img}" -i "${tag}" --turboOCI "${tag_turbo}" -d "${workspace}/turbo_tmp_conv" &> "${output_turbo}"
-    curl -H "Accept: application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.manifest.v1+json" -o "${manifest_turbo}" "${registry}/v2/${img}/manifests/${tag_turbo}" &> /dev/null
+    curl -H "Accept: application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.manifest.v1+json" -o "${manifest_turbo}" "https://${registry}/v2/${img}/manifests/${tag_turbo}" &> /dev/null
     configDigest=$(jq '.config.digest' "${manifest_turbo}")
     configDigest=${configDigest//\"/}
-    curl -o "${config_turbo}" "${registry}/v2/${img}/blobs/${configDigest}" &> /dev/null
+    curl -o "${config_turbo}" "https://${registry}/v2/${img}/blobs/${configDigest}" &> /dev/null
 
     prefix=$(date +%Y%m%d%H%M%S)
 

ci/uconv_reproduce/compare_layers.py

@@ -48,10 +48,10 @@ def main():
     fb = sys.argv[3]
     if not os.path.exists(fa):
         print("file %s not exist" % fa)
-        return -1
+        sys.exit(-1)
     if not os.path.exists(fb):
         print("file %s not exist" % fb)
-        return -1
+        sys.exit(-1)
     fa_conf = json.load(open(fa, 'r'))
     fb_conf = json.load(open(fb, 'r'))
     if ftype == "manifest":

cmd/convertor/builder/builder.go

@@ -83,17 +83,28 @@ func NewOverlayBDBuilder(ctx context.Context, opt BuilderOptions) (Builder, erro
 		TLSClientConfig:       tlsConfig,
 		ExpectContinueTimeout: 5 * time.Second,
 	}
+	client := &http.Client{Transport: transport}
 	resolver := docker.NewResolver(docker.ResolverOptions{
-		Credentials: func(s string) (string, string, error) {
-			if i := strings.IndexByte(opt.Auth, ':'); i > 0 {
-				return opt.Auth[0:i], opt.Auth[i+1:], nil
-			}
-			return "", "", nil
-		},
-		PlainHTTP: opt.PlainHTTP,
-		Client: &http.Client{
-			Transport: transport,
-		},
+		Hosts: docker.ConfigureDefaultRegistries(
+			docker.WithAuthorizer(docker.NewDockerAuthorizer(
+				docker.WithAuthClient(client),
+				docker.WithAuthHeader(make(http.Header)),
+				docker.WithAuthCreds(func(s string) (string, string, error) {
+					if i := strings.IndexByte(opt.Auth, ':'); i > 0 {
+						return opt.Auth[0:i], opt.Auth[i+1:], nil
+					}
+					return "", "", nil
+				}),
+			)),
+			docker.WithClient(client),
+			docker.WithPlainHTTP(func(s string) (bool, error) {
+				if opt.PlainHTTP {
+					return docker.MatchAllHosts(s)
+				} else {
+					return false, nil
+				}
+			}),
+		),
 	})
 	engineBase, err := getBuilderEngineBase(ctx, resolver, opt.Ref, opt.TargetRef)
 	if err != nil {