From 69798e46f3314a99d293d97f8046046e0887f0c3 Mon Sep 17 00:00:00 2001 From: Bowei Zhuang Date: Wed, 24 Jan 2024 19:42:13 +0800 Subject: [PATCH 1/3] ci: lowercase repo Signed-off-by: Bowei Zhuang --- .github/workflows/ci-basic.yml | 5 ++++- .github/workflows/ci-build-image.yml | 5 ++++- .github/workflows/ci-e2e.yml | 6 ++++++ .github/workflows/ci-userspace-convertor.yml | 5 ++++- .github/workflows/ci.yml | 12 ++++++++++++ 5 files changed, 30 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-basic.yml b/.github/workflows/ci-basic.yml index c5b1ecaf..6f3d5f82 100644 --- a/.github/workflows/ci-basic.yml +++ b/.github/workflows/ci-basic.yml @@ -5,6 +5,9 @@ on: image-tag: required: true type: string + github-repository: + required: true + type: string jobs: run-ci-basic: @@ -12,7 +15,7 @@ jobs: runs-on: ubuntu-22.04 timeout-minutes: 10 container: - image: ghcr.io/${{ github.repository }}/overlaybd-ci-images:${{ inputs.image-tag }} + image: ghcr.io/${{ inputs.github-repository }}/overlaybd-ci-images:${{ inputs.image-tag }} credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci-build-image.yml b/.github/workflows/ci-build-image.yml index 27921609..292fe012 100644 --- a/.github/workflows/ci-build-image.yml +++ b/.github/workflows/ci-build-image.yml @@ -8,6 +8,9 @@ on: image-tag: required: true type: string + github-repository: + required: true + type: string env: GO_VERSION: "1.19" @@ -51,7 +54,7 @@ jobs: - name: Build and Push uses: docker/build-push-action@v4 with: - tags: ghcr.io/${{ github.repository }}/overlaybd-ci-images:${{ inputs.image-tag }} + tags: ghcr.io/${{ inputs.github-repository }}/overlaybd-ci-images:${{ inputs.image-tag }} push: true context: . platforms: linux/amd64 diff --git a/.github/workflows/ci-e2e.yml b/.github/workflows/ci-e2e.yml index c3807291..3202290d 100644 --- a/.github/workflows/ci-e2e.yml +++ b/.github/workflows/ci-e2e.yml @@ -8,6 +8,9 @@ on: image-tag: required: true type: string + github-repository: + required: true + type: string concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -19,6 +22,7 @@ jobs: with: commit-hash: ${{ inputs.commit-hash }} image-tag: ${{ inputs.image-tag }} + github-repository: ${{ inputs.github-repository }} secrets: inherit run-ci-basic: @@ -26,6 +30,7 @@ jobs: uses: ./.github/workflows/ci-basic.yml with: image-tag: ${{ inputs.image-tag }} + github-repository: ${{ inputs.github-repository }} secrets: inherit run-ci-userspace-convertor: @@ -34,4 +39,5 @@ jobs: with: commit-hash: ${{ inputs.commit-hash }} image-tag: ${{ inputs.image-tag }} + github-repository: ${{ inputs.github-repository }} secrets: inherit diff --git a/.github/workflows/ci-userspace-convertor.yml b/.github/workflows/ci-userspace-convertor.yml index d2f0992b..92188443 100644 --- a/.github/workflows/ci-userspace-convertor.yml +++ b/.github/workflows/ci-userspace-convertor.yml @@ -8,6 +8,9 @@ on: image-tag: required: true type: string + github-repository: + required: true + type: string jobs: run-ci-userspace-convertor: @@ -15,7 +18,7 @@ jobs: runs-on: ubuntu-22.04 timeout-minutes: 10 container: - image: ghcr.io/${{ github.repository }}/overlaybd-ci-images:${{ inputs.image-tag }} + image: ghcr.io/${{ inputs.github-repository }}/overlaybd-ci-images:${{ inputs.image-tag }} credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 56ab2da4..b56a29cd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,11 +45,23 @@ jobs: run: | sudo GO_TESTFLAGS=-v make test + lowercase-repo: + name: Lowercase Repo + runs-on: ubuntu-22.04 + timeout-minutes: 10 + outputs: + repository: ${{ steps.lowercase_repository.outputs.repository }} + steps: + - id: lowercase_repository + run: echo "repository=${GITHUB_REPOSITORY,,}" >> "$GITHUB_OUTPUT" + e2e-test: + needs: lowercase-repo name: E2E Test if: ${{ contains(github.event.pull_request.labels.*.name, 'ok-to-test') || github.event_name == 'push' }} uses: ./.github/workflows/ci-e2e.yml with: commit-hash: ${{ github.event.pull_request.head.sha || github.sha }} image-tag: ${{ github.event.pull_request.number || 'default' }} + github-repository: ${{ needs.lowercase-repo.outputs.repository }} secrets: inherit From a03d841afa03f044c94c5694d96da5338af33210 Mon Sep 17 00:00:00 2001 From: Bowei Zhuang Date: Wed, 24 Jan 2024 19:32:51 +0800 Subject: [PATCH 2/3] ci: add turbo reproduce test & use local registry Signed-off-by: Bowei Zhuang --- .github/workflows/ci-userspace-convertor.yml | 4 +- ci/build_image/Dockerfile | 11 +++- ci/build_image/start_services.sh | 2 + ci/uconv_reproduce/centos/config-turbo.json | 1 + ci/uconv_reproduce/centos/manifest-turbo.json | 1 + ci/uconv_reproduce/ci-uconv-reproduce.sh | 65 +++++++++++++------ ci/uconv_reproduce/nginx/config-turbo.json | 1 + ci/uconv_reproduce/nginx/manifest-turbo.json | 1 + ci/uconv_reproduce/redis/config-turbo.json | 1 + ci/uconv_reproduce/redis/manifest-turbo.json | 1 + ci/uconv_reproduce/ubuntu/config-turbo.json | 1 + ci/uconv_reproduce/ubuntu/manifest-turbo.json | 1 + .../wordpress/config-turbo.json | 1 + .../wordpress/manifest-turbo.json | 1 + 14 files changed, 69 insertions(+), 23 deletions(-) create mode 100644 ci/uconv_reproduce/centos/config-turbo.json create mode 100644 ci/uconv_reproduce/centos/manifest-turbo.json create mode 100644 ci/uconv_reproduce/nginx/config-turbo.json create mode 100644 ci/uconv_reproduce/nginx/manifest-turbo.json create mode 100644 ci/uconv_reproduce/redis/config-turbo.json create mode 100644 ci/uconv_reproduce/redis/manifest-turbo.json create mode 100644 ci/uconv_reproduce/ubuntu/config-turbo.json create mode 100644 ci/uconv_reproduce/ubuntu/manifest-turbo.json create mode 100644 ci/uconv_reproduce/wordpress/config-turbo.json create mode 100644 ci/uconv_reproduce/wordpress/manifest-turbo.json diff --git a/.github/workflows/ci-userspace-convertor.yml b/.github/workflows/ci-userspace-convertor.yml index 92188443..2a5350db 100644 --- a/.github/workflows/ci-userspace-convertor.yml +++ b/.github/workflows/ci-userspace-convertor.yml @@ -41,7 +41,9 @@ jobs: shell: bash run: | bash start_services.sh - sleep 5s + sleep 3s + docker run -d -p 5000:5000 --restart always --name registry registry:2 + sleep 3s - name: CI - uconv reproduce working-directory: ci/uconv_reproduce diff --git a/ci/build_image/Dockerfile b/ci/build_image/Dockerfile index c806c526..ce4d9b5f 100644 --- a/ci/build_image/Dockerfile +++ b/ci/build_image/Dockerfile @@ -41,8 +41,17 @@ SHELL ["/bin/bash", "-c"] WORKDIR /app COPY --from=builder /tmp/overlaybd-snapshotter_${RELEASE_VERSION}_amd64.deb . COPY ./ci/build_image/start_services.sh . + +RUN apt-get update && apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release software-properties-common && \ + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg && \ + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null && \ + apt-get update && apt-get install -y docker-ce docker-ce-cli containerd.io && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + RUN \ - apt update && apt install -y containerd libnl-3-200 libnl-genl-3-200 libcurl4-openssl-dev libaio-dev wget less kmod && \ + apt update && apt install -y libnl-3-200 libnl-genl-3-200 libcurl4-openssl-dev libaio-dev wget less kmod && \ wget https://github.com/containerd/overlaybd/releases/download/v${OBD_VERSION}/overlaybd-${OBD_VERSION}-0ubuntu1.22.04.x86_64.deb && \ dpkg -i overlaybd-${OBD_VERSION}-0ubuntu1.22.04.x86_64.deb && \ dpkg -i overlaybd-snapshotter_${RELEASE_VERSION}_amd64.deb && \ diff --git a/ci/build_image/start_services.sh b/ci/build_image/start_services.sh index 5f9e5248..32dd4720 100644 --- a/ci/build_image/start_services.sh +++ b/ci/build_image/start_services.sh @@ -20,3 +20,5 @@ fi /opt/overlaybd/snapshotter/overlaybd-snapshotter &>/var/log/overlaybd-snapshotter.log & /sbin/modprobe overlay && /usr/bin/containerd &>/var/log/containerd.log & + +dockerd &>/var/log/dockerd.log & diff --git a/ci/uconv_reproduce/centos/config-turbo.json b/ci/uconv_reproduce/centos/config-turbo.json new file mode 100644 index 00000000..a8234abc --- /dev/null +++ b/ci/uconv_reproduce/centos/config-turbo.json @@ -0,0 +1 @@ +{"created":"2021-09-15T18:20:23.99863383Z","architecture":"amd64","os":"linux","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/bash"],"Labels":{"org.label-schema.build-date":"20201113","org.label-schema.license":"GPLv2","org.label-schema.name":"CentOS Base Image","org.label-schema.schema-version":"1.0","org.label-schema.vendor":"CentOS","org.opencontainers.image.created":"2020-11-13 00:00:00+00:00","org.opencontainers.image.licenses":"GPL-2.0-only","org.opencontainers.image.title":"CentOS Base Image","org.opencontainers.image.vendor":"CentOS"}},"rootfs":{"type":"layers","diff_ids":["sha256:a5cdf182bbffc6992b556633bbb275b559056b6e953133772a80fa261886c07b"]},"history":[{"created":"2021-09-15T18:20:23.417639551Z","created_by":"/bin/sh -c #(nop) ADD file:b3ebbe8bd304723d43b7b44a6d990cd657b63d93d6a2a9293983a30bfc1dfa53 in / "},{"created":"2021-09-15T18:20:23.819893035Z","created_by":"/bin/sh -c #(nop) LABEL org.label-schema.schema-version=1.0 org.label-schema.name=CentOS Base Image org.label-schema.vendor=CentOS org.label-schema.license=GPLv2 org.label-schema.build-date=20201113 org.opencontainers.image.title=CentOS Base Image org.opencontainers.image.vendor=CentOS org.opencontainers.image.licenses=GPL-2.0-only org.opencontainers.image.created=2020-11-13 00:00:00+00:00","empty_layer":true},{"created":"2021-09-15T18:20:23.99863383Z","created_by":"/bin/sh -c #(nop) CMD [\"/bin/bash\"]","empty_layer":true}]} \ No newline at end of file diff --git a/ci/uconv_reproduce/centos/manifest-turbo.json b/ci/uconv_reproduce/centos/manifest-turbo.json new file mode 100644 index 00000000..badf1bd8 --- /dev/null +++ b/ci/uconv_reproduce/centos/manifest-turbo.json @@ -0,0 +1 @@ +{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json","config":{"mediaType":"application/vnd.docker.container.image.v1+json","digest":"sha256:bdeebc045cc88055079dd5ab899cf594614b2757d5b572ec7acc6f4a24031fb1","size":1476},"layers":[{"mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip","digest":"sha256:e91a425b0b5001fa96ae2909db392034ee9eccfdf9347410a1343719db83054e","size":2717102,"annotations":{"containerd.io/snapshot/overlaybd/blob-digest":"sha256:e91a425b0b5001fa96ae2909db392034ee9eccfdf9347410a1343719db83054e","containerd.io/snapshot/overlaybd/blob-size":"2717102","containerd.io/snapshot/overlaybd/turbo-oci/target-digest":"sha256:2d473b07cdd5f0912cd6f1a703352c82b512407db6b05b43f2553732b55df3bc","containerd.io/snapshot/overlaybd/turbo-oci/target-media-type":"application/vnd.docker.image.rootfs.diff.tar.gzip","containerd.io/snapshot/overlaybd/version":"0.1.0-turbo.ociv1"}}]} \ No newline at end of file diff --git a/ci/uconv_reproduce/ci-uconv-reproduce.sh b/ci/uconv_reproduce/ci-uconv-reproduce.sh index abe67d4a..c12bd4cd 100644 --- a/ci/uconv_reproduce/ci-uconv-reproduce.sh +++ b/ci/uconv_reproduce/ci-uconv-reproduce.sh @@ -1,52 +1,75 @@ #!/bin/bash -apt install -y python3 +apt install -y python3 jq convertor="/opt/overlaybd/snapshotter/convertor" images=("centos:centos7.9.2009" "ubuntu:22.04" "redis:7.2.3" "wordpress:6.4.2" "nginx:1.25.3") -repo="registry.hub.docker.com/overlaybd" +registry="localhost:5000" ci_base=$(pwd) result=0 -for image in ${images[@]} +for image in "${images[@]}" do + from_img="registry.hub.docker.com/overlaybd/${image}" + ctr i pull "${from_img}" &> /dev/null + ctr i tag "${from_img}" "${registry}/${image}" &> /dev/null + ctr i push "${registry}/${image}" &> /dev/null + img=${image%%":"*} tag=${image##*":"} - echo ${img} ${tag} + echo "${img} ${tag}" + + workspace="${ci_base}/workspace_${image/:/_}" + + rm -rf "${workspace}" + mkdir -p "${workspace}" - o_tag="${tag}_obd" - tmp_dir="${ci_base}/tmp_conv_${image/:/_}" + tag_obd="${tag}_overlaybd" + tag_turbo="${tag}_turbo" + manifest_obd="${workspace}/manifest.json" + manifest_turbo="${workspace}/manifest-turbo.json" + config_obd="${workspace}/config.json" + config_turbo="${workspace}/config-turbo.json" + output_obd="${workspace}/convert.overlaybd.out" + output_turbo="${workspace}/convert.turbo.out" - rm -rf ${tmp_dir} - mkdir -p ${tmp_dir} + ${convertor} -r "${registry}/${img}" -i "${tag}" --overlaybd "${tag_obd}" -d "${workspace}/overlaybd_tmp_conv" &> "${output_obd}" + curl -H "Accept: application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.manifest.v1+json" -o "${manifest_obd}" "${registry}/v2/${img}/manifests/${tag_obd}" &> /dev/null + configDigest=$(jq '.config.digest' "${manifest_obd}") + configDigest=${configDigest//\"/} + curl -o "${config_obd}" "${registry}/v2/${img}/blobs/${configDigest}" &> /dev/null - ${convertor} -r ${repo}/${img} \ - --reserve --no-upload --dump-manifest \ - -i ${tag} -o ${o_tag} -d ${tmp_dir} &>${tmp_dir}/convert.out + ${convertor} -r "${registry}/${img}" -i "${tag}" --turboOCI "${tag_turbo}" -d "${workspace}/turbo_tmp_conv" &> "${output_turbo}" + curl -H "Accept: application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.manifest.v1+json" -o "${manifest_turbo}" "${registry}/v2/${img}/manifests/${tag_turbo}" &> /dev/null + configDigest=$(jq '.config.digest' "${manifest_turbo}") + configDigest=${configDigest//\"/} + curl -o "${config_turbo}" "${registry}/v2/${img}/blobs/${configDigest}" &> /dev/null prefix=$(date +%Y%m%d%H%M%S) - files=("manifest" "config") + + mode=("manifest" "config" "manifest" "config") + actual=("${manifest_obd}" "${config_obd}" "${manifest_turbo}" "${config_turbo}") + expected=("${ci_base}/${img}/manifest.json" "${ci_base}/${img}/config.json" "${ci_base}/${img}/manifest-turbo.json" "${ci_base}/${img}/config-turbo.json") + conv_res=0 - for file in ${files[@]} - do - fn="${file}.json" - # diff ${tmp_dir}/${fn} ${ci_base}/${img}/${fn} - python3 compare_layers.py ${file} ${tmp_dir}/${fn} ${ci_base}/${img}/${fn} + n=${#mode[@]} + for ((i=0; i Date: Thu, 25 Jan 2024 10:50:41 +0800 Subject: [PATCH 3/3] ci: add userspace convertor E2E Signed-off-by: Bowei Zhuang --- .github/workflows/ci-userspace-convertor.yml | 25 +++++- ci/build_image/Dockerfile | 6 +- ci/build_image/start_services.sh | 2 - ci/scripts/new_registry.sh | 85 ++++++++++++++++++++ ci/scripts/prepare_image.sh | 11 +++ ci/scripts/run_container.sh | 25 ++++++ ci/uconv_reproduce/ci-uconv-reproduce.sh | 15 ++-- ci/uconv_reproduce/compare_layers.py | 4 +- cmd/convertor/builder/builder.go | 31 ++++--- 9 files changed, 173 insertions(+), 31 deletions(-) create mode 100755 ci/scripts/new_registry.sh create mode 100755 ci/scripts/prepare_image.sh create mode 100755 ci/scripts/run_container.sh diff --git a/.github/workflows/ci-userspace-convertor.yml b/.github/workflows/ci-userspace-convertor.yml index 2a5350db..7e4795e1 100644 --- a/.github/workflows/ci-userspace-convertor.yml +++ b/.github/workflows/ci-userspace-convertor.yml @@ -28,6 +28,8 @@ jobs: - /sys/kernel/config:/sys/kernel/config - /mnt:/var/lib/containerd options: --privileged + env: + DOCKER_HOST: "unix:///app/dockerd/docker.sock" steps: - name: Checkout @@ -41,12 +43,29 @@ jobs: shell: bash run: | bash start_services.sh - sleep 3s - docker run -d -p 5000:5000 --restart always --name registry registry:2 - sleep 3s + sleep 5s + + - name: Prepare Local Registry + working-directory: ci/scripts + shell: bash + run: | + bash new_registry.sh + bash prepare_image.sh registry.hub.docker.com/overlaybd/centos:centos7.9.2009 localhost:5000/centos:centos7.9.2009 && \ + bash prepare_image.sh registry.hub.docker.com/overlaybd/ubuntu:22.04 localhost:5000/ubuntu:22.04 && \ + bash prepare_image.sh registry.hub.docker.com/overlaybd/redis:7.2.3 localhost:5000/redis:7.2.3 && \ + bash prepare_image.sh registry.hub.docker.com/overlaybd/wordpress:6.4.2 localhost:5000/wordpress:6.4.2 && \ + bash prepare_image.sh registry.hub.docker.com/overlaybd/nginx:1.25.3 localhost:5000/nginx:1.25.3 - name: CI - uconv reproduce working-directory: ci/uconv_reproduce shell: bash run: | bash ci-uconv-reproduce.sh + + - name: CI - uconv E2E + working-directory: ci/scripts + shell: bash + run: | + /opt/overlaybd/snapshotter/convertor -r localhost:5000/redis -i 7.2.3 --overlaybd 7.2.3_overlaybd --turboOCI 7.2.3_turbo + bash run_container.sh localhost:5000/redis:7.2.3_overlaybd + bash run_container.sh localhost:5000/redis:7.2.3_turbo diff --git a/ci/build_image/Dockerfile b/ci/build_image/Dockerfile index ce4d9b5f..f465b6ec 100644 --- a/ci/build_image/Dockerfile +++ b/ci/build_image/Dockerfile @@ -48,10 +48,8 @@ RUN apt-get update && apt-get install -y apt-transport-https ca-certificates cur "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \ $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null && \ apt-get update && apt-get install -y docker-ce docker-ce-cli containerd.io && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -RUN \ - apt update && apt install -y libnl-3-200 libnl-genl-3-200 libcurl4-openssl-dev libaio-dev wget less kmod && \ + apt-get install -y libnl-3-200 libnl-genl-3-200 libcurl4-openssl-dev libaio-dev wget less kmod && \ + apt-get clean && rm -rf /var/lib/apt/lists/* && \ wget https://github.com/containerd/overlaybd/releases/download/v${OBD_VERSION}/overlaybd-${OBD_VERSION}-0ubuntu1.22.04.x86_64.deb && \ dpkg -i overlaybd-${OBD_VERSION}-0ubuntu1.22.04.x86_64.deb && \ dpkg -i overlaybd-snapshotter_${RELEASE_VERSION}_amd64.deb && \ diff --git a/ci/build_image/start_services.sh b/ci/build_image/start_services.sh index 32dd4720..5f9e5248 100644 --- a/ci/build_image/start_services.sh +++ b/ci/build_image/start_services.sh @@ -20,5 +20,3 @@ fi /opt/overlaybd/snapshotter/overlaybd-snapshotter &>/var/log/overlaybd-snapshotter.log & /sbin/modprobe overlay && /usr/bin/containerd &>/var/log/containerd.log & - -dockerd &>/var/log/dockerd.log & diff --git a/ci/scripts/new_registry.sh b/ci/scripts/new_registry.sh new file mode 100755 index 00000000..806aaad9 --- /dev/null +++ b/ci/scripts/new_registry.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# +# run a HTTPS registry + +set -x + +mkdir -p /app/dockerd/ +echo '{ "hosts": ["unix:///app/dockerd/docker.sock"] }' > /etc/docker/daemon.json +dockerd &>/var/log/dockerd.log & + +sleep 3 + +rm -rf /etc/registry/ +mkdir -p /etc/registry/certs/ +mkdir -p /etc/registry/config/ + +# generate server certifications +cat << EOF > /etc/registry/openssl.cnf +[req] +distinguished_name = req_distinguished_name +x509_extensions = v3_req +prompt = no + +[req_distinguished_name] +C = CN +ST = Beijing +L = Beijing City +O = Alibaba +CN = localhost + +[v3_req] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment +subjectAltName = @alt_names + +[alt_names] +DNS.1 = localhost +IP.1 = 127.0.0.1 +EOF + +openssl req -new -x509 -newkey rsa:2048 -sha256 -nodes -config /etc/registry/openssl.cnf \ + -days 365 -out /etc/registry/certs/server.crt -keyout /etc/registry/certs/server.key + +ls /etc/registry/certs/ +cp /etc/registry/certs/server.crt /usr/local/share/ca-certificates/registry.crt +update-ca-certificates + +# start registry +cat << EOF > /etc/registry/config/config.yml +version: 0.1 +log: + fields: + service: registry +storage: + cache: + blobdescriptor: inmemory + filesystem: + rootdirectory: /var/lib/registry +http: + addr: :5000 + headers: + X-Content-Type-Options: [nosniff] + tls: + certificate: /certs/server.crt + key: /certs/server.key +health: + storagedriver: + enabled: true + interval: 10s + threshold: 3 +EOF + +docker run -d --restart=always --name registry -p 5000:5000 \ + -v /etc/registry/certs:/certs \ + -v /etc/registry/config:/etc/docker/registry/ \ + registry:2 + +sleep 5s + +docker ps -a +apt-get update && apt-get install -y lsof +lsof -i :5000 +curl http://localhost:5000/v2/_catalog +lsof -i :5000 +curl https://localhost:5000/v2/_catalog diff --git a/ci/scripts/prepare_image.sh b/ci/scripts/prepare_image.sh new file mode 100755 index 00000000..9b2484ba --- /dev/null +++ b/ci/scripts/prepare_image.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +from=${1:?} +to=${2:?} + +set -x + +ctr i pull "${from}" +ctr i tag "${from}" "${to}" +ctr i push "${to}" +ctr i rm "${from}" "${to}" diff --git a/ci/scripts/run_container.sh b/ci/scripts/run_container.sh new file mode 100755 index 00000000..f34c45a2 --- /dev/null +++ b/ci/scripts/run_container.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# +# rpull and run on-demand + +image=$1 +container_name=${2:-test} + +exit_code=0 + +/opt/overlaybd/snapshotter/ctr rpull "${image}" +if ! ctr run -d --net-host --snapshotter=overlaybd "${image}" "${container_name}"; then + exit_code=1 +fi +if ! ctr t ls | grep "${container_name}"; then + exit_code=1 +fi +ctr t kill -s 9 "${container_name}" && sleep 5s && ctr t ls +ctr c rm "${container_name}" && ctr c ls +ctr i rm "${image}" + +if [[ ${exit_code} -ne 0 ]]; then + cat /var/log/overlaybd.log +fi + +exit ${exit_code} diff --git a/ci/uconv_reproduce/ci-uconv-reproduce.sh b/ci/uconv_reproduce/ci-uconv-reproduce.sh index c12bd4cd..a0aa1fb6 100644 --- a/ci/uconv_reproduce/ci-uconv-reproduce.sh +++ b/ci/uconv_reproduce/ci-uconv-reproduce.sh @@ -1,6 +1,6 @@ #!/bin/bash -apt install -y python3 jq +apt update && apt install -y python3 jq convertor="/opt/overlaybd/snapshotter/convertor" images=("centos:centos7.9.2009" "ubuntu:22.04" "redis:7.2.3" "wordpress:6.4.2" "nginx:1.25.3") @@ -11,11 +11,6 @@ result=0 for image in "${images[@]}" do - from_img="registry.hub.docker.com/overlaybd/${image}" - ctr i pull "${from_img}" &> /dev/null - ctr i tag "${from_img}" "${registry}/${image}" &> /dev/null - ctr i push "${registry}/${image}" &> /dev/null - img=${image%%":"*} tag=${image##*":"} echo "${img} ${tag}" @@ -35,16 +30,16 @@ do output_turbo="${workspace}/convert.turbo.out" ${convertor} -r "${registry}/${img}" -i "${tag}" --overlaybd "${tag_obd}" -d "${workspace}/overlaybd_tmp_conv" &> "${output_obd}" - curl -H "Accept: application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.manifest.v1+json" -o "${manifest_obd}" "${registry}/v2/${img}/manifests/${tag_obd}" &> /dev/null + curl -H "Accept: application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.manifest.v1+json" -o "${manifest_obd}" "https://${registry}/v2/${img}/manifests/${tag_obd}" &> /dev/null configDigest=$(jq '.config.digest' "${manifest_obd}") configDigest=${configDigest//\"/} - curl -o "${config_obd}" "${registry}/v2/${img}/blobs/${configDigest}" &> /dev/null + curl -o "${config_obd}" "https://${registry}/v2/${img}/blobs/${configDigest}" &> /dev/null ${convertor} -r "${registry}/${img}" -i "${tag}" --turboOCI "${tag_turbo}" -d "${workspace}/turbo_tmp_conv" &> "${output_turbo}" - curl -H "Accept: application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.manifest.v1+json" -o "${manifest_turbo}" "${registry}/v2/${img}/manifests/${tag_turbo}" &> /dev/null + curl -H "Accept: application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.manifest.v1+json" -o "${manifest_turbo}" "https://${registry}/v2/${img}/manifests/${tag_turbo}" &> /dev/null configDigest=$(jq '.config.digest' "${manifest_turbo}") configDigest=${configDigest//\"/} - curl -o "${config_turbo}" "${registry}/v2/${img}/blobs/${configDigest}" &> /dev/null + curl -o "${config_turbo}" "https://${registry}/v2/${img}/blobs/${configDigest}" &> /dev/null prefix=$(date +%Y%m%d%H%M%S) diff --git a/ci/uconv_reproduce/compare_layers.py b/ci/uconv_reproduce/compare_layers.py index 29c04388..0e3d46d5 100644 --- a/ci/uconv_reproduce/compare_layers.py +++ b/ci/uconv_reproduce/compare_layers.py @@ -48,10 +48,10 @@ def main(): fb = sys.argv[3] if not os.path.exists(fa): print("file %s not exist" % fa) - return -1 + sys.exit(-1) if not os.path.exists(fb): print("file %s not exist" % fb) - return -1 + sys.exit(-1) fa_conf = json.load(open(fa, 'r')) fb_conf = json.load(open(fb, 'r')) if ftype == "manifest": diff --git a/cmd/convertor/builder/builder.go b/cmd/convertor/builder/builder.go index 7cb651ed..1ee514a0 100644 --- a/cmd/convertor/builder/builder.go +++ b/cmd/convertor/builder/builder.go @@ -83,17 +83,28 @@ func NewOverlayBDBuilder(ctx context.Context, opt BuilderOptions) (Builder, erro TLSClientConfig: tlsConfig, ExpectContinueTimeout: 5 * time.Second, } + client := &http.Client{Transport: transport} resolver := docker.NewResolver(docker.ResolverOptions{ - Credentials: func(s string) (string, string, error) { - if i := strings.IndexByte(opt.Auth, ':'); i > 0 { - return opt.Auth[0:i], opt.Auth[i+1:], nil - } - return "", "", nil - }, - PlainHTTP: opt.PlainHTTP, - Client: &http.Client{ - Transport: transport, - }, + Hosts: docker.ConfigureDefaultRegistries( + docker.WithAuthorizer(docker.NewDockerAuthorizer( + docker.WithAuthClient(client), + docker.WithAuthHeader(make(http.Header)), + docker.WithAuthCreds(func(s string) (string, string, error) { + if i := strings.IndexByte(opt.Auth, ':'); i > 0 { + return opt.Auth[0:i], opt.Auth[i+1:], nil + } + return "", "", nil + }), + )), + docker.WithClient(client), + docker.WithPlainHTTP(func(s string) (bool, error) { + if opt.PlainHTTP { + return docker.MatchAllHosts(s) + } else { + return false, nil + } + }), + ), }) engineBase, err := getBuilderEngineBase(ctx, resolver, opt.Ref, opt.TargetRef) if err != nil {