chore(Dockerfile): refactor Dockerfile.dapper

Longhorn 10497

Signed-off-by: Derek Su <[email protected]>

Author: derekbit

Dockerfile.dapper

@@ -4,6 +4,8 @@ ARG DAPPER_HOST_ARCH
 ARG http_proxy
 ARG https_proxy
 ENV HOST_ARCH=${DAPPER_HOST_ARCH} ARCH=${DAPPER_HOST_ARCH}
+ARG SRC_BRANCH=main
+ARG SRC_TAG
 
 # Setup environment
 ENV PATH /go/bin:$PATH
@@ -12,25 +14,22 @@ ENV DAPPER_ENV TAG REPO DRONE_REPO DRONE_PULL_REQUEST DRONE_COMMIT_REF
 ENV DAPPER_OUTPUT bin coverage.out
 ENV DAPPER_RUN_ARGS --privileged -v /dev:/host/dev -v /proc:/host/proc -v /sys:/host/sys
 ENV DAPPER_SOURCE /go/src/github.com/longhorn/go-spdk-helper
+ENV SRC_BRANCH ${SRC_BRANCH}
+ENV SRC_TAG ${SRC_TAG}
 
 ENV GOLANGCI_LINT_VERSION="v1.60.3"
 
 WORKDIR ${DAPPER_SOURCE}
 
-ENV SPDK_COMMIT_ID a7421a6e59f1d099294af6f65d73ebec4afebfc5
-ENV LIBJSONC_COMMIT_ID b4c371fa0cbc4dcbaccc359ce9e957a22988fb34
-# Build nvme-cli 2.10.2
-ENV NVME_CLI_COMMIT_ID eeaa08c9a0e9184f3889df0bff3d2a23db6d6294
-
 RUN zypper -n addrepo --refresh https://download.opensuse.org/repositories/system:/snappy/SLE_15/system:snappy.repo && \
     zypper -n addrepo --refresh https://download.opensuse.org/repositories/network:/utilities/SLE_15/network:utilities.repo && \
     zypper -n addrepo --refresh https://download.opensuse.org/repositories/devel:libraries:c_c++/15.6/devel:libraries:c_c++.repo && \
     zypper -n addrepo --refresh https://download.opensuse.org/repositories/devel:languages:python:Factory/15.6/devel:languages:python:Factory.repo && \
     zypper -n addrepo --refresh https://download.opensuse.org/repositories/devel:languages:python:backports/SLE_15/devel:languages:python:backports.repo && \
     zypper --gpg-auto-import-keys ref
 
-RUN zypper -n install cmake wget unzip xsltproc docbook-xsl-stylesheets python311 python311-pip fuse3 libfuse3-3 \
-              e2fsprogs xfsprogs util-linux-systemd device-mapper
+RUN zypper -n install cmake wget unzip xsltproc docbook-xsl-stylesheets python311 python311-pip fuse3-devel \
+              e2fsprogs xfsprogs util-linux-systemd device-mapper jq
 
 # Install Go & tools
 ENV GOLANG_ARCH_amd64=amd64 GOLANG_ARCH_arm64=arm64 GOLANG_ARCH=GOLANG_ARCH_${ARCH} \
@@ -40,48 +39,27 @@ RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/i
 RUN ln -sf /usr/bin/python3.11 /usr/bin/python3 & \
     ln -sf /usr/bin/pip3.11 /usr/bin/pip3
 
-# Build SPDK
-ENV SPDK_DIR /usr/src/spdk
-RUN git clone https://github.com/longhorn/spdk.git ${SPDK_DIR} --recursive && \
-    cd ${SPDK_DIR} && \
-    git checkout ${SPDK_COMMIT_ID} && \
-    git submodule update --init && \
-    sed -i '/python3-pyelftools/d' ./scripts/pkgdep/sles.sh && \
-    sed -i 's/python3-/python311-/g' ./scripts/pkgdep/sles.sh && \
-    ./scripts/pkgdep.sh && \
-    pip3 install -r ./scripts/pkgdep/requirements.txt && \
-    if [ ${ARCH} = "amd64" ]; then \
-    ./configure --target-arch=nehalem --disable-tests --disable-unit-tests --disable-examples --without-nvme-cuse && \
-    make -j$(nproc) && \
-    make install; \
-    elif [ ${ARCH} = "arm64" ]; then \
-    ./configure --target-arch=native --disable-tests --disable-unit-tests --disable-examples --without-nvme-cuse && \
-    DPDKBUILD_FLAGS="-Dplatform=generic" make -j$(nproc) && \
-    make install; \
-    else \
-    echo "Unsupported architecture: ${ARCH}"; \
-    exit 1; \
+RUN git clone https://github.com/longhorn/dep-versions.git -b ${SRC_BRANCH} /usr/src/dep-versions && \
+    cd /usr/src/dep-versions && \
+    if [ -n "${SRC_TAG}" ] && git show-ref --tags ${SRC_TAG} > /dev/null 2>&1; then \
+        echo "Checking out tag ${SRC_TAG}"; \
+        cd /usr/src/dep-versions && git checkout tags/${SRC_TAG}; \
     fi
 
+# Build spdk
+RUN export REPO_OVERRIDE="" && \
+    export COMMIT_ID_OVERRIDE="" && \
+    bash /usr/src/dep-versions/scripts/build-spdk.sh "${REPO_OVERRIDE}" "${COMMIT_ID_OVERRIDE}" "${ARCH}"
+
 # Build libjson-c-devel
-RUN cd /usr/src && \
-    git clone https://github.com/json-c/json-c.git && \
-    cd json-c && \
-    git checkout ${LIBJSONC_COMMIT_ID} && \
-    mkdir .build && \
-    cd .build && \
-    cmake ../ && \
-    make && \
-    make install
+RUN export REPO_OVERRIDE="" && \
+    export COMMIT_ID_OVERRIDE="" && \
+    bash /usr/src/dep-versions/scripts/build-libjsonc.sh "${REPO_OVERRIDE}" "${COMMIT_ID_OVERRIDE}"
 
 # Build nvme-cli
-ENV NVME_CLI_DIR /usr/src/nvme-cli
-RUN git clone https://github.com/linux-nvme/nvme-cli.git ${NVME_CLI_DIR} && \
-    cd ${NVME_CLI_DIR} && \
-    git checkout ${NVME_CLI_COMMIT_ID} && \
-    meson setup --force-fallback-for=libnvme .build && \
-    meson compile -C .build && \
-    meson install -C .build
+RUN export REPO_OVERRIDE="" && \
+    export COMMIT_ID_OVERRIDE="" && \
+    bash /usr/src/dep-versions/scripts/build-nvme-cli.sh "${REPO_OVERRIDE}" "${COMMIT_ID_OVERRIDE}"
 
 RUN ldconfig
 

Makefile

@@ -1,4 +1,6 @@
 TARGETS := $(shell ls scripts)
+export SRC_BRANCH := main
+export SRC_TAG := $(shell git tag --points-at HEAD | head -n 1)
 
 .dapper:
 	@echo Downloading dapper

pkg/spdk/spdk_test.go

@@ -39,14 +39,6 @@ type TestSuite struct{}
 
 var _ = Suite(&TestSuite{})
 
-func GetSPDKDir() string {
-	spdkDir := os.Getenv("SPDK_DIR")
-	if spdkDir != "" {
-		return spdkDir
-	}
-	return filepath.Join(os.Getenv("GOPATH"), "src/github.com/longhorn/spdk")
-}
-
 func LaunchTestSPDKTarget(c *C, execute func(envs []string, binary string, args []string, timeout time.Duration) (string, error)) {
 	targetReady := false
 	if spdkCli, err := client.NewClient(context.Background()); err == nil {
@@ -57,7 +49,7 @@ func LaunchTestSPDKTarget(c *C, execute func(envs []string, binary string, args
 
 	if !targetReady {
 		go func() {
-			err := target.StartTarget(GetSPDKDir(), []string{"2>&1 | tee /tmp/spdk_tgt.log"}, 60*time.Minute, execute)
+			err := target.StartTarget("", []string{"2>&1 | tee /tmp/spdk_tgt.log"}, 60*time.Minute, execute)
 			c.Assert(err, IsNil)
 		}()
 

pkg/spdk/target/target.go

@@ -15,7 +15,7 @@ import (
 const (
 	SPDKScriptsDir  = "scripts"
 	SPDKSetupScript = "setup.sh"
-	SPDKTGTBinary   = "build/bin/spdk_tgt"
+	SPDKTGTBinary   = "spdk_tgt"
 )
 
 // SetupTarget setups the spdk_tgt with the given args
@@ -59,9 +59,14 @@ func StartTarget(spdkDir string, args []string, timeout time.Duration, execute f
 	for _, arg := range args {
 		argsInStr = fmt.Sprintf("%s %s", argsInStr, arg)
 	}
+
+	binary := SPDKTGTBinary
+	if spdkDir == "" {
+		binary = filepath.Join(spdkDir, SPDKTGTBinary)
+	}
 	tgtOpts := []string{
 		"-c",
-		fmt.Sprintf("%s %s", filepath.Join(spdkDir, SPDKTGTBinary), argsInStr),
+		fmt.Sprintf("%s %s", binary, argsInStr),
 	}
 
 	_, err = execute(nil, "sh", tgtOpts, timeout)