feat(retry): wrap everything that depends on external dependencies on…

… a retry with backoff
Kong · Nov 3, 2022 · c54365c · c54365c
1 parent 7c01689
commit c54365c
Show file tree

Hide file tree

Showing 6 changed files with 61 additions and 15 deletions.
diff --git a/build-kong.sh b/build-kong.sh
@@ -2,6 +2,8 @@
 
 set -e
 
+source /common.sh
+
 ROCKS_CONFIG=$(mktemp)
 echo "
 rocks_trees = {
@@ -37,7 +39,7 @@ pushd /kong
     sed -i 's/fips = off/fips = on/g' kong/templates/kong_defaults.lua
   fi
 
-  /usr/local/bin/luarocks make kong-${ROCKSPEC_VERSION}.rockspec \
+  with_backoff /usr/local/bin/luarocks make kong-${ROCKSPEC_VERSION}.rockspec \
     CRYPTO_DIR=/usr/local/kong \
     OPENSSL_DIR=/usr/local/kong \
     YAML_LIBDIR=/tmp/build/usr/local/kong/lib \
@@ -62,7 +64,7 @@ pushd /kong
     cp kong/pluginsocket.proto /tmp/build/usr/local/kong/include/kong
   fi
 
-  curl -fsSLo /tmp/protoc.zip https://github.com/protocolbuffers/protobuf/releases/download/v3.19.0/protoc-3.19.0-linux-x86_64.zip
+  with_backoff curl -fsSLo /tmp/protoc.zip https://github.com/protocolbuffers/protobuf/releases/download/v3.19.0/protoc-3.19.0-linux-x86_64.zip
   unzip -o /tmp/protoc.zip -d /tmp/protoc 'include/*'
   cp -r /tmp/protoc/include/google /tmp/build/usr/local/kong/include/
 popd

diff --git a/dockerfiles/Dockerfile.kong b/dockerfiles/Dockerfile.kong
@@ -20,5 +20,6 @@ WORKDIR /distribution
 RUN --mount=type=secret,id=github-token if [ -f "/distribution/post-install.sh" ] ; then export GITHUB_TOKEN=`cat /run/secrets/github-token` && ./post-install.sh; fi
 
 WORKDIR /kong
+COPY openresty-build-tools/common.sh /common.sh
 COPY build-kong.sh /build-kong.sh
 RUN /build-kong.sh
diff --git a/dockerfiles/Dockerfile.openresty b/dockerfiles/Dockerfile.openresty
@@ -56,6 +56,7 @@ COPY openresty-build-tools /tmp/openresty-build-tools
 COPY openresty-patches /tmp/openresty-patches
 COPY build-openresty.sh /tmp/build-openresty.sh
 
+
 ARG OPENRESTY_PATCHES=1
 ENV OPENRESTY_PATCHES="${OPENRESTY_PATCHES}"
 

diff --git a/openresty-build-tools/common.sh b/openresty-build-tools/common.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+# Retries a command a configurable number of times with backoff.
+#
+# The retry count is given by ATTEMPTS (default 5), the initial backoff
+# timeout is given by TIMEOUT in seconds (default 1.)
+#
+# Successive backoffs double the timeout.
+function with_backoff {
+  local max_attempts=${ATTEMPTS-5}
+  local timeout=${TIMEOUT-5}
+  local attempt=1
+  local exitCode=0
+
+  while (( $attempt < $max_attempts ))
+  do
+    if "$@"
+    then
+      return 0
+    else
+      exitCode=$?
+    fi
+
+    echo "Failure! Retrying in $timeout.." 1>&2
+    sleep $timeout
+    attempt=$(( attempt + 1 ))
+    timeout=$(( timeout * 2 ))
+  done
+
+  if [[ $exitCode != 0 ]]
+  then
+    echo "You've failed me for the last time! ($@)" 1>&2
+  fi
+
+  return $exitCode
+}
diff --git a/openresty-build-tools/kong-ngx-build b/openresty-build-tools/kong-ngx-build
@@ -2,6 +2,9 @@
 
 set -e
 
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+source $SCRIPT_DIR/common.sh
+
 SCRIPT_PATH=
 PREFIX=
 DOWNLOAD_CACHE=work
@@ -247,9 +250,9 @@ main() {
       if [[ ! -f $OPENSSL_INSTALL/bin/openssl && ! -d $OPENSSL_DOWNLOAD ]]; then
         warn "OpenSSL source not found, downloading..."
         set +e
-        curl --fail -sSLO https://www.openssl.org/source/openssl-$OPENSSL_VER.tar.gz
+        with_backoff curl --fail -sSLO https://www.openssl.org/source/openssl-$OPENSSL_VER.tar.gz
         if [[ $? != 0 ]]; then
-          curl --fail -sSLO https://www.openssl.org/source/old/${OPENSSL_VER//[a-z]/}/openssl-$OPENSSL_VER.tar.gz
+          with_backoff curl --fail -sSLO https://www.openssl.org/source/old/${OPENSSL_VER//[a-z]/}/openssl-$OPENSSL_VER.tar.gz
           [[ $? != 0 ]] && err "Could not download OpenSSL"
         fi
         set -e
@@ -270,7 +273,7 @@ main() {
       if [[ ! -d "$BORINGSSL_DOWNLOAD" ]]; then
         warn "BoringSSL source not found, downloading..."
         set +e
-        curl --fail -sSL https://github.com/google/boringssl/archive/${BORINGSSL_VER}.zip -o boringssl-${BORINGSSL_VER}.zip
+        with_backoff curl --fail -sSL https://github.com/google/boringssl/archive/${BORINGSSL_VER}.zip -o boringssl-${BORINGSSL_VER}.zip
         [[ $? != 0 ]] && err "Could not download BoringSSL"
         set -e
 
@@ -298,7 +301,7 @@ main() {
 
       if [ ! -d $OPENRESTY_DOWNLOAD ]; then
         warn "OpenResty source not found, downloading..."
-        curl -sSLO https://openresty.org/download/openresty-$OPENRESTY_VER.tar.gz
+        with_backoff curl -sSLO https://openresty.org/download/openresty-$OPENRESTY_VER.tar.gz
         if [ ! -z ${OPENRESTY_SHA+x} ]; then
           echo "$OPENRESTY_SHA openresty-$OPENRESTY_VER.tar.gz" | sha256sum -c -
         else
@@ -311,7 +314,7 @@ main() {
         if version_eq $OPENRESTY_VER 1.19.3; then
           pushd openresty-$OPENRESTY_VER/bundle
             notice "Updating lua-resty-dns to unreleased version"
-            curl -sSL https://github.com/openresty/lua-resty-dns/tarball/ad4a51c8cae8c3fb8f712fa91fda660ab8a89669 -o lua-resty-dns-0.21.tar.gz
+            with_backoff curl -sSL https://github.com/openresty/lua-resty-dns/tarball/ad4a51c8cae8c3fb8f712fa91fda660ab8a89669 -o lua-resty-dns-0.21.tar.gz
             tar -xzf lua-resty-dns-0.21.tar.gz --keep-newer-files -C lua-resty-dns-0.21 --strip-components 1
             rm -f lua-resty-dns-0.21.tar.gz
           popd
@@ -324,7 +327,7 @@ main() {
             fi
             mkdir "lua-resty-websocket-${RESTY_WEBSOCKET}"
             notice "Downloading Kong/lua-resty-websocket $RESTY_WEBSOCKET"
-            curl -sSL \
+            with_backoff curl -sSL \
               -o lua-resty-websocket.tar.gz \
               "https://github.com/Kong/lua-resty-websocket/archive/refs/tags/${RESTY_WEBSOCKET}.tar.gz"
             tar -xzf lua-resty-websocket.tar.gz \
@@ -342,7 +345,7 @@ main() {
       PCRE_DOWNLOAD=$DOWNLOAD_CACHE/pcre-$PCRE_VER
       if [ ! -d $PCRE_DOWNLOAD ]; then
         warn "PCRE source not found, downloading..."
-        curl -sSLO https://downloads.sourceforge.net/project/pcre/pcre/${PCRE_VER}/pcre-${PCRE_VER}.tar.gz
+        with_backoff curl -sSLO https://downloads.sourceforge.net/project/pcre/pcre/${PCRE_VER}/pcre-${PCRE_VER}.tar.gz
         if [ ! -z ${PCRE_SHA+x} ]; then
           echo "$PCRE_SHA pcre-${PCRE_VER}.tar.gz" | sha256sum -c -
         else
@@ -364,9 +367,9 @@ main() {
         if [ ! -d $LUAROCKS_DOWNLOAD ]; then
           warn "LuaRocks source not found, downloading..."
           set +e
-          curl --fail -sSLO https://luarocks.org/releases/luarocks-$LUAROCKS_VER.tar.gz
+          with_backoff curl --fail -sSLO https://luarocks.org/releases/luarocks-$LUAROCKS_VER.tar.gz
           if [ $? != 0 ]; then
-            curl --fail -sSL https://github.com/luarocks/luarocks/archive/refs/tags/v$LUAROCKS_VER.tar.gz -o luarocks-$LUAROCKS_VER.tar.gz
+            with_backoff curl --fail -sSL https://github.com/luarocks/luarocks/archive/refs/tags/v$LUAROCKS_VER.tar.gz -o luarocks-$LUAROCKS_VER.tar.gz
             [ $? != 0 ] && err "Could not download luarocks source"
           fi
           set -e
@@ -655,7 +658,7 @@ main() {
           llvm_url="https://releases.llvm.org/$clang_version/clang+llvm-$clang_version-x86_64-linux-gnu-ubuntu-16.04.tar.xz"
           llvm_checksum="02ad925add5b2b934d64c3dd5cbd1b2002258059f7d962993ba7f16524c3089c"
           mkdir llvm
-          curl -sSL "$llvm_url" -o llvm.tar.xz
+          with_backoff curl -sSL "$llvm_url" -o llvm.tar.xz
           echo "$llvm_checksum llvm.tar.xz" | sha256sum -c
 
           tar -C llvm --strip-components 1 -Jxf llvm.tar.xz
@@ -667,7 +670,7 @@ main() {
         # golang
           golang_url="https://go.dev/dl/go${golang_version}.linux-amd64.tar.gz"
           golang_checksum="66d83bfb5a9ede000e33c6579a91a29e6b101829ad41fffb5c5bb6c900e109d9"
-          curl -sSL "$golang_url" -o golang.tar.gz
+          with_backoff curl -sSL "$golang_url" -o golang.tar.gz
           echo "$golang_checksum golang.tar.gz" | sha256sum -c
           tar -zxf golang.tar.gz
 
@@ -680,7 +683,7 @@ main() {
         # ninja
           ninja_url="https://github.com/ninja-build/ninja/archive/refs/tags/v$ninja_version.zip"
           ninja_checksum="8e2e654a418373f10c22e4cc9bdbe9baeca8527ace8d572e0b421e9d9b85b7ef"
-          curl -sSL "$ninja_url" -o ninja.zip
+          with_backoff curl -sSL "$ninja_url" -o ninja.zip
           echo "$ninja_checksum ninja.zip" | sha256sum -c
           unzip -o ninja.zip
           pushd ninja-$ninja_version

diff --git a/test/build_container.sh b/test/build_container.sh
@@ -3,6 +3,9 @@
 source test/util.sh
 set -e
 
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+source $SCRIPT_DIR/../openresty-build-tools/common.sh
+
 DOCKER_BUILD_ARGS=()
 
 KONG_TEST_IMAGE_NAME=$DOCKER_RELEASE_REPOSITORY:$ARCHITECTURE-$KONG_TEST_CONTAINER_TAG
@@ -56,7 +59,7 @@ pushd ./docker-kong
     DOCKER_BUILD_ARGS+=(--build-arg EE_PORTS="8002 8445 8003 8446 8004 8447")
   fi
 
-  docker build \
+  with_backoff docker build \
     --progress=${DOCKER_BUILD_PROGRESS:-auto} \
     -t $KONG_TEST_IMAGE_NAME \
     -f Dockerfile.$PACKAGE_TYPE \