feat(kong-ee): adapt kong-build-tools to be more extendable refs: #EN…

…GEN-472 (#452)

* feat(kong-ee): move enterprise specific build steps into kong-ee

* feat(kogn-ee): allow for pre and post build step hooks

* chore(ci): temporarily use a different Kong branch

* chore(ci): nuke the retry

* chore(kong-ee): add the openssl aux module logic

* fix(build): these steps happen after building openresty

* chore(packaging): theses ssh keys were never used

* chore(packaging): move the post build steps to later

* chore(packaging): add some convenience symlinks

* chore(Make): small makefile tweaks

* chore(ci): cleanup the Jenkinsfile

* chore(ci): add enterprise builds to the testing matrix

* fix(packaging): ee installs stuff here but ce does not

* chore(ci): cleanup ci stop building debian:8

* chore(ci): test kong-ee first

* chore(ci): recursive clone so it includes the submodules

* chore(ci): why can't I clone via ssh?

* fix(ci): does a ssh keyscan fix it?

* debug(ci): exactly who am I?

* chore(ci): try using a github key

* fix(ci): envrionment declared in the wrong spot?

* fix(ci): wait for apt to not be running

* chore(ci): re-use this workaround

* fix(c99): use c99 compat mode

* chore(backwards-compatible): keep us backwards compatible

* fix(backwards-compatible): adjustments so we're actually backwards compatible

* fix(cleanup): some adjustments to the submodules

* feat(make): set default target to "package-kong"

* fix(make): plumb GITHUB_TOKEN for openresty build

* fix(make): paths for lmdb luarock & nginx module

* test(enterprise): various enterprise only tests

* fix(jenkins): move GITHUB_TOKEN to kong build

* fix(jenkins): GITHUB_TOKEN jenkinsfile scope

* fix(docker): mv gh token ARG

* fix(docker): rm gh token from oresty dockerfile

* fix(ci): we require pulp credentials to pull the license file

* fix(ci): double check the past run is cleaned up

* fix(build): this file only exists sometimes

* chore(ci): more verbose log messages

* fix(tests): adjust how we cleanup between api and enterprise api tests

* fix(tests): this wait_for seems suspicious

* fix(ci): we require pulp credentials

* tests(portal): copy over the portal tests

* feat(make): announce which makefile we're using

* feat(admin/portal): relocate test + re-add rbac

* fix(ci): ensure perl/zlib1g installed for tests

* fix(make): (cleanup) cleanup-tests container

* fix(make): manage sub-modules

* chore(ci): freshen buildx/docker-machine

* fix(admin/portal): enterprise RBAC test

* feat(docker): accept DOCKER_BUILD_PROGRESS

* feat(bytecompile): optionally bytecompile

* fix(bytecompile): just run script if present

* fix(ci): typo in build-test-container target

* feat(mac): skip systemd pkg tests on Mac OS

* feat(ci): simplify Jenkinsfile make invocation

* chore(jenkins): mv DOCKER_REPOSITORY/KONG_SOURCE

* feat(packaging): adapt so kong-ee can package correctly

* chore(ci): adapt how we infer if Kong is alive

* fix(packaging): default to kong community edition when arguments are empty

* chore(make): print RESTY_IMAGE_{BASE,TAG}

* chore(cleanup): these variables are all bintray related

* fix(make): skip daemon.json write w/o /etc/docker

* feat(©): maybe use post-copyright-manifests.sh

* chore(ci): do the test alpine arm64 build

* fix(jenkins) re-introduce rockylinux 8

* Revert "chore(ci): do the test alpine arm64 build"

This reverts commit 3ecadac.

Seems this hasn't worked for a while. Going to fix it out of band if at all

* chore(ci): leave CI test builds on master

* fix(ci): include the kong/distribution directory in the openresty cache key

* chore(release): need to set the docker repository we're releasing to but default to kong/kong

* Update build-kong.sh

Co-authored-by: Michael Martin <[email protected]>

* test(routes): include a list routes e2e test

* fix(enterprise): we need the enterprise flag when running the release scripts container

* chore(ci): align the test targets with what we release. Add a retry

Co-authored-by: Isa Farnik <[email protected]>
Co-authored-by: Michael Martin <[email protected]>

.ci/setup_ci.sh

@@ -17,20 +17,22 @@ if [ "$RESULT" != "0" ]; then
     sudo apt-get -y -o Dpkg::Options::="--force-confnew" install containerd.io docker-ce
 fi
 
-echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json
+if test -d /etc/docker; then
+    echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json
+fi
 
 docker buildx version
 RESULT=$?
 if [ "$RESULT" != "0" ]; then
-    curl -fsSLo buildx https://github.com/docker/buildx/releases/download/v0.2.2/buildx-v0.2.2.linux-amd64
+    curl -fsSLo buildx https://github.com/docker/buildx/releases/download/v0.8.2/buildx-v0.8.2.linux-amd64
     mkdir -p ~/.docker/cli-plugins/
     chmod +x buildx
     mv buildx ~/.docker/cli-plugins/docker-buildx
     sudo service docker restart
 fi
 
 if ! [ -x "$(command -v docker-machine)" ]; then
-    curl -L https://github.com/docker/machine/releases/download/v0.16.0/docker-machine-$(uname -s)-$(uname -m) >docker-machine
+    curl -L https://github.com/docker/machine/releases/download/v0.16.2/docker-machine-$(uname -s)-$(uname -m) >docker-machine
     sudo install docker-machine /usr/local/bin/docker-machine
 fi
 

.gitmodules

@@ -1,6 +1,6 @@
-[submodule "kong-licensing"]
-	path = kong-licensing
-	url = [email protected]:Kong/kong-licensing.git
 [submodule "lua-kong-nginx-module"]
 	path = lua-kong-nginx-module
 	url = [email protected]:Kong/lua-kong-nginx-module.git
+[submodule "lua-resty-lmdb"]
+	path = lua-resty-lmdb
+	url = [email protected]:Kong/lua-resty-lmdb.git

Jenkinsfile

@@ -13,32 +13,19 @@ pipeline {
         DEBUG = 0
     }
     options {
+        retry(1)
         timeout(time: 120, unit: 'MINUTES')
     }
     stages {
-        stage('Build Kong Test Container') {
-            when {
-                beforeAgent true
-                anyOf {
-                    buildingTag()
-                    branch 'master'
-                    changeRequest target: 'master'
-                }
-            }
-            agent {
-                node {
-                    label 'bionic'
-                }
-            }
-            steps {
-                sh 'echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin || true'
-                sh 'make cleanup'
-                sh 'rm -rf $KONG_SOURCE_LOCATION || true'
-                sh 'git clone --single-branch --branch $KONG_SOURCE https://github.com/Kong/kong.git $KONG_SOURCE_LOCATION'
-                sh 'make kong-test-container'
+        stage('Enteprise Test Builds') {
+            environment {
+                DOCKER_REPOSITORY = "kong/kong-build-tools-private"
+                GITHUB_TOKEN = credentials('github_bot_access_token')
+                KONG_SOURCE = "feat/branch-by-abstraction"
+                PULP = credentials('PULP')
+                PULP_PASSWORD = "${env.PULP_PSW}"
+                PULP_USERNAME = "${env.PULP_USR}"
             }
-        }
-        stage('Test Builds') {
             when {
                 beforeAgent true
                 anyOf {
@@ -48,143 +35,140 @@ pipeline {
                 }
             }
             parallel {
-                stage('AmazonLinux'){
+                stage('Kong Enterprise RPM'){
                     agent {
                         node {
                             label 'bionic'
                         }
                     }
                     environment {
-                        AWS_ACCESS_KEY = credentials('AWS_ACCESS_KEY')
-                        AWS_SECRET_ACCESS_KEY = credentials('AWS_SECRET_ACCESS_KEY')
-                    }
-                    steps {
-                        sh 'echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin || true'
-                        sh 'git clone --single-branch --branch ${KONG_SOURCE} https://github.com/Kong/kong.git ${KONG_SOURCE_LOCATION}'
-                        sh 'export RESTY_IMAGE_BASE=amazonlinux RESTY_IMAGE_TAG=2 PACKAGE_TYPE=rpm && make package-kong && make test && make cleanup'
-                    }
-                }
-                stage('src & Alpine'){
-                    agent {
-                        node {
-                            label 'bionic'
-                        }
+                        GITHUB_SSH_KEY = credentials('github_bot_ssh_key')
+                        PATH = "/home/ubuntu/bin/:${env.PATH}"
+                        PACKAGE_TYPE = "rpm"
                     }
                     steps {
                         sh 'echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin || true'
-                        sh 'git clone --single-branch --branch ${KONG_SOURCE} https://github.com/Kong/kong.git ${KONG_SOURCE_LOCATION}'
-                        sh 'export RESTY_IMAGE_BASE=src RESTY_IMAGE_TAG=src PACKAGE_TYPE=src && make package-kong && make test && make cleanup'
-                        sh 'export RESTY_IMAGE_BASE=alpine RESTY_IMAGE_TAG=3.10 PACKAGE_TYPE=apk CACHE=false UPDATE_CACHE=true DOCKER_MACHINE_ARM64_NAME="jenkins-kong-"`cat /proc/sys/kernel/random/uuid` && make package-kong && make test && make cleanup'
+                        sh 'while /bin/bash -c "ps aux | grep [a]pt-get"; do sleep 5; done'
+                        sh 'curl https://raw.githubusercontent.com/Kong/kong/master/scripts/setup-ci.sh | bash'
+                        sh 'git clone --recursive --single-branch --branch ${KONG_SOURCE} https://github.com/Kong/kong-ee.git ${KONG_SOURCE_LOCATION}'
+                        sh 'make RESTY_IMAGE_BASE=amazonlinux RESTY_IMAGE_TAG=2 package-kong test cleanup'
+                        sh 'make RESTY_IMAGE_BASE=centos      RESTY_IMAGE_TAG=7 package-kong test cleanup'
+                        sh 'make RESTY_IMAGE_BASE=rockylinux  RESTY_IMAGE_TAG=8 package-kong test cleanup'
+                        sh 'make RESTY_IMAGE_BASE=rhel        RESTY_IMAGE_TAG=7 package-kong test cleanup'
+                        sh 'make RESTY_IMAGE_BASE=rhel        RESTY_IMAGE_TAG=8 package-kong test cleanup'
+
                     }
                 }
-                stage('RedHat'){
+                stage('Kong Enterprise src & Alpine'){
                     agent {
                         node {
                             label 'bionic'
                         }
                     }
                     environment {
-                        PACKAGE_TYPE = "rpm"
-                        RESTY_IMAGE_BASE = "rhel"
                         PATH = "/home/ubuntu/bin/:${env.PATH}"
+                        GITHUB_SSH_KEY = credentials('github_bot_ssh_key')
                     }
                     steps {
-                        sh 'mkdir -p /home/ubuntu/bin/'
                         sh 'echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin || true'
-                        sh 'git clone --single-branch --branch ${KONG_SOURCE} https://github.com/Kong/kong.git ${KONG_SOURCE_LOCATION}'
-                        sh 'export RESTY_IMAGE_TAG=7 && make package-kong && make test && make cleanup'
-                        sh 'export RESTY_IMAGE_TAG=8 && make package-kong && make test && make cleanup'
+                        sh 'while /bin/bash -c "ps aux | grep [a]pt-get"; do sleep 5; done'
+                        sh 'curl https://raw.githubusercontent.com/Kong/kong/master/scripts/setup-ci.sh | bash'
+                        sh 'git clone --recursive --single-branch --branch ${KONG_SOURCE} [email protected]:Kong/kong-ee.git ${KONG_SOURCE_LOCATION}'
+                        sh 'make RESTY_IMAGE_BASE=src    RESTY_IMAGE_TAG=src  PACKAGE_TYPE=src package-kong test cleanup'
+                        sh 'make RESTY_IMAGE_BASE=alpine RESTY_IMAGE_TAG=3.10 PACKAGE_TYPE=apk CACHE=false UPDATE_CACHE=true DOCKER_MACHINE_ARM64_NAME="jenkins-kong-"`cat /proc/sys/kernel/random/uuid` package-kong test cleanup'
                     }
                 }
-                stage('CentOS'){
+                stage('Kong Enterprise DEB') {
                     agent {
                         node {
                             label 'bionic'
                         }
                     }
                     environment {
-                        PACKAGE_TYPE = "rpm"
+                        PACKAGE_TYPE = "deb"
                         PATH = "/home/ubuntu/bin/:${env.PATH}"
+                        GITHUB_SSH_KEY = credentials('github_bot_ssh_key')
                     }
                     steps {
-                        sh 'mkdir -p /home/ubuntu/bin/'
                         sh 'echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin || true'
-                        sh 'git clone --single-branch --branch ${KONG_SOURCE} https://github.com/Kong/kong.git ${KONG_SOURCE_LOCATION}'
-                        sh 'export RESTY_IMAGE_BASE=centos RESTY_IMAGE_TAG=7 && make package-kong && make test && make cleanup'
-                        sh 'export RESTY_IMAGE_BASE=rockylinux RESTY_IMAGE_TAG=8 && make package-kong && make test && make cleanup'
+                        sh 'while /bin/bash -c "ps aux | grep [a]pt-get"; do sleep 5; done'
+                        sh 'curl https://raw.githubusercontent.com/Kong/kong/master/scripts/setup-ci.sh | bash'
+                        sh 'git clone --recursive --single-branch --branch ${KONG_SOURCE} [email protected]:Kong/kong-ee.git ${KONG_SOURCE_LOCATION}'
+                        sh 'make RESTY_IMAGE_BASE=debian RESTY_IMAGE_TAG=9     package-kong test cleanup'
+                        sh 'make RESTY_IMAGE_BASE=debian RESTY_IMAGE_TAG=10    package-kong test cleanup'
+                        sh 'make RESTY_IMAGE_BASE=debian RESTY_IMAGE_TAG=11    package-kong test cleanup'
+                        sh 'make RESTY_IMAGE_BASE=ubuntu RESTY_IMAGE_TAG=16.04 package-kong test cleanup'
+                        sh 'make RESTY_IMAGE_BASE=ubuntu RESTY_IMAGE_TAG=18.04 package-kong test cleanup'
+                        sh 'make RESTY_IMAGE_BASE=ubuntu RESTY_IMAGE_TAG=20.04 package-kong test cleanup'
                     }
                 }
-                stage('Debian') {
+            }
+        }
+        stage('OSS Test Builds') {
+            when {
+                beforeAgent true
+                anyOf {
+                    buildingTag()
+                    branch 'master'
+                    changeRequest target: 'master'
+                }
+            }
+            parallel {
+                stage('Kong OSS RPM'){
                     agent {
                         node {
                             label 'bionic'
                         }
                     }
                     environment {
-                        PACKAGE_TYPE = "deb"
-                        RESTY_IMAGE_BASE = "debian"
+                        AWS_ACCESS_KEY = credentials('AWS_ACCESS_KEY')
+                        AWS_SECRET_ACCESS_KEY = credentials('AWS_SECRET_ACCESS_KEY')
                         PATH = "/home/ubuntu/bin/:${env.PATH}"
+                        PACKAGE_TYPE = "rpm"
                     }
                     steps {
-                        sh 'mkdir -p /home/ubuntu/bin/'
                         sh 'echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin || true'
                         sh 'git clone --single-branch --branch ${KONG_SOURCE} https://github.com/Kong/kong.git ${KONG_SOURCE_LOCATION}'
-                        sh 'export RESTY_IMAGE_TAG=9 && make package-kong && make test && make cleanup'
-                        sh 'export RESTY_IMAGE_TAG=10 && make package-kong && make test && make cleanup'
-                        sh 'export RESTY_IMAGE_TAG=11 && make package-kong && make test && make cleanup'
+                        sh 'make RESTY_IMAGE_BASE=amazonlinux RESTY_IMAGE_TAG=2 package-kong test cleanup'
+                        sh 'make RESTY_IMAGE_BASE=centos      RESTY_IMAGE_TAG=7 package-kong test cleanup'
+                        sh 'make RESTY_IMAGE_BASE=rockylinux  RESTY_IMAGE_TAG=8 package-kong test cleanup'
+                        sh 'make RESTY_IMAGE_BASE=rhel        RESTY_IMAGE_TAG=7 package-kong test cleanup'
+                        sh 'make RESTY_IMAGE_BASE=rhel        RESTY_IMAGE_TAG=8 package-kong test cleanup'
                     }
                 }
-                stage('Ubuntu') {
+                stage('Kong OSS src & Alpine'){
                     agent {
                         node {
                             label 'bionic'
                         }
                     }
-                    environment {
-                        PACKAGE_TYPE = "deb"
-                        RESTY_IMAGE_BASE = "ubuntu"
-                        PATH = "/home/ubuntu/bin/:${env.PATH}"
-                        USER = 'jenkins-kbt'
-                        AWS_ACCESS_KEY = credentials('AWS_ACCESS_KEY')
-                        AWS_SECRET_ACCESS_KEY = credentials('AWS_SECRET_ACCESS_KEY')
-                    }
                     steps {
-                        sh 'mkdir -p /home/ubuntu/bin/'
                         sh 'echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin || true'
                         sh 'git clone --single-branch --branch ${KONG_SOURCE} https://github.com/Kong/kong.git ${KONG_SOURCE_LOCATION}'
-                        sh 'export RESTY_IMAGE_TAG=18.04 && make package-kong && make test && make cleanup'
-                        sh 'export RESTY_IMAGE_TAG=20.04 && make package-kong && make test && make cleanup'
-                    }
-                    post {
-                        always {
-                            sh 'make cleanup-build'
-                        }
+                        sh 'make RESTY_IMAGE_BASE=src    RESTY_IMAGE_TAG=src  PACKAGE_TYPE=src package-kong test cleanup'
+                        sh 'make RESTY_IMAGE_BASE=alpine RESTY_IMAGE_TAG=3.10 PACKAGE_TYPE=apk CACHE=false UPDATE_CACHE=true DOCKER_MACHINE_ARM64_NAME="jenkins-kong-"`cat /proc/sys/kernel/random/uuid` package-kong test cleanup'
                     }
                 }
-                stage('Ubuntu Xenial') {
+                stage('Kong OSS DEB') {
                     agent {
                         node {
                             label 'bionic'
                         }
                     }
                     environment {
                         PACKAGE_TYPE = "deb"
-                        RESTY_IMAGE_BASE = "ubuntu"
                         PATH = "/home/ubuntu/bin/:${env.PATH}"
-                        USER = 'jenkins-kbt'
-                        AWS_ACCESS_KEY = credentials('AWS_ACCESS_KEY')
-                        AWS_SECRET_ACCESS_KEY = credentials('AWS_SECRET_ACCESS_KEY')
                     }
                     steps {
                         sh 'mkdir -p /home/ubuntu/bin/'
                         sh 'echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin || true'
                         sh 'git clone --single-branch --branch ${KONG_SOURCE} https://github.com/Kong/kong.git ${KONG_SOURCE_LOCATION}'
-                        sh 'export CACHE=false UPDATE_CACHE=true RESTY_IMAGE_TAG=16.04 DOCKER_MACHINE_ARM64_NAME="jenkins-kong-"`cat /proc/sys/kernel/random/uuid` && make package-kong && make test'
-                    }
-                    post {
-                        always {
-                            sh 'make cleanup-build'
-                        }
+                        sh 'make RESTY_IMAGE_BASE=debian RESTY_IMAGE_TAG=9     package-kong test cleanup'
+                        sh 'make RESTY_IMAGE_BASE=debian RESTY_IMAGE_TAG=10    package-kong test cleanup'
+                        sh 'make RESTY_IMAGE_BASE=debian RESTY_IMAGE_TAG=11    package-kong test cleanup'
+                        sh 'make RESTY_IMAGE_BASE=ubuntu RESTY_IMAGE_TAG=16.04 package-kong test cleanup'
+                        sh 'make RESTY_IMAGE_BASE=ubuntu RESTY_IMAGE_TAG=18.04 package-kong test cleanup'
+                        sh 'make RESTY_IMAGE_BASE=ubuntu RESTY_IMAGE_TAG=20.04 package-kong test cleanup'
                     }
                 }
             }