Merge pull request #134 from UKHomeOffice/drone-v1

Switch to Drone v1 and package updates

Author: KashifSaadat

.drone.yml

@@ -1,46 +1,64 @@
+kind: pipeline
+name: default
+type: kubernetes
+
+platform:
+  os: linux
+  arch: amd64
+
 workspace:
-  base: /workdir
+  path: /workdir
+
+steps:
+- name: build_and_test_image
+  image: 340268328991.dkr.ecr.eu-west-2.amazonaws.com/acp/dind
+  commands:
+  - apk add curl bash wget tar
+  - n=0; while [ "$n" -lt 60 ] && [ ! docker stats --no-stream ]; do n=$(( n + 1 )); sleep 1; done
+  - ./ci-build.sh
+  environment:
+    GEOIP_ACCOUNT_ID:
+      from_secret: geoip_account_id
+    GEOIP_LICENSE_KEY:
+      from_secret: geoip_license_key
+  when:
+    event:
+    - pull_request
+    - push
+    - tag
 
-pipeline:
-  build_and_test_image:
-    image: quay.io/ukhomeofficedigital/centos-base:latest
-    environment:
-      - DOCKER_HOST=tcp://172.17.0.1:2375
-    commands:
-      - yum update -y
-      - yum install -y -q docker openssl wget
-      - ./ci-build.sh
-    when:
-      event: [pull_request, push, tag]
+- name: push_image_to_artifactory
+  image: 340268328991.dkr.ecr.eu-west-2.amazonaws.com/acp/dind
+  commands:
+  - docker login -u="$${DOCKER_USERNAME}" -p="$${DOCKER_PASSWORD}" "$${DOCKER_REPO}"
+  - ./publish.sh "ngx" "$${DOCKER_REPO}$${DOCKER_BASEDIR}$${DOCKER_IMAGE}" "$${DRONE_TAG}"
+  environment:
+    DOCKER_BASEDIR: /
+    DOCKER_IMAGE: nginx-proxy
+    DOCKER_PASSWORD:
+      from_secret: docker_password
+    DOCKER_REPO: artifactory-internal.digital.homeoffice.gov.uk
+    DOCKER_USERNAME: docker-nginx-proxy-robot
+  when:
+    event:
+    - tag
 
-  push_image_to_artifactory:
-    image: docker:17.12.0
-    secrets:
-      - docker_password
-    environment:
-      - DOCKER_HOST=tcp://172.17.0.1:2375
-      - DOCKER_IMAGE=nginx-proxy
-      - DOCKER_REPO=artifactory-internal.digital.homeoffice.gov.uk
-      - DOCKER_BASEDIR=/
-      - DOCKER_USERNAME=docker-nginx-proxy-robot
-    commands:
-      - docker login -u="$${DOCKER_USERNAME}" -p="$${DOCKER_PASSWORD}" "$${DOCKER_REPO}"
-      - ./publish.sh "ngx" "$${DOCKER_REPO}$${DOCKER_BASEDIR}$${DOCKER_IMAGE}" "$${DRONE_TAG}"
-    when:
-      event: tag
+- name: push_image_to_quay
+  image: 340268328991.dkr.ecr.eu-west-2.amazonaws.com/acp/dind
+  commands:
+  - docker login -u="$${DOCKER_USERNAME}" -p="$${DOCKER_QUAY_PASSWORD}" "$${DOCKER_REPO}"
+  - ./publish.sh "ngx" "$${DOCKER_REPO}$${DOCKER_BASEDIR}$${DOCKER_IMAGE}" "$${DRONE_TAG}"
+  environment:
+    DOCKER_BASEDIR: /ukhomeofficedigital/
+    DOCKER_IMAGE: nginx-proxy
+    DOCKER_QUAY_PASSWORD:
+      from_secret: docker_quay_password
+    DOCKER_REPO: quay.io
+    DOCKER_USERNAME: ukhomeofficedigital+nginx_proxy
+  when:
+    event:
+    - tag
 
-  push_image_to_quay:
-    image: docker:17.12.0
-    secrets:
-      - docker_quay_password
-    environment:
-      - DOCKER_HOST=tcp://172.17.0.1:2375
-      - DOCKER_IMAGE=nginx-proxy
-      - DOCKER_REPO=quay.io
-      - DOCKER_BASEDIR=/ukhomeofficedigital/
-      - DOCKER_USERNAME=ukhomeofficedigital+nginx_proxy
-    commands:
-      - docker login -u="$${DOCKER_USERNAME}" -p="$${DOCKER_QUAY_PASSWORD}" "$${DOCKER_REPO}"
-      - ./publish.sh "ngx" "$${DOCKER_REPO}$${DOCKER_BASEDIR}$${DOCKER_IMAGE}" "$${DRONE_TAG}"
-    when:
-      event: tag
+services:
+- name: docker
+  image: 340268328991.dkr.ecr.eu-west-2.amazonaws.com/acp/dind

.travis.yml

@@ -1,5 +1,7 @@
 FROM quay.io/ukhomeofficedigital/centos-base:latest
-MAINTAINER Lewis Marshall <[email protected]>
+
+ARG GEOIP_ACCOUNT_ID
+ARG GEOIP_LICENSE_KEY
 
 WORKDIR /root
 ADD ./build.sh /root/
@@ -8,7 +10,8 @@ RUN ./build.sh
 RUN yum install -y openssl && \
     yum clean all && \
     mkdir -p /etc/keys && \
-    openssl req -x509 -newkey rsa:2048 -keyout /etc/keys/key -out /etc/keys/crt -days 360 -nodes -subj '/CN=test'
+    openssl req -x509 -newkey rsa:2048 -keyout /etc/keys/key -out /etc/keys/crt -days 360 -nodes -subj '/CN=test' && \
+    chmod 644 /etc/keys/key
 
 # This takes a while so best to do it during build
 RUN openssl dhparam -out /usr/local/openresty/nginx/conf/dhparam.pem 2048

Dockerfile

@@ -82,10 +82,9 @@ This is useful when testing or for development instances or when a load-balancer
 * `SSL_PROTOCOLS` - Change the SSL protocols supported default only TLSv1.2
 * `HTTP_LISTEN_PORT` - Change the default inside the container from 10080.
 * `HTTPS_LISTEN_PORT` - Change the default inside the container from 10443.
-* `INTERNAL_LISTEN_PORT` - Change the default inside the container from 10418. Note: This is used for internal processing and is not available externally.
 * `HTTPS_REDIRECT` - Toggle whether or not we force redirects to HTTPS.  Defaults to true.
 * `ALLOW_COUNTRY_CSV` - List of [country codes](http://dev.maxmind.com/geoip/legacy/codes/iso3166/) to allow.
-* `STATSD_METRICS_ENABLED` - Toggle if metrics are logged to statsd (defaults to true)
+* `STATSD_METRICS` - Toggle if metrics are logged to statsd (defaults to true)
 * `STATSD_SERVER` - Server to send statsd metrics to, defaults to 127.0.0.1
 * `DISABLE_SYSDIG_METRICS` - Set to any non-empty string to disable support for Sysdig's metric collection
 

README.md

@@ -4,15 +4,17 @@
 set -eu
 set -o pipefail
 
-GEOIP_CITY_URL='http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz'
-GEOIP_COUNTRY_URL='http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.mmdb.gz'
-GEOIP_MOD_URL='https://github.com/leev/ngx_http_geoip2_module/archive/3.0.tar.gz'
-GEOIP_UPDATE_CLI='https://github.com/maxmind/geoipupdate/releases/download/v3.1.1/geoipupdate-3.1.1.tar.gz'
-GEOIP_URL='https://github.com/maxmind/libmaxminddb/releases/download/1.3.2/libmaxminddb-1.3.2.tar.gz'
-LUAROCKS_URL='http://luarocks.org/releases/luarocks-2.4.2.tar.gz'
-NAXSI_URL='https://github.com/nbs-system/naxsi/archive/0.56.tar.gz'
-OPEN_RESTY_URL='http://openresty.org/download/openresty-1.11.2.4.tar.gz'
-STATSD_URL='https://github.com/UKHomeOffice/nginx-statsd/archive/0.0.1.tar.gz'
+GEOIP_ACCOUNT_ID="${GEOIP_ACCOUNT_ID:-123456}"
+GEOIP_LICENSE_KEY="${GEOIP_LICENSE_KEY:-xxxxxx}"
+GEOIP_CITY_URL="https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=${GEOIP_LICENSE_KEY}&suffix=tar.gz"
+GEOIP_COUNTRY_URL="https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country&license_key=${GEOIP_LICENSE_KEY}&suffix=tar.gz"
+GEOIP_MOD_URL='https://github.com/leev/ngx_http_geoip2_module/archive/3.3.tar.gz'
+GEOIP_UPDATE_CLI='https://github.com/maxmind/geoipupdate/releases/download/v4.7.1/geoipupdate_4.7.1_linux_amd64.tar.gz'
+GEOIP_URL='https://github.com/maxmind/libmaxminddb/releases/download/1.6.0/libmaxminddb-1.6.0.tar.gz'
+LUAROCKS_URL='https://luarocks.github.io/luarocks/releases/luarocks-3.7.0.tar.gz'
+NAXSI_URL='https://github.com/nbs-system/naxsi/archive/1.3.tar.gz'
+OPEN_RESTY_URL='http://openresty.org/download/openresty-1.19.3.1.tar.gz'
+STATSD_URL='https://github.com/UKHomeOffice/nginx-statsd/archive/0.0.1-ngxpatch.tar.gz'
 
 MAXMIND_PATH='/usr/share/GeoIP'
 
@@ -50,20 +52,21 @@ mkdir -p ${MAXMIND_PATH}
 ./configure
 make check install
 echo "/usr/local/lib" >> /etc/ld.so.conf.d/libmaxminddb.conf
-curl -fSL ${GEOIP_COUNTRY_URL} | gzip -d > ${MAXMIND_PATH}/GeoLite2-Country.mmdb
-curl -fSL ${GEOIP_CITY_URL} | gzip -d > ${MAXMIND_PATH}/GeoLite2-City.mmdb
+curl -fSL ${GEOIP_COUNTRY_URL} | tar -xz > ${MAXMIND_PATH}/GeoLite2-Country.mmdb
+curl -fSL ${GEOIP_CITY_URL} | tar -xz > ${MAXMIND_PATH}/GeoLite2-City.mmdb
 chown -R 1000:1000 ${MAXMIND_PATH}
 popd
 
 pushd geoipupdate
-./configure
-make check install
+sed -i 's/YOUR_ACCOUNT_ID_HERE/'"${GEOIP_ACCOUNT_ID}"'/g' GeoIP.conf
+sed -i 's/YOUR_LICENSE_KEY_HERE/'"${GEOIP_LICENSE_KEY}"'/g' GeoIP.conf
+./geoipupdate -f GeoIP.conf -d ${MAXMIND_PATH}
 popd
 
-# check maxmind module
 echo "Checking libmaxminddb module"
 ldconfig && ldconfig -p | grep libmaxminddb
 
+echo "Install openresty"
 pushd openresty
 ./configure --add-dynamic-module="/root/ngx_http_geoip2_module" \
             --add-module="../naxsi/naxsi_src" \
@@ -73,21 +76,23 @@ pushd openresty
 make install
 popd
 
-# Install NAXSI default rules...
+echo "Install NAXSI default rules"
 mkdir -p /usr/local/openresty/naxsi/
 cp "./naxsi/naxsi_config/naxsi_core.rules" /usr/local/openresty/naxsi/
 
+echo "Installing luarocks"
 pushd luarocks
 ./configure --with-lua=/usr/local/openresty/luajit \
             --lua-suffix=jit-2.1.0-beta2 \
             --with-lua-include=/usr/local/openresty/luajit/include/luajit-2.1
 make build install
 popd
 
+echo "Installing luarocks packages"
 luarocks install uuid
 luarocks install luasocket
 
-# Remove the developer tooling
+echo "Removing unnecessary developer tooling"
 rm -fr openresty naxsi nginx-statsd geoip luarocks ngx_http_geoip2_module
 yum -y remove \
     gcc-c++ \

build.sh

@@ -4,10 +4,10 @@ set -e
 
 TAG=ngx
 BUILD_NUMBER="${BUILD_NUMBER:-${DRONE_BUILD_NUMBER}}"
-PORT=$((${BUILD_NUMBER} + 1025))
+PORT="${HTTPS_LISTEN_PORT:-10443}"
 BUILD_NUMBER="${BUILD_NUMBER:-local}"
 START_INSTANCE="docker run "
-DOCKER_HOST_NAME=172.17.0.1
+DOCKER_HOST_NAME="localhost"
 MOCKSERVER="mockserver-${BUILD_NUMBER}"
 SLOWMOCKSERVER="slowmockserver-${BUILD_NUMBER}"
 MUTUAL_TLS="mutual-tls-${BUILD_NUMBER}"
@@ -40,13 +40,17 @@ function clean_up() {
 }
 
 function add_files_to_container() {
+  echo "Copying files to container: $1"
   local CONTAINER=$1
   shift
   while [[ -n $@ ]]; do
     local file=$1
     shift
-    local dest=$1
-    docker cp ${file} ${CONTAINER}:${dest}
+    local rename=$1
+    shift
+    local destdir=$1
+    cp ${file} ${rename}
+    tar -cf - ${rename} --mode u=+rw,g=+r,o=+r --owner root --group root | docker cp - ${CONTAINER}:${destdir}
     shift
   done
 }
@@ -68,7 +72,7 @@ function start_test() {
       files="${files} $1"
       shift
     done
-    echo "Running:$@ --name ${INSTANCE} -p ${PORT}:${HTTPS_LISTEN_PORT} ${TAG}"
+    echo "Running: $@ --name ${INSTANCE} -p ${PORT}:${HTTPS_LISTEN_PORT} ${TAG}"
     bash -c "$@ --name ${INSTANCE} -d -p ${PORT}:${HTTPS_LISTEN_PORT} ${TAG}"
     # if files needed to be mounted in, the container stops immediately so start it again
     if [[ ${files} != "" ]]; then
@@ -87,7 +91,7 @@ echo "========"
 echo "BUILD..."
 echo "========"
 echo "travis_fold:start:BUILD"
-docker build -t ${TAG} .
+docker build --build-arg GEOIP_ACCOUNT_ID=${GEOIP_ACCOUNT_ID} --build-arg GEOIP_LICENSE_KEY=${GEOIP_LICENSE_KEY} -t ${TAG} .
 echo "travis_fold:end:BUILD"
 
 echo "Running mocking-server..."
@@ -114,8 +118,8 @@ echo "TESTING..."
 echo "=========="
 
 start_test "Start with minimal settings" "${STD_CMD} \
-           -e \"PROXY_SERVICE_HOST=http://www.w3.org\" \
-           -e \"PROXY_SERVICE_PORT=80\""
+           -e \"PROXY_SERVICE_HOST=https://www.w3.org\" \
+           -e \"PROXY_SERVICE_PORT=443\""
 
 echo "Test it's up and working..."
 wget -O /dev/null --quiet --no-check-certificate https://${DOCKER_HOST_NAME}:${PORT}/
@@ -208,18 +212,18 @@ curl -s -I -X GET -k --compressed https://${DOCKER_HOST_NAME}:${PORT}/gzip | gre
 start_test "Start with SSL CIPHER set and PROTOCOL" "${STD_CMD} \
            -e \"PROXY_SERVICE_HOST=www.w3.org\" \
            -e \"PROXY_SERVICE_PORT=80\" \
-           -e \"SSL_CIPHERS=RC4-MD5\" \
-           -e \"SSL_PROTOCOLS=TLSv1.1\""
-echo "Test excepts defined protocol and cipher....."
-docker run --link ${INSTANCE}:${INSTANCE} --rm --entrypoint bash ngx -c "echo GET / | /usr/bin/openssl s_client -cipher 'RC4-MD5' -tls1_1 -connect ${INSTANCE}:10443" &> /dev/null;
+           -e \"SSL_CIPHERS=DHE-RSA-AES256-SHA\" \
+           -e \"SSL_PROTOCOLS=TLSv1.2\""
+echo "Test accepts defined protocol and cipher....."
+docker run --link ${INSTANCE}:${INSTANCE} --rm --entrypoint bash ngx -c "echo GET / | /usr/bin/openssl s_client -cipher 'DHE-RSA-AES256-SHA' -tls1_2 -connect ${INSTANCE}:10443" &> /dev/null;
 
 
 
 start_test "Start we auto add a protocol " "${STD_CMD} \
            -e \"PROXY_SERVICE_HOST=www.w3.org\" \
            -e \"PROXY_SERVICE_PORT=80\""
 
-echo "Test It works if we do not define the protocol.."
+echo "Test it works if we do not define the protocol.."
 wget -O /dev/null --quiet --no-check-certificate https://${DOCKER_HOST_NAME}:${PORT}/
 
 
@@ -253,7 +257,7 @@ cd ./client_certs/
 ./sign_client_key_with_ca.sh
 cd ..
 start_test "Start with Client CA, and single proxy. Block unauth for /standards" \
-           "${WORKDIR}/client_certs/ca.crt" "/etc/keys/client-ca" \
+           "${WORKDIR}/client_certs/ca.crt" "client-ca" "/etc/keys/" \
            "${STD_CMD} \
            -e \"PROXY_SERVICE_HOST=http://www.w3.org\" \
            -e \"PROXY_SERVICE_PORT=80\" \
@@ -278,18 +282,20 @@ wget -O /dev/null --quiet --no-check-certificate https://${DOCKER_HOST_NAME}:${P
 echo "Test upstream client certs..."
 docker build -t mutual-tls:latest ${WORKDIR} -f docker-config/Dockerfile.mutual-tls
 ${STD_CMD} -d \
+           -e "HTTP_LISTEN_PORT=10081" \
+           -e "HTTPS_LISTEN_PORT=10444" \
            -e "PROXY_SERVICE_HOST=http://www.w3.org" \
            -e "PROXY_SERVICE_PORT=80" \
            -e "CLIENT_CERT_REQUIRED=TRUE" \
-           --name="${MUTUAL_TLS}" mutual-tls:latest
+           -p 10444:10444 --name="${MUTUAL_TLS}" mutual-tls:latest
+docker run --link "${MUTUAL_TLS}:${MUTUAL_TLS}" --rm martin/wait -p 10444
 
-docker run --link "${MUTUAL_TLS}:${MUTUAL_TLS}" --rm martin/wait
 start_test "Start with upstream client certs" \
-           "${WORKDIR}/client_certs/client.crt" "/etc/keys/upstream-client-crt" \
-           "${WORKDIR}/client_certs/client.key" "/etc/keys/upstream-client-key" \
+           "${WORKDIR}/client_certs/client.crt" "upstream-client-crt" "/etc/keys/" \
+           "${WORKDIR}/client_certs/client.key" "upstream-client-key" "/etc/keys/" \
            "${STD_CMD} \
            -e \"PROXY_SERVICE_HOST=https://${MUTUAL_TLS}\" \
-           -e \"PROXY_SERVICE_PORT=10443\" \
+           -e \"PROXY_SERVICE_PORT=10444\" \
            -e \"DNSMASK=TRUE\" \
            -e \"USE_UPSTREAM_CLIENT_CERT=TRUE\" \
            --link \"${MUTUAL_TLS}:${MUTUAL_TLS}\" "
@@ -301,15 +307,18 @@ tear_down_container "${MUTUAL_TLS}"
 echo "Test failure to verify upstream server cert..."
 docker build -t standard-tls:latest ${WORKDIR} -f docker-config/Dockerfile.standard-tls
 ${STD_CMD} -d \
+           -e "HTTP_LISTEN_PORT=10081" \
+           -e "HTTPS_LISTEN_PORT=10444" \
            -e "PROXY_SERVICE_HOST=http://www.w3.org" \
            -e "PROXY_SERVICE_PORT=80" \
-           --name="${STANDARD_TLS}" standard-tls:latest
-docker run --link "${STANDARD_TLS}:${STANDARD_TLS}" --rm martin/wait
+           -p 10444:10444 --name="${STANDARD_TLS}" standard-tls:latest
+docker run --link "${STANDARD_TLS}:${STANDARD_TLS}" --rm martin/wait -p 10444
+
 start_test "Start with failing upstream server verification" \
-           "${WORKDIR}/client_certs/ca.crt" "/etc/keys/upstream-server-ca" \
+           "${WORKDIR}/client_certs/ca.crt" "upstream-server-ca" "/etc/keys/" \
            "${STD_CMD} \
            -e \"PROXY_SERVICE_HOST=https://${STANDARD_TLS}\" \
-           -e \"PROXY_SERVICE_PORT=10443\" \
+           -e \"PROXY_SERVICE_PORT=10444\" \
            -e \"DNSMASK=TRUE\" \
            -e \"VERIFY_SERVER_CERT=TRUE\" \
            --link \"${STANDARD_TLS}:${STANDARD_TLS}\" "
@@ -328,17 +337,19 @@ cd ./client_certs/
 ./sign_server_key_with_ca.sh
 cd ..
 ${STD_CMD} -d \
+           -e "HTTP_LISTEN_PORT=10081" \
+           -e "HTTPS_LISTEN_PORT=10444" \
            -e "PROXY_SERVICE_HOST=http://www.w3.org" \
            -e "PROXY_SERVICE_PORT=80" \
-           --name="${STANDARD_TLS}" ${TAG}
-
+           -p 10444:10444 --name="${STANDARD_TLS}" ${TAG}
 docker start ${STANDARD_TLS}
-docker run --link "${STANDARD_TLS}:${STANDARD_TLS}" --rm martin/wait
+docker run --link "${STANDARD_TLS}:${STANDARD_TLS}" --rm martin/wait -p 10444
+
 start_test "Start with succeeding upstream server verification" \
-           "${WORKDIR}/client_certs/ca.crt" "/etc/keys/upstream-server-ca" \
+           "${WORKDIR}/client_certs/ca.crt" "upstream-server-ca" "/etc/keys/" \
            "${STD_CMD} \
            -e \"PROXY_SERVICE_HOST=https://${STANDARD_TLS}\" \
-           -e \"PROXY_SERVICE_PORT=10443\" \
+           -e \"PROXY_SERVICE_PORT=10444\" \
            -e \"DNSMASK=TRUE\" \
            -e \"VERIFY_SERVER_CERT=TRUE\" \
            --link \"${STANDARD_TLS}:${STANDARD_TLS}\" "