From bd755fc0f08b31db884674724065e430bd80ced3 Mon Sep 17 00:00:00 2001 From: Tommy Li Date: Mon, 2 Jul 2018 16:17:59 -0700 Subject: [PATCH 1/2] add new travis job for prebuilt images --- .travis.yml | 26 +++++-------------- Makefile | 15 +++++++++++ bin/travis_scripts/test-local-built-images.sh | 20 ++++++++++++++ bin/travis_scripts/test-prebuilt-images.sh | 11 ++++++++ 4 files changed, 52 insertions(+), 20 deletions(-) create mode 100755 bin/travis_scripts/test-local-built-images.sh create mode 100755 bin/travis_scripts/test-prebuilt-images.sh diff --git a/.travis.yml b/.travis.yml index 26fa8299..650fb5fa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,26 +27,12 @@ before_script: # install aws CLI - (sudo pip install -q awscli; which aws > /dev/null) & -script: - # fail fast - - set -e - - export MAKE_ARGS=--no-print-directory - # Open SSH - # - echo travis:$sshpassword | sudo chpasswd - # - sudo sed -i 's/ChallengeResponseAuthentication no/ChallengeResponseAuthentication yes/' /etc/ssh/sshd_config - # - sudo service ssh restart - # - sudo apt-get install sshpass - # - sshpass -p $sshpassword ssh -R 9999:localhost:22 -o StrictHostKeyChecking=no travis@$bouncehostip - # compile and build Docker images - - glide -q install - - make $MAKE_ARGS docker-build-base - - make $MAKE_ARGS gen-certs - - make $MAKE_ARGS build - - make $MAKE_ARGS docker-build - # deploy services - - make $MAKE_ARGS deploy - # submit a test job - - make $MAKE_ARGS test-submit-minikube-ci +jobs: + include: + - stage: test + script: ./bin/travis_scripts/test-local-built-images.sh + - if: type IN (push, api, cron) + script: ./bin/travis_scripts/test-prebuilt-images.sh notifications: email: false diff --git a/Makefile b/Makefile index ebcff728..32ceb914 100644 --- a/Makefile +++ b/Makefile @@ -32,6 +32,10 @@ PUBLIC_IP ?= 127.0.0.1 CI_MINIKUBE_VERSION ?= v0.25.1 CI_KUBECTL_VERSION ?= v1.9.4 +TRAVIS_IMAGE_VERSION ?= v0.1 +TEST_IMAGES = $(addprefix $(DOCKER_NAMESPACE)/, $(TEST_IMAGES_SUFFIX)) +TEST_IMAGES_SUFFIX = ffdl-lcm ffdl-trainer ffdl-metrics ffdl-databroker_s3 ffdl-ui ffdl-restapi ffdl-jobmonitor ffdl-controller log_collector ffdl-databroker_objectstorage tensorboard_extract + AWS_ACCESS_KEY_ID ?= test AWS_SECRET_ACCESS_KEY ?= test @@ -617,6 +621,17 @@ test-s3: echo "s3cmd=$$s3cmd"; \ $$s3cmd ls +pull-prebuilt-images: ## Pull FfDL images from dockerhub +pull-prebuilt-images: $(addprefix pull-, $(TEST_IMAGES)) + +$(addprefix pull-, $(TEST_IMAGES)): pull-%: % + @TRAVIS_IMAGE=$< make .pull-prebuilt-images + +$(TEST_IMAGES): ; # Make targets + +.pull-prebuilt-images: + docker pull $(TRAVIS_IMAGE):$(TRAVIS_IMAGE_VERSION) + .build-service: (cd ./$(SERVICE_NAME)/ && (test ! -e main.go || CGO_ENABLED=0 GOOS=linux go build -ldflags "-s -w" -a -installsuffix cgo -o bin/$(BINARY_NAME))) diff --git a/bin/travis_scripts/test-local-built-images.sh b/bin/travis_scripts/test-local-built-images.sh new file mode 100755 index 00000000..15893d6d --- /dev/null +++ b/bin/travis_scripts/test-local-built-images.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# fail fast +set -e +export MAKE_ARGS=--no-print-directory +# Open SSH +# - echo travis:$sshpassword | sudo chpasswd +# - sudo sed -i 's/ChallengeResponseAuthentication no/ChallengeResponseAuthentication yes/' /etc/ssh/sshd_config +# - sudo service ssh restart +# - sudo apt-get install sshpass +# - sshpass -p $sshpassword ssh -R 9999:localhost:22 -o StrictHostKeyChecking=no travis@$bouncehostip +# compile and build Docker images +glide -q install +make $MAKE_ARGS docker-build-base +make $MAKE_ARGS gen-certs +make $MAKE_ARGS build +make $MAKE_ARGS docker-build +# deploy services +make $MAKE_ARGS deploy +# submit a test job +make $MAKE_ARGS test-submit-minikube-ci diff --git a/bin/travis_scripts/test-prebuilt-images.sh b/bin/travis_scripts/test-prebuilt-images.sh new file mode 100755 index 00000000..95296ef8 --- /dev/null +++ b/bin/travis_scripts/test-prebuilt-images.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# fail fast +set -e +export MAKE_ARGS=--no-print-directory +# get pre-built images +make $MAKE_ARGS pull-prebuilt-images +export IMAGE_TAG=$TRAVIS_IMAGE_VERSION +# deploy services +make $MAKE_ARGS deploy +# submit a test job +make $MAKE_ARGS test-submit-minikube-ci From 13bf86b625dc9a93ba00aed7e7f9ed072adec604 Mon Sep 17 00:00:00 2001 From: Tommy Li Date: Fri, 6 Jul 2018 10:24:01 -0700 Subject: [PATCH 2/2] add list of ffdl microservices into a config file --- Makefile | 5 +++-- bin/ffdl-microservices.txt | 11 +++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 bin/ffdl-microservices.txt diff --git a/Makefile b/Makefile index 32ceb914..d556099e 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ CI_KUBECTL_VERSION ?= v1.9.4 TRAVIS_IMAGE_VERSION ?= v0.1 TEST_IMAGES = $(addprefix $(DOCKER_NAMESPACE)/, $(TEST_IMAGES_SUFFIX)) -TEST_IMAGES_SUFFIX = ffdl-lcm ffdl-trainer ffdl-metrics ffdl-databroker_s3 ffdl-ui ffdl-restapi ffdl-jobmonitor ffdl-controller log_collector ffdl-databroker_objectstorage tensorboard_extract +TEST_IMAGES_SUFFIX = $(shell cat bin/ffdl-microservices.txt) AWS_ACCESS_KEY_ID ?= test AWS_SECRET_ACCESS_KEY ?= test @@ -627,7 +627,8 @@ pull-prebuilt-images: $(addprefix pull-, $(TEST_IMAGES)) $(addprefix pull-, $(TEST_IMAGES)): pull-%: % @TRAVIS_IMAGE=$< make .pull-prebuilt-images -$(TEST_IMAGES): ; # Make targets +# Prebuilt images make targets +$(TEST_IMAGES): ; .pull-prebuilt-images: docker pull $(TRAVIS_IMAGE):$(TRAVIS_IMAGE_VERSION) diff --git a/bin/ffdl-microservices.txt b/bin/ffdl-microservices.txt new file mode 100644 index 00000000..f3ce6343 --- /dev/null +++ b/bin/ffdl-microservices.txt @@ -0,0 +1,11 @@ +ffdl-lcm +ffdl-trainer +ffdl-metrics +ffdl-databroker_s3 +ffdl-ui +ffdl-restapi +ffdl-jobmonitor +ffdl-controller +log_collector +ffdl-databroker_objectstorage +tensorboard_extract