Merge pull request #76 from eth-cscs/dev

Merge from `dev` to `master` for version 1.7.0

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "src/tools/pyfirecrest"]
+	path = src/tools/pyfirecrest
+	url = https://github.com/ekouts/pyfirecrest.git

CONTRIBUTORS.md

@@ -0,0 +1,9 @@
+# Contributors
+
+- Aliaga, Tomas. ETH Zurich - CSCS
+- Cruz, Felipe. ETH Zurich - CSCS
+- Dabin, Alejandro. ETH Zurich - CSCS
+- Dorsch, Juan Pablo. ETH Zurich - CSCS
+- Klein, Mark. ETH Zurich - CSCS
+- Koutsaniti, Eirini. ETH Zurich - CSCS
+- Lezcano, Facundo. UNL-CONICET - CIMEC

ci/dev/Jenkinsfile

@@ -0,0 +1,108 @@
+#!groovy
+//
+//  Copyright (c) 2019-2021, ETH Zurich. All rights reserved.
+//
+//  Please, refer to the LICENSE file in the root directory.
+//  SPDX-License-Identifier: BSD-3-Clause
+//
+pipeline {
+    agent any
+    stages {
+        stage('Info') {
+            steps {
+                echo "Branch Name: ${env.BRANCH_NAME}"
+                echo "BUILD_NUMBER : ${env.BUILD_NUMBER}"
+                echo "BUILD_ID : ${env.BUILD_ID}"
+                echo "JOB_NAME: ${env.JOB_NAME}"
+                echo "BUILD_TAG : ${env.BUILD_TAG}"
+                echo "EXECUTOR_NUMBER : ${env.EXECUTOR_NUMBER}"
+                echo "NODE_NAME: ${env.NODE_NAME}"
+                echo "NODE_LABELS : ${env.NODE_LABELS}"
+                echo "WORKSPACE : ${env.WORKSPACE}"
+                echo "JENKINS_HOME : ${env.JENKINS_HOME}"
+                echo "GIT COMMIT: ${env.GIT_COMMIT}"
+                sh "docker --version"
+                sh "docker-compose --version"
+                sh "bash --version"
+                sh "git --version"
+                sh "pwd"
+                sh "ls -la"
+                sh "git status"
+            }
+        }
+
+        stage('Refresh') {
+            when {
+                anyOf {
+                    // For now, we use these branches to build everythig from scratch
+                    // See refresh.sh
+                    branch 'master';
+                    branch 'dev'
+                }
+            }
+            steps {
+                sh "ci/dev/refresh.sh"
+            }
+        }
+
+        stage('Setup') {
+            steps {
+                sh "ci/dev/setup.sh"
+            }
+        }
+
+        stage('Tests') {
+            steps {
+                sh "ci/dev/test.sh"
+            }
+        }
+    }
+
+    post {
+        always {
+            script {
+                // Save the log files
+                try {
+                    sh "mkdir -p /var/log/jenkins/jobs/${env.JOB_NAME}/builds/${env.BUILD_NUMBER}"
+                    sh "cp -r deploy/test-build/logs/firecrest/* /var/log/jenkins/jobs/${env.JOB_NAME}/builds/${env.BUILD_NUMBER}/."
+                } catch (errCpLogs) {
+                    echo 'Error while saving log files: ' + errCpLogs.toString()
+                }
+            }
+        }
+
+        success {
+            script {
+                // Notify Github on success
+                withCredentials([string(credentialsId: 'firecrestci_access_token', variable: 'accessToken')]) {
+                    sh 'curl -H "Authorization: token ' + "${accessToken}"  + '" "https://api.github.com/repos/eth-cscs/firecrest/statuses/' + "${env.GIT_COMMIT}" + '" \\' +
+                    '-H "Content-Type: application/json" \\' +
+                    '-X POST \\' +
+                    '-d "{\\"state\\": \\"success\\",\\"context\\": \\"continuous-integration/jenkins\\", \\"description\\": \\"Jenkins\\", \\"target_url\\": \\"' + "${env.BUILD_URL}" + '/console\\"}"'
+                }
+            }
+        }
+
+        unsuccessful{
+            script {
+                // Notify Github on failure
+                withCredentials([string(credentialsId: 'firecrestci_access_token', variable: 'accessToken')]) {
+                    sh 'curl -H "Authorization: token ' + "${accessToken}"  + '" "https://api.github.com/repos/eth-cscs/firecrest/statuses/' + "${env.GIT_COMMIT}" + '" \\' +
+                    '-H "Content-Type: application/json" \\' +
+                    '-X POST \\' +
+                    '-d "{\\"state\\": \\"failure\\",\\"context\\": \\"continuous-integration/jenkins\\", \\"description\\": \\"Jenkins\\", \\"target_url\\": \\"' + "${env.BUILD_URL}" + '/console\\"}"'
+                }
+            }
+        }
+
+        cleanup {
+            script {
+                try {
+                    sh "ci/dev/clean.sh"
+                } catch (errCpLogs) {
+                    echo 'Error while trying to clean: ' + errCpLogs.toString()
+                }
+            }
+        }
+    }
+}

ci/dev/README.md

@@ -0,0 +1,41 @@
+# FirecREST Testing
+
+## Requirements
+
+You can run the tests on any linux machine with...
+
+- [bash](https://www.gnu.org/software/bash/) >= `4`
+- [docker](https://docs.docker.com/engine/install/) >= `20.10`
+- [docker-compose](https://docs.docker.com/compose/install/) >= `1.28`
+
+## Usage
+
+Clone this repo and cd into it...
+
+```
+git clone https://github.com/eth-cscs/firecrest
+cd firecrest
+```
+
+To run all tests for the first time simply run...
+
+```
+ci/dev/run.sh
+```
+
+Have a look at that script. It will setup and build everything from scratch and run ALL the dev tests.
+
+If you have already setup everything and just want to re-run some tests without recreating everything
+from scratch, you can call the scripts that `ci/dev/test.sh` is calling.
+
+```
+ci/dev/test.sh
+```
+
+## Debugging
+
+If you want to re-test something specific, you can customize the docker calls of the end of `ci/dev/test.sh`
+to make an ad-hoc pytest call.
+
+If the stdout information is not enough, remember to check either the logs in the logs folder created in `ci/dev/setup.sh`
+or the container logs with [docker logs](https://docs.docker.com/engine/reference/commandline/logs/).

ci/dev/clean.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+##
+##  Copyright (c) 2019-2021, ETH Zurich. All rights reserved.
+##
+##  Please, refer to the LICENSE file in the root directory.
+##  SPDX-License-Identifier: BSD-3-Clause
+##
+set -euo pipefail
+
+echo "starting" $0
+WORKSPACE=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)/../../
+
+rm -rfv ${WORKSPACE}/deploy/test-build/logs/firecrest/* || true
+docker-compose -f ${WORKSPACE}/deploy/test-build/docker-compose.yml down -v
+
+echo "finished" $0

ci/dev/refresh.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+##
+##  Copyright (c) 2019-2021, ETH Zurich. All rights reserved.
+##
+##  Please, refer to the LICENSE file in the root directory.
+##  SPDX-License-Identifier: BSD-3-Clause
+##
+set -euo pipefail
+
+echo "starting" $0
+WORKSPACE=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)/../../
+
+echo "removing potential leftovers"
+rm -rfv ${WORKSPACE}/deploy/test-build/logs/firecrest || true
+docker-compose -f ${WORKSPACE}/deploy/test-build/docker-compose.yml down -v --rmi all --remove-orphans || true
+docker rmi f7t-base f7t-tester || echo "no base image to delete, no problem!"
+
+echo "building images from scratch (no caches)"
+# Building from scratch is slower, but prevents cache-invalidation issues, typical of cached CI machines.
+# You may want to do this only once a while, not for every feature branch.
+docker build -f ${WORKSPACE}/deploy/docker/base/Dockerfile -t f7t-base --no-cache --pull ${WORKSPACE}
+docker build -f ${WORKSPACE}/deploy/docker/tester/Dockerfile -t f7t-tester --no-cache --pull ${WORKSPACE}
+docker-compose -f ${WORKSPACE}/deploy/test-build/docker-compose.yml build --no-cache
+
+echo "finished" $0

ci/dev/retest.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+##
+##  Copyright (c) 2019-2021, ETH Zurich. All rights reserved.
+##
+##  Please, refer to the LICENSE file in the root directory.
+##  SPDX-License-Identifier: BSD-3-Clause
+##
+set -euo pipefail
+
+echo "starting" $0
+WORKSPACE=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)/../../
+
+echo "sleeping..."
+sleep 120
+
+# We start with the reservation tests because other tests still need a proper cleanup step.
+echo "running reservation tests..."
+docker run --rm -u $(id -u):$(id -g) -v ${WORKSPACE}:/firecrest --network f7t-frontend f7t-tester bash \
+    -c 'pytest -m reservations -c test-build.ini'
+
+echo "running unit tests..."
+docker run --rm -u $(id -u):$(id -g) -v ${WORKSPACE}:/firecrest --network f7t-frontend f7t-tester bash \
+    -c 'pytest -m "not reservations" -c test-build.ini unit'
+
+echo "running integration tests..."
+docker run --rm -u $(id -u):$(id -g) -v ${WORKSPACE}:/firecrest --network f7t-frontend f7t-tester bash \
+    -c 'pytest -m "not reservations" -c test-build.ini integration'
+
+echo "finished" $0

ci/dev/run.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+##
+##  Copyright (c) 2019-2021, ETH Zurich. All rights reserved.
+##
+##  Please, refer to the LICENSE file in the root directory.
+##  SPDX-License-Identifier: BSD-3-Clause
+##
+set -euo pipefail
+
+echo "starting" $0
+WORKSPACE=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)
+
+$WORKSPACE/refresh.sh
+$WORKSPACE/setup.sh
+$WORKSPACE/test.sh
+
+echo "finished" $0

ci/dev/setup.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+##
+##  Copyright (c) 2019-2021, ETH Zurich. All rights reserved.
+##
+##  Please, refer to the LICENSE file in the root directory.
+##  SPDX-License-Identifier: BSD-3-Clause
+##
+set -euo pipefail
+
+echo "starting" $0
+WORKSPACE=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)/../../
+
+echo "prepare fresh logs folder..."
+mkdir -pv ${WORKSPACE}/deploy/test-build/logs/firecrest
+chmod 775 ${WORKSPACE}/deploy/test-build/logs/firecrest
+
+echo "adjusting keys..."
+chmod 400 ${WORKSPACE}/deploy/test-build/environment/keys/ca-key
+chmod 400 ${WORKSPACE}/deploy/test-build/environment/keys/user-key
+
+echo "building images (with caches)"
+docker build -f ${WORKSPACE}/deploy/docker/base/Dockerfile -t f7t-base ${WORKSPACE}
+docker build -f ${WORKSPACE}/deploy/docker/tester/Dockerfile -t f7t-tester ${WORKSPACE}
+docker-compose -f ${WORKSPACE}/deploy/test-build/docker-compose.yml build
+
+echo "finished" $0

ci/dev/test.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+##
+##  Copyright (c) 2019-2021, ETH Zurich. All rights reserved.
+##
+##  Please, refer to the LICENSE file in the root directory.
+##  SPDX-License-Identifier: BSD-3-Clause
+##
+set -euo pipefail
+
+echo "starting" $0
+WORKSPACE=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)/../../
+
+${WORKSPACE}/ci/dev/clean.sh
+
+echo "starting containers..."
+docker-compose -f ${WORKSPACE}/deploy/test-build/docker-compose.yml up --build -d
+
+# TODO: Complete the missing endpoints (readinessProbe like) to allow this kind of wait
+#       and remove the sleeps from retest.sh
+# echo "waiting for Firecrest stack to be ready..."
+# attempts=0
+# while [[ "$attempts" -lt 9 && "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:9000)" == "000" ]]; do
+#     let "attempts+=1"
+#     echo "API NOT ready, next attempt in 10 seconds"
+#     sleep 10
+# done
+# if [[ "$attempts" -ge 9 ]]; then
+#     echo "TIMEOUT waiting API. Shutting down cluster..."
+#     docker-compose -f ${WORKSPACE}/deploy/test-build/docker-compose.yml down -v
+#     exit 1
+# fi
+
+${WORKSPACE}/ci/dev/retest.sh
+
+echo "finished" $0