From 460a9b9a265dd73de4a64155bf84b18c071fd3cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=20Erik=20St=C3=B8wer?= Date: Thu, 14 Dec 2023 16:20:23 +0100 Subject: [PATCH] CircleCI config, release scripts, and pom.xml updates. --- .circleci/config.yml | 52 +++++ .circleci/prepare_release | 235 +++++++++++++++++++++++ .circleci/release | 84 ++++++++ .circleci/update_deployment_config | 46 +++++ .gitignore | 1 + pom.xml | 12 +- src/main/java/EnturUpdatePomVersion.java | 135 +++++++++++++ 7 files changed, 559 insertions(+), 6 deletions(-) create mode 100644 .circleci/config.yml create mode 100755 .circleci/prepare_release create mode 100755 .circleci/release create mode 100755 .circleci/update_deployment_config create mode 100644 src/main/java/EnturUpdatePomVersion.java diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000000..c0f2c328971 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,52 @@ +version: 2.1 + +aliases: + - &jfrog-login + name: Rename jfrog environment variable for maven setting.xml + command: | + echo "export JFROG_USER=$ARTIFACTORY_USER" >> $BASH_ENV + echo "export JFROG_PASS=$ARTIFACTORY_PASSWORD" >> $BASH_ENV + - &post-build + name: Update deployment with OTP version + command: | + sudo apt-get update + sudo apt-get install libxml2-utils + chmod u+x .circleci/update_deployment_config + .circleci/update_deployment_config + +jobs: + build: + docker: + - image: cimg/openjdk:21.0-node + environment: + DEBIAN_FRONTEND: "noninteractive" + MAVEN_OPTS: -Xmx6G + resource_class: large + steps: + - checkout + - restore_cache: + keys: + - dep-cache-{{ checksum "pom.xml" }} + # fallback to the most recent cache if there is no exact match for this pom.xml + - dep-cache- + - run: wget https://raw.githubusercontent.com/entur/circleci-toolbox-image-java11/master/tools/m2/settings.xml -O .circleci/settings.xml + - run: *jfrog-login + - run: mvn org.apache.maven.plugins:maven-dependency-plugin:3.1.0:go-offline -s .circleci/settings.xml + - save_cache: + paths: + - ~/.m2 + key: dep-cache-{{ checksum "pom.xml" }} + - run: mvn install -PprettierSkip org.apache.maven.plugins:maven-deploy-plugin:2.8.2:deploy -s .circleci/settings.xml -DaltDeploymentRepository=snapshots::default::https://entur2.jfrog.io/entur2/libs-release-local + - run: *post-build + +workflows: + version: 2 + release: + jobs: + - build: + name: build-release + context: global + filters: + branches: + only: + - otp2_entur_develop \ No newline at end of file diff --git a/.circleci/prepare_release b/.circleci/prepare_release new file mode 100755 index 00000000000..5b2ad0e23c4 --- /dev/null +++ b/.circleci/prepare_release @@ -0,0 +1,235 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ENTUR_DEVELOP=otp2_entur_develop +REMOTE_REPO="`git remote -v | grep "entur/OpenTripPlanner" | grep "push" | awk '{print $1;}'`" +STATUS_FILE=".prepare_release.tmp" +STATUS="" +DRY_RUN="" +OTP_BASE="" + +function main() { + setup "$@" + resumePreviousExecution + resetEnturDevelop + rebaseAndMergeExtBranch otp2_ext_config + rebaseAndMergeExtBranch otp2_ext_hack_sorlandsbanen + logSuccess +} + +function setup() { + if [[ $# -eq 2 && "$1" == "--dryRun" ]] ; then + DRY_RUN="--dryRun" + OTP_BASE="$2" + elif [[ $# -eq 1 ]] ; then + OTP_BASE="$1" + else + printHelp + exit 1 + fi + + echo "" + echo "Options: ${DRY_RUN}" + echo "Git base branch/commit: ${OTP_BASE}" + echo "Entur develop branch: ${ENTUR_DEVELOP}" + echo "Entur remote repo(pull/push): ${REMOTE_REPO}" + echo "" + + if git diff-index --quiet HEAD --; then + echo "" + echo "OK - No local changes, prepare to checkout '${ENTUR_DEVELOP}'" + echo "" + else + echo "" + echo "You have local modification, the script will abort. Nothing done!" + exit 2 + fi + + git fetch ${REMOTE_REPO} +} + +# This script create a status file '.prepare_release.tmp'. This file is used to resume the +# script in the same spot as where is left when the error occurred. This allow us to fix the +# problem (merge conflict or compile error) and re-run the script to complete the proses. +function resumePreviousExecution() { + readStatus + + if [[ -n "${STATUS}" ]] ; then + echo "" + echo "Resume: ${STATUS}?" + echo "" + echo " If all problems are resolved you may continue." + echo " Exit to clear status and start over." + echo "" + + ANSWER="" + while [[ ! "$ANSWER" =~ [yx] ]]; do + echo "Do you want to resume: [y:Yes, x:Exit]" + read ANSWER + done + + if [[ "${ANSWER}" == "x" ]] ; then + exit 0 + fi + fi +} + +function resetEnturDevelop() { + echo "" + echo "## ------------------------------------------------------------------------------------- ##" + echo "## RESET '${ENTUR_DEVELOP}' TO '${OTP_BASE}'" + echo "## ------------------------------------------------------------------------------------- ##" + echo "" + echo "Would you like to reset the '${ENTUR_DEVELOP}' to '${OTP_BASE}'? " + echo "" + + whatDoYouWant + + if [[ "${ANSWER}" == "y" ]] ; then + echo "" + echo "Checkout '${ENTUR_DEVELOP}'" + git checkout ${ENTUR_DEVELOP} + + echo "" + echo "Reset '${ENTUR_DEVELOP}' branch to '${OTP_BASE}' (hard)" + git reset --hard "${OTP_BASE}" + echo "" + fi +} + +function rebaseAndMergeExtBranch() { + EXT_BRANCH="$1" + EXT_STATUS_REBASE="Rebase '${EXT_BRANCH}'" + EXT_STATUS_COMPILE="Compile '${EXT_BRANCH}'" + + echo "" + echo "## ------------------------------------------------------------------------------------- ##" + echo "## REBASE AND MERGE '${EXT_BRANCH}' INTO '${ENTUR_DEVELOP}'" + echo "## ------------------------------------------------------------------------------------- ##" + echo "" + echo "You are about to rebase and merge '${EXT_BRANCH}' into '${ENTUR_DEVELOP}'. Any local" + echo "modification in the '${EXT_BRANCH}' will be lost." + echo "" + + whatDoYouWant + + if [[ "${ANSWER}" == "y" ]] ; then + echo "" + echo "Checkout '${EXT_BRANCH}'" + git checkout "${EXT_BRANCH}" + + echo "" + echo "Reset to '${REMOTE_REPO}/${EXT_BRANCH}'" + git reset --hard "${REMOTE_REPO}/${EXT_BRANCH}" + + echo "" + echo "Top 2 commits in '${EXT_BRANCH}'" + echo "-------------------------------------------------------------------------------------------" + git --no-pager log -2 + echo "-------------------------------------------------------------------------------------------" + echo "" + echo "You are about to rebase the TOP COMMIT ONLY(see above). Check that the " + echo "'${EXT_BRANCH}' only have ONE commit that you want to keep." + echo "" + + whatDoYouWant + + if [[ "${ANSWER}" == "y" ]] ; then + echo "" + echo "Rebase '${EXT_BRANCH}' onto '${ENTUR_DEVELOP}'" + setStatus "${EXT_STATUS_REBASE}" + git rebase --onto ${ENTUR_DEVELOP} HEAD~1 + fi + fi + + if [[ "${STATUS}" == "${EXT_STATUS_REBASE}" || "${STATUS}" == "${EXT_STATUS_COMPILE}" ]] ; then + # Reset status in case the test-compile fails. We need to do this because the status file + # is deleted after reading the status in the setup() function. + setStatus "${EXT_STATUS_COMPILE}" + + mvn clean test-compile + clearStatus + + echo "" + echo "Push '${EXT_BRANCH}'" + if [[ -z "${DRY_RUN}" ]] ; then + git push -f + else + echo "Skip: git push -f (--dryRun)" + fi + + echo "" + echo "Checkout '${ENTUR_DEVELOP}' and merge in '${EXT_BRANCH}'" + git checkout "${ENTUR_DEVELOP}" + git merge "${EXT_BRANCH}" + fi +} + +function logSuccess() { + echo "" + echo "## ------------------------------------------------------------------------------------- ##" + echo "## PREPARE RELEASE DONE -- SUCCESS" + echo "## ------------------------------------------------------------------------------------- ##" + echo " - '${REMOTE_REPO}/${ENTUR_DEVELOP}' reset to '${OTP_BASE}'" + echo " - 'otp2_ext_config' merged" + echo " - 'otp2_ext_hack_sorlandsbanen' merged" + echo "" + echo "" +} + +function whatDoYouWant() { + echo "" + ANSWER="" + + if [[ -n "${STATUS}" ]] ; then + # Skip until process is resumed + ANSWER="s" + else + while [[ ! "$ANSWER" =~ [ysx] ]]; do + echo "Do you want to continue: [y:Yes, s:Skip, x:Exit]" + read ANSWER + done + + if [[ "${ANSWER}" == "x" ]] ; then + exit 0 + fi + fi +} + +function setStatus() { + STATUS="$1" + echo "$STATUS" > "${STATUS_FILE}" +} + +function readStatus() { + if [[ -f "${STATUS_FILE}" ]] ; then + STATUS=`cat $STATUS_FILE` + rm "$STATUS_FILE" + else + STATUS="" + fi +} + +function clearStatus() { + STATUS="" + rm "${STATUS_FILE}" +} + +function printHelp() { + echo "" + echo "This script take ONE argument , the base **branch** or **commit** to use for the" + echo "release. The '${ENTUR_DEVELOP}' branch is reset to this commit and then the extension" + echo "branches is rebased onto that. The 'release' script is used to complete the release." + echo "It tag and push all changes to remote git repo." + echo "" + echo "Options:" + echo " --dryRun : Run script locally, nothing is pushed to remote server." + echo "" + echo "Usage:" + echo " $ .circleci/prepare_release otp/dev-2.x" + echo " $ .circleci/prepare_release --dryRun otp/dev-2.x" + echo "" +} + +main "$@" diff --git a/.circleci/release b/.circleci/release new file mode 100755 index 00000000000..9cac0d97958 --- /dev/null +++ b/.circleci/release @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +set -euo pipefail + +GIT_REMOTE_REPO="`git remote -v | grep "entur/OpenTripPlanner" | grep "push" | awk '{print $1;}'`" +MASTER_BRANCH=otp2_entur_develop +TAGS_FILE=target/git-entur-tags.txt +DRY_RUN="" + +function main() { + setup "$@" + listAllTags + mergeInOldReleaseWithNoChanges + setPomVersion + tagRelease + pushToRemote +} + +function setup() { + echo "" + echo "git fetch ${GIT_REMOTE_REPO}" + git fetch ${GIT_REMOTE_REPO} + + echo "Verify current branch is ${MASTER_BRANCH} " + git status | grep -q "On branch ${MASTER_BRANCH}" + + if [[ "${1+x}" == "--dryRun" ]] ; then + DRY_RUN="--dryRun" + fi +} + +function listAllTags() { + ## List all Entur tags to allow the UpdatePomVersion java program find the next version number + echo "" + echo "Dump all entur tags to ${TAGS_FILE}" + git tag -l | grep entur > ${TAGS_FILE} +} + +function setPomVersion() { + echo "" + echo "Update pom.xml with new version" + javac -d target/classes src/main/java/EnturUpdatePomVersion.java + + VERSION="`java -cp target/classes EnturUpdatePomVersion ${TAGS_FILE}`" + echo "" + echo "New version set: ${VERSION}" + echo "" + + ## Verify everything builds and tests run + echo "" + mvn clean test + + ## Add [ci skip] here before moving this to the CI server + echo "" + echo "Add and commit pom.xml" + git commit -m "Version ${VERSION}" pom.xml +} + +function mergeInOldReleaseWithNoChanges() { + echo "" + echo "Merge the old version of '${GIT_REMOTE_REPO}' into the new version. This only keep " + echo "a reference to the old version, the resulting tree of the merge is that of the new" + echo "branch head, effectively ignoring all changes from the old release." + git merge -s ours "${GIT_REMOTE_REPO}/${MASTER_BRANCH}" -m "Merge old release into '${MASTER_BRANCH}' - NO CHANGES COPIED OVER" +} + + +function tagRelease() { + echo "" + echo "Tag version ${VERSION}" + git tag -a v${VERSION} -m "Version ${VERSION}" +} + +function pushToRemote() { + echo "" + echo "Push pom.xml and new tag" + if [[ -z "${DRY_RUN}" ]] ; then + git push -f ${GIT_REMOTE_REPO} "v${VERSION}" ${MASTER_BRANCH} + else + echo "Skip: push -f ${GIT_REMOTE_REPO} "v${VERSION}" ${MASTER_BRANCH} (--dryRun)" + fi +} + +main "$@" diff --git a/.circleci/update_deployment_config b/.circleci/update_deployment_config new file mode 100755 index 00000000000..64550f19797 --- /dev/null +++ b/.circleci/update_deployment_config @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +## - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ## +## This script run AFTER OTP is build on the CI Server. It checkout the +## deployment config and update the version number, commit and push. This +## trigger the deployment config build witch create and deploy a new OTP2 +## docker image. +## +## Note! There is no need to run this script locally, unless you want to debug it. +## - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ## + + +set -euo pipefail + +OTP_DEPLOYMENT_CONFIG=target/otp-deployment-config +DOCKER_FILE=Dockerfile + +VERSION=$(xmllint --xpath "//*[local-name()='project']/*[local-name()='version']/text()" pom.xml) + +echo "Checkout otp-deplyment-config in ${OTP_DEPLOYMENT_CONFIG}" +git clone -n https://github.com/entur/otp-deployment-config.git --depth 1 ${OTP_DEPLOYMENT_CONFIG} + +pushd ${OTP_DEPLOYMENT_CONFIG} + +echo "Checkout latest version of master Dockerfile" +git checkout master ${DOCKER_FILE} + +echo "Update OTP version number in ${DOCKER_FILE}" +if [[ "$OSTYPE" == "darwin"* ]]; then + sed -E -i '' "s/OTP_VERSION=[\.0-9]+-entur-[0-9]+/OTP_VERSION=${VERSION}/" ${DOCKER_FILE} +else + sed -E -i "s/OTP_VERSION=[\.0-9]+-entur-[0-9]+/OTP_VERSION=${VERSION}/" ${DOCKER_FILE} +fi + +if [[ "$CIRCLE_USERNAME" != "" ]]; then + git config user.email "circleci@entur.no" + git config user.name "circleci ($CIRCLE_USERNAME)" +fi + +echo "Add and commit Dockerfile" +git commit -m "New OTP2 Version ${VERSION}" ${DOCKER_FILE} + +echo "Push otp-deployment-config to GitHub" +git push + +popd \ No newline at end of file diff --git a/.gitignore b/.gitignore index a90f06cdcb0..3bc96654225 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ .project .pydevproject .settings +.prepare_release.tmp .sonar .nvmrc *~ diff --git a/pom.xml b/pom.xml index 66ee281d482..295f39adb79 100644 --- a/pom.xml +++ b/pom.xml @@ -56,7 +56,7 @@ - 148 + EN-0063 30.2 2.51 @@ -84,11 +84,11 @@ - - github - OpenTripPlanner Maven Repository on Github Packages - https://maven.pkg.github.com/${GITHUB_REPOSITORY}/ - + + snapshots + entur2-snapshots + https://entur2.jfrog.io/entur2/libs-snapshot-local + diff --git a/src/main/java/EnturUpdatePomVersion.java b/src/main/java/EnturUpdatePomVersion.java new file mode 100644 index 00000000000..97bd72b3c6e --- /dev/null +++ b/src/main/java/EnturUpdatePomVersion.java @@ -0,0 +1,135 @@ +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.regex.Pattern; + +public class EnturUpdatePomVersion { + + private static final String VERSION_SEP = "-"; + private static final String ENTUR_PREFIX = "entur" + VERSION_SEP; + private static final Path POM_FILE = Path.of("pom.xml"); + + private final List tags = new ArrayList<>(); + private final List pomFile = new ArrayList<>(); + private String mainVersion; + private int versionNumber = 0; + + public static void main(String[] args) { + try { + new EnturUpdatePomVersion().withArgs(args).run(); + } catch (Exception e) { + System.err.println(e.getMessage()); + e.printStackTrace(System.err); + System.exit(10); + } + } + + private EnturUpdatePomVersion withArgs(String[] args) throws IOException { + if (args.length != 1 || args[0].matches("(?i)-h|--help")) { + printHelp(); + System.exit(1); + } + String arg = args[0]; + + if (arg.matches("\\d+")) { + versionNumber = resolveVersionFromNumericString(arg); + } else { + tags.addAll(readTagsFromFile(arg)); + } + return this; + } + + private void run() throws IOException { + readAndReplaceVersion(); + replacePomFile(); + } + + public void readAndReplaceVersion() throws IOException { + var pattern = Pattern.compile( + "(\\s*)(\\d+.\\d+.\\d+)" + + VERSION_SEP + + "(" + + ENTUR_PREFIX + + "\\d+|SNAPSHOT)(\\s*)" + ); + boolean found = false; + int i = 0; + + for (String line : Files.readAllLines(POM_FILE, UTF_8)) { + // Look for the version in the 25 first lines + if (!found) { + var m = pattern.matcher(line); + if (m.matches()) { + mainVersion = m.group(2); + String newVersion = mainVersion + VERSION_SEP + ENTUR_PREFIX + resolveVersionNumber(); + line = m.group(1) + newVersion + m.group(4); + System.out.println(newVersion); + found = true; + } + if (++i == 25) { + throw new IllegalStateException( + "Version not found in first 25 lines of the pom.xml file." + ); + } + } + pomFile.add(line); + } + if (!found) { + throw new IllegalStateException( + "Version not found in 'pom.xml'. Nothing matching pattern: " + pattern + ); + } + } + + public void replacePomFile() throws IOException { + Files.delete(POM_FILE); + Files.write(POM_FILE, pomFile, UTF_8); + } + + private static void printHelp() { + System.err.println( + "Use this small program to replace the OTP version '2.1.0-SNAPSHOT' \n" + + "with a new version number with a Entur qualifier like '2.1.0-entur-1'.\n" + + "\n" + + "Usage:\n" + + " $ java -cp .circleci UpdatePomVersion \n" + ); + } + + private int resolveVersionNumber() { + var pattern = Pattern.compile("v" + mainVersion + VERSION_SEP + ENTUR_PREFIX + "(\\d+)"); + int maxTagVersion = tags + .stream() + .mapToInt(tag -> { + var m = pattern.matcher(tag); + return m.matches() ? Integer.parseInt(m.group(1)) : -999; + }) + .max() + .orElse(-999); + + return 1 + Math.max(maxTagVersion, versionNumber); + } + + public static int resolveVersionFromNumericString(String arg) { + try { + return Integer.parseInt(arg); + } catch (NumberFormatException e) { + throw new IllegalArgumentException( + "Unable to parse input, decimal number expected: '" + arg + "'" + ); + } + } + + private static Collection readTagsFromFile(String arg) throws IOException { + var tags = Files.readAllLines(Path.of(arg)); + if (tags.isEmpty()) { + throw new IllegalStateException("Unable to load git tags from file: " + arg); + } + return tags; + } +}