From 4ef7ff00d4e37f12212d08a1a36d0d502d44f2ce Mon Sep 17 00:00:00 2001 From: Hilario Fernandes Date: Thu, 30 Aug 2018 11:25:17 +0100 Subject: [PATCH 1/2] [BACKLOG-23508] Reworking git usage --- Jenkinsfile | 21 +++++++++++++-------- resources/builders/qat.yaml | 6 +++--- resources/config/buildProperties.yaml | 22 +++++++++++++++++----- 3 files changed, 33 insertions(+), 16 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4360a0a42..da942f25b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -73,7 +73,7 @@ pipeline { } stages { - stage('Configure Pipeline') { + stage('Configure') { // After this stage all properties will be available in the buildProperties Map steps { script { @@ -94,7 +94,7 @@ pipeline { } } - stage('Clean Project Workspaces') { + stage('Clean Projects') { when { expression { buildProperties.CLEAN_SCM_WORKSPACES @@ -107,7 +107,7 @@ pipeline { } } - stage('Checkouts') { + stage('Checkout') { when { expression { buildProperties.RUN_CHECKOUTS @@ -119,6 +119,11 @@ pipeline { } stage('Version') { + when { + expression { + buildProperties.RUN_VERSIONING + } + } steps { doVersioning(buildData) } @@ -137,7 +142,7 @@ pipeline { } } - stage('Unit Test') { + stage('Test') { when { expression { buildProperties.RUN_UNIT_TESTS @@ -150,10 +155,10 @@ pipeline { } } - stage('Push Changes') { + stage('Push') { when { expression { - buildProperties.PUSH_CHANGES && !buildProperties.NOOP + buildProperties.PUSH_CHANGES } } steps { @@ -164,7 +169,7 @@ pipeline { stage('Tag') { when { expression { - !buildProperties.PUSH_CHANGES && !buildProperties.NOOP && buildProperties.TAG_NAME_TYPE != 'NONE' + buildProperties.CREATE_TAG } } steps { @@ -172,7 +177,7 @@ pipeline { } } - stage('Archive Artifacts') { + stage('Archive') { when { expression { buildProperties.ARCHIVE_ARTIFACTS && !buildProperties.NOOP diff --git a/resources/builders/qat.yaml b/resources/builders/qat.yaml index a56852d3e..091e2e856 100644 --- a/resources/builders/qat.yaml +++ b/resources/builders/qat.yaml @@ -18,9 +18,9 @@ buildProperties: SUITE_RELEASE_VERSION : 9.0-QAT RELEASE_MODE : true - # TODO: Change this to SUFFIX when we start wanting to tag - TAG_NAME_TYPE : NONE - TAG_NAME : -RC + # TODO: Enable when we start wanting to tag + CREATE_TAG : false + TAG_NAME : ${RELEASE_VERSION}-RC jobGroups: # Always required diff --git a/resources/config/buildProperties.yaml b/resources/config/buildProperties.yaml index 0ce9e27d0..4e8551fb6 100644 --- a/resources/config/buildProperties.yaml +++ b/resources/config/buildProperties.yaml @@ -33,10 +33,12 @@ BUILD_RETRIES: 1 BUILD_TIMEOUT: 360 PARALLEL_CHECKOUT_CHUNKSIZE: 40 PARALLEL_UNIT_TESTS_CHUNKSIZE: 10 -PUSH_CHANGES: false +RUN_VERSIONING: true RUN_BUILDS: true RUN_CHECKOUTS: true RUN_UNIT_TESTS: true +CREATE_TAG: false +PUSH_CHANGES: false ARCHIVE_ARTIFACTS: true ARCHIVE_ARTIFACTS_PATTERN: '**/dist/*.gz, **/dist/*.zip, **/target/*.gz, **/target/*.zip, **/build/**/*.gz, **/build/**/*.zip, **/build/*.zip' USE_DISTRIBUTED_SOURCE_CACHING: false @@ -51,7 +53,7 @@ PENTAHO_SCM_ROOT: https://github.com/pentaho WEBDETAILS_SCM_ROOT: https://github.com/webdetails DEFAULT_BRANCH: master -CHECKOUT_CREDENTIALS_ID: github-buildguy +SCM_CREDENTIALS_ID: github-buildguy CHECKOUT_DEPTH: 20 CHECKOUT_TIMESTAMP: CURRENT_TIME CHECKOUT_TIMEOUT_MINUTES: 10 @@ -68,10 +70,19 @@ RELEASE_VERSION: 9.0.0.0 SUITE_RELEASE_VERSION: 9.0.0.0 RELEASE_MODE: false ADD_BUILD_ID_VERSION_TAIL: true +VERSION_MESSAGE: '[CLEANUP] updated versions via release version merger' +VERSION_COMMIT_FILES: + - build.properties + - assembly.properties + - manual_assembly.properties + - dev_build.properties + - publish.properties + - pom.xml -# Options: SUFFIX, NONE, PREFIX, FULL -TAG_NAME_TYPE: NONE -TAG_NAME: -RC +# Tagging defaults +TAG_NAME: ${RELEASE_VERSION}-RC +TAG_MESSAGE: Tag created by ${JOB_NAME} ${BUILD_NUMBER} +TAG_SKIP_SNAPSHOT: true # Installer defaults IBUILDER_RESOURCES_ROOT_PATH: /build2/resources/ibuilder-pentaho-8.0-resources @@ -80,6 +91,7 @@ INSTALL_BUILDER_ROOT_DIR: /build2/resources/installbuilder-15.1.0 # Defaults for Job properties JOB_ITEM_DEFAULTS: scmBranch: ${DEFAULT_BRANCH} + scmCredentials: ${SCM_CREDENTIALS_ID} buildFramework: MAVEN buildFile: root: From a7effa08d5bccf49c5b1bf16880a80a5d2732b5e Mon Sep 17 00:00:00 2001 From: Hilario Fernandes Date: Thu, 30 Aug 2018 17:10:26 +0100 Subject: [PATCH 2/2] [CLEANUP] Default properties organizing/commenting --- resources/builders/qat.yaml | 2 +- resources/config/buildProperties.yaml | 47 +++++++++++++++++++-------- 2 files changed, 34 insertions(+), 15 deletions(-) diff --git a/resources/builders/qat.yaml b/resources/builders/qat.yaml index 091e2e856..db95ff746 100644 --- a/resources/builders/qat.yaml +++ b/resources/builders/qat.yaml @@ -13,7 +13,7 @@ buildProperties: MAVEN_PRIVATE_RELEASE_REPO_URL : http://nexus.pentaho.org/content/repositories/private-9.0-qat # Version based control properties - UPDATE_VERSIONS : true + RUN_VERSIONING : true RELEASE_VERSION : 9.0.0.0 SUITE_RELEASE_VERSION : 9.0-QAT RELEASE_MODE : true diff --git a/resources/config/buildProperties.yaml b/resources/config/buildProperties.yaml index 4e8551fb6..778f871b1 100644 --- a/resources/config/buildProperties.yaml +++ b/resources/config/buildProperties.yaml @@ -6,15 +6,20 @@ # 2. Build Properties Defaults (this file) # 3. Build Data File # 4. Pipeline Run Parameters +# 5. OVERRIDE_PARAMS param +# Build Tools JENKINS_ANT_FOR_BUILDS: ant-auto JENKINS_JDK_FOR_BUILDS: Java8_auto JENKINS_MAVEN_FOR_BUILDS: maven3-auto JENKINS_GRADLE_FOR_BUILDS: gradle-auto +# Ant ANT_DEFAULT_COMMAND_OPTIONS: clean-all resolve dist ANT_TEST_TARGETS: test jacoco ANT_DEFAULT_DIRECTIVES: clean-all resolve publish + +# Maven MAVEN_DEFAULT_COMMAND_OPTIONS: -B -e MAVEN_OPTS: -Xms512m MAVEN_RESOLVE_REPO_URL: http://nexus.pentaho.org/content/groups/omni @@ -24,52 +29,59 @@ MAVEN_PRIVATE_RELEASE_REPO_URL: MAVEN_PRIVATE_SNAPSHOT_REPO_URL: MAVEN_TEST_OPTS: -DsurefireArgLine=-Xmx1g MAVEN_DEFAULT_DIRECTIVES: clean install + +# Gradle GRADLE_DEFAULT_DIRECTIVES: --info --console=plain GRADLE_DEFAULT_COMMAND_OPTIONS: clean build GRADLE_TEST_TARGETS: test -NOOP: false -BUILD_RETRIES: 1 -BUILD_TIMEOUT: 360 -PARALLEL_CHECKOUT_CHUNKSIZE: 40 -PARALLEL_UNIT_TESTS_CHUNKSIZE: 10 -RUN_VERSIONING: true +# Stage execution +RUN_VERSIONING: false RUN_BUILDS: true RUN_CHECKOUTS: true RUN_UNIT_TESTS: true CREATE_TAG: false PUSH_CHANGES: false ARCHIVE_ARTIFACTS: true -ARCHIVE_ARTIFACTS_PATTERN: '**/dist/*.gz, **/dist/*.zip, **/target/*.gz, **/target/*.zip, **/build/**/*.gz, **/build/**/*.zip, **/build/*.zip' + +# General +NOOP: false +BUILD_RETRIES: 1 +BUILD_TIMEOUT: 360 +PARALLEL_CHECKOUT_CHUNKSIZE: 40 +PARALLEL_UNIT_TESTS_CHUNKSIZE: 10 USE_DISTRIBUTED_SOURCE_CACHING: false -ARCHIVE_TESTS_PATTERN: '**/bin/**/TEST*.xml, **/target/**/TEST*.xml, **/build/**/*Test.xml' #BUILD_DATA_ROOT_PATH: ${WORKSPACE}/resources/builders #LIB_CACHE_ROOT_PATH: ${WORKSPACE}/caches #BUILDS_ROOT_PATH: ${WORKSPACE}/builds +# SCM +SCM_CREDENTIALS_ID: github-buildguy SCM_HOST_ROOT: https://github.com PENTAHO_SCM_ROOT: https://github.com/pentaho WEBDETAILS_SCM_ROOT: https://github.com/webdetails DEFAULT_BRANCH: master -SCM_CREDENTIALS_ID: github-buildguy +# Checkouts CHECKOUT_DEPTH: 20 CHECKOUT_TIMESTAMP: CURRENT_TIME CHECKOUT_TIMEOUT_MINUTES: 10 CHECKOUT_SLEEP_AVERAGE_SECONDS: 0 +# Cleanups CLEAN_ALL_CACHES: false CLEAN_BUILD_WORKSPACE: false CLEAN_CACHES_REGEX: .*-SNAPSHOT.* CLEAN_SCM_WORKSPACES: false # Version based control properties -UPDATE_VERSIONS: false RELEASE_VERSION: 9.0.0.0 SUITE_RELEASE_VERSION: 9.0.0.0 RELEASE_MODE: false ADD_BUILD_ID_VERSION_TAIL: true + +# Commit message and list of files affected by version changes that are eligible for committing VERSION_MESSAGE: '[CLEANUP] updated versions via release version merger' VERSION_COMMIT_FILES: - build.properties @@ -79,9 +91,11 @@ VERSION_COMMIT_FILES: - publish.properties - pom.xml -# Tagging defaults +# Tagging TAG_NAME: ${RELEASE_VERSION}-RC TAG_MESSAGE: Tag created by ${JOB_NAME} ${BUILD_NUMBER} + +# Skip tagging if the RELEASE_VERSION is a SNAPSHOT TAG_SKIP_SNAPSHOT: true # Installer defaults @@ -113,8 +127,12 @@ SLACK_CHANNEL: 'slack-channel-name' SLACK_TEAM_DOMAIN: 'hitachivantara-eng' SLACK_CREDENTIALS_ID: 'slack-token' -#Specific archiving patterns -#EE +# Archiving +ARCHIVE_ARTIFACTS_PATTERN: '**/dist/*.gz, **/dist/*.zip, **/target/*.gz, **/target/*.zip, **/build/**/*.gz, **/build/**/*.zip, **/build/*.zip' +ARCHIVE_TESTS_PATTERN: '**/bin/**/TEST*.xml, **/target/**/TEST*.xml, **/build/**/*Test.xml' + +# Specific archiving patterns +# EE artifacts INSTALLER_ARTIFACTS_PATTERN_WINDOWS: 'bi-suite/target/windows/pentaho-business-analytics-*.*' INSTALLER_ARTIFACTS_PATTERN_MAC: 'bi-suite/target/mac/pentaho-business-analytics-*.*' INSTALLER_ARTIFACTS_PATTERN_LINUX: 'bi-suite/target/linux/pentaho-business-analytics-*.*, upgrade-utility/target/pentaho-upgrade-utility-*.zip' @@ -136,7 +154,8 @@ PIR_PLUGIN_PATTERN: 'assemblies/plugin/target/pir-plugin-ee-*.zip' PLATFORM_SERVER_ASSEMBLY_EE_PATTERN: 'assemblies/pentaho-server-ee/target/pentaho-server-ee-*.zip, assemblies/pentaho-server-manual-ee/target/pentaho-server-manual-ee-*.zip' BIG_DATA_PLUGIN_ASSEMBLY_EE_PATTERN: 'assemblies/big-data-ee-package/target/pentaho-big-data-ee-package-*.zip, shims/**/assemblies/*-ee-shim/target/pentaho-big-data-ee-*-package-*.zip' FOUNDRY_WORKERS_EE_PATTERN: 'assembly/target/pentaho-workernodes-*.zip, assembly/target/pentaho-workernodes-*.tar.gz' -#CE + +# CE artifacts PAD_ASSEMBLY_PATTERN: 'assemblies/pad-ce/target/pad-ce-*.zip' PDI_ENGINE_CORE_PATTERN: 'assemblies/client/target/pdi-ce-*.zip' BIG_DATA_PLUGIN_PATTERN: 'legacy-plugin/target/pentaho-big-data-plugin-*.zip'