diff --git a/.gitignore b/.gitignore index fb027cfeb4965..4dde0cee36f75 100644 --- a/.gitignore +++ b/.gitignore @@ -61,4 +61,7 @@ documentation/rebuildpdf.bat # release process .build/ -.build_source/ \ No newline at end of file +.build_source/ +# direnv +.envrc +.env \ No newline at end of file diff --git a/.jenkins/scripts/docker_build.sh b/.jenkins/scripts/docker_build.sh index f1e105ce70a7d..9e5d1a22ff879 100755 --- a/.jenkins/scripts/docker_build.sh +++ b/.jenkins/scripts/docker_build.sh @@ -20,20 +20,32 @@ set -xe # Parameters: # $1: docker tag version # $2: should tag as latest +# $3: requested image, if not given, all will be pushed -tag="${1?Missing tag}" -latest="${2:-false}" +_TAG="${1?Missing tag}" +_IS_LATEST="${2-'false'}" +_ONLY_ONE_IMAGE="${3-''}" dockerBuild() { - echo ">> Building and pushing $1:${tag}" - cd "images/${1}-image" - mvn package jib:build@build -Ddocker.talend.image.tag=${tag} - if [[ ${latest} == 'true' ]]; then - mvn package jib:build@build -Ddocker.talend.image.tag=latest + _IMAGE="${1}" + echo ">> Building and push $_IMAGE:${_TAG}" + + mvn package jib:build@build \ + --file "images/${_IMAGE}-image/pom.xml" \ + --define docker.talend.image.tag="${_TAG}" + + if [[ ${_IS_LATEST} == 'true' ]]; then + mvn package jib:build@build \ + --file "images/${_IMAGE}-image/pom.xml" \ + --define docker.talend.image.tag=latest fi - cd ../.. } -dockerBuild "component-server" -dockerBuild "component-starter-server" -dockerBuild "remote-engine-customizer" +if [[ -n "${_ONLY_ONE_IMAGE}" ]]; then + dockerBuild "${_ONLY_ONE_IMAGE}" +else + dockerBuild "component-server" + dockerBuild "component-starter-server" + dockerBuild "remote-engine-customizer" +fi + diff --git a/Jenkinsfile b/Jenkinsfile index cb05e158972f2..f9c38199dc5aa 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -13,8 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +// Imports +import java.time.LocalDateTime +import java.util.regex.Matcher + // Credentials final def ossrhCredentials = usernamePassword(credentialsId: 'ossrh-credentials', usernameVariable: 'OSSRH_USER', passwordVariable: 'OSSRH_PASS') +final def nexusCredentials = usernamePassword(credentialsId: 'nexus-artifact-zl-credentials', usernameVariable: 'NEXUS_USER', passwordVariable: 'NEXUS_PASS') final def jetbrainsCredentials = usernamePassword(credentialsId: 'jetbrains-credentials', usernameVariable: 'JETBRAINS_USER', passwordVariable: 'JETBRAINS_PASS') final def jiraCredentials = usernamePassword(credentialsId: 'jira-credentials', usernameVariable: 'JIRA_USER', passwordVariable: 'JIRA_PASS') final def gitCredentials = usernamePassword(credentialsId: 'github-credentials', usernameVariable: 'GITHUB_USER', passwordVariable: 'GITHUB_PASS') @@ -28,14 +33,24 @@ final String slackChannel = 'components-ci' final Boolean isMasterBranch = env.BRANCH_NAME == "master" final Boolean isStdBranch = (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("maintenance/")) final Boolean hasPostLoginScript = params.POST_LOGIN_SCRIPT != "" -final Boolean hasExtraBuildArgs = params.EXTRA_BUILD_ARGS != "" -final String buildTimestamp = String.format('-%tY% + dev branches are deploying on ''') + booleanParam( + name: 'DOCKER_PUSH', + defaultValue: false, + description: ''' + Force DOCKER push stage for development branches. No effect on master and maintenance. + INFO: master/maintenance and dev branches are deploying on ''') + string( + name: 'VERSION_QUALIFIER', + defaultValue: 'DEFAULT', + description: ''' + Deploy jars with the given version qualifier. No effect on master and maintenance. + - DEFAULT means the qualifier will be the Jira id extracted from the branch name. + From "user/JIRA-12345_some_information" the qualifier will be JIRA-12345.''') string( - name: 'EXTRA_BUILD_ARGS', + name: 'EXTRA_BUILD_PARAMS', defaultValue: '', description: 'Add some extra parameters to maven commands. Applies to all maven calls.') string( @@ -80,7 +109,15 @@ pipeline { defaultValue: '', description: 'Execute a shell command after login. Useful for maintenance.') booleanParam( - name: 'DEBUG_BEFORE_EXITING', + name: 'FORCE_SONAR', + defaultValue: false, + description: 'Force Sonar analysis') + booleanParam( + name: 'FORCE_DOC', + defaultValue: false, + description: 'Force documentation stage for development branches. No effect on master and maintenance.') + booleanParam( + name: 'JENKINS_DEBUG', defaultValue: false, description: 'Add an extra step to the pipeline allowing to keep the pod alive for debug purposes.') } @@ -88,6 +125,9 @@ pipeline { stages { stage('Preliminary steps') { steps { + /////////////////////////////////////////// + // Login tasks + /////////////////////////////////////////// script { withCredentials([gitCredentials]) { sh """ bash .jenkins/scripts/git_login.sh "\${GITHUB_USER}" "\${GITHUB_PASS}" """ @@ -96,16 +136,107 @@ pipeline { sh """ bash .jenkins/scripts/docker_login.sh "${ARTIFACTORY_REGISTRY}" "\${DOCKER_USER}" "\${DOCKER_PASS}" """ } withCredentials([keyImportCredentials]) { - sh """ bash .jenkins/scripts/setup_gpg.sh""" + sh """ bash .jenkins/scripts/setup_gpg.sh """ + } + } + /////////////////////////////////////////// + // asdf install + /////////////////////////////////////////// + script { + println "asdf install the content of repository .tool-versions'\n" + sh 'bash .jenkins/scripts/asdf_install.sh' + } + /////////////////////////////////////////// + // Variables init + /////////////////////////////////////////// + script { + stdBranch_buildOnly = isStdBranch && params.Action != 'RELEASE' + devBranch_mavenDeploy = !isStdBranch && params.MAVEN_DEPLOY + devBranch_dockerPush = !isStdBranch && params.DOCKER_PUSH + + needQualify = devBranch_mavenDeploy || devBranch_dockerPush + + if (needQualify) { + // Qualified version have to be released on talend_repository + // Overwrite the deployOptions + deployOptions = "$skipOptions --activate-profiles private_repository -Denforcer.skip=true" + } + + // By default the doc is skipped for standards branches + Boolean skip_documentation = !( params.FORCE_DOC || isStdBranch ) + extraBuildParams = assemblyExtraBuildParams(skip_documentation) + + } + /////////////////////////////////////////// + // Pom version and Qualifier management + /////////////////////////////////////////// + script{ + final def pom = readMavenPom file: 'pom.xml' + pomVersion = pom.version + + echo 'Manage the version qualifier' + if (!needQualify) { + println """ + No need to add qualifier in followings cases: + - We are on Master or Maintenance branch + - We do not want to deploy on dev branch + """.stripIndent() + finalVersion = pomVersion } + else { + branch_user = "" + branch_ticket = "" + branch_description = "" + if (params.VERSION_QUALIFIER != ("DEFAULT")) { + // If the qualifier is given, use it + println """ + No need to add qualifier, use the given one: "$params.VERSION_QUALIFIER" + """.stripIndent() + } + else { + echo "Validate the branch name" + (branch_user, + branch_ticket, + branch_description) = extract_branch_info("$env.BRANCH_NAME") - def pom = readMavenPom file: 'pom.xml' - env.PROJECT_VERSION = pom.version - try { - EXTRA_BUILD_ARGS = params.EXTRA_BUILD_ARGS - } catch (ignored) { - EXTRA_BUILD_ARGS = "" + // Check only branch_user, because if there is an error all three params are empty. + if (branch_user == ("")) { + println """ + ERROR: The branch name doesn't comply with the format: user/JIRA-1234-Description + It is MANDATORY for artifact management. + You have few options: + - You do not need to deploy, uncheck MAVEN_DEPLOY checkbox + - Change the VERSION_QUALIFIER text box to a personal qualifier, BUT you need to do it on ALL se/ee and cloud-components build + - Rename your branch + """.stripIndent() + currentBuild.description = ("ERROR: The branch name is not correct") + sh """exit 1""" + } + } + + echo "Insert a qualifier in pom version..." + finalVersion = add_qualifier_to_version( + pomVersion, + branch_ticket, + "$params.VERSION_QUALIFIER" as String) + + echo """ + Configure the version qualifier for the curent branche: $env.BRANCH_NAME + requested qualifier: $params.VERSION_QUALIFIER + with User = $branch_user, Ticket = $branch_ticket, Description = $branch_description + Qualified Version = $finalVersion""" + + // On development branches the connectors version shall be edited for deployment + // Maven documentation about maven_version: + // https://docs.oracle.com/middleware/1212/core/MAVEN/maven_version.htm + println "Edit version on dev branches, new version is ${finalVersion}" + sh """\ + #!/usr/bin/env bash + mvn versions:set --define newVersion=${finalVersion} + mvn versions:set --file bom/pom.xml --define newVersion=${finalVersion} + """.stripIndent() } + } /////////////////////////////////////////// // Updating build displayName and description @@ -114,29 +245,42 @@ pipeline { String user_name = currentBuild.getBuildCauses('hudson.model.Cause$UserIdCause').userId[0] if ( user_name == null) { user_name = "auto" } + String deploy_info = '' + if (stdBranch_buildOnly || devBranch_mavenDeploy){ + deploy_info = deploy_info + '+DEPLOY' + } + if (devBranch_dockerPush){ + deploy_info = deploy_info + '+DOCKER' + } + currentBuild.displayName = ( - "#$currentBuild.number-$params.Action: $user_name" + "#$currentBuild.number-$params.Action" + deploy_info + ": $user_name" ) // updating build description - currentBuild.description = (""" - User: $user_name - $params.Action Build - Sonar: $params.FORCE_SONAR - Script: $hasPostLoginScript - Extra args: $hasExtraBuildArgs - Debug: $params.DEBUG_BEFORE_EXITING""".stripIndent() - ) + String description = """ + Version = $finalVersion - $params.Action Build + Sonar forced: $params.FORCE_SONAR - Script: $hasPostLoginScript + Debug: $params.JENKINS_DEBUG + Extra build args: $extraBuildParams""".stripIndent() + job_description_append(description) } - /////////////////////////////////////////// - // asdf install - /////////////////////////////////////////// - script { - println "asdf install the content of repository .tool-versions'\n" - sh 'bash .jenkins/scripts/asdf_install.sh' + } + post { + always { + println "Artifact Poms files for analysis if needed" + archiveArtifacts artifacts: '**/*pom.*', allowEmptyArchive: false, onlyIfSuccessful: false } } } stage('Post login') { steps { - withCredentials([gitCredentials, dockerCredentials, ossrhCredentials, jetbrainsCredentials, jiraCredentials, gpgCredentials]) { + withCredentials([gitCredentials, + dockerCredentials, + ossrhCredentials, + jetbrainsCredentials, + jiraCredentials, + gpgCredentials]) { script { try { sh """\ @@ -151,68 +295,130 @@ pipeline { } } } - stage('Standard maven build') { + stage('Maven validate to install') { when { expression { params.Action != 'RELEASE' } } steps { withCredentials([ossrhCredentials]) { sh """\ #!/usr/bin/env bash - mvn clean install $BUILD_ARGS $EXTRA_BUILD_ARGS -s .jenkins/settings.xml + set -xe + mvn clean install --file bom/pom.xml + mvn clean install $BUILD_ARGS \ + $extraBuildParams \ + --settings .jenkins/settings.xml """.stripIndent() } } + post { + always { + recordIssues( + enabledForFailure: true, + tools: [ + junitParser( + id: 'unit-test', + name: 'Unit Test', + pattern: '**/target/surefire-reports/*.xml' + ) + ] + ) + } + } } - stage('Deploy artifacts') { + stage('Maven deploy') { when { - allOf { - expression { params.Action != 'RELEASE' } - expression { isStdBranch } + anyOf { + expression { stdBranch_buildOnly } + expression { devBranch_mavenDeploy } } } steps { - withCredentials([ossrhCredentials, gpgCredentials]) { - sh """\ + script { + withCredentials([ossrhCredentials, + gpgCredentials, + nexusCredentials]) { + sh """\ #!/usr/bin/env bash - bash mvn deploy $DEPLOY_OPTS $EXTRA_BUILD_ARGS -s .jenkins/settings.xml + set -xe + bash mvn deploy $deployOptions \ + $extraBuildParams \ + --settings .jenkins/settings.xml """.stripIndent() + } + } + // Add description to job + script { + def repo + if (devBranch_mavenDeploy) { + repo = ['artifacts-zl.talend.com', + 'https://artifacts-zl.talend.com/nexus/content/repositories/snapshots/org/talend/sdk/component'] + } else { + repo = ['oss.sonatype.org', + 'https://oss.sonatype.org/content/repositories/snapshots/org/talend/sdk/component/'] + } + + job_description_append("Maven artefact deployed as ${finalVersion} on [${repo[0]}](${repo[1]})") } } } - stage('Docker images') { + stage('Docker build/push') { when { - allOf { - expression { params.Action != 'RELEASE' } - expression { isStdBranch } + anyOf { + expression { stdBranch_buildOnly } + expression { devBranch_dockerPush } } } steps { script { configFileProvider([configFile(fileId: 'maven-settings-nexus-zl', variable: 'MAVEN_SETTINGS')]) { - sh """\ - #!/usr/bin/env bash - bash .jenkins/scripts/docker_build.sh ${env.PROJECT_VERSION}${buildTimestamp} - """.stripIndent() + + String images_options = '' + if (isStdBranch){ + // Build and push all images + job_description_append("Docker images deployed: component-server, component-starter-server and remote-engine-customizer") + } + else{ + images_options = 'false component-server' + job_description_append("Docker images deployed: component-server") + job_description_append("As ${finalVersion}${buildTimestamp} on [artifactory.datapwn.com](https://artifactory.datapwn.com/tlnd-docker-dev/talend/common/tacokit)" as String) + + } + + // Build and push specific image + sh """ + bash .jenkins/scripts/docker_build.sh \ + ${finalVersion}${buildTimestamp} \ + ${images_options} + """ + + job_description_append("docker pull artifactory.datapwn.com/tlnd-docker-dev/talend/common/tacokit/component-server:${finalVersion}${buildTimestamp}") } + } } } stage('Documentation') { when { - allOf { - expression { params.Action != 'RELEASE' } - expression { isMasterBranch } + expression { + params.FORCE_DOC || (params.Action != 'RELEASE' && isMasterBranch) } } steps { withCredentials([ossrhCredentials, gitCredentials]) { sh """\ #!/usr/bin/env bash - cd documentation && mvn verify pre-site -Pgh-pages -Dgpg.skip=true $SKIP_OPTS $EXTRA_BUILD_ARGS -s ../.jenkins/settings.xml && cd - + set -xe + mvn verify pre-site --file documentation/pom.xml \ + --settings .jenkins/settings.xml \ + --activate-profiles gh-pages \ + --define gpg.skip=true \ + $skipOptions \ + $extraBuildParams + """.stripIndent() } } } - stage('Master Post Build Tasks') { + stage('OSS security analysis') { when { expression { params.Action != 'RELEASE' } branch 'master' @@ -222,59 +428,100 @@ pipeline { catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { sh """\ #!/usr/bin/env bash - mvn ossindex:audit-aggregate -pl '!bom' -Dossindex.fail=false -Dossindex.reportFile=target/audit.txt -s .jenkins/settings.xml - mvn versions:dependency-updates-report versions:plugin-updates-report versions:property-updates-report -pl '!bom' + set -xe + mvn ossindex:audit-aggregate -pl '!bom' \ + --define ossindex.fail=false \ + --define ossindex.reportFile=target/audit.txt \ + --settings .jenkins/settings.xml """.stripIndent() } } - withCredentials([sonarCredentials]) { + } + post { + always { + publishHTML( + target: [ + allowMissing : true, + alwaysLinkToLastBuild: false, + keepAll : true, + reportDir : 'target/', + reportFiles : 'audit.txt', + reportName : "security::audit" + ]) + } + } + } + stage('Deps report') { + when { + expression { params.Action != 'RELEASE' } + branch 'master' + } + steps { + withCredentials([ossrhCredentials]) { catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - // TODO https://jira.talendforge.org/browse/TDI-48980 (CI: Reactivate Sonar cache) sh """\ #!/usr/bin/env bash - _JAVA_OPTIONS='--add-opens=java.base/java.lang=ALL-UNNAMED' mvn -Dsonar.host.url=https://sonar-eks.datapwn.com -Dsonar.login='$SONAR_USER' -Dsonar.password='$SONAR_PASS' -Dsonar.branch.name=${env.BRANCH_NAME} -Dsonar.analysisCache.enabled=false sonar:sonar - """.stripIndent() + set -xe + mvn versions:dependency-updates-report versions:plugin-updates-report \ + versions:property-updates-report \ + -pl '!bom' + """.stripIndent() } } } post { always { publishHTML( - target: [ - allowMissing : true, - alwaysLinkToLastBuild: false, - keepAll : true, - reportDir : 'target/', - reportFiles : 'audit.txt', - reportName : "security::audit" - ]) - publishHTML( - target: [ - allowMissing : true, - alwaysLinkToLastBuild: false, - keepAll : true, - reportDir : 'target/site/', - reportFiles : 'property-updates-report.html', - reportName : "outdated::property" - ]) + target: [ + allowMissing : true, + alwaysLinkToLastBuild: false, + keepAll : true, + reportDir : 'target/site/', + reportFiles : 'property-updates-report.html', + reportName : "outdated::property" + ]) publishHTML( - target: [ - allowMissing : true, - alwaysLinkToLastBuild: false, - keepAll : true, - reportDir : 'target/site/', - reportFiles : 'dependency-updates-report.html', - reportName : "outdated::dependency" - ]) + target: [ + allowMissing : true, + alwaysLinkToLastBuild: false, + keepAll : true, + reportDir : 'target/site/', + reportFiles : 'dependency-updates-report.html', + reportName : "outdated::dependency" + ]) publishHTML( - target: [ - allowMissing : true, - alwaysLinkToLastBuild: false, - keepAll : true, - reportDir : 'target/site/', - reportFiles : 'plugin-updates-report.html', - reportName : "outdated::plugins" - ]) + target: [ + allowMissing : true, + alwaysLinkToLastBuild: false, + keepAll : true, + reportDir : 'target/site/', + reportFiles : 'plugin-updates-report.html', + reportName : "outdated::plugins" + ]) + } + } + } + stage('Sonar') { + when { + expression { params.Action != 'RELEASE' } + branch 'master' + } + steps { + withCredentials([sonarCredentials]) { + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + // TODO https://jira.talendforge.org/browse/TDI-48980 (CI: Reactivate Sonar cache) + sh """\ + #!/usr/bin/env bash + set -xe + _JAVA_OPTIONS='--add-opens=java.base/java.lang=ALL-UNNAMED' + mvn sonar:sonar \ + --define sonar.host.url=https://sonar-eks.datapwn.com \ + --define sonar.login='$SONAR_USER' \ + --define sonar.password='$SONAR_PASS' \ + --define sonar.branch.name=${env.BRANCH_NAME} \ + --define sonar.analysisCache.enabled=false + """.stripIndent() + } } } } @@ -291,17 +538,13 @@ pipeline { configFileProvider([configFile(fileId: 'maven-settings-nexus-zl', variable: 'MAVEN_SETTINGS')]) { sh """\ #!/usr/bin/env bash - bash .jenkins/scripts/release.sh ${env.BRANCH_NAME} ${env.PROJECT_VERSION} + bash .jenkins/scripts/release.sh ${env.BRANCH_NAME} ${finalVersion} """.stripIndent() } } } } } - stage('Debug') { - when { expression { return params.DEBUG_BEFORE_EXITING } } - steps { script { input message: 'Finish the job?', ok: 'Yes' } } - } } post { success { @@ -337,7 +580,7 @@ pipeline { //Only post results to Slack for Master and Maintenance branches if (isStdBranch) { //if previous build was a success, ping channel in the Slack message - if ("SUCCESS".equals(currentBuild.previousBuild.result)) { + if ("SUCCESS" == currentBuild.previousBuild.result) { slackSend( color: '#FF0000', message: "@here : NEW FAILURE: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' (${env.BUILD_URL})", @@ -358,11 +601,6 @@ pipeline { recordIssues( enabledForFailure: true, tools: [ - junitParser( - id: 'unit-test', - name: 'Unit Test', - pattern: '**/target/surefire-reports/*.xml' - ), taskScanner( id: 'disabled', name: '@Disabled', @@ -381,10 +619,142 @@ pipeline { ] ) script { - println '====== Archive artifacts' - println "Artifact 1: ${_ARTIFACT_COVERAGE}" - archiveArtifacts artifacts: "${_ARTIFACT_COVERAGE}", allowEmptyArchive: true, onlyIfSuccessful: false + println '====== Archive jacoco reports artifacts' + archiveArtifacts artifacts: "${'**/jacoco-aggregate/**/*.*'}", allowEmptyArchive: true, onlyIfSuccessful: false + } + + script { + if (params.JENKINS_DEBUG) { + jenkinsBreakpoint() + } } } } } + + +/** + * Append a new line to job description + * This is MARKDOWN, do not forget double space at the end of line + * + * @param new line + * @return void + */ +private void job_description_append(String new_line) { + if (currentBuild.description == null) { + println "Create the job description with: \n$new_line" + currentBuild.description = new_line + } else { + println "Edit the job description adding: $new_line" + currentBuild.description = currentBuild.description + '\n' + new_line + } +} + +/** + * Assembly all needed items to put inside extraBuildParams + * + * @param Boolean skip_doc, if set to true documentation build will be skipped + * + * @return extraBuildParams as a string ready for mvn cmd + */ +private String assemblyExtraBuildParams(Boolean skip_doc) { + String extraBuildParams + + println 'Processing extraBuildParams' + final List buildParamsAsArray = [] + + println 'Manage user params' + if ( params.EXTRA_BUILD_PARAMS ) { + buildParamsAsArray.add(params.EXTRA_BUILD_PARAMS as String) + } + + println 'Manage the skip_doc option' + if (skip_doc) { + buildParamsAsArray.add('--projects !documentation') + buildParamsAsArray.add('--define documentation.skip=true') + } + + println 'Construct final params content' + extraBuildParams = buildParamsAsArray.join(' ') + println "extraBuildParams: $extraBuildParams" + + return extraBuildParams +} + +/** + * Implement a simple breakpoint to stop actual job + * Use the method anywhere you need to stop + * The first usage is to keep the pod alive on post stage. + * Change and restore the job description to be more visible + * + * @param none + * @return void + */ +private void jenkinsBreakpoint() { + // Backup the description + String job_description_backup = currentBuild.description + // updating build description + currentBuild.description = "ACTION NEEDED TO CONTINUE \n ${job_description_backup}" + // Request user action + input message: 'Finish the job?', ok: 'Yes' + // updating build description + currentBuild.description = "$job_description_backup" +} + +/** + * create a new version from actual one and given jira ticket or user qualifier + * Priority to user qualifier + * + * The branch name has comply with the format: user/JIRA-1234-Description + * It is MANDATORY for artifact management. + * + * @param String version actual version to edit + * @param GString ticket + * @param GString user_qualifier to be checked as priority qualifier + * + * @return String new_version with added qualifier + */ +private static String add_qualifier_to_version(String version, String ticket, String user_qualifier) { + String new_version + + if (user_qualifier.contains("DEFAULT")) { + if (version.contains("-SNAPSHOT")) { + new_version = version.replace("-SNAPSHOT", "-$ticket-SNAPSHOT" as String) + } else { + new_version = "$version-$ticket".toString() + } + } else { + new_version = version.replace("-SNAPSHOT", "-$user_qualifier-SNAPSHOT" as String) + } + return new_version +} + +/** + * extract given branch information + * + * The branch name has comply with the format: user/JIRA-1234-Description + * It is MANDATORY for artifact management. + * + * @param branch_name row name of the branch + * + * @return A list containing the extracted: [user, ticket, description] + * The method also raise an assert exception in case of wrong branch name + */ +private static ArrayList extract_branch_info(GString branch_name) { + + String branchRegex = /^(?.*)\/(?[A-Z]{2,8}-\d{1,6})[_-](?.*)/ + Matcher branchMatcher = branch_name =~ branchRegex + + try { + assert branchMatcher.matches() + } + catch (AssertionError ignored) { + return ["", "", ""] + } + + String user = branchMatcher.group("user") + String ticket = branchMatcher.group("ticket") + String description = branchMatcher.group("description") + + return [user, ticket, description] +} \ No newline at end of file diff --git a/ci/Jenkinsfile-scan b/ci/Jenkinsfile-scan index 2a36a5f96539c..fee7cd439c37e 100644 --- a/ci/Jenkinsfile-scan +++ b/ci/Jenkinsfile-scan @@ -21,7 +21,7 @@ final def veracodeCredentials = usernamePassword( final def nexusCredentials = usernamePassword( credentialsId: 'nexus-artifact-zl-credentials', usernameVariable: 'NEXUS_USER', - passwordVariable: 'NEXUS_PASSWORD') + passwordVariable: 'NEXUS_PASS') String git_branch_name = "" diff --git a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/ReflectionService.java b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/ReflectionService.java index 2ff65aaccc8b0..a5bb62d152859 100644 --- a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/ReflectionService.java +++ b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/ReflectionService.java @@ -35,6 +35,7 @@ import java.lang.reflect.Type; import java.util.AbstractMap; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; @@ -208,19 +209,29 @@ public Function, Object[]> parameterFactory(final Executable return config -> { final Map notNullConfig = ofNullable(config).orElseGet(Collections::emptyMap); - final PayloadValidator visitor = new PayloadValidator(); - if (!visitor.skip) { - visitor.globalPayload = new PayloadMapper((a, b) -> { - }).visitAndMap(metas, notNullConfig); - final PayloadMapper payloadMapper = new PayloadMapper(visitor); - payloadMapper.setGlobalPayload(visitor.globalPayload); - payloadMapper.visitAndMap(metas, notNullConfig); - visitor.throwIfFailed(); - } + checkPayload(metas, notNullConfig); return factories.stream().map(f -> f.apply(notNullConfig)).toArray(Object[]::new); }; } + public static void checkPayload(final List metas, final Map notNullConfig) { + JsonObject globalPayload = new PayloadMapper((a, b) -> { + }).visitAndMap(metas, notNullConfig); + checkWithPayload(metas, notNullConfig, globalPayload); + } + + public static void checkWithPayload(final List metas, final Map notNullConfig, + final JsonObject payload) { + final PayloadValidator visitor = new PayloadValidator(); + if (!visitor.skip) { + visitor.globalPayload = payload; + final PayloadMapper payloadMapper = new PayloadMapper(visitor); + payloadMapper.setGlobalPayload(payload); + payloadMapper.visitAndMap(metas, notNullConfig); + visitor.throwIfFailed(); + } + } + public Function, Object> createContextualSupplier(final ClassLoader loader) { return supplier -> { final Thread thread = Thread.currentThread(); @@ -319,7 +330,7 @@ private Object createList(final ClassLoader loader, final Function k.startsWith(configName + "."))) { // object - // mapping + // mapping if (paramIdx == 0) { args = findArgsName(itemClass); } @@ -887,6 +898,8 @@ public interface Messages { String uniqueItems(String property); String pattern(String property, String pattern); + + String enumValues(String property, String enums, String val); } @RequiredArgsConstructor @@ -908,7 +921,6 @@ public void onParameter(final ParameterMeta meta, final JsonValue value) { if (!VISIBILITY_SERVICE.build(meta).isVisible(globalPayload)) { return; } - if (Boolean.parseBoolean(meta.getMetadata().get("tcomp::validation::required")) && value == JsonValue.NULL) { errors.add(MESSAGES.required(meta.getPath())); @@ -996,6 +1008,20 @@ public void onParameter(final ParameterMeta meta, final JsonValue value) { } } } + { + final String enumValues = metadata.get("tcomp::validation::enumValues"); + if (enumValues != null) {// value = null or not in enumValues: add error + if (value == null) { + errors.add(MESSAGES.enumValues(meta.getPath(), enumValues, null)); + } else { + final String val = JsonValue.class.cast(value).toString().replace("\"", ""); + String[] enums = enumValues.substring(1, enumValues.length() - 1).split(","); + if (Arrays.stream(enums).noneMatch(s -> s.trim().equalsIgnoreCase(val))) { + errors.add(MESSAGES.enumValues(meta.getPath(), enumValues, val)); + } + } + } + } } private void throwIfFailed() { @@ -1007,7 +1033,7 @@ private void throwIfFailed() { /** * Helper function for creating an instance from a configuration map. - * + * * @param clazz Class of the wanted instance. * @param Type managed * @return function that generate the wanted instance when calling diff --git a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/visibility/PayloadMapper.java b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/visibility/PayloadMapper.java index 2c7ef9861f8e4..20562c5c82463 100644 --- a/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/visibility/PayloadMapper.java +++ b/component-runtime-manager/src/main/java/org/talend/sdk/component/runtime/manager/reflect/visibility/PayloadMapper.java @@ -84,7 +84,7 @@ private void onProperty(final String contextualPrefix, final Collection json.add(name, Boolean.parseBoolean(v))); break; case NUMBER: - final String numberValue = config.get(newPath); + final String numberValue = getValue(config, newPath, definition); if (numberValue == null || numberValue.isEmpty()) { parameterVisitor.onParameter(definition, JsonValue.NULL); } else { @@ -114,7 +114,7 @@ private void onProperty(final String contextualPrefix, final Collection json.add(name, v)); break; @@ -123,6 +123,11 @@ private void onProperty(final String contextualPrefix, final Collection config, final String newPath, + final ParameterMeta definition) { + return config.get(newPath) == null ? config.get(definition.getPath()) : config.get(newPath); + } + private void onObject(final Collection definitions, final ParameterMeta meta, final Map config, final JsonObjectBuilder json, final String name, final String currentPath) { diff --git a/component-runtime-manager/src/main/resources/org/talend/sdk/component/runtime/manager/reflect/Messages.properties b/component-runtime-manager/src/main/resources/org/talend/sdk/component/runtime/manager/reflect/Messages.properties index 0a5a8054100b6..5fb61bf1cd207 100644 --- a/component-runtime-manager/src/main/resources/org/talend/sdk/component/runtime/manager/reflect/Messages.properties +++ b/component-runtime-manager/src/main/resources/org/talend/sdk/component/runtime/manager/reflect/Messages.properties @@ -20,3 +20,4 @@ org.talend.sdk.component.runtime.manager.reflect.ReflectionService$Messages.minI org.talend.sdk.component.runtime.manager.reflect.ReflectionService$Messages.maxItems = Length of property ''{0}'' should be < {1}, got {2}. org.talend.sdk.component.runtime.manager.reflect.ReflectionService$Messages.uniqueItems = ''{0}'' has duplicated items. org.talend.sdk.component.runtime.manager.reflect.ReflectionService$Messages.pattern = ''{0}'' does not match ''{1}''. +org.talend.sdk.component.runtime.manager.reflect.ReflectionService$Messages.enumValues = Invalid value for Property ''{0}'' expected: ''{1}'', got {2}. \ No newline at end of file diff --git a/component-runtime-manager/src/test/java/org/talend/sdk/component/runtime/manager/service/MavenRepositoryResolverTest.java b/component-runtime-manager/src/test/java/org/talend/sdk/component/runtime/manager/service/MavenRepositoryResolverTest.java index c98012157639a..9691d1d2c77a5 100644 --- a/component-runtime-manager/src/test/java/org/talend/sdk/component/runtime/manager/service/MavenRepositoryResolverTest.java +++ b/component-runtime-manager/src/test/java/org/talend/sdk/component/runtime/manager/service/MavenRepositoryResolverTest.java @@ -17,7 +17,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.talend.sdk.component.runtime.manager.service.MavenRepositoryResolver.M2_REPOSITORY; import static org.talend.sdk.component.runtime.manager.service.MavenRepositoryResolver.STUDIO_MVN_REPOSITORY; import static org.talend.sdk.component.runtime.manager.service.MavenRepositoryResolver.TALEND_COMPONENT_MANAGER_M2_REPOSITORY; import static org.talend.sdk.component.runtime.manager.service.MavenRepositoryResolver.TALEND_COMPONENT_MANAGER_M2_SETTINGS; @@ -39,6 +38,8 @@ class MavenRepositoryResolverTest { + static String M2_REPOSITORY = ".m2" + File.separator + "repository"; + private final MavenRepositoryResolver resolver = new MavenRepositoryDefaultResolver(); final MavenRepositoryDefaultResolver mavenSettingsOnlyResolver = new MavenRepositoryDefaultResolver() { @@ -63,7 +64,7 @@ public Path get(final String path) { } }; - private final String fallback = System.getProperty("user.home") + "/" + M2_REPOSITORY; + private final String fallback = System.getProperty("user.home") + File.separator + M2_REPOSITORY; private final Path repository = Paths.get(new File("target/test-classes").getAbsolutePath()); @@ -164,7 +165,8 @@ void discoverFromSettingsWindowsPath() { mavenSettingsOnlyResolver.setHandler(handlerNoExistCheck); final Path m2 = mavenSettingsOnlyResolver.discover(); assertNotNull(m2); - assertEquals("C:/Users/maven/repository", m2.toString()); + assertEquals("C:" + File.separator + "Users" + File.separator + "maven" + File.separator + "repository", + m2.toString()); System.clearProperty(TALEND_COMPONENT_MANAGER_M2_SETTINGS); } @@ -174,7 +176,8 @@ void discoverFromSettingsTildePath() { mavenSettingsOnlyResolver.setHandler(handlerNoExistCheck); final Path m2 = mavenSettingsOnlyResolver.discover(); assertNotNull(m2); - assertEquals(System.getProperty("user.home") + "/mvn_home_dev/repository", m2.toString()); + assertEquals(System.getProperty("user.home") + File.separator + "mvn_home_dev" + File.separator + "repository", + m2.toString()); System.clearProperty(TALEND_COMPONENT_MANAGER_M2_SETTINGS); } @@ -184,7 +187,8 @@ void discoverFromSettingsUserHomeProperty() { mavenSettingsOnlyResolver.setHandler(handlerNoExistCheck); final Path m2 = mavenSettingsOnlyResolver.discover(); assertNotNull(m2); - assertEquals(System.getProperty("user.home") + "/mvn_home_dev/repository", m2.toString()); + assertEquals(System.getProperty("user.home") + File.separator + "mvn_home_dev" + File.separator + "repository", + m2.toString()); System.clearProperty(TALEND_COMPONENT_MANAGER_M2_SETTINGS); } diff --git a/component-server-parent/component-server-model/src/main/java/org/talend/sdk/component/server/front/model/ConfigTypeNode.java b/component-server-parent/component-server-model/src/main/java/org/talend/sdk/component/server/front/model/ConfigTypeNode.java index 8f7990c1ddbc4..80fda57a8d571 100644 --- a/component-server-parent/component-server-model/src/main/java/org/talend/sdk/component/server/front/model/ConfigTypeNode.java +++ b/component-server-parent/component-server-model/src/main/java/org/talend/sdk/component/server/front/model/ConfigTypeNode.java @@ -48,4 +48,7 @@ public class ConfigTypeNode { private Collection actions; + public void visibility(final SimplePropertyDefinition spd) { + // no-op + } } diff --git a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java index 00bfc38b8be53..d7e5969310bd3 100644 --- a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java +++ b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertiesService.java @@ -15,15 +15,18 @@ */ package org.talend.sdk.component.server.service; +import static java.util.Collections.emptyList; import static java.util.Optional.ofNullable; import static java.util.function.Function.identity; import static java.util.stream.Collectors.toSet; import static org.talend.sdk.component.server.lang.CustomCollectors.toLinkedMap; import java.util.AbstractMap; +import java.util.ArrayList; import java.util.Collection; import java.util.Comparator; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -33,13 +36,21 @@ import javax.enterprise.context.ApplicationScoped; import javax.inject.Inject; +import javax.json.JsonException; +import javax.json.JsonObject; +import javax.json.JsonPointer; +import javax.json.JsonString; +import javax.json.JsonValue; import javax.json.bind.Jsonb; +import javax.json.spi.JsonProvider; import org.talend.sdk.component.runtime.internationalization.ParameterBundle; import org.talend.sdk.component.runtime.manager.ParameterMeta; +import org.talend.sdk.component.runtime.manager.reflect.ReflectionService; import org.talend.sdk.component.runtime.manager.reflect.parameterenricher.ValidationParameterEnricher; import org.talend.sdk.component.runtime.manager.util.DefaultValueInspector; import org.talend.sdk.component.server.configuration.ComponentServerConfiguration; +import org.talend.sdk.component.server.front.model.ConfigTypeNode; import org.talend.sdk.component.server.front.model.PropertyValidation; import org.talend.sdk.component.server.front.model.SimplePropertyDefinition; import org.talend.sdk.component.server.service.qualifier.ComponentServer; @@ -67,6 +78,85 @@ public Stream buildProperties(final List from JsonObject(payload). + private Map extractConfig(final ConfigTypeNode configNode, final JsonObject payload) { + Map payloadConfig = new HashMap<>(); + for (SimplePropertyDefinition propertyDefinition : configNode.getProperties()) { + try { + JsonValue value = toPointer(propertyDefinition.getPath()).getValue(payload); + if (value != null && !JsonObject.class.isInstance(value)) { + payloadConfig.put(propertyDefinition.getPath(), getValue(value)); + } + } catch (JsonException e) { + continue; + } + } + + return payloadConfig; + } + + private String getValue(final JsonValue value) { + switch (value.getValueType()) { + case TRUE: + case FALSE: + case NUMBER: + return String.valueOf(value); + case STRING: + return JsonString.class.cast(value).getString(); + default: + return null; + } + } + + private List buildParameterMetas(final List properties) { + List parameterMetaList = new ArrayList<>(); + for (SimplePropertyDefinition propertyDefinition : properties) { + final String path = sanitizePropertyName(propertyDefinition.getPath()); + final String name = sanitizePropertyName(propertyDefinition.getName()); + final ParameterMeta.Type type = findType(propertyDefinition.getType()); + // translate PropertyValidation + final Map sanitizedMetadata = ofNullable(getParamMetadata(propertyDefinition.getMetadata())) + .orElse(new LinkedHashMap<>()); + if (propertyDefinition.getValidation() != null) { + sanitizedMetadata.putAll(propertyValidationService.mapMeta(propertyDefinition.getValidation())); + } + + parameterMetaList.add(new ParameterMeta(null, null, type, path, name, null, + emptyList(), null, sanitizedMetadata, false)); + } + return parameterMetaList; + } + + private static LinkedHashMap getParamMetadata(final Map p) { + return ofNullable(p) + .map(m -> m + .entrySet() + .stream() + .filter(e -> !e.getKey().startsWith(ValidationParameterEnricher.META_PREFIX)) + .collect(toLinkedMap(e -> e.getKey().replace("condition::", "tcomp::condition::"), + Map.Entry::getValue))) + .orElse(null); + } + + private static LinkedHashMap getSanitizedMetadata(final Map p) { + return ofNullable(p) + .map(m -> m + .entrySet() + .stream() + .filter(e -> !e.getKey().startsWith(ValidationParameterEnricher.META_PREFIX)) + .collect(toLinkedMap(e -> e.getKey().replace("tcomp::", ""), Map.Entry::getValue))) + .orElse(null); + } + private Stream buildProperties(final List meta, final ClassLoader loader, final Locale locale, final DefaultValueInspector.Instance rootInstance, final ParameterMeta parent) { return meta.stream().flatMap(p -> { @@ -81,13 +171,7 @@ private Stream buildProperties(final List sanitizedMetadata = ofNullable(p.getMetadata()) - .map(m -> m - .entrySet() - .stream() - .filter(e -> !e.getKey().startsWith(ValidationParameterEnricher.META_PREFIX)) - .collect(toLinkedMap(e -> e.getKey().replace("tcomp::", ""), Map.Entry::getValue))) - .orElse(null); + final Map sanitizedMetadata = getSanitizedMetadata(p.getMetadata()); final Map metadata; if (parent != null) { metadata = sanitizedMetadata; @@ -182,4 +266,21 @@ private String toDefault(final DefaultValueInspector.Instance instance, final Pa private String sanitizePropertyName(final String path) { return path.replace("${index}", ""); } + + private ParameterMeta.Type findType(final String type) { + if (ParameterMeta.Type.STRING.name().equals(type)) { + return ParameterMeta.Type.STRING; + } else if (ParameterMeta.Type.BOOLEAN.name().equals(type)) { + return ParameterMeta.Type.BOOLEAN; + } else if (ParameterMeta.Type.NUMBER.name().equals(type)) { + return ParameterMeta.Type.NUMBER; + } else if (ParameterMeta.Type.ENUM.name().equals(type)) { + return ParameterMeta.Type.ENUM; + } else if (ParameterMeta.Type.ARRAY.name().equals(type)) { + return ParameterMeta.Type.ARRAY; + } else if (ParameterMeta.Type.OBJECT.name().equals(type)) { + return ParameterMeta.Type.OBJECT; + } + return ParameterMeta.Type.OBJECT; + } } diff --git a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertyValidationService.java b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertyValidationService.java index 8479193f4c7e1..6e80d83254b53 100644 --- a/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertyValidationService.java +++ b/component-server-parent/component-server/src/main/java/org/talend/sdk/component/server/service/PropertyValidationService.java @@ -19,6 +19,7 @@ import static java.util.stream.Collectors.toList; import java.util.Collection; +import java.util.HashMap; import java.util.Map; import java.util.function.BiFunction; import java.util.function.Function; @@ -35,6 +36,8 @@ public class PropertyValidationService { private Function, PropertyValidation> propertyValidationCreator; + private Function> propertyMetaCreator; + @PostConstruct private void initMapper() { // precompute the mapping of validations to centralize the convention - note: can be moved to impl for setters @@ -75,9 +78,36 @@ private void initMapper() { } return validation; }; + } public PropertyValidation map(final Map meta) { return propertyValidationCreator.apply(meta); } + + public Map mapMeta(final PropertyValidation propertyValidation) { + final Map metaMap = new HashMap<>(); + + Stream.of(PropertyValidation.class.getDeclaredFields()).filter(field -> { + try { + field.setAccessible(true); + return field.get(propertyValidation) != null; + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + }) + .forEach(field -> { + field.setAccessible(true); + try { + Object value = field.get(propertyValidation); + if (value != null) { + metaMap.put(ValidationParameterEnricher.META_PREFIX + field.getName(), + String.valueOf(value));// f.get(instance).toString()); + } + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + }); + return metaMap; + } } diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ConfigurationTypeResourceImplTest.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ConfigurationTypeResourceImplTest.java index 00c0c53b64c35..b4da2e2e05545 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ConfigurationTypeResourceImplTest.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/front/ConfigurationTypeResourceImplTest.java @@ -182,7 +182,7 @@ void migrateOver8196DefaultByteBuffer() { } private void assertIndex(final ConfigTypeNodes index) { - assertEquals(5, index.getNodes().size()); + assertEquals(8, index.getNodes().size()); index.getNodes().keySet().forEach(Assertions::assertNotNull); // assert no null ids // assert there is at least one parent node assertTrue(index.getNodes().values().stream().anyMatch(n -> n.getParentId() == null)); diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/PropertiesServiceTest.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/PropertiesServiceTest.java index e51a2ca295d3f..7d6120430a4e6 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/PropertiesServiceTest.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/PropertiesServiceTest.java @@ -15,24 +15,7 @@ */ package org.talend.sdk.component.server.service; -import static java.util.Arrays.asList; -import static java.util.Collections.emptyList; -import static java.util.Collections.emptyMap; -import static java.util.Collections.singletonList; -import static java.util.Comparator.comparing; -import static java.util.stream.Collectors.joining; -import static java.util.stream.Collectors.toList; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; - -import javax.inject.Inject; - +import lombok.Data; import org.apache.meecrowave.junit5.MonoMeecrowaveConfig; import org.apache.xbean.propertyeditor.PropertyEditorRegistry; import org.junit.jupiter.api.Test; @@ -42,9 +25,26 @@ import org.talend.sdk.component.runtime.manager.reflect.ParameterModelService; import org.talend.sdk.component.runtime.manager.reflect.parameterenricher.BaseParameterEnricher; import org.talend.sdk.component.runtime.manager.service.LocalConfigurationService; +import org.talend.sdk.component.server.front.model.ConfigTypeNode; +import org.talend.sdk.component.server.front.model.ConfigTypeNodes; import org.talend.sdk.component.server.front.model.SimplePropertyDefinition; -import lombok.Data; +import javax.inject.Inject; +import javax.json.Json; +import javax.json.JsonBuilderFactory; +import javax.json.JsonObject; +import javax.json.JsonValue; +import javax.ws.rs.client.WebTarget; +import java.util.*; + +import static java.util.Arrays.asList; +import static java.util.Collections.*; +import static java.util.Comparator.comparing; +import static java.util.stream.Collectors.joining; +import static java.util.stream.Collectors.toList; +import static javax.ws.rs.core.MediaType.APPLICATION_JSON_TYPE; +import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.*; @MonoMeecrowaveConfig class PropertiesServiceTest { @@ -52,6 +52,11 @@ class PropertiesServiceTest { @Inject private PropertiesService propertiesService; + @Inject + private WebTarget base; + + private ConfigTypeNode connection; + private static void gridLayout(@Option final WithLayout layout) { // no-op } @@ -157,7 +162,8 @@ void buildPropertiesFr() { assertEquals("User Name FR", props.get(4).getDisplayName()); } - @Test // the class BaseConfig don't contains attribute + @Test + // the class BaseConfig don't contains attribute void validateProp() { assertThrows(IllegalArgumentException.class, () -> { ParameterMeta attribute = new ParameterMeta(null, BadConfig.class, ParameterMeta.Type.STRING, @@ -172,6 +178,234 @@ void validateProp() { }); } + @Test + // the class BaseConfig don't contains attribute + void validateConfigNode() { + final JsonBuilderFactory factory = Json.createBuilderFactory(emptyMap()); + final List uniques = Arrays.asList("one", "two", "three"); + final List notUniques = Arrays.asList("one", "two", "three", "one"); + + String activeIfsError = "- Property 'configuration.connection.activedIfs' is required."; + String connectionError = "- Property 'configuration.connection' is required."; + String passwordError = "- Property 'configuration.connection.password' is required."; + String url1Required = "- Property 'configuration.connection.url1' is required."; + String url0Error = "- 'configuration.connection.url0' does not match '^https?://.*'."; + String url1Error = "- 'configuration.connection.url1' does not match '^https?://.*'."; + String usernameError = "- Property 'configuration.connection.username' is required."; + String minError = "- Property 'configuration.limit' should be > 100, got 99."; + String maxError = "- Property 'configuration.limit' should be < 150, got 200."; + String ValueEvalError = + "- Invalid value for Property 'configuration.connection.valueEval' expected: '[VALUE_1, VALUE_2, VALUE_3]', got null."; + + connection = base + .path("configurationtype/details") + .queryParam("identifiers", "Y29tcG9uZW50LXdpdGgtdXNlci1qYXJzI2N1c3RvbSNkYXRhc2V0I2RhdGFzZXQ") + .request(APPLICATION_JSON_TYPE) + .header("Accept-Encoding", "gzip") + .get(ConfigTypeNodes.class) + .getNodes() + .values() + .iterator() + .next(); + + JsonObject payload; + + /** + * min/max + **/ + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("url0", "http://t") + .add("username", "abc") + .add("password", "aaa") + .build()) + .add("limit", 110)) + .build(); + checkErrors(payload, Arrays.asList(connectionError, url1Required)); + + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("url0", "https://t") + .add("url1", "https://t") + .add("username", "abc") + .add("password", "aaa") + .build()) + .add("limit", 99)) + .build(); + checkErrors(payload, Arrays.asList(connectionError, minError)); + + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("username", "abc") + .add("password", "abc") + .add("url0", "https://t") + .add("url1", "https://t") + .build()) + .add("limit", 200)) + .build(); + checkErrors(payload, Arrays.asList(connectionError, maxError)); + + /* + * url0 pattern + */ + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("username", "abc") + .add("password", "abc") + .add("url0", "https://www.talend.com") + .build())) + .build(); + checkErrors(payload, Arrays.asList(connectionError, url1Required)); + + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("username", "abc") + .add("password", "abc") + .add("url0", "mailto://toto@titi.org") + .build())) + .build(); + checkErrors(payload, Arrays.asList(connectionError, url0Error, url1Required)); + + /* + * url1 pattern + */ + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("username", "abc") + .add("password", "abc") + .add("url1", "mailto://toto@titi.org") + .build())) + .build(); + checkErrors(payload, Arrays.asList(connectionError, url1Error)); + + /* + * uniqVals + */ + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("url0", JsonValue.NULL) + .add("url1", JsonValue.NULL) + .add("username", "abc") + .add("password", JsonValue.NULL) + .add("uniqVals", factory.createArrayBuilder(notUniques).build()) + .add("checkbox1", JsonValue.TRUE) + .add("checkbox2", JsonValue.TRUE) + .build()) + .add("limit", 100)) + .build(); + checkErrors(payload, Arrays.asList(activeIfsError, connectionError, url1Required, ValueEvalError)); + + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("url0", JsonValue.NULL) + .add("url1", JsonValue.NULL) + .add("username", JsonValue.NULL) + .add("password", JsonValue.NULL) + .add("uniqVals", factory.createArrayBuilder(uniques).build()) + .add("checkbox1", JsonValue.TRUE) + .add("checkbox2", JsonValue.TRUE) + .build()) + .add("limit", 100)) + .build(); + + checkErrors(payload, + Arrays.asList(activeIfsError, connectionError, url1Required, usernameError, ValueEvalError)); + + /* + * password : @ActiveIf(target = "username", evaluationStrategy = ActiveIf.EvaluationStrategy.CONTAINS, value = + * "undx") + */ + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("username", "undx") + .add("password", JsonValue.NULL) + .build()) + .build()) + .build(); + checkErrors(payload, Arrays.asList(connectionError, passwordError, url1Required)); + + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("username", "abcd") + .add("password", JsonValue.NULL) + .build()) + .build()) + .build(); + checkErrors(payload, Arrays.asList(connectionError, url1Required)); + + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("username", "undx") + .add("password", "abc") + .build()) + .build()) + .build(); + checkErrors(payload, Arrays.asList(connectionError, url1Required)); + + /* + * valueEval @ActiveIf(target = "checkbox1", value = "true") + */ + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("username", "abcd") + .add("checkbox1", JsonValue.TRUE) + .add("valueEval", JsonValue.NULL) + .build()) + .build()) + .build(); + checkErrors(payload, Arrays.asList(connectionError, url1Required, ValueEvalError)); + + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("username", "abcd") + .add("checkbox1", JsonValue.FALSE) + .add("valueEval", JsonValue.NULL) + .build()) + .build()) + .build(); + checkErrors(payload, Arrays.asList(connectionError, url1Required)); + + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("username", "abcd") + .add("checkbox1", JsonValue.TRUE) + .add("valueEval", "VALUE_2") + .build()) + .build()) + .build(); + checkErrors(payload, Arrays.asList(connectionError, url1Required)); + } + + private void checkErrors(JsonObject payload, List expected) { + try { + propertiesService.validate(connection, payload); + if (expected != null && expected.size() > 0) { + fail("There should be errors: " + expected); + } + } catch (Exception errors) { + StringBuffer expectedBuffer = new StringBuffer(); + expected.stream().forEach(e -> { + expectedBuffer.append(e).append("\n"); + }); + expectedBuffer.delete(expectedBuffer.lastIndexOf("\n"), expectedBuffer.length()); + assertEquals(expectedBuffer.toString(), errors.getMessage()); + } + } + @Data public static class BoolBool { diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/ValidationServiceTest.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/ValidationServiceTest.java new file mode 100644 index 0000000000000..a1ca83f4f4a90 --- /dev/null +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/service/ValidationServiceTest.java @@ -0,0 +1,381 @@ +/* + * Copyright (C) 2006-2022 Talend Inc. - www.talend.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + */ +package org.talend.sdk.component.server.service; + +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; +import static java.util.stream.Collectors.toList; +import static javax.ws.rs.core.MediaType.APPLICATION_JSON_TYPE; +import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.stream.Stream; + +import javax.inject.Inject; +import javax.json.Json; +import javax.json.JsonBuilderFactory; +import javax.json.JsonObject; +import javax.json.JsonValue; +import javax.json.bind.Jsonb; +import javax.json.bind.JsonbBuilder; +import javax.ws.rs.client.WebTarget; + +import org.apache.johnzon.jsonschema.JsonSchemaValidator; +import org.apache.johnzon.jsonschema.JsonSchemaValidatorFactory; +import org.apache.meecrowave.junit5.MonoMeecrowaveConfig; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.talend.sdk.component.form.api.Client; +import org.talend.sdk.component.form.api.UiSpecService; +import org.talend.sdk.component.form.internal.converter.PropertyContext; +import org.talend.sdk.component.form.internal.converter.PropertyContext.Configuration; +import org.talend.sdk.component.form.internal.converter.impl.JsonSchemaConverter; +import org.talend.sdk.component.form.internal.lang.CompletionStages; +import org.talend.sdk.component.form.internal.validation.JsonSchemaValidatorFactoryExt; +import org.talend.sdk.component.form.model.Ui; +import org.talend.sdk.component.form.model.jsonschema.JsonSchema; +import org.talend.sdk.component.form.model.uischema.UiSchema; +import org.talend.sdk.component.server.front.model.ConfigTypeNode; +import org.talend.sdk.component.server.front.model.ConfigTypeNodes; +import org.talend.sdk.component.server.test.ComponentClient; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@MonoMeecrowaveConfig +public class ValidationServiceTest { + + @Inject + private WebTarget base; + + @Inject + private ComponentClient client; + + private Jsonb jsonb = JsonbBuilder.create(); + + final String lang = "en"; + + final String family = "custom"; + + final String component = "noop"; + + ConfigTypeNodes details; + + ConfigTypeNode connection; + + JsonSchema jsonSchema; + + private Collection uiSchemas; + + private final UiSpecService uiSpecService = new UiSpecService<>(new Client() { + + @Override // for dynamic_values, just return an empty schema + public CompletionStage> action(final String family, final String type, + final String action, final String lang, final Map params, final Object context) { + final Map result = new HashMap<>(); + result.put("items", emptyList()); + return CompletableFuture.completedFuture(result); + } + + @Override + public void close() { + // no-op + } + }); + + @Inject + private PropertiesService propertiesService; + + @Data + @NoArgsConstructor + @AllArgsConstructor + class Result { + + private Collection errors; + } + + @Data + @NoArgsConstructor + @AllArgsConstructor + static class ValidationError { + + private String field; + + private String message; + + } + + public CompletionStage validate(final ConfigTypeNode config, final JsonObject payload) throws Exception { + try { + propertiesService.validate(connection, payload); + } catch (Exception errors) { + return CompletableFuture + .completedFuture(errors.getMessage()) + .thenApply(e -> getError(e)); + } + return null; + } + + Result getError(final String error) { + Result result = new Result(new ArrayList<>()); + Arrays.stream(error.split("\n")).forEach(e -> { + int index = e.indexOf('\'') + 1; + String property = '/' + e.substring(index, e.indexOf("\'", index)).replace('.', '/'); + result.getErrors().add(new ValidationError(property, e)); + System.err.println("got error of :" + property + ", error : " + e); + }); + return result; + } + + @BeforeEach + void setup() throws Exception { + details = base + .path("configurationtype/details") + .queryParam("identifiers", "Y29tcG9uZW50LXdpdGgtdXNlci1qYXJzI2N1c3RvbSNkYXRhc2V0I2RhdGFzZXQ") + .request(APPLICATION_JSON_TYPE) + .header("Accept-Encoding", "gzip") + .get(ConfigTypeNodes.class); + connection = details.getNodes().values().iterator().next(); + uiSpecService.setConfiguration(new Configuration(true)); + Ui ui = uiSpecService.convert(family, lang, connection, null).toCompletableFuture().get(); + jsonSchema = ui.getJsonSchema(); + uiSchemas = ui.getUiSchema(); + } + + private void checkAsserts(JsonObject payload, List expected) throws Exception { + Result errors = validate(connection, payload).toCompletableFuture().get(); + + System.out.println("==="); + long ecnt = errors.getErrors() + .stream() + .peek(e -> log.warn("[checkAsserts] {}", e)) + .filter(e -> !expected.contains(e)) + .peek(e -> log.error("[checkAsserts]Validation uncaught: {}", e)) + // .map(e-> fail(e)) + .count(); + + expected.stream().forEach(e -> { + System.out.println("expected: " + e); + assertTrue(errors.getErrors().contains(e)); + }); + } + + @Test + void testValidation() throws Exception { + final JsonBuilderFactory factory = Json.createBuilderFactory(emptyMap()); + + final List uniques = Arrays.asList("one", "two", "three"); + final List notUniques = Arrays.asList("one", "two", "three", "one"); + + ValidationError conn = + new ValidationError("/configuration/connection", "- Property 'configuration.connection' is required."); + ValidationError min = new ValidationError("/configuration/limit", + "- Property 'configuration.limit' should be > 100, got 99."); + ValidationError max = new ValidationError("/configuration/limit", + "- Property 'configuration.limit' should be < 150, got 200."); + ValidationError url0 = new ValidationError("/configuration/connection/url0", + "- 'configuration.connection.url0' does not match '^https?://.*'."); + ValidationError url1Patten = new ValidationError("/configuration/connection/url1", + "- 'configuration.connection.url1' does not match '^https?://.*'."); + ValidationError url1Required = new ValidationError("/configuration/connection/url1", + "- Property 'configuration.connection.url1' is required."); + ValidationError username = new ValidationError("/configuration/connection/username", + "- Property 'configuration.connection.username' is required."); + ValidationError valueEval = new ValidationError("/configuration/connection/valueEval", + "- Invalid value for Property 'configuration.connection.valueEval' expected: '[VALUE_1, VALUE_2, VALUE_3]', got null."); + ValidationError password = new ValidationError("/configuration/connection/password", + "- Property 'configuration.connection.password' is required."); + ValidationError activedIfs = new ValidationError("/configuration/connection/activedIfs", + "- Property 'configuration.connection.activedIfs' is required."); + ValidationError uniqVals = new ValidationError("/configuration/connection/uniqVals", "duplicated items: []"); + + JsonObject payload; + /** + * connection is required! + **/ + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", "") + .build()) + .build(); + checkAsserts(payload, Arrays.asList(conn)); + /** + * min/max + **/ + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", "talend") + .add("connection", factory.createObjectBuilder() + .add("url0", "http://t") + .add("username", "abc") + .add("password", "aaa") + .build()) + .add("limit", 110)) + .build(); + checkAsserts(payload, Arrays.asList(conn, url1Required)); + + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("url0", "https://t") + .add("url1", "https://t") + .add("username", "abc") + .add("password", "aaa") + .build()) + .add("limit", 99)) + .build(); + checkAsserts(payload, Arrays.asList(conn, min)); + + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("username", "abc") + .add("password", "abc") + .add("url0", "https://t") + .add("url1", "https://t") + .build()) + .add("limit", 200)) + .build(); + checkAsserts(payload, Arrays.asList(conn, max)); + /* + * url0 pattern + */ + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("username", "abc") + .add("password", "abc") + .add("url0", "mailto://toto@titi.org") + .build())) + .build(); + checkAsserts(payload, Arrays.asList(conn, url1Required)); + + /* + * url1 + */ + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("username", "abc") + .add("password", "abc") + .add("url1", "mailto://toto@titi.org") + .build())) + .build(); + checkAsserts(payload, Arrays.asList(conn, url1Patten)); + + /* + * uniques + */ + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("url0", JsonValue.NULL) + .add("url1", JsonValue.NULL) + .add("username", "abc") + .add("password", JsonValue.NULL) + .add("uniqVals", factory.createArrayBuilder(notUniques).build()) + .add("checkbox1", JsonValue.TRUE) + .add("checkbox2", JsonValue.TRUE) + .build()) + .add("limit", 100)) + .build(); + checkAsserts(payload, Arrays.asList(valueEval, conn, url1Required, activedIfs)); + + /* + * password : @ActiveIf(target = "username", evaluationStrategy = ActiveIf.EvaluationStrategy.CONTAINS, value = + * "undx") + */ + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("username", "undx") + .add("password", JsonValue.NULL) + .build()) + .build()) + .build(); + checkAsserts(payload, Arrays.asList(conn, url1Required, password)); + + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("username", "abcd") + .add("password", JsonValue.NULL) + .build()) + .build()) + .build(); + checkAsserts(payload, Arrays.asList(conn, url1Required)); + + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("username", "undx") + .add("password", "abc") + .build()) + .build()) + .build(); + checkAsserts(payload, Arrays.asList(conn, url1Required)); + + /* + * valueEval @ActiveIf(target = "checkbox1", value = "true") + */ + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("username", "abcd") + .add("checkbox1", JsonValue.TRUE) + .add("valueEval", JsonValue.NULL) + .build()) + .build()) + .build(); + checkAsserts(payload, Arrays.asList(conn, url1Required, valueEval)); + + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("username", "abcd") + .add("checkbox1", JsonValue.FALSE) + .add("valueEval", JsonValue.NULL) + .build()) + .build()) + .build(); + checkAsserts(payload, Arrays.asList(conn, url1Required)); + + payload = factory.createObjectBuilder() + .add("configuration", factory.createObjectBuilder() + .add("connection", factory.createObjectBuilder() + .add("username", "abcd") + .add("checkbox1", JsonValue.TRUE) + .add("valueEval", "VALUE_2") + .build()) + .build()) + .build(); + checkAsserts(payload, Arrays.asList(conn, url1Required)); + + } + +} diff --git a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/custom/CustomProcessor.java b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/custom/CustomProcessor.java index 07e381c511bf8..d635734f5b3e0 100644 --- a/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/custom/CustomProcessor.java +++ b/component-server-parent/component-server/src/test/java/org/talend/sdk/component/server/test/custom/CustomProcessor.java @@ -16,7 +16,21 @@ package org.talend.sdk.component.server.test.custom; import java.io.Serializable; +import java.util.List; +import java.util.Map; +import org.talend.sdk.component.api.component.MigrationHandler; +import org.talend.sdk.component.api.component.Version; +import org.talend.sdk.component.api.configuration.Option; +import org.talend.sdk.component.api.configuration.condition.ActiveIf; +import org.talend.sdk.component.api.configuration.condition.ActiveIfs; +import org.talend.sdk.component.api.configuration.constraint.Max; +import org.talend.sdk.component.api.configuration.constraint.Min; +import org.talend.sdk.component.api.configuration.constraint.Pattern; +import org.talend.sdk.component.api.configuration.constraint.Required; +import org.talend.sdk.component.api.configuration.constraint.Uniques; +import org.talend.sdk.component.api.configuration.type.DataSet; +import org.talend.sdk.component.api.configuration.type.DataStore; import org.talend.sdk.component.api.processor.ElementListener; import org.talend.sdk.component.api.processor.Processor; import org.talend.sdk.component.api.record.Record; @@ -24,8 +38,88 @@ @Processor(family = "custom", name = "noop") public class CustomProcessor implements Serializable { + @Option + DataSetCustom dataset; + + public CustomProcessor(@Option("configuration") final DataSetCustom dataset) { + this.dataset = dataset; + } + @ElementListener public Record onElement(final Record data) { return data; } + + @DataSet("dataset") + static class DataSetCustom { + + @Option + @Required + private Connection connection; + + @Option + @Min(100) + @Max(150) + private int limit; + } + + @Version(migrationHandler = Connection.ConnectionMigration.class) + @DataStore("Connection") + static class Connection { + + @Option + @Pattern("^https?://.*") + private String url0; + + @Option + @Required + @Pattern("^https?://.*") + private String url1; + + @Option + @Required + private String username; + + @Option + @Required + @ActiveIf(target = "username", evaluationStrategy = ActiveIf.EvaluationStrategy.CONTAINS, value = "undx") + private String password; + + @Option + @Uniques + private List uniqVals; + + @Option + private boolean checkbox1; + + @Option + private boolean checkbox2; + + @Option + @ActiveIf(target = "checkbox1", value = "true") + private ValueEval valueEval = ValueEval.VALUE_2; + + @Option + @Required + @ActiveIfs(operator = ActiveIfs.Operator.AND, value = { + @ActiveIf(target = "checkbox1", value = "true"), + @ActiveIf(target = "checkbox2", value = "true") + }) + private String activedIfs; + + enum ValueEval { + VALUE_1, + VALUE_2, + VALUE_3; + } + + static class ConnectionMigration implements MigrationHandler { + + @Override + public Map migrate(final int incomingVersion, final Map incomingData) { + incomingData.put("url", "http://migrated"); + return incomingData; + } + } + } } diff --git a/documentation/src/main/antora/modules/ROOT/pages/_partials/generated_changelog.adoc b/documentation/src/main/antora/modules/ROOT/pages/_partials/generated_changelog.adoc index 799181716dc32..d0c8afc737ca5 100644 --- a/documentation/src/main/antora/modules/ROOT/pages/_partials/generated_changelog.adoc +++ b/documentation/src/main/antora/modules/ROOT/pages/_partials/generated_changelog.adoc @@ -1,5 +1,17 @@ +== Version 1.56.0 + +=== Bug + +- link:https://jira.talendforge.org/browse/TCOMP-2355[TCOMP-2355^]: Error on language support for xx_YY language files link:search.html?query=component-server[component-server^,role='dockey'] + + + +=== Work Item + +- link:https://jira.talendforge.org/browse/TCOMP-2405[TCOMP-2405^]: Upgrade snakeyaml to 2.0 link:search.html?query=build[build^,role='dockey'] + == Version 1.55.0 === Bug @@ -702,6 +714,14 @@ + + + +== Version 1.38.9 + +=== Work Item + +- link:https://jira.talendforge.org/browse/TCOMP-2412[TCOMP-2412^]: Upgrade tomcat to 9.0.69 link:search.html?query=component-server[component-server^,role='dockey'] == Version 1.38.8 @@ -1481,12 +1501,6 @@ - -== Version 1.1.15.2 - -=== Work Item - -- link:https://jira.talendforge.org/browse/TCOMP-1752[TCOMP-1752^]: Make component-runtime class loader find classes in RemoteEngine JobServer == Version 1.1.15 @@ -1521,6 +1535,12 @@ - link:https://jira.talendforge.org/browse/TCOMP-1578[TCOMP-1578^]: Upgrade asciidoctor-pdf to v1.5.0-beta.7 - link:https://jira.talendforge.org/browse/TCOMP-1581[TCOMP-1581^]: Support JUnit5 meta annotations for our extensions +== Version 1.1.15.2 + +=== Work Item + +- link:https://jira.talendforge.org/browse/TCOMP-1752[TCOMP-1752^]: Make component-runtime class loader find classes in RemoteEngine JobServer + == Version 1.1.15.1 === New Feature diff --git a/documentation/src/main/antora/modules/ROOT/pages/_partials/generated_contributors.adoc b/documentation/src/main/antora/modules/ROOT/pages/_partials/generated_contributors.adoc index 5677d2360061e..17355d93380c9 100644 --- a/documentation/src/main/antora/modules/ROOT/pages/_partials/generated_contributors.adoc +++ b/documentation/src/main/antora/modules/ROOT/pages/_partials/generated_contributors.adoc @@ -8,7 +8,7 @@ "name":"Romain Manni-Bucau" }, { - "commits":689, + "commits":695, "description":"Software engineer @Talend.\r\n\r\nComponents team member.\n\nBlog: undx.github.io", "gravatar":"https://avatars.githubusercontent.com/u/265575?v=4", "id":"undx", @@ -50,7 +50,7 @@ "name":"ypiel" }, { - "commits":27, + "commits":28, "description":"", "gravatar":"https://avatars.githubusercontent.com/u/7742508?v=4", "id":"yyin-talend", @@ -70,6 +70,13 @@ "id":"wwang-talend", "name":"wang wei" }, + { + "commits":15, + "description":"QA Automation @ Talend Nantes", + "gravatar":"https://avatars.githubusercontent.com/u/1255625?v=4", + "id":"acatoire", + "name":"Axel CATOIRE" + }, { "commits":10, "description":"", @@ -98,13 +105,6 @@ "id":"jsomsanith-tlnd", "name":"Jimmy Somsanith" }, - { - "commits":8, - "description":"QA Automation @ Talend Nantes", - "gravatar":"https://avatars.githubusercontent.com/u/1255625?v=4", - "id":"acatoire", - "name":"Axel CATOIRE" - }, { "commits":8, "description":"", diff --git a/documentation/src/main/antora/site-template.yml b/documentation/src/main/antora/site-template.yml index 222b35cc4f14f..57e35f1eab826 100644 --- a/documentation/src/main/antora/site-template.yml +++ b/documentation/src/main/antora/site-template.yml @@ -50,6 +50,7 @@ content: - '!component-runtime-1.38.5' - '!component-runtime-1.38.6' - '!component-runtime-1.38.7' + - '!component-runtime-1.38.8' - '!component-runtime-1.39.0' - '!component-runtime-1.39.1' - '!component-runtime-1.39.2' @@ -70,6 +71,10 @@ content: - '!component-runtime-1.50.3' - '!component-runtime-1.51.0' - '!component-runtime-1.51.1' + - '!component-runtime-1.52.0' + - '!component-runtime-1.52.1' + - '!component-runtime-1.53.0' + - '!component-runtime-1.54.0' branches: - HEAD start_path: documentation/src/main/antora diff --git a/images/component-server-image/pom.xml b/images/component-server-image/pom.xml index f801281ce15b1..4d757c8f7e968 100644 --- a/images/component-server-image/pom.xml +++ b/images/component-server-image/pom.xml @@ -231,7 +231,7 @@ ${TALEND_HOME}/connectors v1/tenants-keyrings/decrypt/{x-talend-tenant-id} ${docker.talend.dir.base}component-kit/custom/*:${docker.talend.dir.base}custom/*:${TALEND_HOME}/extensions/*:${docker.talend.dir.app}resources:${docker.talend.dir.app}classes - --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED -Dlog4j.configurationFile=${docker.talend.dir.app}conf/log4j2-component-server-TEXT.xml + --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/sun.net.nio=ALL-UNNAMED -Dlog4j.configurationFile=${docker.talend.dir.app}conf/log4j2-component-server-TEXT.xml docker run -d -p ${docker.image.port}:${docker.image.port} ${docker.image.to} diff --git a/pom.xml b/pom.xml index a00086202a795..7d0c91ef5dcc2 100644 --- a/pom.xml +++ b/pom.xml @@ -1746,6 +1746,19 @@ + + private_repository + + + + talend.snapshots + https://artifacts-zl.talend.com/nexus/content/repositories/snapshots + + + travis diff --git a/reporting/pom.xml b/reporting/pom.xml index ebbf841c0e3ed..94f518117cfd6 100644 --- a/reporting/pom.xml +++ b/reporting/pom.xml @@ -29,6 +29,13 @@ true + + false @@ -187,11 +194,6 @@ remote-engine-customizer-image ${project.version} - - org.talend.sdk.component - documentation - ${project.version} - org.talend.sdk.component talend-component-kit-intellij-plugin @@ -241,4 +243,28 @@ + + + + documentation + + + + documentation.skip + false + + + + + org.talend.sdk.component + documentation + ${project.version} + ${documentation.skip} + + + +