diff --git a/Dockerfile b/Dockerfile index 53a73a36..fe5c262e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,7 +28,7 @@ RUN --mount=target=/root/.ivy2,type=cache sbt "set assembly / test := {}" clean FROM eclipse-temurin:17-jre-focal RUN apt-get update \ - && apt-get install -y --no-install-recommends openssh-client sshpass bash curl git \ + && apt-get install -y --no-install-recommends openssh-client sshpass bash curl git jq \ && rm -rf /var/lib/apt/lists/* ARG VERSION diff --git a/app/controllers/ApiController.scala b/app/controllers/ApiController.scala index 4d50e046..6b1a800d 100644 --- a/app/controllers/ApiController.scala +++ b/app/controllers/ApiController.scala @@ -326,7 +326,7 @@ class ApiController @Inject()(val controllerComponents: SecurityComponents, logger.debug("In ApiController :: updateRulesTxtForSolrIndexAndTargetPlatform") // generate rules.txt(s) - val rulesFiles = rulesTxtDeploymentService.generateRulesTxtContentWithFilenames(SolrIndexId(solrIndexId), targetSystem) + val rulesFiles = rulesTxtDeploymentService.generateRulesTxtContentWithFilenames(SolrIndexId(solrIndexId)) // validate every generated rules.txt rulesTxtDeploymentService.validateCompleteRulesTxts(rulesFiles) match { diff --git a/app/models/FeatureToggleModel.scala b/app/models/FeatureToggleModel.scala index 8545ff2a..0fb21d13 100644 --- a/app/models/FeatureToggleModel.scala +++ b/app/models/FeatureToggleModel.scala @@ -50,10 +50,10 @@ package object FeatureToggleModel extends Logging { private val FEATURE_TOGGLE_UI_LIST_LIMIT_ITEMS_TO = "toggle.ui-list.limit-items-to" private val FEATURE_TOGGLE_RULE_DEPLOYMENT_LOG_RULE_ID = "toggle.rule-deployment.log-rule-id" private val FEATURE_TOGGLE_RULE_DEPLOYMENT_SPLIT_DECOMPOUND_RULES_TXT = "toggle.rule-deployment.split-decompound-rules-txt" - private val FEATURE_TOGGLE_RULE_DEPLOYMENT_SPLIT_DECOMPOUND_RULES_TXT_DST_CP_FILE_TO = "toggle.rule-deployment.split-decompound-rules-txt-DST_CP_FILE_TO" private val FEATURE_TOGGLE_RULE_DEPLOYMENT_PRE_LIVE_PRESENT = "toggle.rule-deployment.pre-live.present" private val FEATURE_TOGGLE_RULE_DEPLOYMENT_CUSTOM_SCRIPT = "toggle.rule-deployment.custom-script" private val FEATURE_TOGGLE_RULE_DEPLOYMENT_CUSTOM_SCRIPT_SMUI2SOLR_SH_PATH = "toggle.rule-deployment.custom-script-SMUI2SOLR-SH_PATH" + private val FEATURE_TOGGLE_RULE_DEPLOYMENT_ENABLE_GIT = "toggle.rule-deployment.git.enable" private val FEATURE_TOGGLE_HEADLINE = "toggle.headline" private val FEATURE_TOGGLE_DEPLOYMENT_LABEL = "toggle.rule-deployment-label" private val FEATURE_TOGGLE_DEPLOYMENT_PRELIVE_LABEL = "toggle.deploy-prelive-fn-label" @@ -173,6 +173,10 @@ package object FeatureToggleModel extends Logging { ) } + def getToggleRuleDeploymentGitEnabled: Boolean = { + appConfig.getOptional[Boolean](FEATURE_TOGGLE_RULE_DEPLOYMENT_ENABLE_GIT).getOrElse(false) + } + def getToggleRuleDeploymentLogRuleId: Boolean = { appConfig.getOptional[Boolean](FEATURE_TOGGLE_RULE_DEPLOYMENT_LOG_RULE_ID).getOrElse(false) } @@ -181,10 +185,6 @@ package object FeatureToggleModel extends Logging { appConfig.getOptional[Boolean](FEATURE_TOGGLE_RULE_DEPLOYMENT_SPLIT_DECOMPOUND_RULES_TXT).getOrElse(false) } - def getToggleRuleDeploymentSplitDecompoundRulesTxtDstCpFileTo: String = { - appConfig.getOptional[String](FEATURE_TOGGLE_RULE_DEPLOYMENT_SPLIT_DECOMPOUND_RULES_TXT_DST_CP_FILE_TO).getOrElse("") - } - def getToggleRuleDeploymentCustomScript: Boolean = { appConfig.getOptional[Boolean](FEATURE_TOGGLE_RULE_DEPLOYMENT_CUSTOM_SCRIPT).getOrElse(false) } diff --git a/app/services/RulesTxtDeploymentService.scala b/app/services/RulesTxtDeploymentService.scala index 3ff2374c..d7511661 100644 --- a/app/services/RulesTxtDeploymentService.scala +++ b/app/services/RulesTxtDeploymentService.scala @@ -20,19 +20,18 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT environment: Environment) extends Logging { case class RulesTxtsForSolrIndex(solrIndexId: SolrIndexId, - regularRules: RulesTxtWithFileNames, - decompoundRules: Option[RulesTxtWithFileNames], - replaceRules: Option[RulesTxtWithFileNames]) { + regularRules: RulesTxtWithFileName, + decompoundRules: Option[RulesTxtWithFileName], + replaceRules: Option[RulesTxtWithFileName]) { - def regularAndDecompoundFiles: List[RulesTxtWithFileNames] = List(regularRules) ++ decompoundRules + def regularAndDecompoundFiles: List[RulesTxtWithFileName] = List(regularRules) ++ decompoundRules - def allFiles: List[RulesTxtWithFileNames] = regularAndDecompoundFiles ++ replaceRules + def allFiles: List[RulesTxtWithFileName] = regularAndDecompoundFiles ++ replaceRules } - case class RulesTxtWithFileNames(content: String, - sourceFileName: String, - destinationFileName: String) + case class RulesTxtWithFileName(content: String, + sourceFileName: String) /** * Generates a list of source to destination filenames containing the rules.txt(s) according to current application settings. @@ -40,78 +39,45 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT * @param solrIndexId Solr Index Id to generate the output for. */ // TODO evaluate, if logDebug should be used to prevent verbose logging of the whole generated rules.txt (for zip download especially) - def generateRulesTxtContentWithFilenames(solrIndexId: SolrIndexId, targetSystem: String, logDebug: Boolean = true): RulesTxtsForSolrIndex = { + def generateRulesTxtContentWithFilenames(solrIndexId: SolrIndexId, logDebug: Boolean = true): RulesTxtsForSolrIndex = { - // SMUI config for (regular) LIVE deployment val SRC_TMP_FILE = appConfig.get[String]("smui2solr.SRC_TMP_FILE") - val DST_CP_FILE_TO = appConfig.get[String]("smui2solr.DST_CP_FILE_TO") val DO_SPLIT_DECOMPOUND_RULES_TXT = featureToggleService.getToggleRuleDeploymentSplitDecompoundRulesTxt - val DECOMPOUND_RULES_TXT_DST_CP_FILE_TO = featureToggleService.getToggleRuleDeploymentSplitDecompoundRulesTxtDstCpFileTo - // (additional) SMUI config for PRELIVE deployment - val SMUI_DEPLOY_PRELIVE_FN_RULES_TXT = appConfig.get[String]("smui2solr.deploy-prelive-fn-rules-txt") - val SMUI_DEPLOY_PRELIVE_FN_DECOMPOUND_TXT = appConfig.get[String]("smui2solr.deploy-prelive-fn-decompound-txt") - - // Replace rules (spelling) - val EXPORT_REPLACE_RULES = featureToggleService.getToggleActivateSpelling + val DO_EXPORT_REPLACE_RULES = featureToggleService.getToggleActivateSpelling val REPLACE_RULES_SRC_TMP_FILE = appConfig.get[String]("smui2solr.replace-rules-tmp-file") - val REPLACE_RULES_DST_CP_FILE_TO = appConfig.get[String]("smui2solr.replace-rules-dst-cp-file-to") - val SMUI_DEPLOY_PRELIVE_FN_REPLACE_TXT = appConfig.get[String]("smui2solr.deploy-prelive-fn-replace-txt") if (logDebug) { logger.debug( s""":: generateRulesTxtContentWithFilenames config |:: SRC_TMP_FILE = $SRC_TMP_FILE - |:: DST_CP_FILE_TO = $DST_CP_FILE_TO |:: DO_SPLIT_DECOMPOUND_RULES_TXT = $DO_SPLIT_DECOMPOUND_RULES_TXT - |:: DECOMPOUND_RULES_TXT_DST_CP_FILE_TO = $DECOMPOUND_RULES_TXT_DST_CP_FILE_TO - |:: SMUI_DEPLOY_PRELIVE_FN_RULES_TXT = $SMUI_DEPLOY_PRELIVE_FN_RULES_TXT - |:: SMUI_DEPLOY_PRELIVE_FN_DECOMPOUND_TXT = $SMUI_DEPLOY_PRELIVE_FN_DECOMPOUND_TXT - |:: EXPORT_REPLACE_RULES = $EXPORT_REPLACE_RULES + |:: EXPORT_REPLACE_RULES = $DO_EXPORT_REPLACE_RULES |:: REPLACE_RULES_SRC_TMP_FILE = $REPLACE_RULES_SRC_TMP_FILE - |:: REPLACE_RULES_DST_CP_FILE_TO = $REPLACE_RULES_DST_CP_FILE_TO - |:: SMUI_DEPLOY_PRELIVE_FN_REPLACE_TXT = $SMUI_DEPLOY_PRELIVE_FN_REPLACE_TXT """.stripMargin) } // generate one rules.txt by default or two separated, if decompound instructions are supposed to be split - // TODO test correct generation in different scenarios (one vs. two rules.txts, etc.) - val dstCpFileTo = if (targetSystem == "PRELIVE") - SMUI_DEPLOY_PRELIVE_FN_RULES_TXT - else // targetSystem == "LIVE" - DST_CP_FILE_TO - - val replaceRulesDstCpFileTo = - if (targetSystem == "PRELIVE") SMUI_DEPLOY_PRELIVE_FN_REPLACE_TXT - else REPLACE_RULES_DST_CP_FILE_TO - - val replaceRules = - if (EXPORT_REPLACE_RULES) { + val optReplaceRules = + if (DO_EXPORT_REPLACE_RULES) { val allCanonicalSpellings = searchManagementRepository.listAllSpellingsWithAlternatives(solrIndexId) - Some(RulesTxtWithFileNames( - QuerqyReplaceRulesGenerator.renderAllCanonicalSpellingsToReplaceRules(allCanonicalSpellings), - REPLACE_RULES_SRC_TMP_FILE, - replaceRulesDstCpFileTo - )) + val spellingsRendered = QuerqyReplaceRulesGenerator.renderAllCanonicalSpellingsToReplaceRules(allCanonicalSpellings) + Some(RulesTxtWithFileName(spellingsRendered, REPLACE_RULES_SRC_TMP_FILE)) } else None if (!DO_SPLIT_DECOMPOUND_RULES_TXT) { RulesTxtsForSolrIndex(solrIndexId, - RulesTxtWithFileNames(querqyRulesTxtGenerator.renderSingleRulesTxt(solrIndexId), SRC_TMP_FILE, dstCpFileTo), + RulesTxtWithFileName(querqyRulesTxtGenerator.renderSingleRulesTxt(solrIndexId), SRC_TMP_FILE), None, - replaceRules + optReplaceRules ) } else { - val decompoundDstCpFileTo = if (targetSystem == "PRELIVE") - SMUI_DEPLOY_PRELIVE_FN_DECOMPOUND_TXT - else // targetSystem == "LIVE" - DECOMPOUND_RULES_TXT_DST_CP_FILE_TO + val rulesWithoutDecompounds = querqyRulesTxtGenerator.renderSeparatedRulesTxts(solrIndexId, renderCompoundsRulesTxt = false) + val decompoundRules = querqyRulesTxtGenerator.renderSeparatedRulesTxts(solrIndexId, renderCompoundsRulesTxt = true) RulesTxtsForSolrIndex(solrIndexId, - RulesTxtWithFileNames(querqyRulesTxtGenerator.renderSeparatedRulesTxts(solrIndexId, renderCompoundsRulesTxt = false), SRC_TMP_FILE, dstCpFileTo), - Some(RulesTxtWithFileNames(querqyRulesTxtGenerator.renderSeparatedRulesTxts(solrIndexId, renderCompoundsRulesTxt = true), - SRC_TMP_FILE + "-2", decompoundDstCpFileTo) - ), - replaceRules + RulesTxtWithFileName(rulesWithoutDecompounds, SRC_TMP_FILE), + Some(RulesTxtWithFileName(decompoundRules, SRC_TMP_FILE + "-2")), + optReplaceRules ) } } @@ -123,7 +89,7 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT def validateCompleteRulesTxts(rulesTxts: RulesTxtsForSolrIndex, logDebug: Boolean = true): List[String] = { val rulesValidation = rulesTxts.regularAndDecompoundFiles.flatMap { rulesFile => if (logDebug) { - logger.debug(":: validateCompleteRulesTxts for src = " + rulesFile.sourceFileName + " dst = " + rulesFile.destinationFileName) + logger.debug(":: validateCompleteRulesTxts for src = " + rulesFile.sourceFileName) logger.debug(":: rulesTxt = <<<" + rulesFile.content + ">>>") } val validationResult = querqyRulesTxtGenerator.validateQuerqyRulesTxtToErrMsg(rulesFile.content) @@ -158,43 +124,34 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT } - def interfaceSmui2SolrSh(scriptPath: String, srcTmpFile: String, dstCpFileTo: String, solrHost: String, - solrCoreName: String, decompoundDstCpFileTo: String, targetSystem: String, - replaceRulesSrcTmpFile: String, replaceRulesDstCpFileTo: String + def interfaceSmui2SolrSh(scriptPath: String, + srcTmpFile: String, + replaceRulesSrcTmpFile: String, + targetSystem: String, + solrCoreName: String ): DeploymentScriptResult = { logger.info( s""":: interfaceSmui2SolrSh |:: scriptPath = $scriptPath |:: srcTmpFile = $srcTmpFile - |:: dstCpFileTo = $dstCpFileTo - |:: solrHost = $solrHost + |:: replaceRulesSrcTmpFile = $replaceRulesSrcTmpFile |:: solrCoreName = $solrCoreName - |:: decompoundDstCpFileTo = $decompoundDstCpFileTo |:: targetSystem = $targetSystem - |:: replaceRulesSrcTmpFile = $replaceRulesSrcTmpFile - |:: replaceRulesDstCpFileTo = $replaceRulesDstCpFileTo """.stripMargin) - val scriptCall = + val scriptCall = List( // define call for regular smui2solr (default or custom script) and add parameters to the script (in expected order, see smui2solr.sh) - scriptPath + " " + + scriptPath, // SRC_TMP_FILE=$1 - srcTmpFile + " " + - // DST_CP_FILE_TO=$2 - dstCpFileTo + " " + - // SOLR_HOST=$3 - solrHost + " " + + srcTmpFile, + // REPLACE_RULES_SRC_TMP_FILE=$2 + replaceRulesSrcTmpFile, + // TARGET_SYSTEM=$3 + targetSystem, // SOLR_CORE_NAME=$4 - solrCoreName + " " + - // DECOMPOUND_DST_CP_FILE_TO=$5 - decompoundDstCpFileTo + " " + - // TARGET_SYSTEM=$6 - targetSystem + " " + - // REPLACE_RULES_SRC_TMP_FILE=$7 - replaceRulesSrcTmpFile + " " + - // REPLACE_RULES_DST_CP_FILE_TO=$8 - replaceRulesDstCpFileTo + solrCoreName + ).mkString(" ") interfaceDeploymentScript(scriptCall) } @@ -233,58 +190,37 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT else environment.rootPath.getAbsolutePath + "/conf/smui2solr.sh" + val deployToGitConfigured = featureToggleService.getToggleRuleDeploymentGitEnabled + val srcTmpFile = rulesTxts.regularRules.sourceFileName - val dstCpFileTo = rulesTxts.regularRules.destinationFileName - val decompoundDstCpFileTo = if (rulesTxts.decompoundRules.isDefined) - rulesTxts.decompoundRules.get.destinationFileName - else - "NONE" - - // host for (optional) core reload - val SMUI_DEPLOY_PRELIVE_SOLR_HOST = appConfig.get[String]("smui2solr.deploy-prelive-solr-host") - val SMUI_DEPLOY_LIVE_SOLR_HOST = appConfig.get[String]("smui2solr.SOLR_HOST") - val solrHost = if (targetSystem == "PRELIVE") - if (SMUI_DEPLOY_PRELIVE_SOLR_HOST.isEmpty) - "NONE" - else - SMUI_DEPLOY_PRELIVE_SOLR_HOST - else // targetSystem == "LIVE" - if (SMUI_DEPLOY_LIVE_SOLR_HOST.isEmpty) - "NONE" - else - SMUI_DEPLOY_LIVE_SOLR_HOST - // core name from repo (optional, for core reload as well) val solrCoreName = searchManagementRepository.getSolrIndexName(rulesTxts.solrIndexId) - val replaceRulesSrcTmpFile = rulesTxts.replaceRules.map(_.sourceFileName).getOrElse("NONE") - val replaceRulesDstCpFileTo = rulesTxts.replaceRules.map(_.destinationFileName).getOrElse("NONE") // execute script // TODO currently only git deployment for LIVE instance available - val deployToGit = targetSystem.equals("LIVE") && appConfig.get[String]("smui2solr.DST_CP_FILE_TO").equals("GIT") - val result = (if(!deployToGit) { + val result = if (deployToGitConfigured) { + if (targetSystem == "LIVE") { + logger.info(":: executeDeploymentScript :: GIT configured calling interfaceSmui2GitSh") + // TODO support further rule files (decompound / replace) and maybe solrCoreName and/or targetSystem for git branch? + interfaceSmui2GitSh( + environment.rootPath.getAbsolutePath + "/conf/smui2git.sh", + srcTmpFile, + featureToggleService.getSmuiDeploymentGitRepoUrl, + featureToggleService.getSmuiDeploymentGitFilenameCommonRulesTxt, + ) + } else { + DeploymentScriptResult(exitCode = -1, output = "Git deployment only support for LIVE target") + } + } else { logger.info(s":: executeDeploymentScript :: regular script configured calling interfaceSmui2SolrSh(scriptPath = '$scriptPath')") interfaceSmui2SolrSh( scriptPath, srcTmpFile, - dstCpFileTo, - solrHost, - solrCoreName, - decompoundDstCpFileTo, - targetSystem, replaceRulesSrcTmpFile, - replaceRulesDstCpFileTo - ) - } else { - logger.info(":: executeDeploymentScript :: GIT configured calling interfaceSmui2GitSh") - // TODO support further rule files (decompound / replace) and maybe solrCoreName and/or targetSystem for git branch? - interfaceSmui2GitSh( - environment.rootPath.getAbsolutePath + "/conf/smui2git.sh", - srcTmpFile, - featureToggleService.getSmuiDeploymentGitRepoUrl, - featureToggleService.getSmuiDeploymentGitFilenameCommonRulesTxt, + targetSystem, + solrCoreName ) - }) + } if (result.success) { logger.info(s"Rules.txt deployment successful:\n${result.output}") } else { @@ -323,7 +259,7 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT try { for (index <- searchManagementRepository.listAllSolrIndexes) { // TODO make targetSystem configurable from ApiController.downloadAllRulesTxtFiles ... go with "LIVE" from now (as there exist no different revisions of the search management content)! - val rules = generateRulesTxtContentWithFilenames(index.id, "LIVE", logDebug = false) + val rules = generateRulesTxtContentWithFilenames(index.id, logDebug = false) zipStream.putNextEntry(new ZipEntry(s"rules_${index.name}.txt")) zipStream.write(rules.regularRules.content.getBytes("UTF-8")) zipStream.closeEntry() diff --git a/build.sbt b/build.sbt index f983f6dd..fb7c2b71 100644 --- a/build.sbt +++ b/build.sbt @@ -1,10 +1,10 @@ import com.typesafe.sbt.GitBranchPrompt name := "search-management-ui" -version := "4.3.1" +version := "4.4.1" maintainer := "Contact productful.io " -scalaVersion := "2.13.14" +scalaVersion := "2.13.15" ThisBuild / evictionErrorLevel := Level.Info diff --git a/conf/application.conf b/conf/application.conf index b8065618..f584d5c1 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -31,17 +31,6 @@ db.default.password=${?SMUI_DB_PASSWORD} smui2solr.SRC_TMP_FILE="/tmp/search-management-ui_rules-txt.tmp" smui2solr.SRC_TMP_FILE=${?SMUI_2SOLR_SRC_TMP_FILE} -smui2solr.DST_CP_FILE_TO="/usr/bin/solr/liveCore/conf/rules.txt" -smui2solr.DST_CP_FILE_TO=${?SMUI_2SOLR_DST_CP_FILE_TO} - -smui2solr.SOLR_HOST="localhost:8983" -smui2solr.SOLR_HOST=${?SMUI_2SOLR_SOLR_HOST} - -smui.deployment.git.repo-url="ssh://git@localhost/git-server/repos/smui_rulestxt_repo.git" -smui.deployment.git.repo-url=${?SMUI_DEPLOYMENT_GIT_REPO_URL} -smui2solr.deployment.git.filename.common-rules-txt="rules.txt" -smui2solr.deployment.git.filename.common-rules-txt=${?SMUI_DEPLOYMENT_GIT_FN_COMMON_RULES_TXT} - # Application Feature Toggles # ~~~~~ toggle.activate-spelling=false @@ -56,30 +45,31 @@ toggle.ui-concept.all-rules.with-solr-fields=${?SMUI_TOGGLE_UI_WITH_SOLR_FIELDS} toggle.ui-list.limit-items-to=-1 # set -1 for no limitation for displayed items in list toggle.ui-list.limit-items-to=${?SMUI_TOGGLE_UI_LIMIT_LIST_ITEMS_TO} +# For rule deployment to a Git repo +smui.deployment.git.repo-url="ssh://git@localhost/git-server/repos/smui_rulestxt_repo.git" +smui.deployment.git.repo-url=${?SMUI_DEPLOYMENT_GIT_REPO_URL} +smui2solr.deployment.git.filename.common-rules-txt="rules.txt" +smui2solr.deployment.git.filename.common-rules-txt=${?SMUI_DEPLOYMENT_GIT_FN_COMMON_RULES_TXT} + +# whether the special Git repo export should happen during rule deployment +toggle.rule-deployment.git.enable=false +toggle.rule-deployment.git.enable=${?SMUI_DEPLOYMENT_GIT_ENABLE} + +# whether the SMUI generated input id should be exported as part of the rules.txt toggle.rule-deployment.log-rule-id=false toggle.rule-deployment.log-rule-id=${?SMUI_TOGGLE_RULE_ID_LOGGING} toggle.rule-deployment.split-decompound-rules-txt=false toggle.rule-deployment.split-decompound-rules-txt=${?SMUI_TOGGLE_DEPL_SPLIT_DECOMPOUND} -toggle.rule-deployment.split-decompound-rules-txt-DST_CP_FILE_TO="/usr/bin/solr/liveCore/conf/decompound-rules.txt" -toggle.rule-deployment.split-decompound-rules-txt-DST_CP_FILE_TO=${?SMUI_TOGGLE_DEPL_DECOMPOUND_DST} smui2solr.replace-rules-tmp-file="/tmp/search-management-ui_replace-rules-txt.tmp" smui2solr.replace-rules-tmp-file=${?SMUI_2SOLR_REPLACE_RULES_TMP_FILE} -smui2solr.replace-rules-dst-cp-file-to="/usr/bin/solr/liveCore/conf/replace-rules.txt" -smui2solr.replace-rules-dst-cp-file-to=${?SMUI_2SOLR_REPLACE_RULES_DST_CP_FILE_TO} toggle.rule-deployment.pre-live.present=false toggle.rule-deployment.pre-live.present=${?SMUI_TOGGLE_DEPL_PRELIVE} # in case pre-live is present, describe PRELIVE deployment -smui2solr.deploy-prelive-fn-rules-txt="/usr/bin/solr/preliveCore/conf/rules.txt" -smui2solr.deploy-prelive-fn-rules-txt=${?SMUI_DEPLOY_PRELIVE_FN_RULES_TXT} smui2solr.deploy-prelive-solr-host="" smui2solr.deploy-prelive-solr-host=${?SMUI_DEPLOY_PRELIVE_SOLR_HOST} -smui2solr.deploy-prelive-fn-decompound-txt="/usr/bin/solr/preliveCore/conf/rules-decompound.txt" -smui2solr.deploy-prelive-fn-decompound-txt=${?SMUI_DEPLOY_PRELIVE_FN_DECOMPOUND_TXT} -smui2solr.deploy-prelive-fn-replace-txt="/usr/bin/solr/preliveCore/conf/replace-rules.txt" -smui2solr.deploy-prelive-fn-replace-txt=${?SMUI_DEPLOY_PRELIVE_FN_REPLACE_TXT} toggle.rule-deployment.custom-script=false toggle.rule-deployment.custom-script=${?SMUI_TOGGLE_DEPL_CUSTOM_SCRIPT} diff --git a/conf/smui2solr.sh b/conf/smui2solr.sh index 711d3341..7bd5f1fb 100755 --- a/conf/smui2solr.sh +++ b/conf/smui2solr.sh @@ -2,86 +2,135 @@ set -euo pipefail -SRC_TMP_FILE=$1 -DST_CP_FILE_TO=$2 -SOLR_HOST=$3 -SOLR_CORE_NAME=$4 -DECOMPOUND_DST_CP_FILE_TO=$5 -TARGET_SYSTEM=$6 -REPLACE_RULES_SRC_TMP_FILE=$7 -REPLACE_RULES_DST_CP_FILE_TO=$8 - -echo "In smui2solr.sh - script performing rules.txt update and core reload" -echo "^-- SRC_TMP_FILE = $SRC_TMP_FILE" -echo "^-- DST_CP_FILE_TO = $DST_CP_FILE_TO" -echo "^-- SOLR_HOST = $SOLR_HOST" -echo "^-- SOLR_CORE_NAME: $SOLR_CORE_NAME" -echo "^-- DECOMPOUND_DST_CP_FILE_TO = $DECOMPOUND_DST_CP_FILE_TO" -echo "^-- TARGET_SYSTEM = $TARGET_SYSTEM" -echo "^-- REPLACE_RULES_SRC_TMP_FILE = $REPLACE_RULES_SRC_TMP_FILE" -echo "^-- REPLACE_RULES_DST_CP_FILE_TO = $REPLACE_RULES_DST_CP_FILE_TO" +# Expected command line parameters +common_rules_file=$1 +decompound_rules_file="${common_rules_file}-2" +replace_rules_file=$2 +target_system=$3 +solr_core_or_collection_name=$4 -# DEPLOYMENT -##### +# Read in environment variables -echo "^-- Perform rules.txt deployment (decompound-rules.txt eventually)" - -# $1 - from_filename -# $2 - to_filename (might be local or remote) -function deploy_rules_txt { - # e.g.: remote_user:remote_pass@remote_host:/path/to/live/solr/defaultCore/conf/rules.txt - if [[ $2 =~ (.*):(.*)@(.*) ]] - then - echo "^-- ... matched remote target (regex). proceeding with remote copy ..." - sshpass -p "${BASH_REMATCH[2]}" scp -v -o StrictHostKeyChecking=no $1 ${BASH_REMATCH[1]}@${BASH_REMATCH[3]} - else - echo "^-- ... no match (regex). proceeding with regular cp ..." - cp $1 $2 - fi -} +# Querqy rewriter names for rules, (optionally when exported separately) the decompound rules and spelling/replacement rules +common_rules_rewriter_name="${SMUI_QUERQY_REWRITER_COMMON_RULES:-common_rules}" +common_rules_decompound_rewriter_name="${SMUI_QUERQY_REWRITER_COMMON_RULES_DECOMPOUND:-common_rules_decompound}" +replace_rules_rewriter_name="${SMUI_QUERQY_REWRITER_REPLACE:-replace_rules}" + +# (legacy) the live Solr host the rules should be deployed to +SMUI_2SOLR_SOLR_HOST=${SMUI_2SOLR_SOLR_HOST:-} + +# (alternatively) the live Solr host the rules should be deployed to +SMUI_DEPLOY_LIVE_SOLR_HOST=${SMUI_DEPLOY_LIVE_SOLR_HOST:-} + +# (optional, if basic authentication is used): Solr user for non-prelive deployment +SMUI_DEPLOY_LIVE_SOLR_USER=${SMUI_DEPLOY_LIVE_SOLR_USER:-} + +# (optional, if basic authentication is used): Solr password for non-prelive deployment +SMUI_DEPLOY_LIVE_SOLR_PASSWORD=${SMUI_DEPLOY_LIVE_SOLR_PASSWORD:-} -echo "^-- ... rules.txt" -deploy_rules_txt $SRC_TMP_FILE $DST_CP_FILE_TO +# (optional): whether certificate validation should be turned off when connecting to (live) Solr host +SMUI_DEPLOY_LIVE_SOLR_ALLOW_INSECURE=${SMUI_DEPLOY_LIVE_SOLR_ALLOW_INSECURE:-} -echo "^-- ... decompound-rules.txt" -if ! [[ $DECOMPOUND_DST_CP_FILE_TO == "NONE" ]] +# (optional, if PRELIVE is used as a deployment target): the prelive Solr host the rules should be deployed to +SMUI_DEPLOY_PRELIVE_SOLR_HOST=${SMUI_DEPLOY_PRELIVE_SOLR_HOST:-} + +# (optional, if basic authentication is used): Prelive Solr user +SMUI_DEPLOY_PRELIVE_SOLR_USER=${SMUI_DEPLOY_PRELIVE_SOLR_USER:-} + +# (optional, if basic authentication is used): Prelive Solr user +SMUI_DEPLOY_PRELIVE_SOLR_PASSWORD=${SMUI_DEPLOY_PRELIVE_SOLR_PASSWORD:-} + +# (optional): whether certificate validation should be turned off when connecting to prelive Solr host +SMUI_DEPLOY_PRELIVE_SOLR_ALLOW_INSECURE=${SMUI_DEPLOY_PRELIVE_SOLR_ALLOW_INSECURE:-} + +# Solr (basic) authentication +solr_host="" +solr_basic_auth_user="" +solr_basic_auth_password="" +curl_solr_insecure_flag="" + +if [ "${target_system}" == "PRELIVE" ] ; then - deploy_rules_txt "$SRC_TMP_FILE-2" $DECOMPOUND_DST_CP_FILE_TO + solr_host="${SMUI_DEPLOY_PRELIVE_SOLR_HOST:-}" + if [ -n "${SMUI_DEPLOY_PRELIVE_SOLR_USER}" ] && [ -n "${SMUI_DEPLOY_PRELIVE_SOLR_PASSWORD}" ]; then + solr_basic_auth_user=${SMUI_DEPLOY_PRELIVE_SOLR_USER} + solr_basic_auth_password=${SMUI_DEPLOY_PRELIVE_SOLR_PASSWORD} + fi + if [ "${SMUI_DEPLOY_PRELIVE_SOLR_ALLOW_INSECURE,,}" == "true" ] ; + then + curl_solr_insecure_flag="--insecure" + fi fi -echo "^-- ... replace-rules.txt" -if ! [[ $REPLACE_RULES_SRC_TMP_FILE == "NONE" && $REPLACE_RULES_DST_CP_FILE_TO == "NONE" ]] +if [ "${target_system}" == "LIVE" ] ; then - deploy_rules_txt $REPLACE_RULES_SRC_TMP_FILE $REPLACE_RULES_DST_CP_FILE_TO + solr_host="${SMUI_2SOLR_SOLR_HOST:-${SMUI_DEPLOY_LIVE_SOLR_HOST:-}}" + if [ -n "${SMUI_DEPLOY_LIVE_SOLR_USER}" ] && [ -n "${SMUI_DEPLOY_LIVE_SOLR_PASSWORD}" ]; then + solr_basic_auth_user=${SMUI_DEPLOY_LIVE_SOLR_USER} + solr_basic_auth_password=${SMUI_DEPLOY_LIVE_SOLR_PASSWORD} + fi + if [ "${SMUI_DEPLOY_LIVE_SOLR_ALLOW_INSECURE,,}" == "true" ] ; + then + curl_solr_insecure_flag="--insecure" + fi fi -# CORE RELOAD +save_rewriter() { + local querqy_rewriter_name="$1" + local querqy_rewriter_data="$2" + + # read into array + IFS=' ' read -r -a curl_command <<< "curl -X POST -H \"Content-Type: application/json\" --fail" + + # add payload + curl_command+=("-d" "${querqy_rewriter_data}") + + # optional basic auth username + password and curl --insecure flag + if [ -n "${solr_basic_auth_user}" ] && [ -n "${solr_basic_auth_password}" ]; then + curl_command+=("--user" "${solr_basic_auth_user}:${solr_basic_auth_password}") + fi + if [ -n "${curl_solr_insecure_flag:-}" ]; then + curl_command+=("${curl_solr_insecure_flag}") + fi + + # add url + curl_command+=("${solr_host}/solr/${solr_core_or_collection_name}/querqy/rewriter/${querqy_rewriter_name}?action=save") + + echo "${curl_command[@]}" + + "${curl_command[@]}" +} + +echo "In smui2solr.sh - script for updating rules in Solr using an API call to the Querqy (>=v5) plugin " +echo "^-- common_rules_file = ${common_rules_file}" +echo "^-- decompound_rules_file = ${common_rules_file}-2" +echo "^-- solr_host = ${solr_host}" +echo "^-- target_system: ${target_system}" +echo "^-- solr_core_or_collection_name = ${solr_core_or_collection_name}" + +# DEPLOYMENT ##### -echo "^-- Perform core reload for SOLR_HOST = $SOLR_HOST, SOLR_CORE_NAME = $SOLR_CORE_NAME" +# common rules +if [ -s "${common_rules_file}" ] && [ -n "${common_rules_rewriter_name}" ]; then + # build the rewriter request payload by embedding the rules text into JSON + common_rules_rewriter_data=$(jq -Rs '{class: "querqy.solr.rewriter.commonrules.CommonRulesRewriterFactory", config: { rules: . }}' "${common_rules_file}") + echo "Deploying common rules for core ${solr_core_or_collection_name} to rewriter ${common_rules_rewriter_name} at ${solr_host}..." + save_rewriter "${common_rules_rewriter_name}" "${common_rules_rewriter_data}" +fi + +# optional: decompound rules to a separate common rules rewriter +if [ -s "${decompound_rules_file}" ] && [ -n "${common_rules_decompound_rewriter_name}" ]; then + common_rules_decompound_rewriter_data=$(jq -Rs '{class: "querqy.solr.rewriter.commonrules.CommonRulesRewriterFactory", config: { rules: . }}' "${decompound_rules_file}") + echo "Deploying decompound common rules for core ${solr_core_or_collection_name} to rewriter ${common_rules_decompound_rewriter_name} at ${solr_host}..." + save_rewriter "${common_rules_decompound_rewriter_name}" "${common_rules_decompound_rewriter_data}" +fi -if ! [[ $SOLR_HOST == "NONE" ]] -then - # TODO only core reload over http possible. make configurable. - SOLR_STATUS=$(curl -s -i -XGET "http://$SOLR_HOST/solr/admin/cores?wt=xml&action=RELOAD&core=$SOLR_CORE_NAME") - - if [ $? -ne 0 ]; then - exit 16 - fi - - if ! [[ $SOLR_STATUS == *"200 OK"* ]] - then - >&2 echo "Error reloading Solr core: $SOLR_STATUS" - exit 17 - fi - - if ! [[ $SOLR_STATUS == *"0"* ]] - then - >&2 echo "Error reloading Solr core: $SOLR_STATUS" - exit 18 - fi +# deploy replace / spelling correction rules to a separate replace rewriter if they exist +if [ -s "${replace_rules_file}" ] && [ -n "${replace_rules_rewriter_name}" ]; then + replace_rules_rewriter_data=$(jq -Rs '{class: "querqy.solr.rewriter.replace.ReplaceRewriterFactory", config: { ignoreCase: true, inputDelimiter: ";", rules: . }}' "${replace_rules_file}") + echo "Deploying replace rules for core ${solr_core_or_collection_name} to rewriter ${replace_rules_rewriter_name} at ${solr_host}..." + save_rewriter "${replace_rules_rewriter_name}" "${replace_rules_rewriter_data}" fi -# all ok -echo "smui2solr.sh - ok" -exit 0 +echo "Done" diff --git a/test/resources/smui2test.sh b/test/resources/smui2test.sh index f11c799c..881f29fb 100755 --- a/test/resources/smui2test.sh +++ b/test/resources/smui2test.sh @@ -8,7 +8,3 @@ echo "\$1 = >>>$1" echo "\$2 = >>>$2" echo "\$3 = >>>$3" echo "\$4 = >>>$4" -echo "\$5 = >>>$5" -echo "\$6 = >>>$6" -echo "\$7 = >>>$7" -echo "\$8 = >>>$8" diff --git a/test/services/RulesTxtDeploymentServiceConfigVariantsSpec.scala b/test/services/RulesTxtDeploymentServiceConfigVariantsSpec.scala index dc7fa1a7..a927c016 100644 --- a/test/services/RulesTxtDeploymentServiceConfigVariantsSpec.scala +++ b/test/services/RulesTxtDeploymentServiceConfigVariantsSpec.scala @@ -63,33 +63,18 @@ class RulesTxtOnlyDeploymentConfigVariantSpec extends AnyFlatSpec with Matchers override protected lazy val additionalAppConfig = Seq( "smui2solr.SRC_TMP_FILE" -> "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp", - "smui2solr.DST_CP_FILE_TO" -> "/deployment-path-live/common-rules.txt", "toggle.rule-deployment.pre-live.present" -> true, - "smui2solr.deploy-prelive-fn-rules-txt" -> "/deployment-path-prelive/common-rules.txt" ) - "RulesTxtDeploymentService" should "provide only the (common) rules.txt for PRELIVE" in { - val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, "PRELIVE", logDebug = false) + "RulesTxtDeploymentService" should "provide only the (common) rules.txt" in { + val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, logDebug = false) deploymentDescriptor.solrIndexId shouldBe core1Id deploymentDescriptor.regularRules.content should include ("aerosmith") // simply cross check content deploymentDescriptor.regularRules.sourceFileName shouldBe "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp" - deploymentDescriptor.regularRules.destinationFileName shouldBe "/deployment-path-prelive/common-rules.txt" deploymentDescriptor.replaceRules shouldBe None deploymentDescriptor.decompoundRules shouldBe None } - - "RulesTxtDeploymentService" should "provide only the (common) rules.txt for LIVE" in { - val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, "LIVE", logDebug = false) - - deploymentDescriptor.solrIndexId shouldBe core1Id - deploymentDescriptor.regularRules.content should include ("aerosmith") // simply cross check content - deploymentDescriptor.regularRules.sourceFileName shouldBe "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp" - deploymentDescriptor.regularRules.destinationFileName shouldBe "/deployment-path-live/common-rules.txt" - deploymentDescriptor.replaceRules shouldBe None - deploymentDescriptor.decompoundRules shouldBe None - } - } class RulesAndReplaceTxtDeploymentConfigVariantSpec extends AnyFlatSpec with Matchers with CommonRulesTxtDeploymentServiceConfigVariantsSpecBase { @@ -98,42 +83,21 @@ class RulesAndReplaceTxtDeploymentConfigVariantSpec extends AnyFlatSpec with Mat override protected lazy val additionalAppConfig = Seq( "smui2solr.SRC_TMP_FILE" -> "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp", - "smui2solr.DST_CP_FILE_TO" -> "/deployment-path-live/common-rules.txt", "toggle.rule-deployment.pre-live.present" -> true, - "smui2solr.deploy-prelive-fn-rules-txt" -> "/deployment-path-prelive/common-rules.txt", // spelling is activated (@see /smui/test/models/ApplicationTestBase.scala) - "smui2solr.replace-rules-tmp-file" -> "/changed-replace-rules-temp-path/search-management-ui_replace-rules-txt.tmp", - "smui2solr.replace-rules-dst-cp-file-to" -> "/deployment-path-live/replace-rules.txt", - "smui2solr.deploy-prelive-fn-replace-txt" -> "/deployment-path-prelive/replace-rules.txt" + "smui2solr.replace-rules-tmp-file" -> "/changed-replace-rules-temp-path/search-management-ui_replace-rules-txt.tmp" ) - "RulesTxtDeploymentService" should "provide the (common) rules.txt and a replace-rules.txt for PRELIVE" in { - val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, "PRELIVE", logDebug = false) + "RulesTxtDeploymentService" should "provide the (common) rules.txt and a replace-rules.txt" in { + val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, logDebug = false) deploymentDescriptor.solrIndexId shouldBe core1Id deploymentDescriptor.regularRules.content should include ("aerosmith") // simply cross check content deploymentDescriptor.regularRules.sourceFileName shouldBe "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp" - deploymentDescriptor.regularRules.destinationFileName shouldBe "/deployment-path-prelive/common-rules.txt" val replaceRules = deploymentDescriptor.replaceRules.get replaceRules.content should include ("freezer") // simply cross check content replaceRules.sourceFileName shouldBe "/changed-replace-rules-temp-path/search-management-ui_replace-rules-txt.tmp" - replaceRules.destinationFileName shouldBe "/deployment-path-prelive/replace-rules.txt" - deploymentDescriptor.decompoundRules shouldBe None - } - - "RulesTxtDeploymentService" should "provide the (common) rules.txt and a replace-rules.txt for LIVE" in { - val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, "LIVE", logDebug = false) - - deploymentDescriptor.solrIndexId shouldBe core1Id - deploymentDescriptor.regularRules.content should include ("aerosmith") // simply cross check content - deploymentDescriptor.regularRules.sourceFileName shouldBe "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp" - deploymentDescriptor.regularRules.destinationFileName shouldBe "/deployment-path-live/common-rules.txt" - - val replaceRules = deploymentDescriptor.replaceRules.get - replaceRules.content should include ("freezer") // simply cross check content - replaceRules.sourceFileName shouldBe "/changed-replace-rules-temp-path/search-management-ui_replace-rules-txt.tmp" - replaceRules.destinationFileName shouldBe "/deployment-path-live/replace-rules.txt" deploymentDescriptor.decompoundRules shouldBe None } @@ -145,12 +109,8 @@ class RulesAndDecompoundTxtDeploymentConfigVariantSpec extends AnyFlatSpec with override protected lazy val additionalAppConfig = Seq( "smui2solr.SRC_TMP_FILE" -> "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp", - "smui2solr.DST_CP_FILE_TO" -> "/deployment-path-live/common-rules.txt", "toggle.rule-deployment.pre-live.present" -> true, - "smui2solr.deploy-prelive-fn-rules-txt" -> "/deployment-path-prelive/common-rules.txt", - "toggle.rule-deployment.split-decompound-rules-txt" -> true, - "toggle.rule-deployment.split-decompound-rules-txt-DST_CP_FILE_TO" -> "/deployment-path-live/decompound-rules.txt", - "smui2solr.deploy-prelive-fn-decompound-txt" -> "/deployment-path-prelive/decompound-rules.txt" + "toggle.rule-deployment.split-decompound-rules-txt" -> true ) override protected def beforeAll(): Unit = { @@ -158,34 +118,17 @@ class RulesAndDecompoundTxtDeploymentConfigVariantSpec extends AnyFlatSpec with createDecompoundRule() } - "RulesTxtDeploymentService" should "provide the (common) rules.txt and a decompound-rules.txt for PRELIVE" in { - val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, "PRELIVE", logDebug = false) + "RulesTxtDeploymentService" should "provide the (common) rules.txt and a decompound-rules.txt" in { + val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, logDebug = false) deploymentDescriptor.solrIndexId shouldBe core1Id deploymentDescriptor.regularRules.content should include ("aerosmith") // simply cross check content deploymentDescriptor.regularRules.sourceFileName shouldBe "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp" - deploymentDescriptor.regularRules.destinationFileName shouldBe "/deployment-path-prelive/common-rules.txt" deploymentDescriptor.replaceRules shouldBe None val decompoundRules = deploymentDescriptor.decompoundRules.get decompoundRules.content should include ("damen") decompoundRules.sourceFileName shouldBe "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp-2" // auto generated (by spec) - decompoundRules.destinationFileName shouldBe "/deployment-path-prelive/decompound-rules.txt" - } - - "RulesTxtDeploymentService" should "provide the (common) rules.txt and a decompound-rules.txt for LIVE" in { - val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, "LIVE", logDebug = false) - - deploymentDescriptor.solrIndexId shouldBe core1Id - deploymentDescriptor.regularRules.content should include ("aerosmith") // simply cross check content - deploymentDescriptor.regularRules.sourceFileName shouldBe "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp" - deploymentDescriptor.regularRules.destinationFileName shouldBe "/deployment-path-live/common-rules.txt" - deploymentDescriptor.replaceRules shouldBe None - - val decompoundRules = deploymentDescriptor.decompoundRules.get - decompoundRules.content should include ("damen") - decompoundRules.sourceFileName shouldBe "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp-2" // auto generated (by spec) - decompoundRules.destinationFileName shouldBe "/deployment-path-live/decompound-rules.txt" } } @@ -197,17 +140,11 @@ class RulesReplaceAndDecompoundTxtDeploymentConfigVariantSpec extends AnyFlatSpe override protected lazy val additionalAppConfig = Seq( // (common) rules.txt config "smui2solr.SRC_TMP_FILE" -> "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp", - "smui2solr.DST_CP_FILE_TO" -> "/deployment-path-live/common-rules.txt", "toggle.rule-deployment.pre-live.present" -> true, - "smui2solr.deploy-prelive-fn-rules-txt" -> "/deployment-path-prelive/common-rules.txt", // replace-rules.txt config "smui2solr.replace-rules-tmp-file" -> "/changed-replace-rules-temp-path/search-management-ui_replace-rules-txt.tmp", - "smui2solr.replace-rules-dst-cp-file-to" -> "/deployment-path-live/replace-rules.txt", - "smui2solr.deploy-prelive-fn-replace-txt" -> "/deployment-path-prelive/replace-rules.txt", // decompound-rules.txt config - "toggle.rule-deployment.split-decompound-rules-txt" -> true, - "toggle.rule-deployment.split-decompound-rules-txt-DST_CP_FILE_TO" -> "/deployment-path-live/decompound-rules.txt", - "smui2solr.deploy-prelive-fn-decompound-txt" -> "/deployment-path-prelive/decompound-rules.txt" + "toggle.rule-deployment.split-decompound-rules-txt" -> true ) override protected def beforeAll(): Unit = { @@ -215,42 +152,20 @@ class RulesReplaceAndDecompoundTxtDeploymentConfigVariantSpec extends AnyFlatSpe createDecompoundRule() } - "RulesTxtDeploymentService" should "provide the (common) rules.txt, replace-rules.txt and a decompound-rules.txt for PRELIVE" in { - val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, "PRELIVE", logDebug = false) - - deploymentDescriptor.solrIndexId shouldBe core1Id - deploymentDescriptor.regularRules.content should include ("aerosmith") // simply cross check content - deploymentDescriptor.regularRules.sourceFileName shouldBe "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp" - deploymentDescriptor.regularRules.destinationFileName shouldBe "/deployment-path-prelive/common-rules.txt" - - val replaceRules = deploymentDescriptor.replaceRules.get - replaceRules.content should include ("freezer") // simply cross check content - replaceRules.sourceFileName shouldBe "/changed-replace-rules-temp-path/search-management-ui_replace-rules-txt.tmp" - replaceRules.destinationFileName shouldBe "/deployment-path-prelive/replace-rules.txt" - - val decompoundRules = deploymentDescriptor.decompoundRules.get - decompoundRules.content should include ("damen") - decompoundRules.sourceFileName shouldBe "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp-2" // auto generated (by spec) - decompoundRules.destinationFileName shouldBe "/deployment-path-prelive/decompound-rules.txt" - } - - "RulesTxtDeploymentService" should "provide the (common) rules.txt, replace-rules.txt and a decompound-rules.txt for LIVE" in { - val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, "LIVE", logDebug = false) + "RulesTxtDeploymentService" should "provide the (common) rules.txt, replace-rules.txt and a decompound-rules.txt" in { + val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, logDebug = false) deploymentDescriptor.solrIndexId shouldBe core1Id deploymentDescriptor.regularRules.content should include ("aerosmith") // simply cross check content deploymentDescriptor.regularRules.sourceFileName shouldBe "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp" - deploymentDescriptor.regularRules.destinationFileName shouldBe "/deployment-path-live/common-rules.txt" val replaceRules = deploymentDescriptor.replaceRules.get replaceRules.content should include ("freezer") // simply cross check content replaceRules.sourceFileName shouldBe "/changed-replace-rules-temp-path/search-management-ui_replace-rules-txt.tmp" - replaceRules.destinationFileName shouldBe "/deployment-path-live/replace-rules.txt" val decompoundRules = deploymentDescriptor.decompoundRules.get decompoundRules.content should include ("damen") decompoundRules.sourceFileName shouldBe "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp-2" // auto generated (by spec) - decompoundRules.destinationFileName shouldBe "/deployment-path-live/decompound-rules.txt" } } @@ -266,20 +181,11 @@ class RulesTxtDeploymentRegularTargetSpec extends AnyFlatSpec with Matchers with override protected lazy val additionalAppConfig = Seq( // (common) rules.txt config "smui2solr.SRC_TMP_FILE" -> "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp", - "smui2solr.DST_CP_FILE_TO" -> "/deployment-path-live/common-rules.txt", "toggle.rule-deployment.pre-live.present" -> true, - "smui2solr.deploy-prelive-fn-rules-txt" -> "/deployment-path-prelive/common-rules.txt", // replace-rules.txt config "smui2solr.replace-rules-tmp-file" -> "/changed-replace-rules-temp-path/search-management-ui_replace-rules-txt.tmp", - "smui2solr.replace-rules-dst-cp-file-to" -> "/deployment-path-live/replace-rules.txt", - "smui2solr.deploy-prelive-fn-replace-txt" -> "/deployment-path-prelive/replace-rules.txt", // decompound-rules.txt config "toggle.rule-deployment.split-decompound-rules-txt" -> true, - "toggle.rule-deployment.split-decompound-rules-txt-DST_CP_FILE_TO" -> "/deployment-path-live/decompound-rules.txt", - "smui2solr.deploy-prelive-fn-decompound-txt" -> "/deployment-path-prelive/decompound-rules.txt", - // Solr host config complete - "smui2solr.SOLR_HOST" -> "live.solr.instance:8983", - "smui2solr.deploy-prelive-solr-host" -> "prelive.solr.instance:8983", // test script "toggle.rule-deployment.custom-script" -> true, "toggle.rule-deployment.custom-script-SMUI2SOLR-SH_PATH" -> "test/resources/smui2test.sh" @@ -291,32 +197,24 @@ class RulesTxtDeploymentRegularTargetSpec extends AnyFlatSpec with Matchers with } "interfaceSmui2SolrSh" should "interface the test script should return all rules.txts for PRELIVE" in { - val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, "PRELIVE", logDebug = false) + val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, logDebug = false) val res = service.executeDeploymentScript(deploymentDescriptor, "PRELIVE") res.success shouldBe true res.output shouldBe s"""$$1 = >>>/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp - |$$2 = >>>/deployment-path-prelive/common-rules.txt - |$$3 = >>>prelive.solr.instance:8983 + |$$2 = >>>/changed-replace-rules-temp-path/search-management-ui_replace-rules-txt.tmp + |$$3 = >>>PRELIVE |$$4 = >>>core1 - |$$5 = >>>/deployment-path-prelive/decompound-rules.txt - |$$6 = >>>PRELIVE - |$$7 = >>>/changed-replace-rules-temp-path/search-management-ui_replace-rules-txt.tmp - |$$8 = >>>/deployment-path-prelive/replace-rules.txt |""".stripMargin } "interfaceSmui2SolrSh" should "interface the test script should return all rules.txts for LIVE" in { - val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, "LIVE", logDebug = false) + val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, logDebug = false) val res = service.executeDeploymentScript(deploymentDescriptor, "LIVE") res.success shouldBe true res.output shouldBe s"""$$1 = >>>/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp - |$$2 = >>>/deployment-path-live/common-rules.txt - |$$3 = >>>live.solr.instance:8983 + |$$2 = >>>/changed-replace-rules-temp-path/search-management-ui_replace-rules-txt.tmp + |$$3 = >>>LIVE |$$4 = >>>core1 - |$$5 = >>>/deployment-path-live/decompound-rules.txt - |$$6 = >>>LIVE - |$$7 = >>>/changed-replace-rules-temp-path/search-management-ui_replace-rules-txt.tmp - |$$8 = >>>/deployment-path-live/replace-rules.txt |""".stripMargin } @@ -328,24 +226,16 @@ class RulesTxtDeploymentGitTargetSpec extends AnyFlatSpec with Matchers with Com override protected lazy val additionalAppConfig = Seq( // switch to GIT for LIVE deployment - "smui2solr.DST_CP_FILE_TO" -> "GIT", + "toggle.rule-deployment.git.enable" -> true, "smui.deployment.git.repo-url" -> "ssh://git@changed-git-server.tld/repos/smui_rulestxt_repo.git", "smui2solr.deployment.git.filename.common-rules-txt" -> "common-rules.txt", // (common) rules.txt config "smui2solr.SRC_TMP_FILE" -> "/changed-common-rules-temp-path/search-management-ui_rules-txt.tmp", "toggle.rule-deployment.pre-live.present" -> true, - "smui2solr.deploy-prelive-fn-rules-txt" -> "/deployment-path-prelive/common-rules.txt", // replace-rules.txt config "smui2solr.replace-rules-tmp-file" -> "/changed-replace-rules-temp-path/search-management-ui_replace-rules-txt.tmp", - "smui2solr.replace-rules-dst-cp-file-to" -> "/deployment-path-live/replace-rules.txt", - "smui2solr.deploy-prelive-fn-replace-txt" -> "/deployment-path-prelive/replace-rules.txt", // decompound-rules.txt config - "toggle.rule-deployment.split-decompound-rules-txt" -> true, - "toggle.rule-deployment.split-decompound-rules-txt-DST_CP_FILE_TO" -> "/deployment-path-live/decompound-rules.txt", - "smui2solr.deploy-prelive-fn-decompound-txt" -> "/deployment-path-prelive/decompound-rules.txt", - // Solr host config complete - "smui2solr.SOLR_HOST" -> "live.solr.instance:8983", - "smui2solr.deploy-prelive-solr-host" -> "prelive.solr.instance:8983" + "toggle.rule-deployment.split-decompound-rules-txt" -> true ) override protected def beforeAll(): Unit = { @@ -358,7 +248,7 @@ class RulesTxtDeploymentGitTargetSpec extends AnyFlatSpec with Matchers with Com "interfaceSmui2GitSh" should "interface the test script should return all rules.txts for LIVE" in { - val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, "LIVE", logDebug = false) + val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, logDebug = false) val res = service.executeDeploymentScript(deploymentDescriptor, "LIVE") // TODO script result itself failed - this is cheesy, but cristal clear, as we don't have a local git server running or a test script instead @@ -375,14 +265,13 @@ class RulesTxtDeploymentGitTargetSpec extends AnyFlatSpec with Matchers with Com "interfacing git configured SMUI" should "stick with file copy deployment configuration PRELIVE" in { - val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, "PRELIVE", logDebug = false) + val deploymentDescriptor = service.generateRulesTxtContentWithFilenames(core1Id, logDebug = false) val res = service.executeDeploymentScript(deploymentDescriptor, "PRELIVE") // TODO script result itself failed - this is cheesy, but cristal clear, as we don't have a local git server running or a test script instead - res.output should include ("/deployment-path-prelive/common-rules.txt") - res.output should include ("/deployment-path-prelive/replace-rules.txt") - res.output should include ("/deployment-path-prelive/decompound-rules.txt") + res.exitCode shouldBe -1 + res.output shouldBe "Git deployment only support for LIVE target" // TODO As of v3.11.7 there is no option: // TODO ... to deploy to different git hosts / repos / branches (it all makes sense) diff --git a/test/services/RulesTxtDeploymentServiceSimpleSpec.scala b/test/services/RulesTxtDeploymentServiceSimpleSpec.scala index 606d4ee8..5886b441 100644 --- a/test/services/RulesTxtDeploymentServiceSimpleSpec.scala +++ b/test/services/RulesTxtDeploymentServiceSimpleSpec.scala @@ -53,21 +53,19 @@ class RulesTxtDeploymentServiceSpec extends AnyFlatSpec with Matchers with Appli |mechine => machine""".stripMargin "RulesTxtDeploymentService" should "generate rules files with correct file names" in { - val rulesTxt = service.generateRulesTxtContentWithFilenames(core1Id, "LIVE", logDebug = false) + val rulesTxt = service.generateRulesTxtContentWithFilenames(core1Id, logDebug = false) rulesTxt.solrIndexId shouldBe core1Id rulesTxt.decompoundRules shouldBe empty rulesTxt.regularRules.content.trim shouldBe rulesFileContent(inputIds) rulesTxt.regularRules.sourceFileName shouldBe "/tmp/search-management-ui_rules-txt.tmp" - rulesTxt.regularRules.destinationFileName shouldBe "/usr/bin/solr/liveCore/conf/rules.txt" rulesTxt.replaceRules.get.content.trim shouldBe replaceRulesFileContent() rulesTxt.replaceRules.get.sourceFileName shouldBe "/tmp/search-management-ui_replace-rules-txt.tmp" - rulesTxt.replaceRules.get.destinationFileName shouldBe "/usr/bin/solr/liveCore/conf/replace-rules.txt" } it should "validate the rules files correctly" in { - val rulesTxt = service.generateRulesTxtContentWithFilenames(core1Id, "LIVE", logDebug = false) + val rulesTxt = service.generateRulesTxtContentWithFilenames(core1Id, logDebug = false) service.validateCompleteRulesTxts(rulesTxt, logDebug = false) shouldBe empty val badRulesTxt = rulesTxt.copy(regularRules = rulesTxt.regularRules.copy(content = "a very bad rules file"))