Change the Solr rule deployment script to support the Querqy 5 rule A…

…PI. The example smui2solr.sh was still using ssh based file upload to deploy Querqy rules, there is a new way with Querqy 5: the Querqy rewriter API. * New smui2solr.sh deployment script that uses the Querqy 5 API. It is triggered with 4 parameters 1) the path to the common rules file 2) the path to the replace rules file 3) the target system, i.e. PRELIVE or LIVE 4) the Solr core or collection name (taken from the rules collection) * The script expects all other configuration as environment variables: - SMUI_QUERQY_REWRITER_COMMON_RULES: the name of the common rules rewriter the rules should be deployed to - SMUI_QUERQY_REWRITER_COMMON_RULES_DECOMPOUND: the name of the common rules rewriter for decompound rules (required, if decompound rules shall be exported separately) - SMUI_QUERQY_REWRITER_REPLACE: the name of the replace rules rewriter (required, if spelling/replacement rules are used) - SMUI_2SOLR_SOLR_HOST or SMUI_DEPLOY_LIVE_SOLR_HOST: the live Solr host - SMUI_DEPLOY_PRELIVE_SOLR_HOST: the prelive Solr host, if prelive deployment is enabled - SMUI_DEPLOY_(PRE)LIVE_SOLR_(USER|PASSWORD): the live/prelive user/password if Solr basic authentication is used - SMUI_DEPLOY_PRELIVE_SOLR_ALLOW_INSECURE: if the --insecure flag should be used when authenticating (i.e. no cert validation) * Removing all configuration related to the *target* file names of the common rules, optional decompound common rules and replace rules file names as they are no longer copied to a target. * Changing the decision to trigger a GIT based rule deployment to a dedicated toggle / environment variable SMUI_DEPLOYMENT_GIT_ENABLE which needs to be set to true. No longer relying on the magic name "GIT" as the rules file name on the deployment target. * Add jq to SMUI Docker image to facilitate JSON based Querqy request recreation in smui2solr.sh script.
querqy · Nov 25, 2024 · f81be6e · f81be6e
1 parent 0afbf10
commit f81be6e
Show file tree

Hide file tree

Showing 10 changed files with 219 additions and 361 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -28,7 +28,7 @@ RUN --mount=target=/root/.ivy2,type=cache sbt "set assembly / test := {}" clean
 FROM eclipse-temurin:17-jre-focal
 
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends openssh-client sshpass bash curl git \
+    && apt-get install -y --no-install-recommends openssh-client sshpass bash curl git jq \
     && rm -rf /var/lib/apt/lists/*
 
 ARG VERSION

diff --git a/app/controllers/ApiController.scala b/app/controllers/ApiController.scala
@@ -326,7 +326,7 @@ class ApiController @Inject()(val controllerComponents: SecurityComponents,
     logger.debug("In ApiController :: updateRulesTxtForSolrIndexAndTargetPlatform")
 
     // generate rules.txt(s)
-    val rulesFiles = rulesTxtDeploymentService.generateRulesTxtContentWithFilenames(SolrIndexId(solrIndexId), targetSystem)
+    val rulesFiles = rulesTxtDeploymentService.generateRulesTxtContentWithFilenames(SolrIndexId(solrIndexId))
 
     // validate every generated rules.txt
     rulesTxtDeploymentService.validateCompleteRulesTxts(rulesFiles) match {

diff --git a/app/models/FeatureToggleModel.scala b/app/models/FeatureToggleModel.scala
@@ -50,10 +50,10 @@ package object FeatureToggleModel extends Logging {
     private val FEATURE_TOGGLE_UI_LIST_LIMIT_ITEMS_TO = "toggle.ui-list.limit-items-to"
     private val FEATURE_TOGGLE_RULE_DEPLOYMENT_LOG_RULE_ID = "toggle.rule-deployment.log-rule-id"
     private val FEATURE_TOGGLE_RULE_DEPLOYMENT_SPLIT_DECOMPOUND_RULES_TXT = "toggle.rule-deployment.split-decompound-rules-txt"
-    private val FEATURE_TOGGLE_RULE_DEPLOYMENT_SPLIT_DECOMPOUND_RULES_TXT_DST_CP_FILE_TO = "toggle.rule-deployment.split-decompound-rules-txt-DST_CP_FILE_TO"
     private val FEATURE_TOGGLE_RULE_DEPLOYMENT_PRE_LIVE_PRESENT = "toggle.rule-deployment.pre-live.present"
     private val FEATURE_TOGGLE_RULE_DEPLOYMENT_CUSTOM_SCRIPT = "toggle.rule-deployment.custom-script"
     private val FEATURE_TOGGLE_RULE_DEPLOYMENT_CUSTOM_SCRIPT_SMUI2SOLR_SH_PATH = "toggle.rule-deployment.custom-script-SMUI2SOLR-SH_PATH"
+    private val FEATURE_TOGGLE_RULE_DEPLOYMENT_ENABLE_GIT = "toggle.rule-deployment.git.enable"
     private val FEATURE_TOGGLE_HEADLINE = "toggle.headline"
     private val FEATURE_TOGGLE_DEPLOYMENT_LABEL = "toggle.rule-deployment-label"
     private val FEATURE_TOGGLE_DEPLOYMENT_PRELIVE_LABEL = "toggle.deploy-prelive-fn-label"
@@ -173,6 +173,10 @@ package object FeatureToggleModel extends Logging {
       )
     }
 
+    def getToggleRuleDeploymentGitEnabled: Boolean = {
+      appConfig.getOptional[Boolean](FEATURE_TOGGLE_RULE_DEPLOYMENT_ENABLE_GIT).getOrElse(false)
+    }
+
     def getToggleRuleDeploymentLogRuleId: Boolean = {
       appConfig.getOptional[Boolean](FEATURE_TOGGLE_RULE_DEPLOYMENT_LOG_RULE_ID).getOrElse(false)
     }
@@ -181,10 +185,6 @@ package object FeatureToggleModel extends Logging {
       appConfig.getOptional[Boolean](FEATURE_TOGGLE_RULE_DEPLOYMENT_SPLIT_DECOMPOUND_RULES_TXT).getOrElse(false)
     }
 
-    def getToggleRuleDeploymentSplitDecompoundRulesTxtDstCpFileTo: String = {
-      appConfig.getOptional[String](FEATURE_TOGGLE_RULE_DEPLOYMENT_SPLIT_DECOMPOUND_RULES_TXT_DST_CP_FILE_TO).getOrElse("")
-    }
-
     def getToggleRuleDeploymentCustomScript: Boolean = {
       appConfig.getOptional[Boolean](FEATURE_TOGGLE_RULE_DEPLOYMENT_CUSTOM_SCRIPT).getOrElse(false)
     }

diff --git a/app/services/RulesTxtDeploymentService.scala b/app/services/RulesTxtDeploymentService.scala
@@ -20,98 +20,64 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT
                                            environment: Environment) extends Logging {
 
   case class RulesTxtsForSolrIndex(solrIndexId: SolrIndexId,
-                                   regularRules: RulesTxtWithFileNames,
-                                   decompoundRules: Option[RulesTxtWithFileNames],
-                                   replaceRules: Option[RulesTxtWithFileNames]) {
+                                   regularRules: RulesTxtWithFileName,
+                                   decompoundRules: Option[RulesTxtWithFileName],
+                                   replaceRules: Option[RulesTxtWithFileName]) {
 
-    def regularAndDecompoundFiles: List[RulesTxtWithFileNames] = List(regularRules) ++ decompoundRules
+    def regularAndDecompoundFiles: List[RulesTxtWithFileName] = List(regularRules) ++ decompoundRules
 
-    def allFiles: List[RulesTxtWithFileNames] = regularAndDecompoundFiles ++ replaceRules
+    def allFiles: List[RulesTxtWithFileName] = regularAndDecompoundFiles ++ replaceRules
 
   }
 
-  case class RulesTxtWithFileNames(content: String,
-                                   sourceFileName: String,
-                                   destinationFileName: String)
+  case class RulesTxtWithFileName(content: String,
+                                  sourceFileName: String)
 
   /**
     * Generates a list of source to destination filenames containing the rules.txt(s) according to current application settings.
     *
     * @param solrIndexId Solr Index Id to generate the output for.
     */
   // TODO evaluate, if logDebug should be used to prevent verbose logging of the whole generated rules.txt (for zip download especially)
-  def generateRulesTxtContentWithFilenames(solrIndexId: SolrIndexId, targetSystem: String, logDebug: Boolean = true): RulesTxtsForSolrIndex = {
+  def generateRulesTxtContentWithFilenames(solrIndexId: SolrIndexId, logDebug: Boolean = true): RulesTxtsForSolrIndex = {
 
-    // SMUI config for (regular) LIVE deployment
     val SRC_TMP_FILE = appConfig.get[String]("smui2solr.SRC_TMP_FILE")
-    val DST_CP_FILE_TO = appConfig.get[String]("smui2solr.DST_CP_FILE_TO")
     val DO_SPLIT_DECOMPOUND_RULES_TXT = featureToggleService.getToggleRuleDeploymentSplitDecompoundRulesTxt
-    val DECOMPOUND_RULES_TXT_DST_CP_FILE_TO = featureToggleService.getToggleRuleDeploymentSplitDecompoundRulesTxtDstCpFileTo
-    // (additional) SMUI config for PRELIVE deployment
-    val SMUI_DEPLOY_PRELIVE_FN_RULES_TXT = appConfig.get[String]("smui2solr.deploy-prelive-fn-rules-txt")
-    val SMUI_DEPLOY_PRELIVE_FN_DECOMPOUND_TXT = appConfig.get[String]("smui2solr.deploy-prelive-fn-decompound-txt")
-
-    // Replace rules (spelling)
-    val EXPORT_REPLACE_RULES = featureToggleService.getToggleActivateSpelling
+    val DO_EXPORT_REPLACE_RULES = featureToggleService.getToggleActivateSpelling
     val REPLACE_RULES_SRC_TMP_FILE = appConfig.get[String]("smui2solr.replace-rules-tmp-file")
-    val REPLACE_RULES_DST_CP_FILE_TO = appConfig.get[String]("smui2solr.replace-rules-dst-cp-file-to")
-    val SMUI_DEPLOY_PRELIVE_FN_REPLACE_TXT = appConfig.get[String]("smui2solr.deploy-prelive-fn-replace-txt")
 
     if (logDebug) {
       logger.debug(
         s""":: generateRulesTxtContentWithFilenames config
            |:: SRC_TMP_FILE = $SRC_TMP_FILE
-           |:: DST_CP_FILE_TO = $DST_CP_FILE_TO
            |:: DO_SPLIT_DECOMPOUND_RULES_TXT = $DO_SPLIT_DECOMPOUND_RULES_TXT
-           |:: DECOMPOUND_RULES_TXT_DST_CP_FILE_TO = $DECOMPOUND_RULES_TXT_DST_CP_FILE_TO
-           |:: SMUI_DEPLOY_PRELIVE_FN_RULES_TXT = $SMUI_DEPLOY_PRELIVE_FN_RULES_TXT
-           |:: SMUI_DEPLOY_PRELIVE_FN_DECOMPOUND_TXT = $SMUI_DEPLOY_PRELIVE_FN_DECOMPOUND_TXT
-           |:: EXPORT_REPLACE_RULES = $EXPORT_REPLACE_RULES
+           |:: EXPORT_REPLACE_RULES = $DO_EXPORT_REPLACE_RULES
            |:: REPLACE_RULES_SRC_TMP_FILE = $REPLACE_RULES_SRC_TMP_FILE
-           |:: REPLACE_RULES_DST_CP_FILE_TO = $REPLACE_RULES_DST_CP_FILE_TO
-           |:: SMUI_DEPLOY_PRELIVE_FN_REPLACE_TXT = $SMUI_DEPLOY_PRELIVE_FN_REPLACE_TXT
       """.stripMargin)
     }
 
     // generate one rules.txt by default or two separated, if decompound instructions are supposed to be split
 
-    // TODO test correct generation in different scenarios (one vs. two rules.txts, etc.)
-    val dstCpFileTo = if (targetSystem == "PRELIVE")
-      SMUI_DEPLOY_PRELIVE_FN_RULES_TXT
-    else // targetSystem == "LIVE"
-      DST_CP_FILE_TO
-
-    val replaceRulesDstCpFileTo =
-      if (targetSystem == "PRELIVE") SMUI_DEPLOY_PRELIVE_FN_REPLACE_TXT
-      else REPLACE_RULES_DST_CP_FILE_TO
-
-    val replaceRules =
-      if (EXPORT_REPLACE_RULES) {
+    val optReplaceRules =
+      if (DO_EXPORT_REPLACE_RULES) {
         val allCanonicalSpellings = searchManagementRepository.listAllSpellingsWithAlternatives(solrIndexId)
-        Some(RulesTxtWithFileNames(
-          QuerqyReplaceRulesGenerator.renderAllCanonicalSpellingsToReplaceRules(allCanonicalSpellings),
-          REPLACE_RULES_SRC_TMP_FILE,
-          replaceRulesDstCpFileTo
-        ))
+        val spellingsRendered =  QuerqyReplaceRulesGenerator.renderAllCanonicalSpellingsToReplaceRules(allCanonicalSpellings)
+        Some(RulesTxtWithFileName(spellingsRendered, REPLACE_RULES_SRC_TMP_FILE))
       } else None
 
     if (!DO_SPLIT_DECOMPOUND_RULES_TXT) {
       RulesTxtsForSolrIndex(solrIndexId,
-        RulesTxtWithFileNames(querqyRulesTxtGenerator.renderSingleRulesTxt(solrIndexId), SRC_TMP_FILE, dstCpFileTo),
+        RulesTxtWithFileName(querqyRulesTxtGenerator.renderSingleRulesTxt(solrIndexId), SRC_TMP_FILE),
         None,
-        replaceRules
+        optReplaceRules
       )
     } else {
-      val decompoundDstCpFileTo = if (targetSystem == "PRELIVE")
-        SMUI_DEPLOY_PRELIVE_FN_DECOMPOUND_TXT
-      else // targetSystem == "LIVE"
-        DECOMPOUND_RULES_TXT_DST_CP_FILE_TO
+      val rulesWithoutDecompounds = querqyRulesTxtGenerator.renderSeparatedRulesTxts(solrIndexId, renderCompoundsRulesTxt = false)
+      val decompoundRules = querqyRulesTxtGenerator.renderSeparatedRulesTxts(solrIndexId, renderCompoundsRulesTxt = true)
       RulesTxtsForSolrIndex(solrIndexId,
-        RulesTxtWithFileNames(querqyRulesTxtGenerator.renderSeparatedRulesTxts(solrIndexId, renderCompoundsRulesTxt = false), SRC_TMP_FILE, dstCpFileTo),
-        Some(RulesTxtWithFileNames(querqyRulesTxtGenerator.renderSeparatedRulesTxts(solrIndexId, renderCompoundsRulesTxt = true),
-          SRC_TMP_FILE + "-2", decompoundDstCpFileTo)
-        ),
-        replaceRules
+        RulesTxtWithFileName(rulesWithoutDecompounds, SRC_TMP_FILE),
+        Some(RulesTxtWithFileName(decompoundRules, SRC_TMP_FILE + "-2")),
+        optReplaceRules
       )
     }
   }
@@ -123,7 +89,7 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT
   def validateCompleteRulesTxts(rulesTxts: RulesTxtsForSolrIndex, logDebug: Boolean = true): List[String] = {
     val rulesValidation = rulesTxts.regularAndDecompoundFiles.flatMap { rulesFile =>
       if (logDebug) {
-        logger.debug(":: validateCompleteRulesTxts for src = " + rulesFile.sourceFileName + " dst = " + rulesFile.destinationFileName)
+        logger.debug(":: validateCompleteRulesTxts for src = " + rulesFile.sourceFileName)
         logger.debug(":: rulesTxt = <<<" + rulesFile.content + ">>>")
       }
       val validationResult = querqyRulesTxtGenerator.validateQuerqyRulesTxtToErrMsg(rulesFile.content)
@@ -158,43 +124,34 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT
 
     }
 
-    def interfaceSmui2SolrSh(scriptPath: String, srcTmpFile: String, dstCpFileTo: String, solrHost: String,
-                             solrCoreName: String, decompoundDstCpFileTo: String, targetSystem: String,
-                             replaceRulesSrcTmpFile: String, replaceRulesDstCpFileTo: String
+    def interfaceSmui2SolrSh(scriptPath: String,
+                             srcTmpFile: String,
+                             replaceRulesSrcTmpFile: String,
+                             targetSystem: String,
+                             solrCoreName: String
                             ): DeploymentScriptResult = {
 
       logger.info(
         s""":: interfaceSmui2SolrSh
            |:: scriptPath = $scriptPath
            |:: srcTmpFile = $srcTmpFile
-           |:: dstCpFileTo = $dstCpFileTo
-           |:: solrHost = $solrHost
+           |:: replaceRulesSrcTmpFile = $replaceRulesSrcTmpFile
            |:: solrCoreName = $solrCoreName
-           |:: decompoundDstCpFileTo = $decompoundDstCpFileTo
            |:: targetSystem = $targetSystem
-           |:: replaceRulesSrcTmpFile = $replaceRulesSrcTmpFile
-           |:: replaceRulesDstCpFileTo = $replaceRulesDstCpFileTo
       """.stripMargin)
 
-      val scriptCall =
+      val scriptCall = List(
         // define call for regular smui2solr (default or custom script) and add parameters to the script (in expected order, see smui2solr.sh)
-        scriptPath + " " +
+        scriptPath,
         // SRC_TMP_FILE=$1
-        srcTmpFile + " " +
-        // DST_CP_FILE_TO=$2
-        dstCpFileTo + " " +
-        // SOLR_HOST=$3
-        solrHost + " " +
+        srcTmpFile,
+        // REPLACE_RULES_SRC_TMP_FILE=$2
+        replaceRulesSrcTmpFile,
+        // TARGET_SYSTEM=$3
+        targetSystem,
         // SOLR_CORE_NAME=$4
-        solrCoreName + " " +
-        // DECOMPOUND_DST_CP_FILE_TO=$5
-        decompoundDstCpFileTo + " " +
-        // TARGET_SYSTEM=$6
-        targetSystem + " " +
-        // REPLACE_RULES_SRC_TMP_FILE=$7
-        replaceRulesSrcTmpFile + " " +
-        // REPLACE_RULES_DST_CP_FILE_TO=$8
-        replaceRulesDstCpFileTo
+        solrCoreName
+      ).mkString(" ")
 
       interfaceDeploymentScript(scriptCall)
     }
@@ -233,58 +190,37 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT
     else
       environment.rootPath.getAbsolutePath + "/conf/smui2solr.sh"
 
+    val deployToGitConfigured = featureToggleService.getToggleRuleDeploymentGitEnabled
+
     val srcTmpFile = rulesTxts.regularRules.sourceFileName
-    val dstCpFileTo = rulesTxts.regularRules.destinationFileName
-    val decompoundDstCpFileTo = if (rulesTxts.decompoundRules.isDefined)
-      rulesTxts.decompoundRules.get.destinationFileName
-    else
-      "NONE"
-
-    // host for (optional) core reload
-    val SMUI_DEPLOY_PRELIVE_SOLR_HOST = appConfig.get[String]("smui2solr.deploy-prelive-solr-host")
-    val SMUI_DEPLOY_LIVE_SOLR_HOST = appConfig.get[String]("smui2solr.SOLR_HOST")
-    val solrHost = if (targetSystem == "PRELIVE")
-      if (SMUI_DEPLOY_PRELIVE_SOLR_HOST.isEmpty)
-        "NONE"
-      else
-        SMUI_DEPLOY_PRELIVE_SOLR_HOST
-    else // targetSystem == "LIVE"
-      if (SMUI_DEPLOY_LIVE_SOLR_HOST.isEmpty)
-        "NONE"
-      else
-        SMUI_DEPLOY_LIVE_SOLR_HOST
-    // core name from repo (optional, for core reload as well)
     val solrCoreName = searchManagementRepository.getSolrIndexName(rulesTxts.solrIndexId)
-
     val replaceRulesSrcTmpFile = rulesTxts.replaceRules.map(_.sourceFileName).getOrElse("NONE")
-    val replaceRulesDstCpFileTo = rulesTxts.replaceRules.map(_.destinationFileName).getOrElse("NONE")
 
     // execute script
     // TODO currently only git deployment for LIVE instance available
-    val deployToGit = targetSystem.equals("LIVE") && appConfig.get[String]("smui2solr.DST_CP_FILE_TO").equals("GIT")
-    val result = (if(!deployToGit) {
+    val result = if (deployToGitConfigured) {
+      if (targetSystem == "LIVE") {
+        logger.info(":: executeDeploymentScript :: GIT configured calling interfaceSmui2GitSh")
+        // TODO support further rule files (decompound / replace) and maybe solrCoreName and/or targetSystem for git branch?
+        interfaceSmui2GitSh(
+          environment.rootPath.getAbsolutePath + "/conf/smui2git.sh",
+          srcTmpFile,
+          featureToggleService.getSmuiDeploymentGitRepoUrl,
+          featureToggleService.getSmuiDeploymentGitFilenameCommonRulesTxt,
+        )
+      } else {
+        DeploymentScriptResult(exitCode = -1, output = "Git deployment only support for LIVE target")
+      }
+    } else {
       logger.info(s":: executeDeploymentScript :: regular script configured calling interfaceSmui2SolrSh(scriptPath = '$scriptPath')")
       interfaceSmui2SolrSh(
         scriptPath,
         srcTmpFile,
-        dstCpFileTo,
-        solrHost,
-        solrCoreName,
-        decompoundDstCpFileTo,
-        targetSystem,
         replaceRulesSrcTmpFile,
-        replaceRulesDstCpFileTo
-      )
-    } else {
-      logger.info(":: executeDeploymentScript :: GIT configured calling interfaceSmui2GitSh")
-      // TODO support further rule files (decompound / replace) and maybe solrCoreName and/or targetSystem for git branch?
-      interfaceSmui2GitSh(
-        environment.rootPath.getAbsolutePath + "/conf/smui2git.sh",
-        srcTmpFile,
-        featureToggleService.getSmuiDeploymentGitRepoUrl,
-        featureToggleService.getSmuiDeploymentGitFilenameCommonRulesTxt,
+        targetSystem,
+        solrCoreName
       )
-    })
+    }
     if (result.success) {
       logger.info(s"Rules.txt deployment successful:\n${result.output}")
     } else {
@@ -323,7 +259,7 @@ class RulesTxtDeploymentService @Inject() (querqyRulesTxtGenerator: QuerqyRulesT
     try {
       for (index <- searchManagementRepository.listAllSolrIndexes) {
         // TODO make targetSystem configurable from ApiController.downloadAllRulesTxtFiles ... go with "LIVE" from now (as there exist no different revisions of the search management content)!
-        val rules = generateRulesTxtContentWithFilenames(index.id, "LIVE", logDebug = false)
+        val rules = generateRulesTxtContentWithFilenames(index.id, logDebug = false)
         zipStream.putNextEntry(new ZipEntry(s"rules_${index.name}.txt"))
         zipStream.write(rules.regularRules.content.getBytes("UTF-8"))
         zipStream.closeEntry()

diff --git a/build.sbt b/build.sbt
@@ -1,10 +1,10 @@
 import com.typesafe.sbt.GitBranchPrompt
 
 name := "search-management-ui"
-version := "4.3.1"
+version := "4.4.1"
 maintainer := "Contact productful.io <[email protected]>"
 
-scalaVersion := "2.13.14"
+scalaVersion := "2.13.15"
 
 ThisBuild / evictionErrorLevel := Level.Info