Merge branch 'shyamsai/GlobalParamObject2' of https://github.com/sss0…

…4/SynapseML into shyamsai/GlobalParamObject2

cognitive/src/main/scala/com/microsoft/azure/synapse/ml/services/openai/OpenAI.scala

@@ -104,11 +104,12 @@ trait HasOpenAIEmbeddingParams extends HasOpenAISharedParams with HasAPIVersion
 }
 
 trait HasOpenAITextParams extends HasOpenAISharedParams {
-
   val maxTokens: ServiceParam[Int] = new ServiceParam[Int](
     this, "maxTokens",
     "The maximum number of tokens to generate. Has minimum of 0.",
-    isRequired = false)
+    isRequired = false){
+    override val payloadName: String = "max_tokens"
+  }
 
   def getMaxTokens: Int = getScalarParam(maxTokens)
 
@@ -153,7 +154,9 @@ trait HasOpenAITextParams extends HasOpenAISharedParams {
       " So 0.1 means only the tokens comprising the top 10 percent probability mass are considered." +
       " We generally recommend using this or `temperature` but not both." +
       " Minimum of 0 and maximum of 1 allowed.",
-    isRequired = false)
+    isRequired = false) {
+    override val payloadName: String = "top_p"
+  }
 
   def getTopP: Double = getScalarParam(topP)
 
@@ -182,7 +185,9 @@ trait HasOpenAITextParams extends HasOpenAISharedParams {
       " So for example, if `logprobs` is 10, the API will return a list of the 10 most likely tokens." +
       " If `logprobs` is 0, only the chosen tokens will have logprobs returned." +
       " Minimum of 0 and maximum of 100 allowed.",
-    isRequired = false)
+    isRequired = false) {
+    override val payloadName: String = "logprobs"
+  }
 
   def getLogProbs: Int = getScalarParam(logProbs)
 
@@ -208,7 +213,9 @@ trait HasOpenAITextParams extends HasOpenAISharedParams {
   val cacheLevel: ServiceParam[Int] = new ServiceParam[Int](
     this, "cacheLevel",
     "can be used to disable any server-side caching, 0=no cache, 1=prompt prefix enabled, 2=full cache",
-    isRequired = false)
+    isRequired = false){
+    override val payloadName: String = "cache_level"
+  }
 
   def getCacheLevel: Int = getScalarParam(cacheLevel)
 
@@ -222,7 +229,9 @@ trait HasOpenAITextParams extends HasOpenAISharedParams {
     this, "presencePenalty",
     "How much to penalize new tokens based on their existing frequency in the text so far." +
       " Decreases the likelihood of the model to repeat the same line verbatim. Has minimum of -2 and maximum of 2.",
-    isRequired = false)
+    isRequired = false){
+    override val payloadName: String = "presence_penalty"
+  }
 
   def getPresencePenalty: Double = getScalarParam(presencePenalty)
 
@@ -236,7 +245,9 @@ trait HasOpenAITextParams extends HasOpenAISharedParams {
     this, "frequencyPenalty",
     "How much to penalize new tokens based on whether they appear in the text so far." +
       " Increases the likelihood of the model to talk about new topics.",
-    isRequired = false)
+    isRequired = false){
+    override val payloadName: String = "frequency_penalty"
+  }
 
   def getFrequencyPenalty: Double = getScalarParam(frequencyPenalty)
 
@@ -250,7 +261,9 @@ trait HasOpenAITextParams extends HasOpenAISharedParams {
     this, "bestOf",
     "How many generations to create server side, and display only the best." +
       " Will not stream intermediate progress if best_of > 1. Has maximum value of 128.",
-    isRequired = false)
+    isRequired = false){
+    override val payloadName: String = "best_of"
+  }
 
   def getBestOf: Int = getScalarParam(bestOf)
 
@@ -260,24 +273,27 @@ trait HasOpenAITextParams extends HasOpenAISharedParams {
 
   def setBestOfCol(v: String): this.type = setVectorParam(bestOf, v)
 
+  // list of shared text parameters. In method getOptionalParams, we will iterate over these parameters
+  // to compute the optional parameters. Since this list never changes, we can create it once and reuse it.
+  private val sharedTextParams = Seq(
+    maxTokens,
+    temperature,
+    topP,
+    user,
+    n,
+    echo,
+    stop,
+    cacheLevel,
+    presencePenalty,
+    frequencyPenalty,
+    bestOf,
+    logProbs
+  )
+
   private[ml] def getOptionalParams(r: Row): Map[String, Any] = {
-    Seq(
-      maxTokens,
-      temperature,
-      topP,
-      user,
-      n,
-      echo,
-      stop,
-      cacheLevel,
-      presencePenalty,
-      frequencyPenalty,
-      bestOf
-    ).flatMap(param =>
-      getValueOpt(r, param).map(v => (GenerationUtils.camelToSnake(param.name), v))
-    ).++(Seq(
-      getValueOpt(r, logProbs).map(v => ("logprobs", v))
-    ).flatten).toMap
+    sharedTextParams.flatMap { param =>
+      getValueOpt(r, param).map { value => param.payloadName -> value }
+    }.toMap
   }
 }
 

pipeline.yaml

@@ -143,8 +143,10 @@ jobs:
         TEST-CLASS: "com.microsoft.azure.synapse.ml.nbtest.DatabricksCPUTests"
       databricks-gpu:
         TEST-CLASS: "com.microsoft.azure.synapse.ml.nbtest.DatabricksGPUTests"
-      databricks-rapids:
-        TEST-CLASS: "com.microsoft.azure.synapse.ml.nbtest.DatabricksRapidsTests"
+#      databricks-rapids tests have been disabled because these tests are failing.
+#      This test will be re-enabled once the issue is fixed.
+#      databricks-rapids:
+#        TEST-CLASS: "com.microsoft.azure.synapse.ml.nbtest.DatabricksRapidsTests"
       synapse:
         TEST-CLASS: "com.microsoft.azure.synapse.ml.nbtest.SynapseTests"
 #      ${{ if eq(parameters.runSynapseExtensionE2ETests, true) }}:
@@ -264,7 +266,7 @@ jobs:
         chmod +x git-chglog_linux_amd64
         ./git-chglog_linux_amd64 -o CHANGELOG.md $TAG
       condition: and(eq(variables.isMaster, true), startsWith(variables['tag'], 'v'))
-    - task: GitHubRelease@0
+    - task: GitHubRelease@1
       condition: and(eq(variables.isMaster, true), startsWith(variables['tag'], 'v'))
       inputs:
         gitHubConnection: 'MMLSpark Github'
@@ -294,7 +296,7 @@ jobs:
         conda env create --force -f environment.yml -v
       condition: and(eq(variables.isMaster, true), and(startsWith(variables['tag'], 'v'), eq(variables.CONDA_CACHE_RESTORED, 'false')))
       displayName: Create Anaconda environment
-    - task: AzureKeyVault@1
+    - task: AzureKeyVault@2
       condition: and(eq(variables.isMaster, true), startsWith(variables['tag'], 'v'))
       inputs:
         azureSubscription: 'SynapseML Build'

templates/kv.yml

@@ -1,5 +1,5 @@
 steps:
-  - task: AzureKeyVault@1
+  - task: AzureKeyVault@2
     retryCountOnTaskFailure: 3
     inputs:
       azureSubscription: 'SynapseML Build'