Esrp release

pipeline.yaml

@@ -95,6 +95,8 @@ jobs:
         sbt publishBlob publishDocs publishR publishPython uploadNotebooks
         sbt genBuildInfo
         echo "##vso[task.uploadsummary]$(pwd)/target/Build.md"
+        sbt publishLocalSigned
+        python tools/esrp/prepare_jar.py
       displayName: Publish Artifacts
       env:
         STORAGE-KEY: $(storage-key)
@@ -103,7 +105,7 @@ jobs:
         PGP-PRIVATE: $(pgp-private)
         PGP-PUBLIC: $(pgp-public)
         PGP-PW: $(pgp-pw)
-        SYNAPSEML-ENABLE-PUBLISH: true
+        SYNAPSEML_ENABLE_PUBLISH: true
     - bash: |
         set -e
         sbt aetherDeploy
@@ -112,7 +114,7 @@ jobs:
         ADO-FEED-TOKEN: $(ado-feed-token)
         STORAGE-KEY: $(storage-key)
         PUBLISH-TO-FEED: true
-        SYNAPSEML-ENABLE-PUBLISH: true
+        SYNAPSEML_ENABLE_PUBLISH: true
     - bash: |
         set -e
         sbt publishBadges
@@ -125,7 +127,7 @@ jobs:
         PGP-PRIVATE: $(pgp-private)
         PGP-PUBLIC: $(pgp-public)
         PGP-PW: $(pgp-pw)
-        SYNAPSEML-ENABLE-PUBLISH: true
+        SYNAPSEML_ENABLE_PUBLISH: true
 
 - job: E2E
   timeoutInMinutes: 120
@@ -161,7 +163,7 @@ jobs:
         PGP-PRIVATE: $(pgp-private)
         PGP-PUBLIC: $(pgp-public)
         PGP-PW: $(pgp-pw)
-        SYNAPSEML-ENABLE-PUBLISH: true
+        SYNAPSEML_ENABLE_PUBLISH: true
     - task: AzureCLI@2
       displayName: 'E2E'
       inputs:
@@ -318,13 +320,13 @@ jobs:
         PGP-PUBLIC: $(pgp-public)
         PGP-PW: $(pgp-pw)
         PYPI-API-TOKEN: $(pypi-api-token)
-        SYNAPSEML-ENABLE-PUBLISH: true
+        SYNAPSEML_ENABLE_PUBLISH: true
       displayName: 'publish python package to pypi'
     - bash: |
         set -e
         source activate synapseml
-        sbt publishSigned
-        sbt sonatypeBundleRelease
+        sbt publishLocalSigned
+        python tools/esrp/prepare_jar.py
       condition: and(eq(variables.isMaster, true), startsWith(variables['tag'], 'v'))
       env:
         STORAGE-KEY: $(storage-key)
@@ -333,8 +335,22 @@ jobs:
         PGP-PRIVATE: $(pgp-private)
         PGP-PUBLIC: $(pgp-public)
         PGP-PW: $(pgp-pw)
-        SYNAPSEML-ENABLE-PUBLISH: true
+        SYNAPSEML_ENABLE_PUBLISH: true
       displayName: 'publish jar package to maven central'
+    - task: EsrpRelease@3
+      inputs:
+        ConnectedServiceName: 'DataScienceESRPRelease'
+        Intent: 'PackageDistribution'
+        ContentType: 'Maven'
+        PackageLocation: '/home/vsts/.ivy2/local/com.microsoft.azure/'
+        Owners: '[email protected],[email protected],[email protected],[email protected],[email protected],[email protected]'
+        Approvers: '[email protected],[email protected],[email protected]'
+        ServiceEndpointUrl: 'https://api.esrp.microsoft.com'
+        MainPublisher: 'synapseml'
+        DomainTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47'
+      displayName: 'ESRP Publish Package'
+      condition: and(eq(variables.isMaster, true), startsWith(variables['tag'], 'v'))
+
 
 - job: PythonTests
   timeoutInMinutes: 120
@@ -451,7 +467,7 @@ jobs:
 #          (timeout 5m sbt setup) || (echo "retrying" && timeout 5m sbt setup) || (echo "retrying" && timeout 5m sbt setup)
 #          (sbt "project $(PACKAGE)" coverage testDotnet) || (sbt "project $(PACKAGE)" coverage testDotnet) || (sbt "project $(PACKAGE)" coverage testDotnet)
 #      env:
-#        SYNAPSEML-ENABLE-PUBLISH: true
+#        SYNAPSEML_ENABLE_PUBLISH: true
 #    - task: PublishTestResults@2
 #      displayName: 'Publish Test Results'
 #      inputs:

project/Secrets.scala

@@ -14,7 +14,7 @@ object Secrets {
   private val KvName = "mmlspark-keys"
   private val SubscriptionID = "e342c2c0-f844-4b18-9208-52c8c234c30e"
   private val PgpFileExtension = ".asc"
-  private val EnablePublishEnvVar = "SYNAPSEML-ENABLE-PUBLISH"
+  private val EnablePublishEnvVar = "SYNAPSEML_ENABLE_PUBLISH"
 
   lazy private val publishingEnabled: Boolean = sys.env.getOrElse(EnablePublishEnvVar, "false").toBoolean
 

tools/esrp/prepare_jar.py

@@ -0,0 +1,80 @@
+import getpass
+import os
+import shutil
+import glob
+
+current_username = getpass.getuser()
+
+root_dir = f"/home/{current_username}/.ivy2/local/com.microsoft.azure/"
+
+
+def find_second_level_folder(root):
+    # Walk through the root directory
+    for foldername, subfolders, filenames in os.walk(root):
+        # Check if the current folder is a second-level folder by comparing its depth to the root's depth
+        if foldername.count(os.path.sep) == root.count(os.path.sep) + 1:
+            # Return the name of the second-level folder
+            return os.path.basename(foldername)
+    # Return None if no such folder is found
+    return None
+
+
+version = find_second_level_folder(root_dir)
+
+
+def flatten_dir(top_dir):
+    # Collect directories to delete
+    directories_to_delete = []
+
+    # Walk through all subdirectories
+    for foldername, subfolders, filenames in os.walk(top_dir, topdown=False):
+
+        # If we are not in the top-level directory, move files to the top-level directory
+        if foldername != top_dir:
+            for filename in filenames:
+                source = os.path.join(foldername, filename)
+                destination = os.path.join(top_dir, filename)
+
+                # Check if a file with the same name already exists in the top-level directory
+                if os.path.exists(destination):
+                    base, ext = os.path.splitext(filename)
+                    counter = 1
+                    new_destination = os.path.join(top_dir, f"{base}_{counter}{ext}")
+
+                    # Find a new destination path that does not exist yet
+                    while os.path.exists(new_destination):
+                        counter += 1
+                        new_destination = os.path.join(
+                            top_dir, f"{base}_{counter}{ext}"
+                        )
+
+                    destination = new_destination
+
+                # Move file
+                shutil.move(source, destination)
+                print(f"Moved: {source} to {destination}")
+
+            # Add the foldername to the list of directories to delete
+            directories_to_delete.append(foldername)
+
+    # Delete the old subdirectories
+    for directory in directories_to_delete:
+        os.rmdir(directory)
+        print(f"Deleted: {directory}")
+
+
+for top_dir in os.listdir(root_dir):
+    path_to_jars = os.path.join(root_dir, top_dir)
+    flatten_dir(path_to_jars)
+
+    for file in os.listdir(path_to_jars):
+        if "_2.12" in file and version not in file:
+            old_file_path = os.path.join(path_to_jars, file)
+            name_parts = file.split("_2.12")
+            if name_parts[1].startswith(".") or name_parts[1].startswith("-"):
+                sep_char = ""
+            else:
+                sep_char = "-"
+            new_file = f"{name_parts[0]}_2.12-{version}{sep_char}{name_parts[1]}"
+            new_file_path = os.path.join(path_to_jars, new_file)
+            shutil.move(old_file_path, new_file_path)