From 7e7d1a80eb73ebdfb15ae1eed51aaf9b84354e8e Mon Sep 17 00:00:00 2001 From: Mark Hamilton Date: Tue, 26 Sep 2023 14:54:26 -0400 Subject: [PATCH 1/8] chore: add esrp pipeline --- pipeline.yaml | 13 ++++++++ tools/esrp/prepare_jar.py | 65 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 tools/esrp/prepare_jar.py diff --git a/pipeline.yaml b/pipeline.yaml index b7a1bc4532..87f1c7cdd5 100644 --- a/pipeline.yaml +++ b/pipeline.yaml @@ -87,6 +87,19 @@ jobs: inputs: artifactsFeeds: SynapseML_PublicPackages mavenServiceConnections: SynapseML_PublicPackages-Feed-Connection + - bash: | + set -e + sbt publishSigned + python tools/esrp/prepare_jar.py + displayName: ESRP release + env: + STORAGE-KEY: $(storage-key) + NEXUS-UN: $(nexus-un) + NEXUS-PW: $(nexus-pw) + PGP-PRIVATE: $(pgp-private) + PGP-PUBLIC: $(pgp-public) + PGP-PW: $(pgp-pw) + SYNAPSEML-ENABLE-PUBLISH: true - bash: | set -e sudo apt-get install graphviz doxygen -y diff --git a/tools/esrp/prepare_jar.py b/tools/esrp/prepare_jar.py new file mode 100644 index 0000000000..4d1e6d6426 --- /dev/null +++ b/tools/esrp/prepare_jar.py @@ -0,0 +1,65 @@ +import getpass +import os +import shutil + +current_username = getpass.getuser() + +root_dir = f"/home/{current_username}/.ivy2/local/com.microsoft.azure/" + + +def flatten_dir(top_dir): + # Collect directories to delete + directories_to_delete = [] + + # Walk through all subdirectories + for foldername, subfolders, filenames in os.walk(top_dir, topdown=False): + + # If we are not in the top-level directory, move files to the top-level directory + if foldername != top_dir: + for filename in filenames: + source = os.path.join(foldername, filename) + destination = os.path.join(top_dir, filename) + + # Check if a file with the same name already exists in the top-level directory + if os.path.exists(destination): + base, ext = os.path.splitext(filename) + counter = 1 + new_destination = os.path.join(top_dir, f"{base}_{counter}{ext}") + + # Find a new destination path that does not exist yet + while os.path.exists(new_destination): + counter += 1 + new_destination = os.path.join(top_dir, f"{base}_{counter}{ext}") + + destination = new_destination + + # Move file + shutil.move(source, destination) + print(f"Moved: {source} to {destination}") + + # Add the foldername to the list of directories to delete + directories_to_delete.append(foldername) + + # Delete the old subdirectories + for directory in directories_to_delete: + os.rmdir(directory) + print(f"Deleted: {directory}") + + +for top_dir in os.listdir(root_dir): + path_to_jars = os.path.join(root_dir, top_dir) + flatten_dir(path_to_jars) + + for file in os.listdir(path_to_jars): + if "_2.12" in file and top_dir not in file: + old_file_path = os.path.join(path_to_jars, file) + name_parts = file.split("_2.12") + if name_parts[1].startswith(".") or name_parts[1].startswith("-"): + sep_char = "" + else: + sep_char = "-" + new_file = f"{name_parts[0]}_2.12-{top_dir}{sep_char}{name_parts[1]}" + new_file_path = os.path.join(path_to_jars, new_file) + shutil.move(old_file_path, new_file_path) + + From 3a7c5a75d19b39aa09fc3f4ae4f23a446b8f1d02 Mon Sep 17 00:00:00 2001 From: Mark Hamilton Date: Tue, 26 Sep 2023 14:59:43 -0400 Subject: [PATCH 2/8] wip --- pipeline.yaml | 16 ++++++++-------- project/Secrets.scala | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pipeline.yaml b/pipeline.yaml index 87f1c7cdd5..e9795c17c4 100644 --- a/pipeline.yaml +++ b/pipeline.yaml @@ -99,7 +99,7 @@ jobs: PGP-PRIVATE: $(pgp-private) PGP-PUBLIC: $(pgp-public) PGP-PW: $(pgp-pw) - SYNAPSEML-ENABLE-PUBLISH: true + SYNAPSEML_ENABLE_PUBLISH: true - bash: | set -e sudo apt-get install graphviz doxygen -y @@ -116,7 +116,7 @@ jobs: PGP-PRIVATE: $(pgp-private) PGP-PUBLIC: $(pgp-public) PGP-PW: $(pgp-pw) - SYNAPSEML-ENABLE-PUBLISH: true + SYNAPSEML_ENABLE_PUBLISH: true - bash: | set -e sbt aetherDeploy @@ -125,7 +125,7 @@ jobs: ADO-FEED-TOKEN: $(ado-feed-token) STORAGE-KEY: $(storage-key) PUBLISH-TO-FEED: true - SYNAPSEML-ENABLE-PUBLISH: true + SYNAPSEML_ENABLE_PUBLISH: true - bash: | set -e sbt publishBadges @@ -138,7 +138,7 @@ jobs: PGP-PRIVATE: $(pgp-private) PGP-PUBLIC: $(pgp-public) PGP-PW: $(pgp-pw) - SYNAPSEML-ENABLE-PUBLISH: true + SYNAPSEML_ENABLE_PUBLISH: true - job: E2E timeoutInMinutes: 120 @@ -174,7 +174,7 @@ jobs: PGP-PRIVATE: $(pgp-private) PGP-PUBLIC: $(pgp-public) PGP-PW: $(pgp-pw) - SYNAPSEML-ENABLE-PUBLISH: true + SYNAPSEML_ENABLE_PUBLISH: true - task: AzureCLI@2 displayName: 'E2E' inputs: @@ -331,7 +331,7 @@ jobs: PGP-PUBLIC: $(pgp-public) PGP-PW: $(pgp-pw) PYPI-API-TOKEN: $(pypi-api-token) - SYNAPSEML-ENABLE-PUBLISH: true + SYNAPSEML_ENABLE_PUBLISH: true displayName: 'publish python package to pypi' - bash: | set -e @@ -346,7 +346,7 @@ jobs: PGP-PRIVATE: $(pgp-private) PGP-PUBLIC: $(pgp-public) PGP-PW: $(pgp-pw) - SYNAPSEML-ENABLE-PUBLISH: true + SYNAPSEML_ENABLE_PUBLISH: true displayName: 'publish jar package to maven central' - job: PythonTests @@ -464,7 +464,7 @@ jobs: # (timeout 5m sbt setup) || (echo "retrying" && timeout 5m sbt setup) || (echo "retrying" && timeout 5m sbt setup) # (sbt "project $(PACKAGE)" coverage testDotnet) || (sbt "project $(PACKAGE)" coverage testDotnet) || (sbt "project $(PACKAGE)" coverage testDotnet) # env: -# SYNAPSEML-ENABLE-PUBLISH: true +# SYNAPSEML_ENABLE_PUBLISH: true # - task: PublishTestResults@2 # displayName: 'Publish Test Results' # inputs: diff --git a/project/Secrets.scala b/project/Secrets.scala index c31433af0b..1fc12361c3 100644 --- a/project/Secrets.scala +++ b/project/Secrets.scala @@ -14,7 +14,7 @@ object Secrets { private val KvName = "mmlspark-keys" private val SubscriptionID = "e342c2c0-f844-4b18-9208-52c8c234c30e" private val PgpFileExtension = ".asc" - private val EnablePublishEnvVar = "SYNAPSEML-ENABLE-PUBLISH" + private val EnablePublishEnvVar = "SYNAPSEML_ENABLE_PUBLISH" lazy private val publishingEnabled: Boolean = sys.env.getOrElse(EnablePublishEnvVar, "false").toBoolean From b012372ea6852c6eed71dcae7a6aefdf94bcb2a5 Mon Sep 17 00:00:00 2001 From: Mark Hamilton Date: Tue, 26 Sep 2023 15:05:57 -0400 Subject: [PATCH 3/8] wip --- tools/esrp/prepare_jar.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/esrp/prepare_jar.py b/tools/esrp/prepare_jar.py index 4d1e6d6426..77edddec7f 100644 --- a/tools/esrp/prepare_jar.py +++ b/tools/esrp/prepare_jar.py @@ -1,11 +1,15 @@ import getpass import os import shutil +import glob current_username = getpass.getuser() root_dir = f"/home/{current_username}/.ivy2/local/com.microsoft.azure/" +version = list(glob.glob(os.path.join(root_dir, "*", "*")))[0].split("/")[-2] + +print(version) def flatten_dir(top_dir): # Collect directories to delete @@ -51,14 +55,14 @@ def flatten_dir(top_dir): flatten_dir(path_to_jars) for file in os.listdir(path_to_jars): - if "_2.12" in file and top_dir not in file: + if "_2.12" in file and version not in file: old_file_path = os.path.join(path_to_jars, file) name_parts = file.split("_2.12") if name_parts[1].startswith(".") or name_parts[1].startswith("-"): sep_char = "" else: sep_char = "-" - new_file = f"{name_parts[0]}_2.12-{top_dir}{sep_char}{name_parts[1]}" + new_file = f"{name_parts[0]}_2.12-{version}{sep_char}{name_parts[1]}" new_file_path = os.path.join(path_to_jars, new_file) shutil.move(old_file_path, new_file_path) From de5529f92a911cf271d90bffd4d03a7043d1f5d6 Mon Sep 17 00:00:00 2001 From: Mark Hamilton Date: Tue, 26 Sep 2023 15:19:56 -0400 Subject: [PATCH 4/8] wip --- tools/esrp/prepare_jar.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tools/esrp/prepare_jar.py b/tools/esrp/prepare_jar.py index 77edddec7f..853ca7e31e 100644 --- a/tools/esrp/prepare_jar.py +++ b/tools/esrp/prepare_jar.py @@ -7,9 +7,20 @@ root_dir = f"/home/{current_username}/.ivy2/local/com.microsoft.azure/" -version = list(glob.glob(os.path.join(root_dir, "*", "*")))[0].split("/")[-2] -print(version) +def find_second_level_folder(root): + # Walk through the root directory + for foldername, subfolders, filenames in os.walk(root): + # Check if the current folder is a second-level folder by comparing its depth to the root's depth + if foldername.count(os.path.sep) == root.count(os.path.sep) + 1: + # Return the name of the second-level folder + return os.path.basename(foldername) + # Return None if no such folder is found + return None + + +version = find_second_level_folder(root_dir) + def flatten_dir(top_dir): # Collect directories to delete @@ -65,5 +76,3 @@ def flatten_dir(top_dir): new_file = f"{name_parts[0]}_2.12-{version}{sep_char}{name_parts[1]}" new_file_path = os.path.join(path_to_jars, new_file) shutil.move(old_file_path, new_file_path) - - From 7ebec2c5af06b68862c4f4d767ff7a9f284331f6 Mon Sep 17 00:00:00 2001 From: Mark Hamilton Date: Tue, 26 Sep 2023 15:24:35 -0400 Subject: [PATCH 5/8] wip --- pipeline.yaml | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/pipeline.yaml b/pipeline.yaml index e9795c17c4..e4ea26e0ee 100644 --- a/pipeline.yaml +++ b/pipeline.yaml @@ -89,9 +89,12 @@ jobs: mavenServiceConnections: SynapseML_PublicPackages-Feed-Connection - bash: | set -e - sbt publishSigned + sudo apt-get install graphviz doxygen -y + source activate synapseml + sbt packagePython + sbt publishLocalSigned python tools/esrp/prepare_jar.py - displayName: ESRP release + displayName: ESRP release prep env: STORAGE-KEY: $(storage-key) NEXUS-UN: $(nexus-un) @@ -100,11 +103,20 @@ jobs: PGP-PUBLIC: $(pgp-public) PGP-PW: $(pgp-pw) SYNAPSEML_ENABLE_PUBLISH: true + - task: EsrpRelease@3 + inputs: + ConnectedServiceName: 'DataScienceESRPRelease' + Intent: 'PackageDistribution' + ContentType: 'Maven' + PackageLocation: '/home/vsts/.ivy2/local/com.microsoft.azure/' + Owners: 'admukund@microsoft.com,richwyd@microsoft.com,taniaarya@microsoft.com,alsavelv@microsoft.com,marcozo@microsoft.com,romanbat@microsoft.com' + Approvers: 'romanbat@microsoft.com,markus.weimer@microsoft.com,negust@microsoft.com' + ServiceEndpointUrl: 'https://api.esrp.microsoft.com' + MainPublisher: 'synapseml' + DomainTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + displayName: 'ESRP Publish Package' - bash: | set -e - sudo apt-get install graphviz doxygen -y - source activate synapseml - sbt packagePython sbt publishBlob publishDocs publishR publishPython uploadNotebooks sbt genBuildInfo echo "##vso[task.uploadsummary]$(pwd)/target/Build.md" From 4b1fb63aa00ee81279599242a5e4afdd254edb58 Mon Sep 17 00:00:00 2001 From: Mark Hamilton Date: Tue, 26 Sep 2023 15:29:46 -0400 Subject: [PATCH 6/8] wip --- pipeline.yaml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/pipeline.yaml b/pipeline.yaml index e4ea26e0ee..bfb8349c3d 100644 --- a/pipeline.yaml +++ b/pipeline.yaml @@ -103,18 +103,6 @@ jobs: PGP-PUBLIC: $(pgp-public) PGP-PW: $(pgp-pw) SYNAPSEML_ENABLE_PUBLISH: true - - task: EsrpRelease@3 - inputs: - ConnectedServiceName: 'DataScienceESRPRelease' - Intent: 'PackageDistribution' - ContentType: 'Maven' - PackageLocation: '/home/vsts/.ivy2/local/com.microsoft.azure/' - Owners: 'admukund@microsoft.com,richwyd@microsoft.com,taniaarya@microsoft.com,alsavelv@microsoft.com,marcozo@microsoft.com,romanbat@microsoft.com' - Approvers: 'romanbat@microsoft.com,markus.weimer@microsoft.com,negust@microsoft.com' - ServiceEndpointUrl: 'https://api.esrp.microsoft.com' - MainPublisher: 'synapseml' - DomainTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' - displayName: 'ESRP Publish Package' - bash: | set -e sbt publishBlob publishDocs publishR publishPython uploadNotebooks @@ -348,8 +336,8 @@ jobs: - bash: | set -e source activate synapseml - sbt publishSigned - sbt sonatypeBundleRelease + sbt publishLocalSigned + python tools/esrp/prepare_jar.py condition: and(eq(variables.isMaster, true), startsWith(variables['tag'], 'v')) env: STORAGE-KEY: $(storage-key) @@ -360,6 +348,18 @@ jobs: PGP-PW: $(pgp-pw) SYNAPSEML_ENABLE_PUBLISH: true displayName: 'publish jar package to maven central' + - task: EsrpRelease@3 + inputs: + ConnectedServiceName: 'DataScienceESRPRelease' + Intent: 'PackageDistribution' + ContentType: 'Maven' + PackageLocation: '/home/vsts/.ivy2/local/com.microsoft.azure/' + Owners: 'admukund@microsoft.com,richwyd@microsoft.com,taniaarya@microsoft.com,alsavelv@microsoft.com,marcozo@microsoft.com,romanbat@microsoft.com' + Approvers: 'romanbat@microsoft.com,markus.weimer@microsoft.com,negust@microsoft.com' + ServiceEndpointUrl: 'https://api.esrp.microsoft.com' + MainPublisher: 'synapseml' + DomainTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + displayName: 'ESRP Publish Package' - job: PythonTests timeoutInMinutes: 120 From ccdb50a3720350d3133e0e18a7c01a8423146dc9 Mon Sep 17 00:00:00 2001 From: Mark Hamilton Date: Tue, 26 Sep 2023 15:31:13 -0400 Subject: [PATCH 7/8] wip --- pipeline.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pipeline.yaml b/pipeline.yaml index bfb8349c3d..bcfa3e0840 100644 --- a/pipeline.yaml +++ b/pipeline.yaml @@ -103,6 +103,18 @@ jobs: PGP-PUBLIC: $(pgp-public) PGP-PW: $(pgp-pw) SYNAPSEML_ENABLE_PUBLISH: true + - task: EsrpRelease@3 + inputs: + ConnectedServiceName: 'DataScienceESRPRelease' + Intent: 'PackageDistribution' + ContentType: 'Maven' + PackageLocation: '/home/vsts/.ivy2/local/com.microsoft.azure/' + Owners: 'admukund@microsoft.com,richwyd@microsoft.com,taniaarya@microsoft.com,alsavelv@microsoft.com,marcozo@microsoft.com,romanbat@microsoft.com' + Approvers: 'romanbat@microsoft.com,markus.weimer@microsoft.com,negust@microsoft.com' + ServiceEndpointUrl: 'https://api.esrp.microsoft.com' + MainPublisher: 'synapseml' + DomainTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + displayName: 'ESRP Publish Package' - bash: | set -e sbt publishBlob publishDocs publishR publishPython uploadNotebooks @@ -360,6 +372,8 @@ jobs: MainPublisher: 'synapseml' DomainTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' displayName: 'ESRP Publish Package' + condition: and(eq(variables.isMaster, true), startsWith(variables['tag'], 'v')) + - job: PythonTests timeoutInMinutes: 120 From 5963d2ecd4e0fa8c96ae6946742de0ab2d724799 Mon Sep 17 00:00:00 2001 From: Mark Hamilton Date: Tue, 26 Sep 2023 16:26:13 -0400 Subject: [PATCH 8/8] wip --- pipeline.yaml | 27 ++------------------------- tools/esrp/prepare_jar.py | 4 +++- 2 files changed, 5 insertions(+), 26 deletions(-) diff --git a/pipeline.yaml b/pipeline.yaml index bcfa3e0840..c2dc70c165 100644 --- a/pipeline.yaml +++ b/pipeline.yaml @@ -92,34 +92,11 @@ jobs: sudo apt-get install graphviz doxygen -y source activate synapseml sbt packagePython - sbt publishLocalSigned - python tools/esrp/prepare_jar.py - displayName: ESRP release prep - env: - STORAGE-KEY: $(storage-key) - NEXUS-UN: $(nexus-un) - NEXUS-PW: $(nexus-pw) - PGP-PRIVATE: $(pgp-private) - PGP-PUBLIC: $(pgp-public) - PGP-PW: $(pgp-pw) - SYNAPSEML_ENABLE_PUBLISH: true - - task: EsrpRelease@3 - inputs: - ConnectedServiceName: 'DataScienceESRPRelease' - Intent: 'PackageDistribution' - ContentType: 'Maven' - PackageLocation: '/home/vsts/.ivy2/local/com.microsoft.azure/' - Owners: 'admukund@microsoft.com,richwyd@microsoft.com,taniaarya@microsoft.com,alsavelv@microsoft.com,marcozo@microsoft.com,romanbat@microsoft.com' - Approvers: 'romanbat@microsoft.com,markus.weimer@microsoft.com,negust@microsoft.com' - ServiceEndpointUrl: 'https://api.esrp.microsoft.com' - MainPublisher: 'synapseml' - DomainTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' - displayName: 'ESRP Publish Package' - - bash: | - set -e sbt publishBlob publishDocs publishR publishPython uploadNotebooks sbt genBuildInfo echo "##vso[task.uploadsummary]$(pwd)/target/Build.md" + sbt publishLocalSigned + python tools/esrp/prepare_jar.py displayName: Publish Artifacts env: STORAGE-KEY: $(storage-key) diff --git a/tools/esrp/prepare_jar.py b/tools/esrp/prepare_jar.py index 853ca7e31e..26cadd0953 100644 --- a/tools/esrp/prepare_jar.py +++ b/tools/esrp/prepare_jar.py @@ -44,7 +44,9 @@ def flatten_dir(top_dir): # Find a new destination path that does not exist yet while os.path.exists(new_destination): counter += 1 - new_destination = os.path.join(top_dir, f"{base}_{counter}{ext}") + new_destination = os.path.join( + top_dir, f"{base}_{counter}{ext}" + ) destination = new_destination