Refactor conda build script to publish all binaries from ubuntu machi…

…ne (#2561) * refactor conda build * refactor conda build * change workflow to use only ubuntu * changed logic to skip conversion for linux-64 * changed logic to skip conversion for linux-64 * changed to use dry run * added link to conda convert * review comment * review comment --------- Co-authored-by: agunapal <[email protected]>
pytorch · Aug 31, 2023 · 16242d6 · 16242d6
1 parent 242895c
commit 16242d6
Show file tree

Hide file tree

Showing 3 changed files with 30 additions and 25 deletions.
diff --git a/.github/workflows/official_release.yml b/.github/workflows/official_release.yml
@@ -4,16 +4,10 @@ on: workflow_dispatch
 
 jobs:
   official-release:
-    # creates workflows for the 3 OS: ubuntu, macOS & windows
-    runs-on: ${{ matrix.os }}
-
+    runs-on: ubuntu-20.04
     # The official-release environment requires 2 manual approvals
     environment:
       name: official-release
-    strategy:
-      fail-fast: true
-      matrix:
-        os: [ubuntu-20.04, macOS-latest, windows-latest]
     steps:
       - name: Setup Conda
         uses: s-weigand/setup-conda@v1
@@ -41,21 +35,17 @@ jobs:
       - name: Push PyPI binaries
         env:
           TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
-        if: matrix.os == 'ubuntu-20.04'
         run: python binaries/upload.py --upload-pypi-packages
       - name: Login to Docker
         env:
           DOCKER_PASSWORD: ${{secrets.DOCKER_PASSWORD}}
-        if: matrix.os == 'ubuntu-20.04'
         run: docker login --username pytorchbot --password "$DOCKER_PASSWORD"
       - name: Build & Upload pytorch/torchserve Docker images
-        if: matrix.os == 'ubuntu-20.04'
         run: |
           cd docker
           python build_upload_release.py
           cd ..
       - name: Build & Upload pytorch/torchserve-kfs Docker images
-        if: matrix.os == 'ubuntu-20.04'
         run: |
           cd kubernetes/kserve
           python build_upload_release.py

diff --git a/.github/workflows/torchserve-nightly-build.yml b/.github/workflows/torchserve-nightly-build.yml
@@ -6,12 +6,7 @@ on:
     - cron:  '15 11 * * *'
 jobs:
   nightly:
-    # creates workflows for the 3 OS: ubuntu, macOS & windows
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: true
-      matrix:
-        os: [ubuntu-20.04, macOS-latest, windows-latest]
+    runs-on: ubuntu-20.04
     steps:
       - name: Setup Conda
         uses: s-weigand/setup-conda@v1
@@ -37,8 +32,6 @@ jobs:
       - name: Push PyPI nightly binaries
         env:
           TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
-        # skip pushing PyPI nightly binaries for macOS & windows as the binaries have the same name
-        if: matrix.os == 'ubuntu-20.04'
         run: python binaries/upload.py --upload-pypi-packages
       - name: Open issue on failure
         if: ${{ failure() && github.event_name  == 'schedule' }}

diff --git a/binaries/conda/build_packages.py b/binaries/conda/build_packages.py
@@ -15,6 +15,15 @@
     else f"$HOME/miniconda/condabin/conda"
 )
 
+CONDA_PACKAGES_PATH = os.path.join(REPO_ROOT, "binaries", "conda", "output")
+CONDA_LINUX_PACKAGES_PATH = os.path.join(
+    REPO_ROOT, "binaries", "conda", "output", "linux-64"
+)
+PACKAGES = ["torchserve", "torch-model-archiver", "torch-workflow-archiver"]
+
+# conda convert supported platforms https://docs.conda.io/projects/conda-build/en/stable/resources/commands/conda-convert.html
+PLATFORMS = ["linux-64", "osx-64", "win-64"]  # Add a new platform here
+
 if os.name == "nt":
     # Assumes miniconda is installed in windows
     CONDA_BINARY = "conda"
@@ -122,18 +131,31 @@ def conda_build(
     os.environ["PYTHON"] = "python"
 
     python_versions = ["3.8", "3.9", "3.10"]
-    packages = [
-        os.path.join(conda_build_dir, pkg)
-        for pkg in ["torchserve", "torch-model-archiver", "torch-workflow-archiver"]
-    ]
+    packages = [os.path.join(conda_build_dir, pkg) for pkg in PACKAGES]
 
+    # Generate conda binaries for linux-64
     for pkg in packages:
         for pyv in python_versions:
             output_dir = os.path.join(conda_build_dir, "output")
             cmd = f"{CONDA_BINARY} build --output-folder {output_dir} --python={pyv} {pkg}"
             print(f"## In directory: {os.getcwd()}; Executing command: {cmd}")
-            if not dry_run:
-                os.system(cmd)
+            try_and_handle(cmd, dry_run)
+
+    # Generate conda binaries for other platforms
+    for file in os.listdir(CONDA_LINUX_PACKAGES_PATH):
+        file_path = os.path.join(CONDA_LINUX_PACKAGES_PATH, file)
+        # Identify *.tar.bz2 files to convert
+        if any(
+            package_name in file_path for package_name in PACKAGES
+        ) and file_path.endswith("tar.bz2"):
+            for platform in PLATFORMS:
+                # Skip linux-64 since it already exists
+                if platform == "linux-64":
+                    continue
+                cmd = f"{CONDA_BINARY} convert {file_path} -p {platform} -o {CONDA_PACKAGES_PATH}"
+                print(f"## In directory: {os.getcwd()}; Executing command: {cmd}")
+                try_and_handle(cmd, dry_run)
+
     return 0  # Used for sys.exit(0) --> to indicate successful system exit