Fix CI for migration (#764)

Signed-off-by: Clement Fuji Tsang <[email protected]>
NVIDIAGameWorks · Oct 31, 2023 · 8c331cc · 8c331cc
1 parent 0e0a802
commit 8c331cc
Show file tree

Hide file tree

Showing 11 changed files with 92 additions and 31 deletions.
diff --git a/ci/gitlab_jenkins_templates/ubuntu_build_CI.jenkins b/ci/gitlab_jenkins_templates/ubuntu_build_CI.jenkins
@@ -56,12 +56,12 @@ spec:
                   trackingSubmodules: false
               ]],
               userRemoteConfigs: [[
-                  credentialsId: 'gitlab-credentials',
+                  credentialsId: 'kaolin-gitlab-access-token-as-password',
                   url: "${repoUrl}"
               ]]
           ])
         }
-        docker.withRegistry("https://${docker_registry_server}", 'gitlab-credentials') {
+        docker.withRegistry("https://${docker_registry_server}", 'kaolin-gitlab-access-token-as-password') {
           stage("Build") {
             targetImage = docker.build(
                 "${targetImageTag}",

diff --git a/ci/gitlab_jenkins_templates/ubuntu_cpuonly_CI.jenkins b/ci/gitlab_jenkins_templates/ubuntu_cpuonly_CI.jenkins
@@ -39,6 +39,8 @@ spec:
 ''') {
   node(POD_LABEL) {
     container("docker") {
+      // This is to let the time for the docker-daemon to get initialized.
+      sleep 10
       try {
         stage("Checkout") {
           checkout([
@@ -53,7 +55,7 @@ spec:
                   trackingSubmodules: false
               ]],
               userRemoteConfigs: [[
-                  credentialsId: 'gitlab-credentials',
+                  credentialsId: 'kaolin-gitlab-access-token-as-password',
                   url: "${repoUrl}"
               ]]
           ])

diff --git a/ci/gitlab_jenkins_templates/ubuntu_custom_build_CI.jenkins b/ci/gitlab_jenkins_templates/ubuntu_custom_build_CI.jenkins
@@ -65,12 +65,12 @@ spec:
                     trackingSubmodules: false
                 ]],
                 userRemoteConfigs: [[
-                    credentialsId: 'gitlab-credentials',
+                    credentialsId: 'kaolin-gitlab-access-token-as-password',
                     url: "${repoUrl}"
                 ]]
             ])
           }
-          docker.withRegistry("https://${docker_registry_server}", 'gitlab-credentials') {
+          docker.withRegistry("https://${docker_registry_server}", 'kaolin-gitlab-access-token-as-password') {
             stage("Build") {
               baseImage = docker.build(
                   "${targetImageTag}-base",

diff --git a/ci/gitlab_jenkins_templates/ubuntu_test_CI.jenkins b/ci/gitlab_jenkins_templates/ubuntu_test_CI.jenkins
@@ -27,6 +27,10 @@ if (arch == "MULTI") {
     """
 }
 
+node_blacklist = """
+            - "a4u8g-0031.ipp1u1.colossus"
+"""
+
 gitlabCommitStatus("test-${configName}-${arch}") {
 
 podTemplate(
@@ -54,7 +58,7 @@ spec:
       - mountPath: /mnt
         name: pvc-mount
   imagePullSecrets:
-  - name: gitlabregcred
+  - name: gitlabcred
   nodeSelector:
     kubernetes.io/os: linux
   affinity:
@@ -65,6 +69,9 @@ spec:
           - key: "nvidia.com/gpu_type"
             operator: "In"
             values:${gpu_list}
+          - key: "kubernetes.io/hostname"
+            operator: "NotIn"
+            values:${node_blacklist}
 """) {
   node(POD_LABEL) {
     container("docker") {
@@ -78,6 +85,8 @@ spec:
           stage('Disp info') {
             sh 'nvidia-smi'
             sh 'python --version'
+            sh 'lscpu'
+            sh 'free -h --si'
           }
         } catch(e) {
           build_passed = false
@@ -91,7 +100,11 @@ spec:
             export KAOLIN_TEST_SHAPENETV2_PATH=/mnt/data/ci_shapenetv2
             export KAOLIN_TEST_MODELNET_PATH=/mnt/data/ModelNet
             export KAOLIN_TEST_SHREC16_PATH=/mnt/data/ci_shrec16
-            pytest --import-mode=importlib -rs --cov=/kaolin/kaolin /kaolin/tests/python/kaolin/
+            pytest --durations=50 --import-mode=importlib -rs --cov=/kaolin/kaolin \
+                --log-disable=PIL.PngImagePlugin \
+                --log-disable=PIL.TiffImagePlugin \
+                --log-disable=kaolin.rep.surface_mesh \
+                /kaolin/tests/python/kaolin
             '''
           }
         } catch(e) {
@@ -123,6 +136,23 @@ spec:
           build_passed = false
           echo e.toString()
         }
+        // TUTORIALS
+        try {
+          stage("BBox Tutorial") {
+            sh 'cd /kaolin/examples/tutorial && ipython bbox_tutorial.ipynb'
+          }
+        } catch(e) {
+          build_passed = false
+          echo e.toString()
+        }
+        try {
+          stage("Camera and Rasterization Tutorial") {
+            sh 'cd /kaolin/examples/tutorial && ipython camera_and_rasterization.ipynb'
+          }
+        } catch(e) {
+          build_passed = false
+          echo e.toString()
+        }
         try {
           stage("DIB-R Tutorial") {
             sh 'cd /kaolin/examples/tutorial && ipython dibr_tutorial.ipynb'
@@ -132,32 +162,32 @@ spec:
           echo e.toString()
         }
         try {
-          stage("DMTet Tutorial") {
-            sh 'cd /kaolin/examples/tutorial && ipython dmtet_tutorial.ipynb'
+          stage("Diffuse lighting Tutorial") {
+            sh 'cd /kaolin/examples/tutorial && ipython diffuse_lighting.ipynb'
           }
         } catch(e) {
           build_passed = false
           echo e.toString()
         }
         try {
-          stage("Understanding SPCs Tutorial") {
-            sh 'cd /kaolin/examples/tutorial && ipython understanding_spcs_tutorial.ipynb'
+          stage("DMTet Tutorial") {
+            sh 'cd /kaolin/examples/tutorial && ipython dmtet_tutorial.ipynb'
           }
         } catch(e) {
           build_passed = false
           echo e.toString()
         }
         try {
-          stage("Camera and Rasterization Tutorial") {
-            sh 'cd /kaolin/examples/tutorial && ipython camera_and_rasterization.ipynb'
+          stage("GLTF Visualizer") {
+            sh 'cd /kaolin/examples/tutorial && ipython gltf_viz.ipynb'
           }
         } catch(e) {
           build_passed = false
           echo e.toString()
         }
         try {
-          stage("Diffuse lighting Tutorial") {
-            sh 'cd /kaolin/examples/tutorial && ipython diffuse_lighting.ipynb'
+          stage("Interactive Visualizer") {
+            sh 'cd /kaolin/examples/tutorial && ipython interactive_visualizer.ipynb'
           }
         } catch(e) {
           build_passed = false
@@ -172,13 +202,23 @@ spec:
           echo e.toString()
         }
         try {
-          stage("Interactive Visualizer") {
-            sh 'cd /kaolin/examples/tutorial && ipython interactive_visualizer.ipynb'
+          stage("Understanding SPCs Tutorial") {
+            sh 'cd /kaolin/examples/tutorial && ipython understanding_spcs_tutorial.ipynb'
+          }
+        } catch(e) {
+          build_passed = false
+          echo e.toString()
+        }
+        try {
+          stage("Working with meshes Tutorial") {
+            sh 'cd /kaolin/examples/tutorial && ipython working_with_meshes.ipynb'
           }
         } catch(e) {
           build_passed = false
           echo e.toString()
         }
+
+        // RECIPES
         try {
           stage("SPC from Pointcloud Recipe") {
             sh 'cd /kaolin/examples/recipes/dataload/ && python spc_from_pointcloud.py'

diff --git a/ci/gitlab_jenkins_templates/windows_build_CI.jenkins b/ci/gitlab_jenkins_templates/windows_build_CI.jenkins
@@ -24,7 +24,7 @@ currentBuild.description = sourceBranch + ": " + commitHash
 gitlabCommitStatus("build-${configName}") {
 
 podTemplate(
-cloud:'sc-ipp-blossom-prod',
+cloud:'sc-ipp-blossom-116',
 envVars:[envVar(key:"JENKINS_URL", value:"${env.JENKINS_URL}")],
 yaml:'''
 apiVersion: v1
@@ -53,7 +53,7 @@ spec:
         limits:
           memory: 32Gi
   imagePullSecrets:
-  - name: gitlabregcred
+  - name: gitlabcred
   nodeSelector:
     kubernetes.io/os: windows
 ''') 
@@ -75,12 +75,12 @@ spec:
                   trackingSubmodules: false
               ]],
               userRemoteConfigs: [[
-                  credentialsId: 'gitlab-credentials',
+                  credentialsId: 'kaolin-gitlab-access-token-as-password',
                   url: "${repoUrl}"
               ]]
           ])
         }
-        docker.withRegistry("https://${docker_registry_server}", 'gitlab-credentials') {
+        docker.withRegistry("https://${docker_registry_server}", 'kaolin-gitlab-access-token-as-password') {
           stage("Build") {
             cudaUrl = cuda_version_url[cudaVer]
             targetImage = docker.build(

diff --git a/ci/gitlab_jenkins_templates/windows_test_CI.jenkins b/ci/gitlab_jenkins_templates/windows_test_CI.jenkins
@@ -44,7 +44,7 @@ spec:
       - mountPath: c:/mnt
         name: pvc-mount
   imagePullSecrets:
-  - name: gitlabregcred
+  - name: gitlabcred
   nodeSelector:
     kubernetes.io/os: windows
     nvidia.com/node_type: ${arch}

diff --git a/examples/tutorial/bbox_tutorial.ipynb b/examples/tutorial/bbox_tutorial.ipynb
@@ -16,15 +16,23 @@
     "Before starting the tutorial uncompress the training data found in [examples/samples/rendered_clock.zip](examples/samples/rendered_clock.zip)."
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "98082a4a-21da-4774-a5f5-02c17f8e102a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!pip install -q matplotlib"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 1,
    "id": "736eb3ef",
    "metadata": {},
    "outputs": [],
    "source": [
-    "!pip install -q matplotlib\n",
-    "%matplotlib notebook\n",
     "\n",
     "import glob\n",
     "import os\n",
@@ -36,6 +44,14 @@
     "import torch\n",
     "from torch import Tensor\n",
     "from matplotlib import pyplot as plt\n",
+    "\n",
+    "try:\n",
+    "    ipy_str = str(type(get_ipython()))\n",
+    "    if 'zmqshell' in ipy_str:\n",
+    "        %matplotlib notebook\n",
+    "finally:\n",
+    "    pass\n",
+    "\n",
     "from torch.utils.data import DataLoader\n",
     "\n",
     "import kaolin"
@@ -1590,7 +1606,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -1604,7 +1620,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.13"
+   "version": "3.8.16"
   }
  },
  "nbformat": 4,

diff --git a/tests/python/kaolin/ops/spc/test_spc.py b/tests/python/kaolin/ops/spc/test_spc.py
@@ -376,11 +376,12 @@ def test_to_dense(self, batch_size, max_level, feature_dim):
             grad_out, torch.any(feature_grids != 0, dim=1))
         assert torch.equal(coalescent_features.grad, coalescent_expected_grad)
 
-@pytest.mark.parametrize('device', ['cpu','cuda'])
+@pytest.mark.parametrize('device,height,width,depth,threshold', [
+                         ('cpu', 2, 2, 2, 0.1),
+                         ('cuda', 2, 2, 2, 0.1),
+                         ('cuda', 113, 251, 251, 0.9)])
 @pytest.mark.parametrize('batch_size', [1, 5])
 @pytest.mark.parametrize('feature_dim', [1, 3])
-@pytest.mark.parametrize('height,width,depth,threshold',
-                         [(2, 2, 2, 0.1), (113, 251, 251, 0.9)])
 @pytest.mark.parametrize('dtype', [torch.float])
 class TestCycleConversionsFeatureGrids:
     @pytest.fixture(autouse=True)

diff --git a/tests/samples/rep/textures/toppings.png b/tests/samples/rep/textures/toppings.png
@@ -0,0 +1 @@
+../../../../sample_data/meshes/textures/toppings.png
diff --git a/tools/ci_requirements.txt b/tools/ci_requirements.txt
@@ -3,7 +3,7 @@ flake8-bugbear==20.1.4
 flake8-comprehensions==3.2.2
 flake8-mypy==17.8.0
 flake8-pyi==19.3.0
-pytest==7.1.0
+pytest==7.3.0
 pytest-cov==3.0.0
 nbmake==1.4.1
 jupyter
diff --git a/tools/linux/Dockerfile.base_cpuonly b/tools/linux/Dockerfile.base_cpuonly
@@ -1,4 +1,4 @@
-FROM ubuntu:20.04
+FROM ubuntu:18.04
 # used for cross-compilation in docker build
 #
 ARG PYTHON_VERSION=3.9
@@ -53,4 +53,5 @@ RUN conda list > conda_build.txt
 ### Install Dash3D Requirements ###
 RUN npm install -g [email protected]
 COPY package.json package-lock.json ./
+RUN chown -R root package.json package-lock.json
 RUN npm install
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		../../../../sample_data/meshes/textures/toppings.png