Merge branch 'main' into feat/status-history

lablup · Sep 22, 2023 · f625634 · f625634
2 parents 1d5c6e2 + 81fbc60
commit f625634
Show file tree

Hide file tree

Showing 620 changed files with 45,851 additions and 26,154 deletions.
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -2,3 +2,11 @@
 Please precisely, concisely, and concretely describe what this PR changes, the rationale behind codes,
 and how it affects the users and other developers.
 -->
+
+**Checklist:** (if applicable)
+
+- [ ] Milestone metadata specifying the target backport version
+- [ ] Mention to the original issue
+- [ ] Documentation
+- [ ] API server-client counterparts (e.g., manager API -> client SDK)
+- [ ] Test case(s) to demonstrate the difference of before/after
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -0,0 +1,9 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    allow:
+      - dependency-name: actions/*
+    directory: "/"
+    schedule:
+      interval: "weekly"
+      day: "saturday"
diff --git a/.github/workflows/add-to-project.yml b/.github/workflows/add-to-project.yml
@@ -10,13 +10,13 @@ jobs:
     name: Add issue to project
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/add-to-project@v0.4.0
+      - uses: actions/add-to-project@v0.5.0
         with:
           # You can target a repository in a different organization
           # to the issue
           project-url: https://github.com/orgs/lablup/projects/11
           github-token: ${{ secrets.ADD_TO_PROJECT_PAT }}
-      - uses: actions/add-to-project@v0.4.0
+      - uses: actions/add-to-project@v0.5.0
         with:
           project-url: https://github.com/orgs/lablup/projects/20
           github-token: ${{ secrets.ADD_TO_PROJECT_PAT }}
diff --git a/.github/workflows/alembic-head-check.yml b/.github/workflows/alembic-head-check.yml
@@ -10,7 +10,7 @@ jobs:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'require:db-migration') }}
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 2
     - name: Parse versions from config

diff --git a/.github/workflows/auto-label.yml b/.github/workflows/auto-label.yml
@@ -1,7 +1,7 @@
 name: 'Auto Label'
 
 on:
-  pull_request:
+  pull_request_target:
     types: [labeled, unlabeled, opened, synchronize, reopened]
 
 permissions:
@@ -11,4 +11,4 @@ jobs:
   auto-label:
     runs-on: ubuntu-latest
     steps:
-      - uses: lablup/auto-label-in-issue@1.3.0
+      - uses: lablup/auto-label-in-issue@1.4.0
diff --git a/.github/workflows/check-requirement.yml b/.github/workflows/check-requirement.yml
@@ -7,7 +7,7 @@ jobs:
   check-requirement:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Check if a precompiled binary file exists in dependencies
         run: |
             cat python.lock | grep -v '^//' | jq -r '.locked_resolves[].locked_requirements[]

diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
@@ -12,9 +12,9 @@ on:
 jobs:
   test-coverage:
     if: ${{ contains(fromJson('["schedule", "workflow_dispatch"]'), github.event_name) || (!contains(github.event.pull_request.labels.*.name, 'skip:ci') && github.event.pull_request.merged == true) }}
-    runs-on: ubuntu-latest-8-cores
+    runs-on: [ubuntu-latest-8-cores, self-hosted]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 2
     - name: Create LFS file hash list
@@ -47,9 +47,9 @@ jobs:
       timeout-minutes: 15
       run: |
         # configure redis sentinel cluster hostnames for testing
-        echo "127.0.0.1 node01" | sudo tee -a /etc/hosts
-        echo "127.0.0.1 node02" | sudo tee -a /etc/hosts
-        echo "127.0.0.1 node03" | sudo tee -a /etc/hosts
+        grep -q "127.0.0.1 node01" /etc/hosts || echo "127.0.0.1 node01" | sudo tee -a /etc/hosts
+        grep -q "127.0.0.1 node02" /etc/hosts || echo "127.0.0.1 node02" | sudo tee -a /etc/hosts
+        grep -q "127.0.0.1 node03" /etc/hosts || echo "127.0.0.1 node03" | sudo tee -a /etc/hosts
         pants test --use-coverage tests:: -- -m 'not integration' -v
     - name: Upload coverage report
       uses: codecov/codecov-action@v3

diff --git a/.github/workflows/default.yml b/.github/workflows/default.yml
@@ -13,7 +13,7 @@ concurrency:
 jobs:
   lint:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
-    runs-on: ubuntu-latest
+    runs-on: arc-runner-set
     steps:
     - name: Calculate the fetch depth
       run: |
@@ -22,7 +22,7 @@ jobs:
         else
           echo "GIT_FETCH_DEPTH=2" >> "${GITHUB_ENV}"
         fi
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: ${{ env.GIT_FETCH_DEPTH }}
     - name: Extract Python version from pants.toml
@@ -34,12 +34,20 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: ${{ env.PROJECT_PYTHON_VERSION }}
+    - name: Set up remote cache backend (if applicable)
+      run: |
+        echo "PANTS_REMOTE_STORE_ADDRESS=${REMOTE_CACHE_BACKEND_ENDPOINT}" >> $GITHUB_ENV
+        echo "PANTS_REMOTE_CACHE_READ=true" >> $GITHUB_ENV
+        echo "PANTS_REMOTE_CACHE_WRITE=true" >> $GITHUB_ENV
+        echo "PANTS_REMOTE_INSTANCE_NAME=main" >> $GITHUB_ENV
+      env:
+        REMOTE_CACHE_BACKEND_ENDPOINT: ${{ secrets.PANTS_REMOTE_CACHE_ENDPOINT_ARC }}
+      if: ${{ env.REMOTE_CACHE_BACKEND_ENDPOINT != '' }}
     - name: Bootstrap Pants
-      uses: pantsbuild/actions/init-pants@v5-scie-pants
+      uses: ./actions/init-pants
       # See: github.com/pantsbuild/actions/tree/main/init-pants/
       # ref) https://github.com/pantsbuild/example-python/blob/main/.github/workflows/pants.yaml#L30-L49
       with:
-        gha-cache-key: pants-cache-main-1-lint-py${{ env.PROJECT_PYTHON_VERSION }}-${{ runner.os }}-${{ runner.arch }}
         named-caches-hash: ${{ hashFiles('python*.lock', 'tools/*.lock') }}
         cache-lmdb-store: 'true'
     - name: Check BUILD files
@@ -74,7 +82,7 @@ jobs:
 
   typecheck:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
-    runs-on: ubuntu-latest-8-cores
+    runs-on: arc-runner-set
     steps:
     - name: Calculate the fetch depth
       run: |
@@ -83,7 +91,7 @@ jobs:
         else
           echo "GIT_FETCH_DEPTH=2" >> "${GITHUB_ENV}"
         fi
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: ${{ env.GIT_FETCH_DEPTH }}
     - name: Extract Python version from pants.toml
@@ -95,10 +103,18 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: ${{ env.PROJECT_PYTHON_VERSION }}
+    - name: Set up remote cache backend (if applicable)
+      run: |
+        echo "PANTS_REMOTE_STORE_ADDRESS=${REMOTE_CACHE_BACKEND_ENDPOINT}" >> $GITHUB_ENV
+        echo "PANTS_REMOTE_CACHE_READ=true" >> $GITHUB_ENV
+        echo "PANTS_REMOTE_CACHE_WRITE=true" >> $GITHUB_ENV
+        echo "PANTS_REMOTE_INSTANCE_NAME=main" >> $GITHUB_ENV
+      env:
+        REMOTE_CACHE_BACKEND_ENDPOINT: ${{ secrets.PANTS_REMOTE_CACHE_ENDPOINT_ARC }}
+      if: ${{ env.REMOTE_CACHE_BACKEND_ENDPOINT != '' }}
     - name: Bootstrap Pants
-      uses: pantsbuild/actions/init-pants@v5-scie-pants
+      uses: ./actions/init-pants
       with:
-        gha-cache-key: pants-cache-main-1-typecheck-py${{ env.PROJECT_PYTHON_VERSION }}-${{ runner.os }}-${{ runner.arch }}
         named-caches-hash: ${{ hashFiles('python*.lock', 'tools/*.lock') }}
         cache-lmdb-store: 'true'
     - name: Typecheck
@@ -129,7 +145,7 @@ jobs:
 
   test:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
-    runs-on: ubuntu-latest-8-cores
+    runs-on: [ubuntu-latest-8-cores]
     steps:
     - name: Calculate the fetch depth
       run: |
@@ -138,13 +154,13 @@ jobs:
         else
           echo "GIT_FETCH_DEPTH=2" >> "${GITHUB_ENV}"
         fi
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: ${{ env.GIT_FETCH_DEPTH }}
     - name: Create LFS file hash list
       run: git lfs ls-files -l | cut -d ' ' -f1 | sort > .lfs-assets-id
     - name: Restore LFS cache
-      uses: actions/cache@v3
+      uses: maxnowack/local-cache@v1
       id: lfs-cache
       with:
           path: .git/lfs
@@ -160,19 +176,27 @@ jobs:
       uses: actions/setup-python@v4
       with:
         python-version: ${{ env.PROJECT_PYTHON_VERSION }}
+    - name: Set up remote cache backend (if applicable)
+      run: |
+        echo "PANTS_REMOTE_STORE_ADDRESS=${REMOTE_CACHE_BACKEND_ENDPOINT}" >> $GITHUB_ENV
+        echo "PANTS_REMOTE_CACHE_READ=true" >> $GITHUB_ENV
+        echo "PANTS_REMOTE_CACHE_WRITE=true" >> $GITHUB_ENV
+        echo "PANTS_REMOTE_INSTANCE_NAME=main" >> $GITHUB_ENV
+      env:
+        REMOTE_CACHE_BACKEND_ENDPOINT: ${{ secrets.PANTS_REMOTE_CACHE_ENDPOINT }}
+      if: ${{ env.REMOTE_CACHE_BACKEND_ENDPOINT != '' }}
     - name: Bootstrap Pants
-      uses: pantsbuild/actions/init-pants@v5-scie-pants
+      uses: ./actions/init-pants
       with:
-        gha-cache-key: pants-cache-main-1-test-py${{ env.PROJECT_PYTHON_VERSION }}-${{ runner.os }}-${{ runner.arch }}
         named-caches-hash: ${{ hashFiles('python*.lock', 'tools/*.lock') }}
         cache-lmdb-store: 'true'
     - name: Test
       timeout-minutes: 15
       run: |
         # configure redis sentinel cluster hostnames for testing
-        echo "127.0.0.1 node01" | sudo tee -a /etc/hosts
-        echo "127.0.0.1 node02" | sudo tee -a /etc/hosts
-        echo "127.0.0.1 node03" | sudo tee -a /etc/hosts
+        grep -q "127.0.0.1 node01" /etc/hosts || echo "127.0.0.1 node01" | sudo tee -a /etc/hosts
+        grep -q "127.0.0.1 node02" /etc/hosts || echo "127.0.0.1 node02" | sudo tee -a /etc/hosts
+        grep -q "127.0.0.1 node03" /etc/hosts || echo "127.0.0.1 node03" | sudo tee -a /etc/hosts
         if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then
           [ -n "$GITHUB_BASE_REF" ] && BASE_REF_SHORT="${GITHUB_BASE_REF}" || BASE_REF_SHORT="main"
           BASE_REF="origin/${BASE_REF_SHORT}"
@@ -198,7 +222,7 @@ jobs:
     runs-on: ubuntu-latest
     environment: deploy-to-pypi
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Fetch remote tags
       run: git fetch origin 'refs/tags/*:refs/tags/*' -f
     - name: Create LFS file hash list
@@ -295,7 +319,7 @@ jobs:
     env:
       GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Create LFS file hash list
       run: git lfs ls-files -l | cut -d ' ' -f1 | sort > .lfs-assets-id
     - name: Restore LFS cache
@@ -350,10 +374,10 @@ jobs:
 
   build-client-image:
     needs: [build-wheels]
-    runs-on: ubuntu-latest
+    runs-on: [ubuntu-latest-8-cores, self-hosted]
     environment: deploy-to-docker-hub
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
       - name: Set up Docker Buildx

diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -11,7 +11,7 @@ jobs:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 2
     - name: Create LFS file hash list

diff --git a/.github/workflows/pr-number-assign.yml b/.github/workflows/pr-number-assign.yml
@@ -12,10 +12,9 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 2
-        token: ${{ secrets.WORKFLOW_PAT }}
 
     - name: Extract Python version from pants.toml
       run: |

diff --git a/.github/workflows/sbom.yml b/.github/workflows/sbom.yml
@@ -7,7 +7,7 @@ jobs:
     if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 2
     - name: Extract Python version from pants.toml

diff --git a/.github/workflows/timeline-check.yml b/.github/workflows/timeline-check.yml
@@ -18,7 +18,7 @@ jobs:
     steps:
     - name: Get PR's fetch depth
       run: echo "PR_FETCH_DEPTH=$(( ${{ github.event.pull_request.commits }} + 1 ))" >> "${GITHUB_ENV}"
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         ref: ${{ github.event.pull_request.head.sha }}
         fetch-depth: ${{ env.PR_FETCH_DEPTH }}

diff --git a/BUILD b/BUILD
@@ -17,7 +17,6 @@ python_requirements(
     },
     type_stubs_module_mapping={
         "types-aiofiles": ["aiofiles"],
-        "types-click": ["click"],
         "types-cachetools": ["cachetools"],
         "types-Jinja2": ["jinja2"],
         "types-PyYAML": ["yaml"],