From b010ca60f9811964d444afe5fe2ea90339cd5026 Mon Sep 17 00:00:00 2001 From: Victor Chang Date: Wed, 6 Sep 2023 17:20:19 -0700 Subject: [PATCH] Add hello-world sample and update docs, init script --- deploy/helm-charts/README | 112 +--------------- deploy/helm-charts/docs/01.installation.md | 123 ++++++++++++++++++ .../helm-charts/docs/02.RegisterWorkflows.md | 49 +++++++ deploy/helm-charts/docs/03.TriggerNewJobs.md | 47 +++++++ deploy/helm-charts/docs/04.Uninstallation.md | 34 +++++ .../hello-world-argo-template.yml | 65 +++++++++ .../files/sample-workflows/hello-world.json | 65 +++++++++ deploy/helm-charts/files/welcome.sh | 8 +- .../templates/00-serviceaccount.yaml | 55 ++++++++ deploy/helm-charts/values.yaml | 11 +- 10 files changed, 454 insertions(+), 115 deletions(-) create mode 100644 deploy/helm-charts/docs/01.installation.md create mode 100644 deploy/helm-charts/docs/02.RegisterWorkflows.md create mode 100644 deploy/helm-charts/docs/03.TriggerNewJobs.md create mode 100644 deploy/helm-charts/docs/04.Uninstallation.md create mode 100644 deploy/helm-charts/files/sample-workflows/hello-world-argo-template.yml create mode 100644 deploy/helm-charts/files/sample-workflows/hello-world.json diff --git a/deploy/helm-charts/README b/deploy/helm-charts/README index 66b15e4..2c088b9 100644 --- a/deploy/helm-charts/README +++ b/deploy/helm-charts/README @@ -1,110 +1,10 @@ # MONAI Deploy Helm Charts +The MONAI Deploy Helm Charts are designed to be used with Kubernetes and use Argo Workflow to execute your MAPs. For development, you may use [MONAI Deploy Express](../monai-deploy-express/) for a more straightforward setup based on Docker Engine. -## Prerequisites +Use the following guidelines to install MONAI Deploy Helm Charts. -### Install Tools - -```bash -# Install kubernetes tools -sudo apt-get install -y -q kubelet kubectl kubeadm -sudo apt-mark hold kubelet kubeadm kubectl - -# Install Helm -curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -``` - -### Install Kubernetes -Select one of the following Kubernetes distribution: -- [k3s](https://k3s.io/) [[Requirements](https://docs.k3s.io/installation/requirements)] - ```bash - curl -sfL https://get.k3s.io | sh -s - --flannel-backend host-gw --service-node-port-range 104-32767 --flannel-external-ip - - # Copy default configuration - mkdir -p $HOME/.kube - sudo cp -i /etc/rancher/k3s/k3s.yaml $HOME/.kube/config - sudo chown $(id -u):$(id -g) $HOME/.kube/config - ``` - -- [K8s](https://kubernetes.io/) - For detail installation instructions with GPU support, see [cloud-native-stack](https://github.com/NVIDIA/cloud-native-stack/tree/master/install-guides). - - ```bash - kubeadm init --pod-network-cidr=192.168.0.0/16 - # Copy default configuration - mkdir -p $HOME/.kube - sudo cp -i /etc/rancher/k3s/k3s.yaml $HOME/.kube/config - sudo chown $(id -u):$(id -g) $HOME/.kube/config - - kubectl taint nodes --all node-role.kubernetes.io/control-plane- - ``` - - Note: in order to use the default DIMSE port 104, you must update `/etc/kubernetes/manifests/kube-apiserver.yaml`: - - ```bash - sudo nano /etc/kubernetes/manifests/kube-apiserver.yaml - ``` - - Insert, `--service-node-port-range=104-32767` under `spec.containers.command`. - -## Install MONAI Deploy - -### Build & download dependencies - -```bash -helm dependency build -``` - -If you are reinstalling, run the following to remove existing `local-path-provisioner` Storage Class: - -```bash -kubectl delete StorageClass local-path -``` - -### Install MONAI Deploy: - -```bash -helm upgrade -i monai-deploy . -``` - -Upon successfully installation, optinally follow the on screen instructions to initialize Informatics Gateway & Orthanc. - -### Uninstall MONAI Deploy: -``` -helm uninstall monai-deploy -``` - - -### Advanced Configuration - -Most of the configurations may be found in the following files: - -- `values.yaml`: contains container image repo, application secrets, ports, etc... -- `files/informatics-gateway.json`: Informatics Gateway specific configurations... -- `files/workflow-manager.json`: Workflow Manager specific configurations... -- `files/task-manager.json`: Task Manager specific configurations... -- `files/orthanc.json`: Orthanc specific configurations... - -## Uninstall - -### Uninstall Kubernetes - -- [k3s](https://k3s.io/) - ```bash - kubectl delete StorageClass local-path - /usr/local/bin/k3s-uninstall.sh - ``` - -- [K8s](https://kubernetes.io/) - ```bash - kubectl delete StorageClass local-path - sudo kubeadm reset - ``` - -### Uninstall Tools - -```bash -sudo apt-get purge -y kubeadm kubectl kubelet kubernetes-cni kube* helm -sudo apt-get autoremove -y -sudo rm -rf ~/.kube -``` \ No newline at end of file +1. [Installation](./docs/01.installation.md) +2. [Register Workflows](./docs/02.RegisterWorkflows.md) +3. [Trigger Jobs](./docs/03.TriggerNewJobs.md) +4. [Uninstallation](./docs/04.Uninstallation.md) \ No newline at end of file diff --git a/deploy/helm-charts/docs/01.installation.md b/deploy/helm-charts/docs/01.installation.md new file mode 100644 index 0000000..5d6a09f --- /dev/null +++ b/deploy/helm-charts/docs/01.installation.md @@ -0,0 +1,123 @@ +# MONAI Deploy Helm Charts - Installation + +## Prerequisites + +```bash +# Install apt-transport-https curl jq +sudo apt-get update && sudo apt-get install -y apt-transport-https curl jq + +# Install kubernetes tools +sudo apt-get install -y -q kubelet kubectl kubeadm +sudo apt-mark hold kubelet kubeadm kubectl + +# Install Helm +curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash + +# Install Argo CLI +curl -sLO https://github.com/argoproj/argo-workflows/releases/download/v3.4.10/argo-linux-amd64.gz +gunzip argo-linux-amd64.gz +chmod +x argo-linux-amd64 +sudo mv ./argo-linux-amd64 /usr/local/bin/argo +``` + +## Install Kubernetes + +Select one of the following Kubernetes distribution: + +### [k3s](https://k3s.io/) + +See [Requirements](https://docs.k3s.io/installation/requirements) for K3s for hardware requirements. + +```bash +curl -sfL https://get.k3s.io | sh -s - --flannel-backend host-gw --service-node-port-range 104-32767 --flannel-external-ip + +# Copy default configuration +mkdir -p $HOME/.kube +sudo cp -i /etc/rancher/k3s/k3s.yaml $HOME/.kube/config +sudo chown $(id -u):$(id -g) $HOME/.kube/config +``` + +### [K8s](https://kubernetes.io/) + +For detail installation instructions with GPU support, see [cloud-native-stack](https://github.com/NVIDIA/cloud-native-stack/tree/master/install-guides). + +```bash +sudo kubeadm init --pod-network-cidr=192.168.0.0/16 +# Copy default configuration +mkdir -p $HOME/.kube +sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config +sudo chown $(id -u):$(id -g) $HOME/.kube/config + +# Install Calico +kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.1/manifests/tigera-operator.yaml +kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.1/manifests/custom-resources.yaml + +kubectl taint nodes --all node-role.kubernetes.io/control-plane- +``` + +Note: in order to use the default DIMSE port 104, you must update `/etc/kubernetes/manifests/kube-apiserver.yaml`: + +```bash +sudo nano /etc/kubernetes/manifests/kube-apiserver.yaml +``` + +Insert, `--service-node-port-range=104-32767` under `spec.containers.command` and wait for K8s to restart. + + +```bash +# The following error denotes that the Kubernetes node is restarting. +$ kubectl get nodes +The connection to the server 10.20.12.243:6443 was refused - did you specify the right host or port? + +# And when the node is ready... +$ kubectl get nodes +NAME STATUS ROLES AGE VERSION +my-system Ready control-plane 73s v1.28.1 +``` + +If modifying the port range is not an option, update the port numbers inside `values.yaml` to be in the range. + +## Build & download MONAI Deploy dependencies + +```bash +helm dependency build +``` + +If you are reinstalling, run the following to remove existing `local-path-provisioner` Storage Class: + +```bash +kubectl delete StorageClass local-path +``` + +## Install MONAI Deploy + +Use the following commands to install MONAI Deploy Helm charts and its dependencies: + +- MONAI Deploy Informatics Gateway +- MONAI Deploy Workflow Manager +- MONAI Deploy Task Manager +- MinIO +- MongoDB +- Orthanc +- RabbitMQ +- [Argo Workflows](https://github.com/argoproj/argo-helm/tree/main/charts/argo-workflows) +- [local-path-provisioner](https://github.com/rancher/local-path-provisioner) + +```bash +helm upgrade -i monai-deploy . # default/current namespace +helm upgrade -i monai-deploy -n my-space . # install in namespace "my-namespace" +``` + +Upon successfully installation, optionally follow the on screen instructions to initialize Informatics Gateway & Orthanc. + +## Advanced Configuration + +Most of the configurations may be found in the following files: + +- `values.yaml`: contains container image repo, application secrets, ports, etc... +- `files/informatics-gateway.json`: Informatics Gateway specific configurations... +- `files/workflow-manager.json`: Workflow Manager specific configurations... +- `files/task-manager.json`: Task Manager specific configurations... +- `files/orthanc.json`: Orthanc specific configurations... + +[Next - Register Workflows ➡️](./02.RegisterWorkflows.md) \ No newline at end of file diff --git a/deploy/helm-charts/docs/02.RegisterWorkflows.md b/deploy/helm-charts/docs/02.RegisterWorkflows.md new file mode 100644 index 0000000..a62ce16 --- /dev/null +++ b/deploy/helm-charts/docs/02.RegisterWorkflows.md @@ -0,0 +1,49 @@ +# MONAI Deploy Helm Charts - Register Workflows + +This section includes three sample workflows: + +1. Hello World (small) +2. *tbd* (medium) +3. *tbd* (large) + + +## Hello World + +### Create the Hello World Argo Tempalte + +```bash +$ argo template create ./files/sample-workflows//hello-world-argo-template.yml + +Name: hello-world-template +Namespace: default +Created: Tue Sep 05 09:47:50 -0700 (now) +``` + +### Submit Hello World Workflow Definition + +If the helm installation name used in the previous section is not `monai-deploy`, open `hello-world.json`, find `server_url` and change the value to the name of the Argo workflow server. Similarly, if it was installed in a namespace other than `default`, change the value of `namespace` property to the namespace used. + +Use the following command to find the name of the Argo workflow server: + +```bash +$ kubectl get svc -l app.kubernetes.io/name=argo-workflows-server + +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +monai-deploy-argo-workflows-server ClusterIP 10.109.42.166 2746/TCP 11m +``` + +```bash +# Submit the workflow definition +MWM_API_IP=$(kubectl get services/mwm -o jsonpath="{.spec.clusterIP}") +MWM_API_PORT=$(kubectl get -o jsonpath="{.spec.ports[0].port}" services mwm) +curl --request POST --header 'Content-Type: application/json' --data "@files/sample-workflows/hello-world.json" http://${MWM_API_IP}:${MWM_API_PORT}/workflows | jq + +# `workflow_id` printed with a successful API call. +{ + "workflow_id": "a25a43cb-b586-45b4-9c27-25e53e1570a2" +} +``` + + + +[Next - Trigger New Jobs ➡️](./03.TriggerNewJobs.md) diff --git a/deploy/helm-charts/docs/03.TriggerNewJobs.md b/deploy/helm-charts/docs/03.TriggerNewJobs.md new file mode 100644 index 0000000..2652844 --- /dev/null +++ b/deploy/helm-charts/docs/03.TriggerNewJobs.md @@ -0,0 +1,47 @@ +# MONAI Deploy Helm Charts - Trigger New Jobs + +## Download DICOM Datasets + +In the last section, you have registered some sample workflows. It's now time to trigger some new jobs using those workflows. + +First, download & unzip the following DICOM datasets and upload them to Orthanc. + +- [Chest CT dataset](https://drive.google.com/file/d/1IGXUgZ7NQCwsix57cdSgr-iYErevqETO/view?usp=sharing) +- [Abdomen CT dataset](https://drive.google.com/file/d/1d8Scm3q-kHTqr_-KfnXH0rPnCgKld2Iy/view?usp=sharing) + +### Upload DICOM Datasets + +Navigate to Orthanc UI, and click *Upload* on the top right-hand corner. + +Use the following command to locate the Orthanc service: + +```bash +$ kubectl get svc orthanc + +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +orthanc NodePort 10.43.151.152 8042:27011/TCP,4242:8898/TCP 93m +``` + +The UI can be accessed via http://10.43.151.152:8042 or http://localhost:27011. + +Drag & drop any DICOM files (no folders) to the blue box on the page and then click *Start the upload*. + +Navigate to the home page and click *All studies* to confirm data's been uploaded. + +### Triggering New Jobs + +With the sample DICOM dataset uploaded to Orthanc, trigger a new job by navigating to *All studies* page and then +click on a study, from the menu on the left, click on *Send to DICOM modality* and then click on *MONAI-DEPOLOY*. + +After a minute or two, depending on the size of the DICOM dataset, you may find the job listed in Argo. + +Note: The jobs get deleted from the page after a while by Argo GC. + +Enable UI (http://localhost:2746) access with the following command to forward port 2476: + +```bash +kubectl port-forward services/monai-deploy-argo-workflows-server 2746:2746 +``` + + +[Next - Uninstallation ➡️](./04.Uninstallation.md) diff --git a/deploy/helm-charts/docs/04.Uninstallation.md b/deploy/helm-charts/docs/04.Uninstallation.md new file mode 100644 index 0000000..fcadd17 --- /dev/null +++ b/deploy/helm-charts/docs/04.Uninstallation.md @@ -0,0 +1,34 @@ + +# MONAI Deploy Helm Charts - Uninstallation + +## Uninstall MONAI Deploy + +```bash +helm uninstall monai-deploy +``` + +## Uninstall Kubernetes + +### [k3s](https://k3s.io/) + +```bash +kubectl delete StorageClass local-path +/usr/local/bin/k3s-uninstall.sh +sudo rm -rf ~/.kube +``` + +### [K8s](https://kubernetes.io/) + +```bash +kubectl delete StorageClass local-path +kubectl drain $(kubectl get node -o name) +sudo kubeadm reset +sudo rm -rf ~/.kube +``` + +## Uninstall Tools + +```bash +sudo apt-get purge -y kubeadm kubectl kubelet kubernetes-cni kube* helm +sudo apt-get autoremove -y +``` diff --git a/deploy/helm-charts/files/sample-workflows/hello-world-argo-template.yml b/deploy/helm-charts/files/sample-workflows/hello-world-argo-template.yml new file mode 100644 index 0000000..7b557fd --- /dev/null +++ b/deploy/helm-charts/files/sample-workflows/hello-world-argo-template.yml @@ -0,0 +1,65 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: hello-world-template + generateName: hello-world- +spec: + entrypoint: hello-world-pipeline + podGC: + strategy: OnWorkflowSuccess + # Uncomment for Argo 3.5 + # deleteDelayDuration: 300s + arguments: {} + templates: + - name: hello-world-pipeline + inputs: {} + outputs: {} + metadata: {} + steps: + - - name: hello-world-step + template: hello-world + arguments: + artifacts: + - name: input + s3: + accessKeySecret: + name: hello-world-minio-secrets + key: accesskey + secretKeySecret: + name: hello-world-minio-secrets + key: secretkey + key: argo/inputs/artifacts/input + - name: hello-world + inputs: + artifacts: + - name: input + path: /var/monai/input + s3: + accessKeySecret: + name: hello-world-minio-secrets + key: accesskey + secretKeySecret: + name: hello-world-minio-secrets + key: secretkey + key: argo/inputs/artifacts/input + outputs: + artifacts: + - name: output + path: /var/monai/output + s3: + bucket: hello-world-bucket + accessKeySecret: + name: hello-world-minio-secrets + key: accesskey + secretKeySecret: + name: hello-world-minio-secrets + key: secretkey + metadata: {} + container: + name: '' + image: alpine:latest + command: + - /bin/sh + args: + - '-c' + - date && cp -r /var/monai/input/* /var/monai/output && date \ No newline at end of file diff --git a/deploy/helm-charts/files/sample-workflows/hello-world.json b/deploy/helm-charts/files/sample-workflows/hello-world.json new file mode 100644 index 0000000..43b502c --- /dev/null +++ b/deploy/helm-charts/files/sample-workflows/hello-world.json @@ -0,0 +1,65 @@ +{ + "name": "hello-world", + "version": "1.0.0", + "description": "Hello MONAI Deploy!", + "informatics_gateway": { + "ae_title": "MONAI-DEPLOY", + "data_origins": [ + "ORTHANC" + ], + "export_destinations": [ + "ORTHANC" + ] + }, + "tasks": [ + { + "id": "copy-file-list", + "description": "Copes files from input directory", + "type": "argo", + "args": { + "namespace": "default", + "workflow_template_name": "hello-world-template", + "server_url": "http://monai-deploy-argo-workflows-server:2746", + "allow_insecure": "true" + }, + "task_destinations": [ + { + "name": "export-data" + } + ], + "artifacts": { + "input": [ + { + "name": "input", + "value": "{{ context.input.dicom }}", + "mandatory": true + } + ], + "output": [ + { + "name": "output" + } + ] + } + }, + { + "id": "export-data", + "description": "Export data back to Orthanc", + "type": "export", + "export_destinations": [ + { + "name": "ORTHANC" + } + ], + "artifacts": { + "input": [ + { + "name": "output", + "value": "{{ context.executions.copy-file-list.artifacts.output }}", + "mandatory": true + } + ] + } + } + ] +} \ No newline at end of file diff --git a/deploy/helm-charts/files/welcome.sh b/deploy/helm-charts/files/welcome.sh index 0a793f6..5becd0b 100755 --- a/deploy/helm-charts/files/welcome.sh +++ b/deploy/helm-charts/files/welcome.sh @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -echo Waiting for MONAI Deploy service to be ready... -kubectl wait --timeout=180s --for=condition=Ready po -l 'app in (mig,mwm,mtm)' >/dev/null +echo Waiting for MONAI Deploy services to be ready... +kubectl wait --timeout=180s --for=condition=Ready po -l 'role in (external-svcs,internal-svcs)' >/dev/null sleep 3 echo "" @@ -86,11 +86,11 @@ echo - POD: $(kubectl get po --namespace $1 -l app=rabbitmq -o jsonpath= echo - API: http://$NODE_IP:$RABBITMQ_API_PORT echo - Console: http://$NODE_IP:$RABBITMQ_CONSOLE_PORT echo ==================================== -echo MongoDB: +echo MongoDB: echo - POD: $(kubectl get po --namespace $1 -l app=mongodb -o jsonpath={..metadata.name}) echo - POrt: $MONGO_PORT echo ==================================== -echo Argo Workflow: +echo Argo Workflow: echo - POD: $(kubectl get po --namespace $1 -l app=mongodb -o jsonpath={..metadata.name}) echo - Console: http://$ARGO_IP:$ARGO_PORT echo - Commands: diff --git a/deploy/helm-charts/templates/00-serviceaccount.yaml b/deploy/helm-charts/templates/00-serviceaccount.yaml index 254e8eb..db1ee41 100644 --- a/deploy/helm-charts/templates/00-serviceaccount.yaml +++ b/deploy/helm-charts/templates/00-serviceaccount.yaml @@ -1,3 +1,31 @@ + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + namespace: {{ .Release.Namespace }} + name: pods-patch-role +rules: +- apiGroups: [""] # "" indicates the core API group + resources: ["pods"] + verbs: ["patch"] + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: pods-patch-role-binding +subjects: +- kind: ServiceAccount + namespace: {{ .Release.Namespace }} + name: default +roleRef: + kind: ClusterRole + name: pods-patch-role + apiGroup: rbac.authorization.k8s.io + +--- + {{- if .Values.serviceAccount.create -}} apiVersion: v1 kind: ServiceAccount @@ -9,4 +37,31 @@ metadata: annotations: {{- toYaml . | nindent 4 }} {{- end }} + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "local.serviceAccountName" . }}-Role-Secrets +rules: +- apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "local.serviceAccountName" . }}-RoleBinding-Secrets +subjects: +- kind: ServiceAccount + name: {{ include "local.serviceAccountName" . }} +roleRef: + kind: Role + name: {{ include "local.serviceAccountName" . }}-Role-Secrets + apiGroup: rbac.authorization.k8s.io {{- end }} + +--- diff --git a/deploy/helm-charts/values.yaml b/deploy/helm-charts/values.yaml index b16dfe8..5eded20 100644 --- a/deploy/helm-charts/values.yaml +++ b/deploy/helm-charts/values.yaml @@ -13,7 +13,7 @@ mig: imagePullSecrets: [] serviceType: NodePort dimsePort: 104 - apiPort: 5000 + apiPort: 6000 databaseSize: 1Gi logsSize: 1Gi payloadSize: 10Gi @@ -37,11 +37,11 @@ mwm: name: workflow-manager serviceName: mwm repository: ghcr.io/project-monai/monai-deploy-workflow-manager - tag: develop-latest + tag: 2023.5.0-beta.161 pullPolicy: IfNotPresent imagePullSecrets: [] serviceType: NodePort - apiPort: 5000 + apiPort: 6000 resources: {} # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little @@ -58,11 +58,11 @@ mtm: name: task-manager serviceName: mtm repository: ghcr.io/project-monai/monai-deploy-task-manager - tag: develop-latest + tag: 2023.5.0-beta.161 pullPolicy: IfNotPresent imagePullSecrets: [] serviceType: NodePort - apiPort: 5000 + apiPort: 6000 resources: {} # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little @@ -237,6 +237,7 @@ argo-workflows: create: true controller: workflowNamespaces: + # include your namespace here - default server: extraArgs: