diff --git a/404.html b/404.html index 89940b769..214f36503 100644 --- a/404.html +++ b/404.html @@ -16,13 +16,13 @@ - - + +
Skip to main content

Page Not Found

We could not find what you were looking for.

Please contact the owner of the site that linked you to the original URL and let them know their link is broken.

- - + + \ No newline at end of file diff --git a/acknowledgement/index.html b/acknowledgement/index.html index ba697250e..e13a5aaa5 100644 --- a/acknowledgement/index.html +++ b/acknowledgement/index.html @@ -16,13 +16,13 @@ - - + +
Skip to main content

Acknowledging the DSRI in your publications

If you are planning to present or publish your work which was made possible by using the DSRI, we encourage you to acknowledge the use of DSRI. For this purpose we propose to add the following sentence to your publication:

"This research was made possible, in part, using the Data Science Research Infrastructure (DSRI) hosted at Maastricht University."

Citations

- - + + \ No newline at end of file diff --git a/assets/js/09722083.9371d9b4.js b/assets/js/09722083.60be4374.js similarity index 98% rename from assets/js/09722083.9371d9b4.js rename to assets/js/09722083.60be4374.js index 67fa6f588..bb97652b9 100644 --- a/assets/js/09722083.9371d9b4.js +++ b/assets/js/09722083.60be4374.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[9676],{5680:(e,t,r)=>{r.d(t,{xA:()=>l,yg:()=>g});var n=r(6540);function a(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function o(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function i(e){for(var t=1;t=0||(a[r]=e[r]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(a[r]=e[r])}return a}var c=n.createContext({}),p=function(e){var t=n.useContext(c),r=t;return e&&(r="function"==typeof e?e(t):i(i({},t),e)),r},l=function(e){var t=p(e.components);return n.createElement(c.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},m=n.forwardRef((function(e,t){var r=e.components,a=e.mdxType,o=e.originalType,c=e.parentName,l=s(e,["components","mdxType","originalType","parentName"]),m=p(r),g=a,y=m["".concat(c,".").concat(g)]||m[g]||u[g]||o;return r?n.createElement(y,i(i({ref:t},l),{},{components:r})):n.createElement(y,i({ref:t},l))}));function g(e,t){var r=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=r.length,i=new Array(o);i[0]=m;var s={};for(var c in t)hasOwnProperty.call(t,c)&&(s[c]=t[c]);s.originalType=e,s.mdxType="string"==typeof e?e:a,i[1]=s;for(var p=2;p{r.r(t),r.d(t,{assets:()=>l,contentTitle:()=>c,default:()=>g,frontMatter:()=>s,metadata:()=>p,toc:()=>u});var n=r(9668),a=r(1367),o=(r(6540),r(5680)),i=["components"],s={id:"catalog-genomics",title:"Genomics"},c=void 0,p={unversionedId:"catalog-genomics",id:"catalog-genomics",title:"Genomics",description:"Feel free to propose new services using pull requests, or to request them by creating new issues.",source:"@site/docs/catalog-genomics.md",sourceDirName:".",slug:"/catalog-genomics",permalink:"/docs/catalog-genomics",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/catalog-genomics.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"catalog-genomics",title:"Genomics"},sidebar:"docs",previous:{title:"Neuroscience research",permalink:"/docs/neuroscience"},next:{title:"Imaging softwares",permalink:"/docs/catalog-imaging"}},l={},u=[{value:"Trinity RNA Seq",id:"trinity-rna-seq",level:2}],m={toc:u};function g(e){var t=e.components,r=(0,a.A)(e,i);return(0,o.yg)("wrapper",(0,n.A)({},m,r,{components:t,mdxType:"MDXLayout"}),(0,o.yg)("p",null,"Feel free to propose new services using ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/pulls"},"pull requests"),", or to request them by creating ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/issues"},"new issues"),"."),(0,o.yg)("h2",{id:"trinity-rna-seq"},"Trinity RNA Seq"),(0,o.yg)("p",null,"Trinity assembles transcript sequences from Illumina RNA-Seq data. It represents a novel method for the efficient and robust de novo reconstruction of transcriptomes from RNA-seq data. See ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/trinityrnaseq/trinityrnaseq/wiki"},"their documentation"),"."),(0,o.yg)("p",null,"You can start a container using the ",(0,o.yg)("strong",{parentName:"p"},"Trinity RNA-Seq")," template in the ",(0,o.yg)("a",{parentName:"p",href:"https://console-openshift-console.apps.dsri2.unimaas.nl/catalog"},"Catalog web UI")," (make sure the ",(0,o.yg)("strong",{parentName:"p"},"Templates")," checkbox is checked)"),(0,o.yg)("img",{src:"/img/screenshot_trinityrnaseq.png",alt:"Deploy Trinity RNA Seq",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("p",null,"This template uses the Trinity RNA-Seq image hosted in the ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/orgs/maastrichtu-ids/packages/container/package/trinityrnaseq"},"UM IDS GitHub Container Registry")," "),(0,o.yg)("admonition",{title:"Persistent data folder",type:"info"},(0,o.yg)("p",{parentName:"admonition"},"\ud83d\udcc2 Use the ",(0,o.yg)("inlineCode",{parentName:"p"},"/usr/local/src/work")," folder (home of the root user) to store your data in the existing persistent storage. You can find the persistent volumes in the DSRI web UI, go to the ",(0,o.yg)("strong",{parentName:"p"},"Administrator")," view > ",(0,o.yg)("strong",{parentName:"p"},"Storage")," > ",(0,o.yg)("strong",{parentName:"p"},"Persistent Volume Claims"),".")),(0,o.yg)("p",null,"We enabled the port ",(0,o.yg)("inlineCode",{parentName:"p"},"8787")," in the container, if you need to deploy applications."))}g.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[9676],{5680:(e,t,r)=>{r.d(t,{xA:()=>l,yg:()=>g});var n=r(6540);function a(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function o(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function i(e){for(var t=1;t=0||(a[r]=e[r]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(a[r]=e[r])}return a}var c=n.createContext({}),p=function(e){var t=n.useContext(c),r=t;return e&&(r="function"==typeof e?e(t):i(i({},t),e)),r},l=function(e){var t=p(e.components);return n.createElement(c.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},m=n.forwardRef((function(e,t){var r=e.components,a=e.mdxType,o=e.originalType,c=e.parentName,l=s(e,["components","mdxType","originalType","parentName"]),m=p(r),g=a,y=m["".concat(c,".").concat(g)]||m[g]||u[g]||o;return r?n.createElement(y,i(i({ref:t},l),{},{components:r})):n.createElement(y,i({ref:t},l))}));function g(e,t){var r=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=r.length,i=new Array(o);i[0]=m;var s={};for(var c in t)hasOwnProperty.call(t,c)&&(s[c]=t[c]);s.originalType=e,s.mdxType="string"==typeof e?e:a,i[1]=s;for(var p=2;p{r.r(t),r.d(t,{assets:()=>l,contentTitle:()=>c,default:()=>g,frontMatter:()=>s,metadata:()=>p,toc:()=>u});var n=r(9668),a=r(1367),o=(r(6540),r(5680)),i=["components"],s={id:"catalog-genomics",title:"Genomics"},c=void 0,p={unversionedId:"catalog-genomics",id:"catalog-genomics",title:"Genomics",description:"Feel free to propose new services using pull requests, or to request them by creating new issues.",source:"@site/docs/catalog-genomics.md",sourceDirName:".",slug:"/catalog-genomics",permalink:"/docs/catalog-genomics",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/catalog-genomics.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"catalog-genomics",title:"Genomics"},sidebar:"docs",previous:{title:"Neuroscience research",permalink:"/docs/neuroscience"},next:{title:"Imaging softwares",permalink:"/docs/catalog-imaging"}},l={},u=[{value:"Trinity RNA Seq",id:"trinity-rna-seq",level:2}],m={toc:u};function g(e){var t=e.components,r=(0,a.A)(e,i);return(0,o.yg)("wrapper",(0,n.A)({},m,r,{components:t,mdxType:"MDXLayout"}),(0,o.yg)("p",null,"Feel free to propose new services using ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/pulls"},"pull requests"),", or to request them by creating ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/issues"},"new issues"),"."),(0,o.yg)("h2",{id:"trinity-rna-seq"},"Trinity RNA Seq"),(0,o.yg)("p",null,"Trinity assembles transcript sequences from Illumina RNA-Seq data. It represents a novel method for the efficient and robust de novo reconstruction of transcriptomes from RNA-seq data. See ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/trinityrnaseq/trinityrnaseq/wiki"},"their documentation"),"."),(0,o.yg)("p",null,"You can start a container using the ",(0,o.yg)("strong",{parentName:"p"},"Trinity RNA-Seq")," template in the ",(0,o.yg)("a",{parentName:"p",href:"https://console-openshift-console.apps.dsri2.unimaas.nl/catalog"},"Catalog web UI")," (make sure the ",(0,o.yg)("strong",{parentName:"p"},"Templates")," checkbox is checked)"),(0,o.yg)("img",{src:"/img/screenshot_trinityrnaseq.png",alt:"Deploy Trinity RNA Seq",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("p",null,"This template uses the Trinity RNA-Seq image hosted in the ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/orgs/maastrichtu-ids/packages/container/package/trinityrnaseq"},"UM IDS GitHub Container Registry")," "),(0,o.yg)("admonition",{title:"Persistent data folder",type:"info"},(0,o.yg)("p",{parentName:"admonition"},"\ud83d\udcc2 Use the ",(0,o.yg)("inlineCode",{parentName:"p"},"/usr/local/src/work")," folder (home of the root user) to store your data in the existing persistent storage. You can find the persistent volumes in the DSRI web UI, go to the ",(0,o.yg)("strong",{parentName:"p"},"Administrator")," view > ",(0,o.yg)("strong",{parentName:"p"},"Storage")," > ",(0,o.yg)("strong",{parentName:"p"},"Persistent Volume Claims"),".")),(0,o.yg)("p",null,"We enabled the port ",(0,o.yg)("inlineCode",{parentName:"p"},"8787")," in the container, if you need to deploy applications."))}g.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/09f56f30.33a4eb59.js b/assets/js/09f56f30.73bb2c6c.js similarity index 99% rename from assets/js/09f56f30.33a4eb59.js rename to assets/js/09f56f30.73bb2c6c.js index 7ae46d01d..c57a33b36 100644 --- a/assets/js/09f56f30.33a4eb59.js +++ b/assets/js/09f56f30.73bb2c6c.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[3360],{5680:(e,t,n)=>{n.d(t,{xA:()=>c,yg:()=>d});var a=n(6540);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function r(e){for(var t=1;t=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var s=a.createContext({}),p=function(e){var t=a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},c=function(e){var t=p(e.components);return a.createElement(s.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},m=a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,i=e.originalType,s=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),m=p(n),d=o,y=m["".concat(s,".").concat(d)]||m[d]||u[d]||i;return n?a.createElement(y,r(r({ref:t},c),{},{components:n})):a.createElement(y,r({ref:t},c))}));function d(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var i=n.length,r=new Array(i);r[0]=m;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:o,r[1]=l;for(var p=2;p{n.r(t),n.d(t,{assets:()=>c,contentTitle:()=>s,default:()=>d,frontMatter:()=>l,metadata:()=>p,toc:()=>u});var a=n(9668),o=n(1367),i=(n(6540),n(5680)),r=["components"],l={id:"anatomy-of-an-application",title:"Anatomy of a DSRI application"},s=void 0,p={unversionedId:"anatomy-of-an-application",id:"anatomy-of-an-application",title:"Anatomy of a DSRI application",description:"This page will present you how an applications is typically built using an OpenShift template. This will also help you understand more in general the different objects that needs to be defined when deploying an application on a Kubernetes cluster. Even if OpenShift templates can only be deployed to OpenShift, the objects they define are the same as in Kubernetes (apart from the Route which becomes Ingress).",source:"@site/docs/anatomy-of-an-application.md",sourceDirName:".",slug:"/anatomy-of-an-application",permalink:"/docs/anatomy-of-an-application",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/anatomy-of-an-application.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"anatomy-of-an-application",title:"Anatomy of a DSRI application"},sidebar:"docs",previous:{title:"Deploy from a Docker image",permalink:"/docs/deploy-from-docker"},next:{title:"Install from Helm charts",permalink:"/docs/helm"}},c={},u=[{value:"Application walkthrough",id:"application-walkthrough",level:2},{value:"Parameters",id:"parameters",level:3},{value:"Image",id:"image",level:3},{value:"Create storage",id:"create-storage",level:3},{value:"Secret",id:"secret",level:3},{value:"Deployment",id:"deployment",level:3},{value:"Pod spec",id:"pod-spec",level:3},{value:"Environment variables in the container",id:"environment-variables-in-the-container",level:3},{value:"Mount storage",id:"mount-storage",level:3},{value:"Security context",id:"security-context",level:3},{value:"Service",id:"service",level:3},{value:"Route",id:"route",level:3},{value:"The complete application",id:"the-complete-application",level:2},{value:"Add a configuration file",id:"add-a-configuration-file",level:2},{value:"Add automated health checks",id:"add-automated-health-checks",level:2},{value:"Define resource limits",id:"define-resource-limits",level:2},{value:"Build your own application template",id:"build-your-own-application-template",level:2}],m={toc:u};function d(e){var t=e.components,n=(0,o.A)(e,r);return(0,i.yg)("wrapper",(0,a.A)({},m,n,{components:t,mdxType:"MDXLayout"}),(0,i.yg)("p",null,"This page will present you how an applications is typically built using an ",(0,i.yg)("strong",{parentName:"p"},"OpenShift template"),". This will also help you understand more in general the different objects that needs to be defined when ",(0,i.yg)("strong",{parentName:"p"},"deploying an application on a Kubernetes cluster"),". Even if OpenShift templates can only be deployed to OpenShift, the objects they define are the same as in Kubernetes (apart from the Route which becomes Ingress)."),(0,i.yg)("p",null,"There are other ways to describe applications on OpenShift cluster (here the DSRI), such as Helm or Operators. But OpenShift templates are the easiest and quickest way to build an application that can be deployed from the DSRI web UI catalog in a few clicks, and by providing a few parameters."),(0,i.yg)("p",null,"It is better to have a basic understanding of what a docker container is to fully understand this walkthrough, but it should already gives a good idea of the different objects deployed with each DSRI application."),(0,i.yg)("p",null,"We will use the template used to deploy JupyterLab as example, and we will describe the goal, importance and caveats of each parts of the application definition. But the same template and instructions can be easily reused for other applications with a web UI to access."),(0,i.yg)("p",null,"Checkout the ",(0,i.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/blob/master/applications/templates/template-jupyterlab-root.yml"},"complete JupyterLab template here")," (it will be slightly different with a bit more comments, but there are globally the same)"),(0,i.yg)("p",null,"You will see that deploying on Kubernetes (and by extension, here OpenShift), is just about defining objects in a YAML file, like a complex ",(0,i.yg)("inlineCode",{parentName:"p"},"docker-compose.yml")," file. "),(0,i.yg)("admonition",{title:"Do you got what it takes?",type:"info"},(0,i.yg)("p",{parentName:"admonition"},"The amount of objects might seems a bit overwhelming at first, but this is what it takes to automatically deploy a complex application on a large cluster, automatically available through a generated URL, with ",(0,i.yg)("inlineCode",{parentName:"p"},"HTTPS")," encryption to protect your passwords when you log to a web UI!")),(0,i.yg)("h2",{id:"application-walkthrough"},"Application walkthrough"),(0,i.yg)("p",null,"First, you need to create your ",(0,i.yg)("strong",{parentName:"p"},"Template")," objects, this will be the main object we will create here as all other objects defined will be deployed by this template. "),(0,i.yg)("p",null,"In this part we mainly just provide the description and information that will be shown to users when deploying the application from the DSRI web UI catalog."),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},"---\nkind: Template\napiVersion: template.openshift.io/v1\nlabels:\n template: jupyterlab-root\nmetadata:\n name: jupyterlab-root\n annotations:\n openshift.io/display-name: JupyterLab\n description: |-\n Start JupyterLab images as the `jovyan` user, with sudo privileges to install anything you need. \n \ud83d\udcc2 Use the `/home/jovyan` folder (workspace of the JupyterLab UI) to store your data in the persistent storage automatically created\n You can find the persistent storage in the DSRI web UI, go to Administrator view > Storage > Persistent Volume Claims\n You can use any image based on the official Jupyter docker stack https://github.com/jupyter/docker-stacks\n - jupyter/tensorflow-notebook\n - jupyter/r-notebook\n - jupyter/all-spark-notebook\n - ghcr.io/maastrichtu-ids/jupyterlab (with Java and SPARQL kernels)\n Or build your own! Checkout https://github.com/MaastrichtU-IDS/jupyterlab for an example of custom image\n Once JupyterLab is deployed you can install any pip packages, JupyterLab extensions, and apt packages.\n iconClass: icon-python\n tags: python,jupyter,notebook\n openshift.io/provider-display-name: Institute of Data Science, UM\n openshift.io/documentation-url: https://maastrichtu-ids.github.io/dsri-documentation/docs/deploy-jupyter\n openshift.io/support-url: https://maastrichtu-ids.github.io/dsri-documentation/help\n")),(0,i.yg)("h3",{id:"parameters"},"Parameters"),(0,i.yg)("p",null,"Then define the ",(0,i.yg)("strong",{parentName:"p"},"parameters")," the user will be able to define in the DSRI catalog web UI when instantiating the application. ",(0,i.yg)("inlineCode",{parentName:"p"},"APPLICATION_NAME")," is the most important as it will be used everywhere to create the objects and identify the application."),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},"parameters:\n- name: APPLICATION_NAME\n displayName: Name for the application\n description: Must be without spaces (use -), and unique in the project.\n value: jupyterlab\n required: true\n- name: PASSWORD\n displayName: JupyterLab UI Password\n description: The password/token to access the JupyterLab web UI\n required: true\n- name: APPLICATION_IMAGE\n displayName: Jupyter notebook Docker image\n value: ghcr.io/maastrichtu-ids/jupyterlab:latest\n required: true\n description: You can use any image based on https://github.com/jupyter/docker-stacks\n- name: STORAGE_SIZE\n displayName: Storage size\n description: Size of the storage allocated to the notebook persistent storage in `/home/jovyan`.\n value: 5Gi\n required: true\n")),(0,i.yg)("p",null,"We can then refer to those parameters value (filled by the users of the template) in the rest of the template using this syntax: ",(0,i.yg)("inlineCode",{parentName:"p"},"${APPLICATION_NAME}")),(0,i.yg)("p",null,"We will now ",(0,i.yg)("strong",{parentName:"p"},"describe all objects deployed")," when we instantiate this template (to start an application). "),(0,i.yg)("h3",{id:"image"},"Image"),(0,i.yg)("p",null,"First we define the ",(0,i.yg)("strong",{parentName:"p"},"ImageStream")," object to import the Docker image(s) of your application(s) on the DSRI cluster"),(0,i.yg)("p",null,"Setting the ",(0,i.yg)("inlineCode",{parentName:"p"},"importPolicy: scheduled")," to ",(0,i.yg)("inlineCode",{parentName:"p"},"true")," will have the DSRI to automatically check for new version of this image, which can be useful if you want to always have the latest published version of an applications. Visit the ",(0,i.yg)("a",{parentName:"p",href:"https://docs.openshift.com/container-platform/4.6/openshift_images/image-streams-manage.html"},"OpenShift ImageStreams documentation")," for more details. Be careful as enabling this feature without real need will cause the DSRI to query DockerHub more, which might require you to login to DockerHub to increase your pull request quota."),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},'objects:\n- kind: "ImageStream"\n apiVersion: image.openshift.io/v1\n metadata:\n name: ${APPLICATION_NAME}\n labels:\n app: ${APPLICATION_NAME}\n spec:\n tags:\n - name: latest\n from:\n kind: DockerImage\n name: ${APPLICATION_IMAGE}\n importPolicy:\n scheduled: true\n lookupPolicy:\n local: true\n')),(0,i.yg)("h3",{id:"create-storage"},"Create storage"),(0,i.yg)("p",null,"Then we define the ",(0,i.yg)("strong",{parentName:"p"},"PersistentVolumeClaim"),", which is a persistent storage on which we will mount the ",(0,i.yg)("inlineCode",{parentName:"p"},"/home/jovyan")," folder to avoid loosing data if our application is restarted."),(0,i.yg)("p",null,"Any file outside of a persistent volume can be lost at any moment if the pod restart, usually it only consists in temporary file if you are properly working in the persistent volume folder. This can be useful also if your application is crashing, stopping and restarting your pod (application) might fix it."),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},'- kind: "PersistentVolumeClaim"\n apiVersion: "v1"\n metadata:\n name: ${APPLICATION_NAME}\n labels:\n app: ${APPLICATION_NAME}\n spec:\n accessModes:\n - "ReadWriteMany"\n resources:\n requests:\n storage: ${STORAGE_SIZE}\n')),(0,i.yg)("h3",{id:"secret"},"Secret"),(0,i.yg)("p",null,"Then the ",(0,i.yg)("strong",{parentName:"p"},"Secret")," to store the password"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},'- kind: "Secret"\n apiVersion: v1\n metadata:\n name: "${APPLICATION_NAME}"\n labels:\n app: ${APPLICATION_NAME}\n stringData:\n application-password: "${PASSWORD}"\n')),(0,i.yg)("h3",{id:"deployment"},"Deployment"),(0,i.yg)("p",null,"Then the ",(0,i.yg)("strong",{parentName:"p"},"DeploymentConfig")," (aka. Deployment) define how to deploy the JupyterLab image, if you want to deploy another application alongside JupyterLab you can do it by adding as many deployments as you want! (and use the same, or different, persistent volume claims for storage). Checkout the ",(0,i.yg)("a",{parentName:"p",href:"https://docs.openshift.com/container-platform/4.6/applications/deployments/what-deployments-are.html"},"OpenShift Deployments documentation")," for more details."),(0,i.yg)("p",null,"In this first block we will define the strategy to update and recreate our applications if you change the YAML configuration, or when a new latest docker image is updated, allowing your service to always use the latest up-to-date version of a software without any intervention from you. "),(0,i.yg)("p",null,"We chose the ",(0,i.yg)("inlineCode",{parentName:"p"},"Recreate")," release option to make sure the container is properly recreated and avoid unnecessary resources consumption, but you can also use ",(0,i.yg)("inlineCode",{parentName:"p"},"Rolling")," to have a downtime free transition between deployments."),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},'- kind: "DeploymentConfig"\n apiVersion: v1\n metadata:\n name: "${APPLICATION_NAME}"\n labels:\n app: "${APPLICATION_NAME}"\n spec:\n replicas: 1\n strategy:\n type: "Recreate"\n triggers:\n - type: "ConfigChange"\n - type: "ImageChange"\n imageChangeParams:\n automatic: true\n containerNames:\n - jupyter-notebook\n from:\n kind: ImageStreamTag\n name: ${APPLICATION_NAME}:latest\n selector:\n app: "${APPLICATION_NAME}"\n deploymentconfig: "${APPLICATION_NAME}"\n')),(0,i.yg)("h3",{id:"pod-spec"},"Pod spec"),(0,i.yg)("p",null,"Then we define the spec of the ",(0,i.yg)("strong",{parentName:"p"},"pod")," that will be deployed by this DeploymentConfig."),(0,i.yg)("p",null,"Setting the ",(0,i.yg)("inlineCode",{parentName:"p"},"serviceAccountName: anyuid")," is required for most Docker containers as it allows to run a container using any user ID (e.g. root). Otherwise OpenShift expect to use a random user ID, which is require to build the Docker image especially to work with random user IDs."),(0,i.yg)("p",null,"We then create the ",(0,i.yg)("inlineCode",{parentName:"p"},"containers:")," array which is where we will define the containers deployed in the pod. It is recommended to deploy 1 container per pod, as it enables a better separation and management of the applications, apart if you know what you are doing. You can also provide the command to run at the start of the container to overwrite the default one, and define the exposed ports (here 8080)."),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},' template:\n metadata:\n labels:\n app: "${APPLICATION_NAME}"\n deploymentconfig: "${APPLICATION_NAME}"\n spec:\n serviceAccountName: "anyuid"\n containers:\n - name: "jupyter-notebook"\n image: "${APPLICATION_NAME}:latest"\n command:\n - "start-notebook.sh"\n - "--no-browser"\n - "--ip=0.0.0.0"\n ports:\n - containerPort: 8888\n protocol: TCP\n')),(0,i.yg)("h3",{id:"environment-variables-in-the-container"},"Environment variables in the container"),(0,i.yg)("p",null,"Then define the ",(0,i.yg)("strong",{parentName:"p"},"environment variables")," used in your container, usually the password and most parameters are set here, such as enabling ",(0,i.yg)("inlineCode",{parentName:"p"},"sudo")," in the container."),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},' env:\n - name: JUPYTER_TOKEN\n valueFrom:\n secretKeyRef:\n key: "application-password"\n name: "${APPLICATION_NAME}"\n - name: JUPYTER_ENABLE_LAB\n value: "yes"\n - name: GRANT_SUDO\n value: "yes"\n')),(0,i.yg)("h3",{id:"mount-storage"},"Mount storage"),(0,i.yg)("p",null,"Then we need to mount the previously created ",(0,i.yg)("strong",{parentName:"p"},"PersistentVolume")," on ",(0,i.yg)("inlineCode",{parentName:"p"},"/home/jovyan")," , the workspace of JupyterLab. Be careful: ",(0,i.yg)("inlineCode",{parentName:"p"},"volumeMounts")," is in the ",(0,i.yg)("inlineCode",{parentName:"p"},"containers:")," object, and ",(0,i.yg)("inlineCode",{parentName:"p"},"volumes")," is defined in the ",(0,i.yg)("inlineCode",{parentName:"p"},"spec:")," object"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},' volumeMounts:\n - name: data\n mountPath: "/home/jovyan"\n volumes:\n - name: data\n persistentVolumeClaim:\n claimName: "${APPLICATION_NAME}"\n')),(0,i.yg)("h3",{id:"security-context"},"Security context"),(0,i.yg)("p",null,"Then we define the ",(0,i.yg)("strong",{parentName:"p"},"securityContext")," to allow JupyterLab to run as root, this is ",(0,i.yg)("strong",{parentName:"p"},"not required for most applications"),", just a specificity of the official Jupyter images to run with root privileges."),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"}," securityContext:\n runAsUser: 0\n supplementalGroups:\n - 100\n automountServiceAccountToken: false\n")),(0,i.yg)("h3",{id:"service"},"Service"),(0,i.yg)("p",null,"Then we create the ",(0,i.yg)("strong",{parentName:"p"},"Service")," to expose the port 8888 of our JupyterLab container on the project network. This means that the JupyterLab web UI will reachable by all other application deployed in your project using its application name as hostname (e.g. ",(0,i.yg)("inlineCode",{parentName:"p"},"jupyterlab"),")"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},'- kind: "Service"\n apiVersion: v1\n metadata:\n name: "${APPLICATION_NAME}"\n labels:\n app: ${APPLICATION_NAME}\n spec:\n ports:\n - name: 8888-tcp\n protocol: TCP\n port: 8888\n targetPort: 8888\n selector:\n app: ${APPLICATION_NAME}\n deploymentconfig: "${APPLICATION_NAME}"\n type: ClusterIP\n')),(0,i.yg)("h3",{id:"route"},"Route"),(0,i.yg)("p",null,"Finally, we define the ",(0,i.yg)("strong",{parentName:"p"},"Route")," which will automatically generate a URL for the service of your application based following this template: ",(0,i.yg)("inlineCode",{parentName:"p"},"APPLICATION_NAME-PROJECT_ID-DSRI_URL")),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},'- kind: "Route"\n apiVersion: v1\n metadata:\n name: "${APPLICATION_NAME}"\n labels:\n app: ${APPLICATION_NAME}\n spec:\n host: \'\'\n to:\n kind: Service\n name: "${APPLICATION_NAME}"\n weight: 100\n port:\n targetPort: 8888-tcp\n tls:\n termination: edge\n insecureEdgeTerminationPolicy: Redirect\n')),(0,i.yg)("h2",{id:"the-complete-application"},"The complete application"),(0,i.yg)("p",null,"Here is a complete file to describe the JupyterLab deployment template, you can add it to your project catalog by going to ",(0,i.yg)("strong",{parentName:"p"},"+Add")," in the DSRI web UI, then click on the option to add a ",(0,i.yg)("strong",{parentName:"p"},"YAML")," file content, and copy paste the template YAML."),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},'---\nkind: Template\napiVersion: template.openshift.io/v1\nlabels:\n template: jupyterlab-root\nmetadata:\n name: jupyterlab-root\n annotations:\n openshift.io/display-name: JupyterLab\n description: |-\n Start JupyterLab images as the `jovyan` user, with sudo privileges to install anything you need. \n \ud83d\udcc2 Use the `/home/jovyan` folder (workspace of the JupyterLab UI) to store your data in the persistent storage automatically created\n You can find the persistent storage in the DSRI web UI, go to Administrator view > Storage > Persistent Volume Claims\n You can use any image based on the official Jupyter docker stack https://github.com/jupyter/docker-stacks\n - jupyter/tensorflow-notebook\n - jupyter/r-notebook\n - jupyter/all-spark-notebook\n - ghcr.io/maastrichtu-ids/jupyterlab (with Java and SPARQL kernels)\n Or build your own! Checkout https://github.com/MaastrichtU-IDS/jupyterlab for an example of custom image\n Once JupyterLab is deployed you can install any pip packages, JupyterLab extensions, and apt packages.\n iconClass: icon-python\n tags: python,jupyter,notebook\n openshift.io/provider-display-name: Institute of Data Science, UM\n openshift.io/documentation-url: https://maastrichtu-ids.github.io/dsri-documentation/docs/deploy-jupyter\n openshift.io/support-url: https://maastrichtu-ids.github.io/dsri-documentation/help\n \nparameters:\n- name: APPLICATION_NAME\n displayName: Name for the application\n description: Must be without spaces (use -), and unique in the project.\n value: jupyterlab\n required: true\n- name: PASSWORD\n displayName: JupyterLab UI Password\n description: The password/token to access the JupyterLab web UI\n required: true\n- name: APPLICATION_IMAGE\n displayName: Jupyter notebook Docker image\n value: ghcr.io/maastrichtu-ids/jupyterlab:latest\n required: true\n description: You can use any image based on https://github.com/jupyter/docker-stacks\n- name: STORAGE_SIZE\n displayName: Storage size\n description: Size of the storage allocated to the notebook persistent storage in `/home/jovyan`.\n value: 5Gi\n required: true\n \nobjects:\n- kind: "ImageStream"\n apiVersion: image.openshift.io/v1\n metadata:\n name: ${APPLICATION_NAME}\n labels:\n app: ${APPLICATION_NAME}\n spec:\n tags:\n - name: latest\n from:\n kind: DockerImage\n name: ${APPLICATION_IMAGE}\n lookupPolicy:\n local: true\n\n- kind: "PersistentVolumeClaim"\n apiVersion: "v1"\n metadata:\n name: ${APPLICATION_NAME}\n labels:\n app: ${APPLICATION_NAME}\n spec:\n accessModes:\n - "ReadWriteMany"\n resources:\n requests:\n storage: ${STORAGE_SIZE}\n\n- kind: "Secret"\n apiVersion: v1\n metadata:\n name: "${APPLICATION_NAME}"\n labels:\n app: ${APPLICATION_NAME}\n stringData:\n application-password: "${PASSWORD}"\n\n- kind: "DeploymentConfig"\n apiVersion: v1\n metadata:\n name: "${APPLICATION_NAME}"\n labels:\n app: "${APPLICATION_NAME}"\n spec:\n replicas: 1\n strategy:\n type: Recreate\n triggers:\n - type: ConfigChange\n - type: ImageChange\n imageChangeParams:\n automatic: true\n containerNames:\n - jupyter-notebook\n from:\n kind: ImageStreamTag\n name: ${APPLICATION_NAME}:latest\n selector:\n app: "${APPLICATION_NAME}"\n deploymentconfig: "${APPLICATION_NAME}"\n\n template:\n metadata:\n labels:\n app: "${APPLICATION_NAME}"\n deploymentconfig: "${APPLICATION_NAME}"\n spec:\n serviceAccountName: "anyuid"\n containers:\n - name: jupyter-notebook\n image: "${APPLICATION_NAME}:latest"\n command:\n - "start-notebook.sh"\n - "--no-browser"\n - "--ip=0.0.0.0"\n ports:\n - containerPort: 8888\n protocol: TCP\n\n env:\n - name: "JUPYTER_TOKEN"\n valueFrom:\n secretKeyRef:\n key: application-password\n name: "${APPLICATION_NAME}"\n - name: JUPYTER_ENABLE_LAB\n value: "yes"\n - name: GRANT_SUDO\n value: "yes"\n\n volumeMounts:\n - name: data\n mountPath: "/home/jovyan"\n volumes:\n - name: data\n persistentVolumeClaim:\n claimName: "${APPLICATION_NAME}"\n\n securityContext:\n runAsUser: 0\n supplementalGroups:\n - 100\n automountServiceAccountToken: false\n\n- kind: "Service"\n apiVersion: v1\n metadata:\n name: "${APPLICATION_NAME}"\n labels:\n app: ${APPLICATION_NAME}\n spec:\n ports:\n - name: 8888-tcp\n protocol: TCP\n port: 8888\n targetPort: 8888\n selector:\n app: ${APPLICATION_NAME}\n deploymentconfig: "${APPLICATION_NAME}"\n type: ClusterIP\n\n- kind: "Route"\n apiVersion: v1\n metadata:\n name: "${APPLICATION_NAME}"\n labels:\n app: ${APPLICATION_NAME}\n spec:\n host: \'\'\n to:\n kind: Service\n name: "${APPLICATION_NAME}"\n weight: 100\n port:\n targetPort: 8888-tcp\n tls:\n termination: edge\n insecureEdgeTerminationPolicy: Redirect\n')),(0,i.yg)("h2",{id:"add-a-configuration-file"},"Add a configuration file"),(0,i.yg)("p",null,"This practice is more advanced, and is not required for most deployments, but you can easily create a ",(0,i.yg)("strong",{parentName:"p"},"ConfigMap")," object to define any file to be provided at runtime to the application."),(0,i.yg)("p",null,"For example here we are going to define a python script that will be run when starting JupyterLab (",(0,i.yg)("inlineCode",{parentName:"p"},"jupyter_notebook_config.py"),"). It will clone the git repository URL, provided by the user when creating the template, at the start of JupyterLab in the workspace. If this repo contains files with list of packages in the root folder (",(0,i.yg)("inlineCode",{parentName:"p"},"requirements.txt")," and ",(0,i.yg)("inlineCode",{parentName:"p"},"packages.txt"),"), they will be installed at start"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},"- kind: ConfigMap\n apiVersion: v1\n metadata:\n name: \"${APPLICATION_NAME}-cfg\"\n labels:\n app: \"${APPLICATION_NAME}\"\n data:\n # Clone git repo, then install requirements.txt and packages.txt\n jupyter_notebook_config.py: |\n import os\n git_url = os.environ.get('GIT_URL')\n home_dir = os.environ.get('HOME')\n os.chdir(home_dir)\n if git_url:\n repo_id = git_url.rsplit('/', 1)[-1]\n os.system('git clone --quiet --recursive ' + git_url)\n os.chdir(repo_id)\n if os.path.exists('packages.txt'):\n os.system('sudo apt-get update')\n os.system('cat packages.txt | xargs sudo apt-get install -y')\n if os.path.exists('requirements.txt'):\n os.system('pip install -r requirements.txt')\n os.chdir(home_dir)\n")),(0,i.yg)("p",null,"We will then need to mount this config file like a persistent volume in the path we want it to be (here ",(0,i.yg)("inlineCode",{parentName:"p"},"/etc/jupyter/openshift"),"), change the ",(0,i.yg)("strong",{parentName:"p"},"volumes")," and ",(0,i.yg)("strong",{parentName:"p"},"volumeMounts")," of your ",(0,i.yg)("strong",{parentName:"p"},"DeploymentConfig"),":"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},' volumeMounts:\n - name: data\n mountPath: "/home/jovyan"\n - name: configs\n mountPath: "/etc/jupyter/openshift"\n automountServiceAccountToken: false\n volumes:\n - name: data\n persistentVolumeClaim:\n claimName: "${APPLICATION_NAME}"\n - name: configs\n configMap:\n name: "${APPLICATION_NAME}-cfg"\n')),(0,i.yg)("p",null,"Then change the ",(0,i.yg)("inlineCode",{parentName:"p"},"jupyter-notebook")," container start command to include this config file:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},' command:\n - "start-notebook.sh"\n - "--no-browser"\n - "--ip=0.0.0.0"\n - "--config=/etc/jupyter/openshift/jupyter_notebook_config.py"\n')),(0,i.yg)("p",null,"Add the ",(0,i.yg)("strong",{parentName:"p"},"optional parameter")," to get the git URL to clone when the user create the template:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},"parameters:\n- name: GIT_URL\n displayName: URL of the git repository to clone (optional)\n required: false\n description: Source code will be automatically cloned, then requirements.txt and packages.txt content will be automatically installed if presents\n")),(0,i.yg)("p",null,"Finally, add the git URL parameter provided by the user as ",(0,i.yg)("strong",{parentName:"p"},"environment variable")," of the container, so that it is picked up by the config script when running at the start of JupyterLab:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},' env:\n - name: GIT_URL\n value: "${GIT_URL}"\n')),(0,i.yg)("h2",{id:"add-automated-health-checks"},"Add automated health checks"),(0,i.yg)("p",null,"You can add ",(0,i.yg)("strong",{parentName:"p"},"readiness and liveness probes")," to a container to automatically check if the web application is up and ready. This will allow to wait for the JupyterLab web UI to be accessible before showing the application as ready in the Topology. Useful if you are cloning a repository and installing packages, which will take more time to start JupyterLab."),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"}," containers:\n - name: jupyter-notebook\n readinessProbe: \n tcpSocket:\n port: 8888\n livenessProbe: \n initialDelaySeconds: 15 \n tcpSocket: \n port: 8888 \n failureThreshold: 40\n periodSeconds: 10\n timeoutSeconds: 2\n")),(0,i.yg)("p",null,"Checkout the ",(0,i.yg)("a",{parentName:"p",href:"https://docs.openshift.com/container-platform/4.6/applications/application-health.html"},"OpenShift Application health documentation")," for more details."),(0,i.yg)("h2",{id:"define-resource-limits"},"Define resource limits"),(0,i.yg)("p",null,"You can also define resources request and limits for each ",(0,i.yg)("strong",{parentName:"p"},"DeploymentConfig"),", in ",(0,i.yg)("inlineCode",{parentName:"p"},"spec:")),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},' spec:\n resources:\n requests: \n cpu: "1"\n memory: "2Gi"\n limits:\n cpu: "128"\n memory: "300Gi"\n')),(0,i.yg)("h2",{id:"build-your-own-application-template"},"Build your own application template"),(0,i.yg)("p",null,"The easiest way to build a template for a new application is to start from this JupyterLab template:"),(0,i.yg)("ul",null,(0,i.yg)("li",{parentName:"ul"},"Replace ",(0,i.yg)("inlineCode",{parentName:"li"},"jupyterlab-root")," by your application name"),(0,i.yg)("li",{parentName:"ul"},"Replace ",(0,i.yg)("inlineCode",{parentName:"li"},"8888")," by your application"),(0,i.yg)("li",{parentName:"ul"},"Change the template and parameters descriptions to match your application"),(0,i.yg)("li",{parentName:"ul"},"Remove the ",(0,i.yg)("inlineCode",{parentName:"li"},"securityContext")," part, and other objects you do not need")),(0,i.yg)("p",null,"If you need to start multiple containers, copy/paste the objects you need to create and edit them"))}d.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[3360],{5680:(e,t,n)=>{n.d(t,{xA:()=>c,yg:()=>d});var a=n(6540);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function r(e){for(var t=1;t=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var s=a.createContext({}),p=function(e){var t=a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},c=function(e){var t=p(e.components);return a.createElement(s.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},m=a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,i=e.originalType,s=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),m=p(n),d=o,y=m["".concat(s,".").concat(d)]||m[d]||u[d]||i;return n?a.createElement(y,r(r({ref:t},c),{},{components:n})):a.createElement(y,r({ref:t},c))}));function d(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var i=n.length,r=new Array(i);r[0]=m;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:o,r[1]=l;for(var p=2;p{n.r(t),n.d(t,{assets:()=>c,contentTitle:()=>s,default:()=>d,frontMatter:()=>l,metadata:()=>p,toc:()=>u});var a=n(9668),o=n(1367),i=(n(6540),n(5680)),r=["components"],l={id:"anatomy-of-an-application",title:"Anatomy of a DSRI application"},s=void 0,p={unversionedId:"anatomy-of-an-application",id:"anatomy-of-an-application",title:"Anatomy of a DSRI application",description:"This page will present you how an applications is typically built using an OpenShift template. This will also help you understand more in general the different objects that needs to be defined when deploying an application on a Kubernetes cluster. Even if OpenShift templates can only be deployed to OpenShift, the objects they define are the same as in Kubernetes (apart from the Route which becomes Ingress).",source:"@site/docs/anatomy-of-an-application.md",sourceDirName:".",slug:"/anatomy-of-an-application",permalink:"/docs/anatomy-of-an-application",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/anatomy-of-an-application.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"anatomy-of-an-application",title:"Anatomy of a DSRI application"},sidebar:"docs",previous:{title:"Deploy from a Docker image",permalink:"/docs/deploy-from-docker"},next:{title:"Install from Helm charts",permalink:"/docs/helm"}},c={},u=[{value:"Application walkthrough",id:"application-walkthrough",level:2},{value:"Parameters",id:"parameters",level:3},{value:"Image",id:"image",level:3},{value:"Create storage",id:"create-storage",level:3},{value:"Secret",id:"secret",level:3},{value:"Deployment",id:"deployment",level:3},{value:"Pod spec",id:"pod-spec",level:3},{value:"Environment variables in the container",id:"environment-variables-in-the-container",level:3},{value:"Mount storage",id:"mount-storage",level:3},{value:"Security context",id:"security-context",level:3},{value:"Service",id:"service",level:3},{value:"Route",id:"route",level:3},{value:"The complete application",id:"the-complete-application",level:2},{value:"Add a configuration file",id:"add-a-configuration-file",level:2},{value:"Add automated health checks",id:"add-automated-health-checks",level:2},{value:"Define resource limits",id:"define-resource-limits",level:2},{value:"Build your own application template",id:"build-your-own-application-template",level:2}],m={toc:u};function d(e){var t=e.components,n=(0,o.A)(e,r);return(0,i.yg)("wrapper",(0,a.A)({},m,n,{components:t,mdxType:"MDXLayout"}),(0,i.yg)("p",null,"This page will present you how an applications is typically built using an ",(0,i.yg)("strong",{parentName:"p"},"OpenShift template"),". This will also help you understand more in general the different objects that needs to be defined when ",(0,i.yg)("strong",{parentName:"p"},"deploying an application on a Kubernetes cluster"),". Even if OpenShift templates can only be deployed to OpenShift, the objects they define are the same as in Kubernetes (apart from the Route which becomes Ingress)."),(0,i.yg)("p",null,"There are other ways to describe applications on OpenShift cluster (here the DSRI), such as Helm or Operators. But OpenShift templates are the easiest and quickest way to build an application that can be deployed from the DSRI web UI catalog in a few clicks, and by providing a few parameters."),(0,i.yg)("p",null,"It is better to have a basic understanding of what a docker container is to fully understand this walkthrough, but it should already gives a good idea of the different objects deployed with each DSRI application."),(0,i.yg)("p",null,"We will use the template used to deploy JupyterLab as example, and we will describe the goal, importance and caveats of each parts of the application definition. But the same template and instructions can be easily reused for other applications with a web UI to access."),(0,i.yg)("p",null,"Checkout the ",(0,i.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/blob/master/applications/templates/template-jupyterlab-root.yml"},"complete JupyterLab template here")," (it will be slightly different with a bit more comments, but there are globally the same)"),(0,i.yg)("p",null,"You will see that deploying on Kubernetes (and by extension, here OpenShift), is just about defining objects in a YAML file, like a complex ",(0,i.yg)("inlineCode",{parentName:"p"},"docker-compose.yml")," file. "),(0,i.yg)("admonition",{title:"Do you got what it takes?",type:"info"},(0,i.yg)("p",{parentName:"admonition"},"The amount of objects might seems a bit overwhelming at first, but this is what it takes to automatically deploy a complex application on a large cluster, automatically available through a generated URL, with ",(0,i.yg)("inlineCode",{parentName:"p"},"HTTPS")," encryption to protect your passwords when you log to a web UI!")),(0,i.yg)("h2",{id:"application-walkthrough"},"Application walkthrough"),(0,i.yg)("p",null,"First, you need to create your ",(0,i.yg)("strong",{parentName:"p"},"Template")," objects, this will be the main object we will create here as all other objects defined will be deployed by this template. "),(0,i.yg)("p",null,"In this part we mainly just provide the description and information that will be shown to users when deploying the application from the DSRI web UI catalog."),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},"---\nkind: Template\napiVersion: template.openshift.io/v1\nlabels:\n template: jupyterlab-root\nmetadata:\n name: jupyterlab-root\n annotations:\n openshift.io/display-name: JupyterLab\n description: |-\n Start JupyterLab images as the `jovyan` user, with sudo privileges to install anything you need. \n \ud83d\udcc2 Use the `/home/jovyan` folder (workspace of the JupyterLab UI) to store your data in the persistent storage automatically created\n You can find the persistent storage in the DSRI web UI, go to Administrator view > Storage > Persistent Volume Claims\n You can use any image based on the official Jupyter docker stack https://github.com/jupyter/docker-stacks\n - jupyter/tensorflow-notebook\n - jupyter/r-notebook\n - jupyter/all-spark-notebook\n - ghcr.io/maastrichtu-ids/jupyterlab (with Java and SPARQL kernels)\n Or build your own! Checkout https://github.com/MaastrichtU-IDS/jupyterlab for an example of custom image\n Once JupyterLab is deployed you can install any pip packages, JupyterLab extensions, and apt packages.\n iconClass: icon-python\n tags: python,jupyter,notebook\n openshift.io/provider-display-name: Institute of Data Science, UM\n openshift.io/documentation-url: https://maastrichtu-ids.github.io/dsri-documentation/docs/deploy-jupyter\n openshift.io/support-url: https://maastrichtu-ids.github.io/dsri-documentation/help\n")),(0,i.yg)("h3",{id:"parameters"},"Parameters"),(0,i.yg)("p",null,"Then define the ",(0,i.yg)("strong",{parentName:"p"},"parameters")," the user will be able to define in the DSRI catalog web UI when instantiating the application. ",(0,i.yg)("inlineCode",{parentName:"p"},"APPLICATION_NAME")," is the most important as it will be used everywhere to create the objects and identify the application."),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},"parameters:\n- name: APPLICATION_NAME\n displayName: Name for the application\n description: Must be without spaces (use -), and unique in the project.\n value: jupyterlab\n required: true\n- name: PASSWORD\n displayName: JupyterLab UI Password\n description: The password/token to access the JupyterLab web UI\n required: true\n- name: APPLICATION_IMAGE\n displayName: Jupyter notebook Docker image\n value: ghcr.io/maastrichtu-ids/jupyterlab:latest\n required: true\n description: You can use any image based on https://github.com/jupyter/docker-stacks\n- name: STORAGE_SIZE\n displayName: Storage size\n description: Size of the storage allocated to the notebook persistent storage in `/home/jovyan`.\n value: 5Gi\n required: true\n")),(0,i.yg)("p",null,"We can then refer to those parameters value (filled by the users of the template) in the rest of the template using this syntax: ",(0,i.yg)("inlineCode",{parentName:"p"},"${APPLICATION_NAME}")),(0,i.yg)("p",null,"We will now ",(0,i.yg)("strong",{parentName:"p"},"describe all objects deployed")," when we instantiate this template (to start an application). "),(0,i.yg)("h3",{id:"image"},"Image"),(0,i.yg)("p",null,"First we define the ",(0,i.yg)("strong",{parentName:"p"},"ImageStream")," object to import the Docker image(s) of your application(s) on the DSRI cluster"),(0,i.yg)("p",null,"Setting the ",(0,i.yg)("inlineCode",{parentName:"p"},"importPolicy: scheduled")," to ",(0,i.yg)("inlineCode",{parentName:"p"},"true")," will have the DSRI to automatically check for new version of this image, which can be useful if you want to always have the latest published version of an applications. Visit the ",(0,i.yg)("a",{parentName:"p",href:"https://docs.openshift.com/container-platform/4.6/openshift_images/image-streams-manage.html"},"OpenShift ImageStreams documentation")," for more details. Be careful as enabling this feature without real need will cause the DSRI to query DockerHub more, which might require you to login to DockerHub to increase your pull request quota."),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},'objects:\n- kind: "ImageStream"\n apiVersion: image.openshift.io/v1\n metadata:\n name: ${APPLICATION_NAME}\n labels:\n app: ${APPLICATION_NAME}\n spec:\n tags:\n - name: latest\n from:\n kind: DockerImage\n name: ${APPLICATION_IMAGE}\n importPolicy:\n scheduled: true\n lookupPolicy:\n local: true\n')),(0,i.yg)("h3",{id:"create-storage"},"Create storage"),(0,i.yg)("p",null,"Then we define the ",(0,i.yg)("strong",{parentName:"p"},"PersistentVolumeClaim"),", which is a persistent storage on which we will mount the ",(0,i.yg)("inlineCode",{parentName:"p"},"/home/jovyan")," folder to avoid loosing data if our application is restarted."),(0,i.yg)("p",null,"Any file outside of a persistent volume can be lost at any moment if the pod restart, usually it only consists in temporary file if you are properly working in the persistent volume folder. This can be useful also if your application is crashing, stopping and restarting your pod (application) might fix it."),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},'- kind: "PersistentVolumeClaim"\n apiVersion: "v1"\n metadata:\n name: ${APPLICATION_NAME}\n labels:\n app: ${APPLICATION_NAME}\n spec:\n accessModes:\n - "ReadWriteMany"\n resources:\n requests:\n storage: ${STORAGE_SIZE}\n')),(0,i.yg)("h3",{id:"secret"},"Secret"),(0,i.yg)("p",null,"Then the ",(0,i.yg)("strong",{parentName:"p"},"Secret")," to store the password"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},'- kind: "Secret"\n apiVersion: v1\n metadata:\n name: "${APPLICATION_NAME}"\n labels:\n app: ${APPLICATION_NAME}\n stringData:\n application-password: "${PASSWORD}"\n')),(0,i.yg)("h3",{id:"deployment"},"Deployment"),(0,i.yg)("p",null,"Then the ",(0,i.yg)("strong",{parentName:"p"},"DeploymentConfig")," (aka. Deployment) define how to deploy the JupyterLab image, if you want to deploy another application alongside JupyterLab you can do it by adding as many deployments as you want! (and use the same, or different, persistent volume claims for storage). Checkout the ",(0,i.yg)("a",{parentName:"p",href:"https://docs.openshift.com/container-platform/4.6/applications/deployments/what-deployments-are.html"},"OpenShift Deployments documentation")," for more details."),(0,i.yg)("p",null,"In this first block we will define the strategy to update and recreate our applications if you change the YAML configuration, or when a new latest docker image is updated, allowing your service to always use the latest up-to-date version of a software without any intervention from you. "),(0,i.yg)("p",null,"We chose the ",(0,i.yg)("inlineCode",{parentName:"p"},"Recreate")," release option to make sure the container is properly recreated and avoid unnecessary resources consumption, but you can also use ",(0,i.yg)("inlineCode",{parentName:"p"},"Rolling")," to have a downtime free transition between deployments."),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},'- kind: "DeploymentConfig"\n apiVersion: v1\n metadata:\n name: "${APPLICATION_NAME}"\n labels:\n app: "${APPLICATION_NAME}"\n spec:\n replicas: 1\n strategy:\n type: "Recreate"\n triggers:\n - type: "ConfigChange"\n - type: "ImageChange"\n imageChangeParams:\n automatic: true\n containerNames:\n - jupyter-notebook\n from:\n kind: ImageStreamTag\n name: ${APPLICATION_NAME}:latest\n selector:\n app: "${APPLICATION_NAME}"\n deploymentconfig: "${APPLICATION_NAME}"\n')),(0,i.yg)("h3",{id:"pod-spec"},"Pod spec"),(0,i.yg)("p",null,"Then we define the spec of the ",(0,i.yg)("strong",{parentName:"p"},"pod")," that will be deployed by this DeploymentConfig."),(0,i.yg)("p",null,"Setting the ",(0,i.yg)("inlineCode",{parentName:"p"},"serviceAccountName: anyuid")," is required for most Docker containers as it allows to run a container using any user ID (e.g. root). Otherwise OpenShift expect to use a random user ID, which is require to build the Docker image especially to work with random user IDs."),(0,i.yg)("p",null,"We then create the ",(0,i.yg)("inlineCode",{parentName:"p"},"containers:")," array which is where we will define the containers deployed in the pod. It is recommended to deploy 1 container per pod, as it enables a better separation and management of the applications, apart if you know what you are doing. You can also provide the command to run at the start of the container to overwrite the default one, and define the exposed ports (here 8080)."),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},' template:\n metadata:\n labels:\n app: "${APPLICATION_NAME}"\n deploymentconfig: "${APPLICATION_NAME}"\n spec:\n serviceAccountName: "anyuid"\n containers:\n - name: "jupyter-notebook"\n image: "${APPLICATION_NAME}:latest"\n command:\n - "start-notebook.sh"\n - "--no-browser"\n - "--ip=0.0.0.0"\n ports:\n - containerPort: 8888\n protocol: TCP\n')),(0,i.yg)("h3",{id:"environment-variables-in-the-container"},"Environment variables in the container"),(0,i.yg)("p",null,"Then define the ",(0,i.yg)("strong",{parentName:"p"},"environment variables")," used in your container, usually the password and most parameters are set here, such as enabling ",(0,i.yg)("inlineCode",{parentName:"p"},"sudo")," in the container."),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},' env:\n - name: JUPYTER_TOKEN\n valueFrom:\n secretKeyRef:\n key: "application-password"\n name: "${APPLICATION_NAME}"\n - name: JUPYTER_ENABLE_LAB\n value: "yes"\n - name: GRANT_SUDO\n value: "yes"\n')),(0,i.yg)("h3",{id:"mount-storage"},"Mount storage"),(0,i.yg)("p",null,"Then we need to mount the previously created ",(0,i.yg)("strong",{parentName:"p"},"PersistentVolume")," on ",(0,i.yg)("inlineCode",{parentName:"p"},"/home/jovyan")," , the workspace of JupyterLab. Be careful: ",(0,i.yg)("inlineCode",{parentName:"p"},"volumeMounts")," is in the ",(0,i.yg)("inlineCode",{parentName:"p"},"containers:")," object, and ",(0,i.yg)("inlineCode",{parentName:"p"},"volumes")," is defined in the ",(0,i.yg)("inlineCode",{parentName:"p"},"spec:")," object"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},' volumeMounts:\n - name: data\n mountPath: "/home/jovyan"\n volumes:\n - name: data\n persistentVolumeClaim:\n claimName: "${APPLICATION_NAME}"\n')),(0,i.yg)("h3",{id:"security-context"},"Security context"),(0,i.yg)("p",null,"Then we define the ",(0,i.yg)("strong",{parentName:"p"},"securityContext")," to allow JupyterLab to run as root, this is ",(0,i.yg)("strong",{parentName:"p"},"not required for most applications"),", just a specificity of the official Jupyter images to run with root privileges."),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"}," securityContext:\n runAsUser: 0\n supplementalGroups:\n - 100\n automountServiceAccountToken: false\n")),(0,i.yg)("h3",{id:"service"},"Service"),(0,i.yg)("p",null,"Then we create the ",(0,i.yg)("strong",{parentName:"p"},"Service")," to expose the port 8888 of our JupyterLab container on the project network. This means that the JupyterLab web UI will reachable by all other application deployed in your project using its application name as hostname (e.g. ",(0,i.yg)("inlineCode",{parentName:"p"},"jupyterlab"),")"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},'- kind: "Service"\n apiVersion: v1\n metadata:\n name: "${APPLICATION_NAME}"\n labels:\n app: ${APPLICATION_NAME}\n spec:\n ports:\n - name: 8888-tcp\n protocol: TCP\n port: 8888\n targetPort: 8888\n selector:\n app: ${APPLICATION_NAME}\n deploymentconfig: "${APPLICATION_NAME}"\n type: ClusterIP\n')),(0,i.yg)("h3",{id:"route"},"Route"),(0,i.yg)("p",null,"Finally, we define the ",(0,i.yg)("strong",{parentName:"p"},"Route")," which will automatically generate a URL for the service of your application based following this template: ",(0,i.yg)("inlineCode",{parentName:"p"},"APPLICATION_NAME-PROJECT_ID-DSRI_URL")),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},'- kind: "Route"\n apiVersion: v1\n metadata:\n name: "${APPLICATION_NAME}"\n labels:\n app: ${APPLICATION_NAME}\n spec:\n host: \'\'\n to:\n kind: Service\n name: "${APPLICATION_NAME}"\n weight: 100\n port:\n targetPort: 8888-tcp\n tls:\n termination: edge\n insecureEdgeTerminationPolicy: Redirect\n')),(0,i.yg)("h2",{id:"the-complete-application"},"The complete application"),(0,i.yg)("p",null,"Here is a complete file to describe the JupyterLab deployment template, you can add it to your project catalog by going to ",(0,i.yg)("strong",{parentName:"p"},"+Add")," in the DSRI web UI, then click on the option to add a ",(0,i.yg)("strong",{parentName:"p"},"YAML")," file content, and copy paste the template YAML."),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},'---\nkind: Template\napiVersion: template.openshift.io/v1\nlabels:\n template: jupyterlab-root\nmetadata:\n name: jupyterlab-root\n annotations:\n openshift.io/display-name: JupyterLab\n description: |-\n Start JupyterLab images as the `jovyan` user, with sudo privileges to install anything you need. \n \ud83d\udcc2 Use the `/home/jovyan` folder (workspace of the JupyterLab UI) to store your data in the persistent storage automatically created\n You can find the persistent storage in the DSRI web UI, go to Administrator view > Storage > Persistent Volume Claims\n You can use any image based on the official Jupyter docker stack https://github.com/jupyter/docker-stacks\n - jupyter/tensorflow-notebook\n - jupyter/r-notebook\n - jupyter/all-spark-notebook\n - ghcr.io/maastrichtu-ids/jupyterlab (with Java and SPARQL kernels)\n Or build your own! Checkout https://github.com/MaastrichtU-IDS/jupyterlab for an example of custom image\n Once JupyterLab is deployed you can install any pip packages, JupyterLab extensions, and apt packages.\n iconClass: icon-python\n tags: python,jupyter,notebook\n openshift.io/provider-display-name: Institute of Data Science, UM\n openshift.io/documentation-url: https://maastrichtu-ids.github.io/dsri-documentation/docs/deploy-jupyter\n openshift.io/support-url: https://maastrichtu-ids.github.io/dsri-documentation/help\n \nparameters:\n- name: APPLICATION_NAME\n displayName: Name for the application\n description: Must be without spaces (use -), and unique in the project.\n value: jupyterlab\n required: true\n- name: PASSWORD\n displayName: JupyterLab UI Password\n description: The password/token to access the JupyterLab web UI\n required: true\n- name: APPLICATION_IMAGE\n displayName: Jupyter notebook Docker image\n value: ghcr.io/maastrichtu-ids/jupyterlab:latest\n required: true\n description: You can use any image based on https://github.com/jupyter/docker-stacks\n- name: STORAGE_SIZE\n displayName: Storage size\n description: Size of the storage allocated to the notebook persistent storage in `/home/jovyan`.\n value: 5Gi\n required: true\n \nobjects:\n- kind: "ImageStream"\n apiVersion: image.openshift.io/v1\n metadata:\n name: ${APPLICATION_NAME}\n labels:\n app: ${APPLICATION_NAME}\n spec:\n tags:\n - name: latest\n from:\n kind: DockerImage\n name: ${APPLICATION_IMAGE}\n lookupPolicy:\n local: true\n\n- kind: "PersistentVolumeClaim"\n apiVersion: "v1"\n metadata:\n name: ${APPLICATION_NAME}\n labels:\n app: ${APPLICATION_NAME}\n spec:\n accessModes:\n - "ReadWriteMany"\n resources:\n requests:\n storage: ${STORAGE_SIZE}\n\n- kind: "Secret"\n apiVersion: v1\n metadata:\n name: "${APPLICATION_NAME}"\n labels:\n app: ${APPLICATION_NAME}\n stringData:\n application-password: "${PASSWORD}"\n\n- kind: "DeploymentConfig"\n apiVersion: v1\n metadata:\n name: "${APPLICATION_NAME}"\n labels:\n app: "${APPLICATION_NAME}"\n spec:\n replicas: 1\n strategy:\n type: Recreate\n triggers:\n - type: ConfigChange\n - type: ImageChange\n imageChangeParams:\n automatic: true\n containerNames:\n - jupyter-notebook\n from:\n kind: ImageStreamTag\n name: ${APPLICATION_NAME}:latest\n selector:\n app: "${APPLICATION_NAME}"\n deploymentconfig: "${APPLICATION_NAME}"\n\n template:\n metadata:\n labels:\n app: "${APPLICATION_NAME}"\n deploymentconfig: "${APPLICATION_NAME}"\n spec:\n serviceAccountName: "anyuid"\n containers:\n - name: jupyter-notebook\n image: "${APPLICATION_NAME}:latest"\n command:\n - "start-notebook.sh"\n - "--no-browser"\n - "--ip=0.0.0.0"\n ports:\n - containerPort: 8888\n protocol: TCP\n\n env:\n - name: "JUPYTER_TOKEN"\n valueFrom:\n secretKeyRef:\n key: application-password\n name: "${APPLICATION_NAME}"\n - name: JUPYTER_ENABLE_LAB\n value: "yes"\n - name: GRANT_SUDO\n value: "yes"\n\n volumeMounts:\n - name: data\n mountPath: "/home/jovyan"\n volumes:\n - name: data\n persistentVolumeClaim:\n claimName: "${APPLICATION_NAME}"\n\n securityContext:\n runAsUser: 0\n supplementalGroups:\n - 100\n automountServiceAccountToken: false\n\n- kind: "Service"\n apiVersion: v1\n metadata:\n name: "${APPLICATION_NAME}"\n labels:\n app: ${APPLICATION_NAME}\n spec:\n ports:\n - name: 8888-tcp\n protocol: TCP\n port: 8888\n targetPort: 8888\n selector:\n app: ${APPLICATION_NAME}\n deploymentconfig: "${APPLICATION_NAME}"\n type: ClusterIP\n\n- kind: "Route"\n apiVersion: v1\n metadata:\n name: "${APPLICATION_NAME}"\n labels:\n app: ${APPLICATION_NAME}\n spec:\n host: \'\'\n to:\n kind: Service\n name: "${APPLICATION_NAME}"\n weight: 100\n port:\n targetPort: 8888-tcp\n tls:\n termination: edge\n insecureEdgeTerminationPolicy: Redirect\n')),(0,i.yg)("h2",{id:"add-a-configuration-file"},"Add a configuration file"),(0,i.yg)("p",null,"This practice is more advanced, and is not required for most deployments, but you can easily create a ",(0,i.yg)("strong",{parentName:"p"},"ConfigMap")," object to define any file to be provided at runtime to the application."),(0,i.yg)("p",null,"For example here we are going to define a python script that will be run when starting JupyterLab (",(0,i.yg)("inlineCode",{parentName:"p"},"jupyter_notebook_config.py"),"). It will clone the git repository URL, provided by the user when creating the template, at the start of JupyterLab in the workspace. If this repo contains files with list of packages in the root folder (",(0,i.yg)("inlineCode",{parentName:"p"},"requirements.txt")," and ",(0,i.yg)("inlineCode",{parentName:"p"},"packages.txt"),"), they will be installed at start"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},"- kind: ConfigMap\n apiVersion: v1\n metadata:\n name: \"${APPLICATION_NAME}-cfg\"\n labels:\n app: \"${APPLICATION_NAME}\"\n data:\n # Clone git repo, then install requirements.txt and packages.txt\n jupyter_notebook_config.py: |\n import os\n git_url = os.environ.get('GIT_URL')\n home_dir = os.environ.get('HOME')\n os.chdir(home_dir)\n if git_url:\n repo_id = git_url.rsplit('/', 1)[-1]\n os.system('git clone --quiet --recursive ' + git_url)\n os.chdir(repo_id)\n if os.path.exists('packages.txt'):\n os.system('sudo apt-get update')\n os.system('cat packages.txt | xargs sudo apt-get install -y')\n if os.path.exists('requirements.txt'):\n os.system('pip install -r requirements.txt')\n os.chdir(home_dir)\n")),(0,i.yg)("p",null,"We will then need to mount this config file like a persistent volume in the path we want it to be (here ",(0,i.yg)("inlineCode",{parentName:"p"},"/etc/jupyter/openshift"),"), change the ",(0,i.yg)("strong",{parentName:"p"},"volumes")," and ",(0,i.yg)("strong",{parentName:"p"},"volumeMounts")," of your ",(0,i.yg)("strong",{parentName:"p"},"DeploymentConfig"),":"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},' volumeMounts:\n - name: data\n mountPath: "/home/jovyan"\n - name: configs\n mountPath: "/etc/jupyter/openshift"\n automountServiceAccountToken: false\n volumes:\n - name: data\n persistentVolumeClaim:\n claimName: "${APPLICATION_NAME}"\n - name: configs\n configMap:\n name: "${APPLICATION_NAME}-cfg"\n')),(0,i.yg)("p",null,"Then change the ",(0,i.yg)("inlineCode",{parentName:"p"},"jupyter-notebook")," container start command to include this config file:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},' command:\n - "start-notebook.sh"\n - "--no-browser"\n - "--ip=0.0.0.0"\n - "--config=/etc/jupyter/openshift/jupyter_notebook_config.py"\n')),(0,i.yg)("p",null,"Add the ",(0,i.yg)("strong",{parentName:"p"},"optional parameter")," to get the git URL to clone when the user create the template:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},"parameters:\n- name: GIT_URL\n displayName: URL of the git repository to clone (optional)\n required: false\n description: Source code will be automatically cloned, then requirements.txt and packages.txt content will be automatically installed if presents\n")),(0,i.yg)("p",null,"Finally, add the git URL parameter provided by the user as ",(0,i.yg)("strong",{parentName:"p"},"environment variable")," of the container, so that it is picked up by the config script when running at the start of JupyterLab:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},' env:\n - name: GIT_URL\n value: "${GIT_URL}"\n')),(0,i.yg)("h2",{id:"add-automated-health-checks"},"Add automated health checks"),(0,i.yg)("p",null,"You can add ",(0,i.yg)("strong",{parentName:"p"},"readiness and liveness probes")," to a container to automatically check if the web application is up and ready. This will allow to wait for the JupyterLab web UI to be accessible before showing the application as ready in the Topology. Useful if you are cloning a repository and installing packages, which will take more time to start JupyterLab."),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"}," containers:\n - name: jupyter-notebook\n readinessProbe: \n tcpSocket:\n port: 8888\n livenessProbe: \n initialDelaySeconds: 15 \n tcpSocket: \n port: 8888 \n failureThreshold: 40\n periodSeconds: 10\n timeoutSeconds: 2\n")),(0,i.yg)("p",null,"Checkout the ",(0,i.yg)("a",{parentName:"p",href:"https://docs.openshift.com/container-platform/4.6/applications/application-health.html"},"OpenShift Application health documentation")," for more details."),(0,i.yg)("h2",{id:"define-resource-limits"},"Define resource limits"),(0,i.yg)("p",null,"You can also define resources request and limits for each ",(0,i.yg)("strong",{parentName:"p"},"DeploymentConfig"),", in ",(0,i.yg)("inlineCode",{parentName:"p"},"spec:")),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},' spec:\n resources:\n requests: \n cpu: "1"\n memory: "2Gi"\n limits:\n cpu: "128"\n memory: "300Gi"\n')),(0,i.yg)("h2",{id:"build-your-own-application-template"},"Build your own application template"),(0,i.yg)("p",null,"The easiest way to build a template for a new application is to start from this JupyterLab template:"),(0,i.yg)("ul",null,(0,i.yg)("li",{parentName:"ul"},"Replace ",(0,i.yg)("inlineCode",{parentName:"li"},"jupyterlab-root")," by your application name"),(0,i.yg)("li",{parentName:"ul"},"Replace ",(0,i.yg)("inlineCode",{parentName:"li"},"8888")," by your application"),(0,i.yg)("li",{parentName:"ul"},"Change the template and parameters descriptions to match your application"),(0,i.yg)("li",{parentName:"ul"},"Remove the ",(0,i.yg)("inlineCode",{parentName:"li"},"securityContext")," part, and other objects you do not need")),(0,i.yg)("p",null,"If you need to start multiple containers, copy/paste the objects you need to create and edit them"))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/0e4359fd.f43d495f.js b/assets/js/0e4359fd.f1e4698c.js similarity index 99% rename from assets/js/0e4359fd.f43d495f.js rename to assets/js/0e4359fd.f1e4698c.js index 352c47f25..ebfb4488f 100644 --- a/assets/js/0e4359fd.f43d495f.js +++ b/assets/js/0e4359fd.f1e4698c.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[2957],{5680:(e,a,t)=>{t.d(a,{xA:()=>m,yg:()=>g});var l=t(6540);function n(e,a,t){return a in e?Object.defineProperty(e,a,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[a]=t,e}function r(e,a){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);a&&(l=l.filter((function(a){return Object.getOwnPropertyDescriptor(e,a).enumerable}))),t.push.apply(t,l)}return t}function o(e){for(var a=1;a=0||(n[t]=e[t]);return n}(e,a);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(l=0;l=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(n[t]=e[t])}return n}var i=l.createContext({}),p=function(e){var a=l.useContext(i),t=a;return e&&(t="function"==typeof e?e(a):o(o({},a),e)),t},m=function(e){var a=p(e.components);return l.createElement(i.Provider,{value:a},e.children)},c={inlineCode:"code",wrapper:function(e){var a=e.children;return l.createElement(l.Fragment,{},a)}},h=l.forwardRef((function(e,a){var t=e.components,n=e.mdxType,r=e.originalType,i=e.parentName,m=s(e,["components","mdxType","originalType","parentName"]),h=p(t),g=n,u=h["".concat(i,".").concat(g)]||h[g]||c[g]||r;return t?l.createElement(u,o(o({ref:a},m),{},{components:t})):l.createElement(u,o({ref:a},m))}));function g(e,a){var t=arguments,n=a&&a.mdxType;if("string"==typeof e||n){var r=t.length,o=new Array(r);o[0]=h;var s={};for(var i in a)hasOwnProperty.call(a,i)&&(s[i]=a[i]);s.originalType=e,s.mdxType="string"==typeof e?e:n,o[1]=s;for(var p=2;p{t.r(a),t.d(a,{assets:()=>m,contentTitle:()=>i,default:()=>g,frontMatter:()=>s,metadata:()=>p,toc:()=>c});var l=t(9668),n=t(1367),r=(t(6540),t(5680)),o=["components"],s={id:"helm",title:"Install from Helm charts"},i=void 0,p={unversionedId:"helm",id:"helm",title:"Install from Helm charts",description:"Helm is a popular package manager for Kubernetes. A Helm chart is a bundle of parameterizable YAML resources for Kubernetes/OpenShift.",source:"@site/docs/helm.md",sourceDirName:".",slug:"/helm",permalink:"/docs/helm",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/helm.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"helm",title:"Install from Helm charts"},sidebar:"docs",previous:{title:"Anatomy of a DSRI application",permalink:"/docs/anatomy-of-an-application"},next:{title:"Install from Operators",permalink:"/docs/operators"}},m={},c=[{value:"Install the Helm client",id:"install-the-helm-client",level:2},{value:"Install Golang",id:"install-golang",level:3},{value:"Install Helm",id:"install-helm",level:3},{value:"Install on Linux",id:"install-on-linux",level:4},{value:"Install on MacOS",id:"install-on-macos",level:4},{value:"Install on Windows",id:"install-on-windows",level:4},{value:"Check Helm installation",id:"check-helm-installation",level:3},{value:"Install a Helm chart",id:"install-a-helm-chart",level:2},{value:"Start a MySQL database with Helm",id:"start-a-mysql-database-with-helm",level:3},{value:"Uninstall the application",id:"uninstall-the-application",level:3},{value:"Set deployment parameters",id:"set-deployment-parameters",level:3}],h={toc:c};function g(e){var a=e.components,t=(0,n.A)(e,o);return(0,r.yg)("wrapper",(0,l.A)({},h,t,{components:a,mdxType:"MDXLayout"}),(0,r.yg)("p",null,(0,r.yg)("a",{parentName:"p",href:"https://helm.sh/"},"Helm")," is a popular package manager for ",(0,r.yg)("a",{parentName:"p",href:"https://kubernetes.io/"},"Kubernetes"),". A Helm chart is a bundle of parameterizable YAML resources for Kubernetes/OpenShift."),(0,r.yg)("admonition",{title:"Difference with Operators",type:"caution"},(0,r.yg)("p",{parentName:"admonition"},"Helm charts can be defined as ",(0,r.yg)("a",{parentName:"p",href:"/docs/operators"},"Operators")," (if they are packaged using the ",(0,r.yg)("inlineCode",{parentName:"p"},"operator-sdk"),"), but they are not all Operators.")),(0,r.yg)("p",null,"See the official ",(0,r.yg)("a",{parentName:"p",href:"https://docs.openshift.com/container-platform/4.6/cli_reference/helm_cli/getting-started-with-helm-on-openshift-container-platform.html"},"documentation for Helm on OpenShift"),"."),(0,r.yg)("h2",{id:"install-the-helm-client"},"Install the Helm client"),(0,r.yg)("h3",{id:"install-golang"},"Install Golang"),(0,r.yg)("p",null,"Go lang is required to run Helm. Install ",(0,r.yg)("inlineCode",{parentName:"p"},"go 1.14.4")," on Linux, you can find instructions for MacOS, Windows and newer versions at ",(0,r.yg)("a",{parentName:"p",href:"https://golang.org/dl"},"https://golang.org/dl")),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},'wget https://dl.google.com/go/go1.14.4.linux-amd64.tar.gz\n\n# Extract to /usr/local\ntar -C /usr/local -xzf go1.14.4.linux-amd64.tar.gz\n\n# Add Go to path in .profile\necho "export PATH=$PATH:/usr/local/go/bin" >> ~/.profile\n# Or in .zshrc if you use ZSH\necho "export PATH=$PATH:/usr/local/go/bin" >> ~/.zshrc\n')),(0,r.yg)("blockquote",null,(0,r.yg)("p",{parentName:"blockquote"},"Restart your laptop for the changes to take effects or execute ",(0,r.yg)("inlineCode",{parentName:"p"},"source ~/.profile"))),(0,r.yg)("h3",{id:"install-helm"},"Install Helm"),(0,r.yg)("p",null,"You can also use the ",(0,r.yg)("a",{parentName:"p",href:"https://helm.sh/docs/intro/install/"},"official documentation to install Helm")," on your machine."),(0,r.yg)("h4",{id:"install-on-linux"},"Install on Linux"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash\n")),(0,r.yg)("blockquote",null,(0,r.yg)("p",{parentName:"blockquote"},"See ",(0,r.yg)("a",{parentName:"p",href:"https://helm.sh/docs/intro/install/#from-the-binary-releases"},"Helm documentation for Linux"),".")),(0,r.yg)("h4",{id:"install-on-macos"},"Install on MacOS"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"brew install helm\n")),(0,r.yg)("blockquote",null,(0,r.yg)("p",{parentName:"blockquote"},"See ",(0,r.yg)("a",{parentName:"p",href:"https://helm.sh/docs/intro/install/#from-homebrew-macos"},"Helm documentation for MacOS"),".")),(0,r.yg)("h4",{id:"install-on-windows"},"Install on Windows"),(0,r.yg)("p",null,"Install using ",(0,r.yg)("a",{parentName:"p",href:"https://chocolatey.org/"},"Chocolatey"),"."),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"choco install kubernetes-helm\n")),(0,r.yg)("blockquote",null,(0,r.yg)("p",{parentName:"blockquote"},"See ",(0,r.yg)("a",{parentName:"p",href:"https://helm.sh/docs/intro/install/#from-chocolatey-windows"},"Helm documentation for Windows"),".")),(0,r.yg)("h3",{id:"check-helm-installation"},"Check Helm installation"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"helm version\n")),(0,r.yg)("h2",{id:"install-a-helm-chart"},"Install a Helm chart"),(0,r.yg)("p",null,"Explore published Helm charts at ",(0,r.yg)("a",{parentName:"p",href:"https://hub.helm.sh"},"https://hub.helm.sh \u26f5")),(0,r.yg)("h3",{id:"start-a-mysql-database-with-helm"},"Start a MySQL database with Helm"),(0,r.yg)("p",null,"Example from the ",(0,r.yg)("a",{parentName:"p",href:"https://docs.openshift.com/container-platform/4.3/cli_reference/helm_cli/getting-started-with-helm-on-openshift-container-platform.html"},"OpenShift 4.3 documentation"),". See also the ",(0,r.yg)("a",{parentName:"p",href:"https://helm.sh/docs/intro/using_helm/"},"official Helm documentation"),"."),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},"Add the repository of official Helm charts to your local Helm client:")),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"helm repo add stable https://kubernetes-charts.storage.googleapis.com/\n")),(0,r.yg)("ol",{start:2},(0,r.yg)("li",{parentName:"ol"},"Update the repository:")),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"helm repo update\n")),(0,r.yg)("ol",{start:3},(0,r.yg)("li",{parentName:"ol"},"Install an example MySQL chart, and start the application named ",(0,r.yg)("inlineCode",{parentName:"li"},"example-mysql"),":")),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"helm install example-mysql stable/mysql\n")),(0,r.yg)("admonition",{title:"Password",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"The instructions to retrieve the admin password and connect to the database will be displayed in the terminal. "),(0,r.yg)("p",{parentName:"admonition"},"Retrieve the database password with this command (N.B.: ",(0,r.yg)("inlineCode",{parentName:"p"},"kubectl")," can also be used in place of ",(0,r.yg)("inlineCode",{parentName:"p"},"oc"),"):"),(0,r.yg)("pre",{parentName:"admonition"},(0,r.yg)("code",{parentName:"pre",className:"language-bash"},'oc get secret example-mysql -o jsonpath="{.data.mysql-root-password}" | base64 --decode; echo\n'))),(0,r.yg)("ol",{start:4},(0,r.yg)("li",{parentName:"ol"},"Verify that the chart has installed successfully:")),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"helm list\n")),(0,r.yg)("ol",{start:5},(0,r.yg)("li",{parentName:"ol"},"Expose the MySQL service as a route:")),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc expose service example-mysql\noc get routes\n")),(0,r.yg)("p",null,"Or port-forward to http://localhost:3306"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc port-forward svc/example-mysql 3306\n")),(0,r.yg)("h3",{id:"uninstall-the-application"},"Uninstall the application"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"helm uninstall example-mysql\n")),(0,r.yg)("h3",{id:"set-deployment-parameters"},"Set deployment parameters"),(0,r.yg)("p",null,"You can also define deployment parameters when installing a Helm chart, such as the ",(0,r.yg)("strong",{parentName:"p"},"service account")," and ",(0,r.yg)("strong",{parentName:"p"},"node selector"),". "),(0,r.yg)("p",null,"For example, here we make sure the application will run on DSRI CPU nodes and use the ",(0,r.yg)("inlineCode",{parentName:"p"},"anyuid")," service account:"),(0,r.yg)("p",null,"Add Bitnami repository:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"helm repo add bitnami https://charts.bitnami.com/bitnami\n")),(0,r.yg)("p",null,"Install and start Postgresql:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"helm install postgresql-db bitnami/postgresql --set nodeSelector.dsri.unimaas.nl/cpu=true --set serviceAccount.name=anyuid\n")))}g.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[2957],{5680:(e,a,t)=>{t.d(a,{xA:()=>m,yg:()=>g});var l=t(6540);function n(e,a,t){return a in e?Object.defineProperty(e,a,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[a]=t,e}function r(e,a){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);a&&(l=l.filter((function(a){return Object.getOwnPropertyDescriptor(e,a).enumerable}))),t.push.apply(t,l)}return t}function o(e){for(var a=1;a=0||(n[t]=e[t]);return n}(e,a);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(l=0;l=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(n[t]=e[t])}return n}var i=l.createContext({}),p=function(e){var a=l.useContext(i),t=a;return e&&(t="function"==typeof e?e(a):o(o({},a),e)),t},m=function(e){var a=p(e.components);return l.createElement(i.Provider,{value:a},e.children)},c={inlineCode:"code",wrapper:function(e){var a=e.children;return l.createElement(l.Fragment,{},a)}},h=l.forwardRef((function(e,a){var t=e.components,n=e.mdxType,r=e.originalType,i=e.parentName,m=s(e,["components","mdxType","originalType","parentName"]),h=p(t),g=n,u=h["".concat(i,".").concat(g)]||h[g]||c[g]||r;return t?l.createElement(u,o(o({ref:a},m),{},{components:t})):l.createElement(u,o({ref:a},m))}));function g(e,a){var t=arguments,n=a&&a.mdxType;if("string"==typeof e||n){var r=t.length,o=new Array(r);o[0]=h;var s={};for(var i in a)hasOwnProperty.call(a,i)&&(s[i]=a[i]);s.originalType=e,s.mdxType="string"==typeof e?e:n,o[1]=s;for(var p=2;p{t.r(a),t.d(a,{assets:()=>m,contentTitle:()=>i,default:()=>g,frontMatter:()=>s,metadata:()=>p,toc:()=>c});var l=t(9668),n=t(1367),r=(t(6540),t(5680)),o=["components"],s={id:"helm",title:"Install from Helm charts"},i=void 0,p={unversionedId:"helm",id:"helm",title:"Install from Helm charts",description:"Helm is a popular package manager for Kubernetes. A Helm chart is a bundle of parameterizable YAML resources for Kubernetes/OpenShift.",source:"@site/docs/helm.md",sourceDirName:".",slug:"/helm",permalink:"/docs/helm",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/helm.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"helm",title:"Install from Helm charts"},sidebar:"docs",previous:{title:"Anatomy of a DSRI application",permalink:"/docs/anatomy-of-an-application"},next:{title:"Install from Operators",permalink:"/docs/operators"}},m={},c=[{value:"Install the Helm client",id:"install-the-helm-client",level:2},{value:"Install Golang",id:"install-golang",level:3},{value:"Install Helm",id:"install-helm",level:3},{value:"Install on Linux",id:"install-on-linux",level:4},{value:"Install on MacOS",id:"install-on-macos",level:4},{value:"Install on Windows",id:"install-on-windows",level:4},{value:"Check Helm installation",id:"check-helm-installation",level:3},{value:"Install a Helm chart",id:"install-a-helm-chart",level:2},{value:"Start a MySQL database with Helm",id:"start-a-mysql-database-with-helm",level:3},{value:"Uninstall the application",id:"uninstall-the-application",level:3},{value:"Set deployment parameters",id:"set-deployment-parameters",level:3}],h={toc:c};function g(e){var a=e.components,t=(0,n.A)(e,o);return(0,r.yg)("wrapper",(0,l.A)({},h,t,{components:a,mdxType:"MDXLayout"}),(0,r.yg)("p",null,(0,r.yg)("a",{parentName:"p",href:"https://helm.sh/"},"Helm")," is a popular package manager for ",(0,r.yg)("a",{parentName:"p",href:"https://kubernetes.io/"},"Kubernetes"),". A Helm chart is a bundle of parameterizable YAML resources for Kubernetes/OpenShift."),(0,r.yg)("admonition",{title:"Difference with Operators",type:"caution"},(0,r.yg)("p",{parentName:"admonition"},"Helm charts can be defined as ",(0,r.yg)("a",{parentName:"p",href:"/docs/operators"},"Operators")," (if they are packaged using the ",(0,r.yg)("inlineCode",{parentName:"p"},"operator-sdk"),"), but they are not all Operators.")),(0,r.yg)("p",null,"See the official ",(0,r.yg)("a",{parentName:"p",href:"https://docs.openshift.com/container-platform/4.6/cli_reference/helm_cli/getting-started-with-helm-on-openshift-container-platform.html"},"documentation for Helm on OpenShift"),"."),(0,r.yg)("h2",{id:"install-the-helm-client"},"Install the Helm client"),(0,r.yg)("h3",{id:"install-golang"},"Install Golang"),(0,r.yg)("p",null,"Go lang is required to run Helm. Install ",(0,r.yg)("inlineCode",{parentName:"p"},"go 1.14.4")," on Linux, you can find instructions for MacOS, Windows and newer versions at ",(0,r.yg)("a",{parentName:"p",href:"https://golang.org/dl"},"https://golang.org/dl")),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},'wget https://dl.google.com/go/go1.14.4.linux-amd64.tar.gz\n\n# Extract to /usr/local\ntar -C /usr/local -xzf go1.14.4.linux-amd64.tar.gz\n\n# Add Go to path in .profile\necho "export PATH=$PATH:/usr/local/go/bin" >> ~/.profile\n# Or in .zshrc if you use ZSH\necho "export PATH=$PATH:/usr/local/go/bin" >> ~/.zshrc\n')),(0,r.yg)("blockquote",null,(0,r.yg)("p",{parentName:"blockquote"},"Restart your laptop for the changes to take effects or execute ",(0,r.yg)("inlineCode",{parentName:"p"},"source ~/.profile"))),(0,r.yg)("h3",{id:"install-helm"},"Install Helm"),(0,r.yg)("p",null,"You can also use the ",(0,r.yg)("a",{parentName:"p",href:"https://helm.sh/docs/intro/install/"},"official documentation to install Helm")," on your machine."),(0,r.yg)("h4",{id:"install-on-linux"},"Install on Linux"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash\n")),(0,r.yg)("blockquote",null,(0,r.yg)("p",{parentName:"blockquote"},"See ",(0,r.yg)("a",{parentName:"p",href:"https://helm.sh/docs/intro/install/#from-the-binary-releases"},"Helm documentation for Linux"),".")),(0,r.yg)("h4",{id:"install-on-macos"},"Install on MacOS"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"brew install helm\n")),(0,r.yg)("blockquote",null,(0,r.yg)("p",{parentName:"blockquote"},"See ",(0,r.yg)("a",{parentName:"p",href:"https://helm.sh/docs/intro/install/#from-homebrew-macos"},"Helm documentation for MacOS"),".")),(0,r.yg)("h4",{id:"install-on-windows"},"Install on Windows"),(0,r.yg)("p",null,"Install using ",(0,r.yg)("a",{parentName:"p",href:"https://chocolatey.org/"},"Chocolatey"),"."),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"choco install kubernetes-helm\n")),(0,r.yg)("blockquote",null,(0,r.yg)("p",{parentName:"blockquote"},"See ",(0,r.yg)("a",{parentName:"p",href:"https://helm.sh/docs/intro/install/#from-chocolatey-windows"},"Helm documentation for Windows"),".")),(0,r.yg)("h3",{id:"check-helm-installation"},"Check Helm installation"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"helm version\n")),(0,r.yg)("h2",{id:"install-a-helm-chart"},"Install a Helm chart"),(0,r.yg)("p",null,"Explore published Helm charts at ",(0,r.yg)("a",{parentName:"p",href:"https://hub.helm.sh"},"https://hub.helm.sh \u26f5")),(0,r.yg)("h3",{id:"start-a-mysql-database-with-helm"},"Start a MySQL database with Helm"),(0,r.yg)("p",null,"Example from the ",(0,r.yg)("a",{parentName:"p",href:"https://docs.openshift.com/container-platform/4.3/cli_reference/helm_cli/getting-started-with-helm-on-openshift-container-platform.html"},"OpenShift 4.3 documentation"),". See also the ",(0,r.yg)("a",{parentName:"p",href:"https://helm.sh/docs/intro/using_helm/"},"official Helm documentation"),"."),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},"Add the repository of official Helm charts to your local Helm client:")),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"helm repo add stable https://kubernetes-charts.storage.googleapis.com/\n")),(0,r.yg)("ol",{start:2},(0,r.yg)("li",{parentName:"ol"},"Update the repository:")),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"helm repo update\n")),(0,r.yg)("ol",{start:3},(0,r.yg)("li",{parentName:"ol"},"Install an example MySQL chart, and start the application named ",(0,r.yg)("inlineCode",{parentName:"li"},"example-mysql"),":")),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"helm install example-mysql stable/mysql\n")),(0,r.yg)("admonition",{title:"Password",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"The instructions to retrieve the admin password and connect to the database will be displayed in the terminal. "),(0,r.yg)("p",{parentName:"admonition"},"Retrieve the database password with this command (N.B.: ",(0,r.yg)("inlineCode",{parentName:"p"},"kubectl")," can also be used in place of ",(0,r.yg)("inlineCode",{parentName:"p"},"oc"),"):"),(0,r.yg)("pre",{parentName:"admonition"},(0,r.yg)("code",{parentName:"pre",className:"language-bash"},'oc get secret example-mysql -o jsonpath="{.data.mysql-root-password}" | base64 --decode; echo\n'))),(0,r.yg)("ol",{start:4},(0,r.yg)("li",{parentName:"ol"},"Verify that the chart has installed successfully:")),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"helm list\n")),(0,r.yg)("ol",{start:5},(0,r.yg)("li",{parentName:"ol"},"Expose the MySQL service as a route:")),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc expose service example-mysql\noc get routes\n")),(0,r.yg)("p",null,"Or port-forward to http://localhost:3306"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc port-forward svc/example-mysql 3306\n")),(0,r.yg)("h3",{id:"uninstall-the-application"},"Uninstall the application"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"helm uninstall example-mysql\n")),(0,r.yg)("h3",{id:"set-deployment-parameters"},"Set deployment parameters"),(0,r.yg)("p",null,"You can also define deployment parameters when installing a Helm chart, such as the ",(0,r.yg)("strong",{parentName:"p"},"service account")," and ",(0,r.yg)("strong",{parentName:"p"},"node selector"),". "),(0,r.yg)("p",null,"For example, here we make sure the application will run on DSRI CPU nodes and use the ",(0,r.yg)("inlineCode",{parentName:"p"},"anyuid")," service account:"),(0,r.yg)("p",null,"Add Bitnami repository:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"helm repo add bitnami https://charts.bitnami.com/bitnami\n")),(0,r.yg)("p",null,"Install and start Postgresql:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"helm install postgresql-db bitnami/postgresql --set nodeSelector.dsri.unimaas.nl/cpu=true --set serviceAccount.name=anyuid\n")))}g.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/11da9ee4.8d2f38ab.js b/assets/js/11da9ee4.7651d68e.js similarity index 99% rename from assets/js/11da9ee4.8d2f38ab.js rename to assets/js/11da9ee4.7651d68e.js index aa5b387bb..ce995b7da 100644 --- a/assets/js/11da9ee4.8d2f38ab.js +++ b/assets/js/11da9ee4.7651d68e.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[4809],{5680:(t,e,a)=>{a.d(e,{xA:()=>u,yg:()=>h});var r=a(6540);function n(t,e,a){return e in t?Object.defineProperty(t,e,{value:a,enumerable:!0,configurable:!0,writable:!0}):t[e]=a,t}function o(t,e){var a=Object.keys(t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(t);e&&(r=r.filter((function(e){return Object.getOwnPropertyDescriptor(t,e).enumerable}))),a.push.apply(a,r)}return a}function i(t){for(var e=1;e=0||(n[a]=t[a]);return n}(t,e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(t);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(t,a)&&(n[a]=t[a])}return n}var l=r.createContext({}),s=function(t){var e=r.useContext(l),a=e;return t&&(a="function"==typeof t?t(e):i(i({},e),t)),a},u=function(t){var e=s(t.components);return r.createElement(l.Provider,{value:e},t.children)},c={inlineCode:"code",wrapper:function(t){var e=t.children;return r.createElement(r.Fragment,{},e)}},m=r.forwardRef((function(t,e){var a=t.components,n=t.mdxType,o=t.originalType,l=t.parentName,u=p(t,["components","mdxType","originalType","parentName"]),m=s(a),h=n,d=m["".concat(l,".").concat(h)]||m[h]||c[h]||o;return a?r.createElement(d,i(i({ref:e},u),{},{components:a})):r.createElement(d,i({ref:e},u))}));function h(t,e){var a=arguments,n=e&&e.mdxType;if("string"==typeof t||n){var o=a.length,i=new Array(o);i[0]=m;var p={};for(var l in e)hasOwnProperty.call(e,l)&&(p[l]=e[l]);p.originalType=t,p.mdxType="string"==typeof t?t:n,i[1]=p;for(var s=2;s{a.r(e),a.d(e,{assets:()=>u,contentTitle:()=>l,default:()=>h,frontMatter:()=>p,metadata:()=>s,toc:()=>c});var r=a(9668),n=a(1367),o=(a(6540),a(5680)),i=["components"],p={id:"catalog-opendatahub",title:"OpenDataHub"},l=void 0,s={unversionedId:"catalog-opendatahub",id:"catalog-opendatahub",title:"OpenDataHub",description:"Deploying an OpenDataHub cluster is a work in progress on the DSRI, contact us if you are interested in trying it out.",source:"@site/docs/catalog-opendatahub.md",sourceDirName:".",slug:"/catalog-opendatahub",permalink:"/docs/catalog-opendatahub",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/catalog-opendatahub.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"catalog-opendatahub",title:"OpenDataHub"}},u={},c=[{value:"Components available on DSRI",id:"components-available-on-dsri",level:2},{value:"Start Spark with JupyterHub",id:"start-spark-with-jupyterhub",level:3},{value:"All components",id:"all-components",level:2}],m={toc:c};function h(t){var e=t.components,a=(0,n.A)(t,i);return(0,o.yg)("wrapper",(0,r.A)({},m,a,{components:e,mdxType:"MDXLayout"}),(0,o.yg)("admonition",{title:"Work in progress",type:"warning"},(0,o.yg)("p",{parentName:"admonition"},"Deploying an ",(0,o.yg)("a",{parentName:"p",href:"https://opendatahub.io"},"OpenDataHub")," cluster is a ",(0,o.yg)("strong",{parentName:"p"},"work in progress")," on the DSRI, ",(0,o.yg)("strong",{parentName:"p"},(0,o.yg)("a",{parentName:"strong",href:"/help"},"contact us"))," if you are interested in trying it out.")),(0,o.yg)("p",null,(0,o.yg)("a",{parentName:"p",href:"https://opendatahub.io"},"OpenDataHub")," is a project to orchestrate the deployment of Data Science applications on OpenShift, based on KubeFlow."),(0,o.yg)("h2",{id:"components-available-on-dsri"},"Components available on DSRI"),(0,o.yg)("p",null,"Those components have been tested on the DSRI:"),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/odh-manifests/blob/master/jupyterhub/README.md"},"JupyterHub")),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/odh-manifests/blob/master/radanalyticsio/README.md"},"Spark Operator")," from ",(0,o.yg)("a",{parentName:"li",href:"https://radanalytics.io/"},"radanalytics"))),(0,o.yg)("h3",{id:"start-spark-with-jupyterhub"},"Start Spark with JupyterHub"),(0,o.yg)("ol",null,(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Checkout the ",(0,o.yg)("a",{parentName:"p",href:"https://opendatahub.io/docs/getting-started/quick-installation.html"},"official documentation to start an instance of OpenDataHub")," (note that the Operator has already been installed)")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Then visit the ",(0,o.yg)("a",{parentName:"p",href:"https://opendatahub.io/docs/getting-started/basic-tutorial.html"},"documentation to reach the Spark cluster")," from a Jupyter notebook."))),(0,o.yg)("h2",{id:"all-components"},"All components"),(0,o.yg)("p",null,"Here are all the components that can be deployed as part of an ",(0,o.yg)("a",{parentName:"p",href:"https://opendatahub.io"},"OpenDataHub"),":"),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/odh-manifests/blob/master/jupyterhub/README.md"},"JupyterHub")),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/odh-manifests/blob/master/airflow/README.md"},"Airflow")),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/odh-manifests/blob/master/odhargo/README.md"},"Argo")),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/odh-manifests/blob/master/grafana/README.md"},"Grafana")," & ",(0,o.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/odh-manifests/blob/master/prometheus/README.md"},"Prometheus")," for data/logs visualization"),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/odh-manifests/blob/master/radanalyticsio/README.md"},"Spark Operator")," from ",(0,o.yg)("a",{parentName:"li",href:"https://radanalytics.io/"},"radanalytics")),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/odh-manifests/blob/master/kafka/README.md"},"Kafka"),"/Strimzi for streaming applications"),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/odh-manifests/blob/master/superset/README.md"},"Superset")," for data visualization"),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/odh-manifests/blob/master/ai-library/README.md"},"AI Library")," (Seldon to publish AI models)")),(0,o.yg)("p",null,"Let us know if you need help to deploy one of those components on the DSRI."))}h.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[4809],{5680:(t,e,a)=>{a.d(e,{xA:()=>u,yg:()=>h});var r=a(6540);function n(t,e,a){return e in t?Object.defineProperty(t,e,{value:a,enumerable:!0,configurable:!0,writable:!0}):t[e]=a,t}function o(t,e){var a=Object.keys(t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(t);e&&(r=r.filter((function(e){return Object.getOwnPropertyDescriptor(t,e).enumerable}))),a.push.apply(a,r)}return a}function i(t){for(var e=1;e=0||(n[a]=t[a]);return n}(t,e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(t);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(t,a)&&(n[a]=t[a])}return n}var l=r.createContext({}),s=function(t){var e=r.useContext(l),a=e;return t&&(a="function"==typeof t?t(e):i(i({},e),t)),a},u=function(t){var e=s(t.components);return r.createElement(l.Provider,{value:e},t.children)},c={inlineCode:"code",wrapper:function(t){var e=t.children;return r.createElement(r.Fragment,{},e)}},m=r.forwardRef((function(t,e){var a=t.components,n=t.mdxType,o=t.originalType,l=t.parentName,u=p(t,["components","mdxType","originalType","parentName"]),m=s(a),h=n,d=m["".concat(l,".").concat(h)]||m[h]||c[h]||o;return a?r.createElement(d,i(i({ref:e},u),{},{components:a})):r.createElement(d,i({ref:e},u))}));function h(t,e){var a=arguments,n=e&&e.mdxType;if("string"==typeof t||n){var o=a.length,i=new Array(o);i[0]=m;var p={};for(var l in e)hasOwnProperty.call(e,l)&&(p[l]=e[l]);p.originalType=t,p.mdxType="string"==typeof t?t:n,i[1]=p;for(var s=2;s{a.r(e),a.d(e,{assets:()=>u,contentTitle:()=>l,default:()=>h,frontMatter:()=>p,metadata:()=>s,toc:()=>c});var r=a(9668),n=a(1367),o=(a(6540),a(5680)),i=["components"],p={id:"catalog-opendatahub",title:"OpenDataHub"},l=void 0,s={unversionedId:"catalog-opendatahub",id:"catalog-opendatahub",title:"OpenDataHub",description:"Deploying an OpenDataHub cluster is a work in progress on the DSRI, contact us if you are interested in trying it out.",source:"@site/docs/catalog-opendatahub.md",sourceDirName:".",slug:"/catalog-opendatahub",permalink:"/docs/catalog-opendatahub",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/catalog-opendatahub.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"catalog-opendatahub",title:"OpenDataHub"}},u={},c=[{value:"Components available on DSRI",id:"components-available-on-dsri",level:2},{value:"Start Spark with JupyterHub",id:"start-spark-with-jupyterhub",level:3},{value:"All components",id:"all-components",level:2}],m={toc:c};function h(t){var e=t.components,a=(0,n.A)(t,i);return(0,o.yg)("wrapper",(0,r.A)({},m,a,{components:e,mdxType:"MDXLayout"}),(0,o.yg)("admonition",{title:"Work in progress",type:"warning"},(0,o.yg)("p",{parentName:"admonition"},"Deploying an ",(0,o.yg)("a",{parentName:"p",href:"https://opendatahub.io"},"OpenDataHub")," cluster is a ",(0,o.yg)("strong",{parentName:"p"},"work in progress")," on the DSRI, ",(0,o.yg)("strong",{parentName:"p"},(0,o.yg)("a",{parentName:"strong",href:"/help"},"contact us"))," if you are interested in trying it out.")),(0,o.yg)("p",null,(0,o.yg)("a",{parentName:"p",href:"https://opendatahub.io"},"OpenDataHub")," is a project to orchestrate the deployment of Data Science applications on OpenShift, based on KubeFlow."),(0,o.yg)("h2",{id:"components-available-on-dsri"},"Components available on DSRI"),(0,o.yg)("p",null,"Those components have been tested on the DSRI:"),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/odh-manifests/blob/master/jupyterhub/README.md"},"JupyterHub")),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/odh-manifests/blob/master/radanalyticsio/README.md"},"Spark Operator")," from ",(0,o.yg)("a",{parentName:"li",href:"https://radanalytics.io/"},"radanalytics"))),(0,o.yg)("h3",{id:"start-spark-with-jupyterhub"},"Start Spark with JupyterHub"),(0,o.yg)("ol",null,(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Checkout the ",(0,o.yg)("a",{parentName:"p",href:"https://opendatahub.io/docs/getting-started/quick-installation.html"},"official documentation to start an instance of OpenDataHub")," (note that the Operator has already been installed)")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Then visit the ",(0,o.yg)("a",{parentName:"p",href:"https://opendatahub.io/docs/getting-started/basic-tutorial.html"},"documentation to reach the Spark cluster")," from a Jupyter notebook."))),(0,o.yg)("h2",{id:"all-components"},"All components"),(0,o.yg)("p",null,"Here are all the components that can be deployed as part of an ",(0,o.yg)("a",{parentName:"p",href:"https://opendatahub.io"},"OpenDataHub"),":"),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/odh-manifests/blob/master/jupyterhub/README.md"},"JupyterHub")),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/odh-manifests/blob/master/airflow/README.md"},"Airflow")),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/odh-manifests/blob/master/odhargo/README.md"},"Argo")),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/odh-manifests/blob/master/grafana/README.md"},"Grafana")," & ",(0,o.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/odh-manifests/blob/master/prometheus/README.md"},"Prometheus")," for data/logs visualization"),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/odh-manifests/blob/master/radanalyticsio/README.md"},"Spark Operator")," from ",(0,o.yg)("a",{parentName:"li",href:"https://radanalytics.io/"},"radanalytics")),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/odh-manifests/blob/master/kafka/README.md"},"Kafka"),"/Strimzi for streaming applications"),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/odh-manifests/blob/master/superset/README.md"},"Superset")," for data visualization"),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/odh-manifests/blob/master/ai-library/README.md"},"AI Library")," (Seldon to publish AI models)")),(0,o.yg)("p",null,"Let us know if you need help to deploy one of those components on the DSRI."))}h.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/1dc85e61.5719cfe3.js b/assets/js/1dc85e61.b06c6fc0.js similarity index 98% rename from assets/js/1dc85e61.5719cfe3.js rename to assets/js/1dc85e61.b06c6fc0.js index a4ef5ca79..ece7ecdb5 100644 --- a/assets/js/1dc85e61.5719cfe3.js +++ b/assets/js/1dc85e61.b06c6fc0.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[9328],{5680:(e,n,t)=>{t.d(n,{xA:()=>c,yg:()=>m});var r=t(6540);function a(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function o(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);n&&(r=r.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,r)}return t}function i(e){for(var n=1;n=0||(a[t]=e[t]);return a}(e,n);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(a[t]=e[t])}return a}var u=r.createContext({}),p=function(e){var n=r.useContext(u),t=n;return e&&(t="function"==typeof e?e(n):i(i({},n),e)),t},c=function(e){var n=p(e.components);return r.createElement(u.Provider,{value:n},e.children)},s={inlineCode:"code",wrapper:function(e){var n=e.children;return r.createElement(r.Fragment,{},n)}},g=r.forwardRef((function(e,n){var t=e.components,a=e.mdxType,o=e.originalType,u=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),g=p(t),m=a,y=g["".concat(u,".").concat(m)]||g[m]||s[m]||o;return t?r.createElement(y,i(i({ref:n},c),{},{components:t})):r.createElement(y,i({ref:n},c))}));function m(e,n){var t=arguments,a=n&&n.mdxType;if("string"==typeof e||a){var o=t.length,i=new Array(o);i[0]=g;var l={};for(var u in n)hasOwnProperty.call(n,u)&&(l[u]=n[u]);l.originalType=e,l.mdxType="string"==typeof e?e:a,i[1]=l;for(var p=2;p{t.r(n),t.d(n,{assets:()=>c,contentTitle:()=>u,default:()=>m,frontMatter:()=>l,metadata:()=>p,toc:()=>s});var r=t(9668),a=t(1367),o=(t(6540),t(5680)),i=["components"],l={},u=void 0,p={unversionedId:"deploy-gitlab-runner",id:"deploy-gitlab-runner",title:"deploy-gitlab-runner",description:"First, obtain gitlab runner registration token via the gitlab webinterface",source:"@site/docs/deploy-gitlab-runner.md",sourceDirName:".",slug:"/deploy-gitlab-runner",permalink:"/docs/deploy-gitlab-runner",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/deploy-gitlab-runner.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{}},c={},s=[],g={toc:s};function m(e){var n=e.components,t=(0,a.A)(e,i);return(0,o.yg)("wrapper",(0,r.A)({},g,t,{components:n,mdxType:"MDXLayout"}),(0,o.yg)("p",null,"First, obtain gitlab runner registration token via the gitlab webinterface"),(0,o.yg)("p",null,"TODO: add screenshot"),(0,o.yg)("p",null,'Add "GitLab Runner" operator to your project from the Operators --\x3e OperatorHub page.\nMake sure you choose the "certified" GitLab Runner (v1.4.0) The community runner (v1.10.0) is a bit more up to date, but currently does not work.'),(0,o.yg)("p",null,"Install in a specific namespace on the cluster. Choose your namespace in the dropdown."),(0,o.yg)("p",null,"Create registration token secret:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-yaml"},"---\napiVersion: v1\nkind: Secret\nmetadata:\n name: gitlab-runner-secret\ntype: Opaque\nstringData:\n runner-registration-token: \n")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre"},"oc create -f gitlab-runner-secret.yaml\n")),(0,o.yg)("p",null,"Although, this should also work:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre"},"oc create secret generic gitlab-runner-secret --from-literal=runner-registration-token=\n")),(0,o.yg)("p",null,"Add the following to the ConfigMap of the GitLab Runner operator:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-yaml"},'[[runners]]\n executor = "kubernetes"\n [runners.kubernetes]\n [runners.kubernetes.volumes]\n [[runners.kubernetes.volumes.empty_dir]]\n name = "empty-dir"\n mount_path = "/"\n medium = "Memory"\n')),(0,o.yg)("p",null,"Create the configmap:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre"},"oc create configmap custom-config-toml --from-file config.toml=/tmp/customconfig \n")),(0,o.yg)("p",null,"Create the gitlab runner Custom Resource Definition:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-yaml"},"apiVersion: apps.gitlab.com/v1beta2\nkind: Runner\nmetadata:\n name: gitlab-runner\nspec:\n gitlabUrl: https://gitlab.maastrichtuniversity.nl\n token: gitlab-runner-secret\n config: custom-config-toml\n tags: openshift\n")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-yaml"},"--- other stuff dont use!\napiVersion: apps.gitlab.com/v1beta2\nkind: Runner\nmetadata:\n name: gitlab-runner\nspec:\n gitlabUrl: https://gitlab.maastrichtuniversity.nl\n buildImage: alpine\n token: gitlab-runner-secret\n tags: openshift\n")))}m.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[9328],{5680:(e,n,t)=>{t.d(n,{xA:()=>c,yg:()=>m});var r=t(6540);function a(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function o(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);n&&(r=r.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,r)}return t}function i(e){for(var n=1;n=0||(a[t]=e[t]);return a}(e,n);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(a[t]=e[t])}return a}var u=r.createContext({}),p=function(e){var n=r.useContext(u),t=n;return e&&(t="function"==typeof e?e(n):i(i({},n),e)),t},c=function(e){var n=p(e.components);return r.createElement(u.Provider,{value:n},e.children)},s={inlineCode:"code",wrapper:function(e){var n=e.children;return r.createElement(r.Fragment,{},n)}},g=r.forwardRef((function(e,n){var t=e.components,a=e.mdxType,o=e.originalType,u=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),g=p(t),m=a,y=g["".concat(u,".").concat(m)]||g[m]||s[m]||o;return t?r.createElement(y,i(i({ref:n},c),{},{components:t})):r.createElement(y,i({ref:n},c))}));function m(e,n){var t=arguments,a=n&&n.mdxType;if("string"==typeof e||a){var o=t.length,i=new Array(o);i[0]=g;var l={};for(var u in n)hasOwnProperty.call(n,u)&&(l[u]=n[u]);l.originalType=e,l.mdxType="string"==typeof e?e:a,i[1]=l;for(var p=2;p{t.r(n),t.d(n,{assets:()=>c,contentTitle:()=>u,default:()=>m,frontMatter:()=>l,metadata:()=>p,toc:()=>s});var r=t(9668),a=t(1367),o=(t(6540),t(5680)),i=["components"],l={},u=void 0,p={unversionedId:"deploy-gitlab-runner",id:"deploy-gitlab-runner",title:"deploy-gitlab-runner",description:"First, obtain gitlab runner registration token via the gitlab webinterface",source:"@site/docs/deploy-gitlab-runner.md",sourceDirName:".",slug:"/deploy-gitlab-runner",permalink:"/docs/deploy-gitlab-runner",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/deploy-gitlab-runner.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{}},c={},s=[],g={toc:s};function m(e){var n=e.components,t=(0,a.A)(e,i);return(0,o.yg)("wrapper",(0,r.A)({},g,t,{components:n,mdxType:"MDXLayout"}),(0,o.yg)("p",null,"First, obtain gitlab runner registration token via the gitlab webinterface"),(0,o.yg)("p",null,"TODO: add screenshot"),(0,o.yg)("p",null,'Add "GitLab Runner" operator to your project from the Operators --\x3e OperatorHub page.\nMake sure you choose the "certified" GitLab Runner (v1.4.0) The community runner (v1.10.0) is a bit more up to date, but currently does not work.'),(0,o.yg)("p",null,"Install in a specific namespace on the cluster. Choose your namespace in the dropdown."),(0,o.yg)("p",null,"Create registration token secret:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-yaml"},"---\napiVersion: v1\nkind: Secret\nmetadata:\n name: gitlab-runner-secret\ntype: Opaque\nstringData:\n runner-registration-token: \n")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre"},"oc create -f gitlab-runner-secret.yaml\n")),(0,o.yg)("p",null,"Although, this should also work:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre"},"oc create secret generic gitlab-runner-secret --from-literal=runner-registration-token=\n")),(0,o.yg)("p",null,"Add the following to the ConfigMap of the GitLab Runner operator:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-yaml"},'[[runners]]\n executor = "kubernetes"\n [runners.kubernetes]\n [runners.kubernetes.volumes]\n [[runners.kubernetes.volumes.empty_dir]]\n name = "empty-dir"\n mount_path = "/"\n medium = "Memory"\n')),(0,o.yg)("p",null,"Create the configmap:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre"},"oc create configmap custom-config-toml --from-file config.toml=/tmp/customconfig \n")),(0,o.yg)("p",null,"Create the gitlab runner Custom Resource Definition:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-yaml"},"apiVersion: apps.gitlab.com/v1beta2\nkind: Runner\nmetadata:\n name: gitlab-runner\nspec:\n gitlabUrl: https://gitlab.maastrichtuniversity.nl\n token: gitlab-runner-secret\n config: custom-config-toml\n tags: openshift\n")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-yaml"},"--- other stuff dont use!\napiVersion: apps.gitlab.com/v1beta2\nkind: Runner\nmetadata:\n name: gitlab-runner\nspec:\n gitlabUrl: https://gitlab.maastrichtuniversity.nl\n buildImage: alpine\n token: gitlab-runner-secret\n tags: openshift\n")))}m.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/21dd1498.b86a930c.js b/assets/js/21dd1498.8eb4482d.js similarity index 99% rename from assets/js/21dd1498.b86a930c.js rename to assets/js/21dd1498.8eb4482d.js index bdb0acb53..f5b3f4ac7 100644 --- a/assets/js/21dd1498.b86a930c.js +++ b/assets/js/21dd1498.8eb4482d.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[4896],{5680:(e,t,a)=>{a.d(t,{xA:()=>g,yg:()=>d});var n=a(6540);function r(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function o(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function i(e){for(var t=1;t=0||(r[a]=e[a]);return r}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(r[a]=e[a])}return r}var p=n.createContext({}),l=function(e){var t=n.useContext(p),a=t;return e&&(a="function"==typeof e?e(t):i(i({},t),e)),a},g=function(e){var t=l(e.components);return n.createElement(p.Provider,{value:t},e.children)},m={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},y=n.forwardRef((function(e,t){var a=e.components,r=e.mdxType,o=e.originalType,p=e.parentName,g=s(e,["components","mdxType","originalType","parentName"]),y=l(a),d=r,c=y["".concat(p,".").concat(d)]||y[d]||m[d]||o;return a?n.createElement(c,i(i({ref:t},g),{},{components:a})):n.createElement(c,i({ref:t},g))}));function d(e,t){var a=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var o=a.length,i=new Array(o);i[0]=y;var s={};for(var p in t)hasOwnProperty.call(t,p)&&(s[p]=t[p]);s.originalType=e,s.mdxType="string"==typeof e?e:r,i[1]=s;for(var l=2;l{a.r(t),a.d(t,{assets:()=>g,contentTitle:()=>p,default:()=>d,frontMatter:()=>s,metadata:()=>l,toc:()=>m});var n=a(9668),r=a(1367),o=(a(6540),a(5680)),i=["components"],s={id:"openshift-storage",title:"Data storage"},p=void 0,l={unversionedId:"openshift-storage",id:"openshift-storage",title:"Data storage",description:"Different storages can be used when running services on the DSRI:",source:"@site/docs/openshift-storage.md",sourceDirName:".",slug:"/openshift-storage",permalink:"/docs/openshift-storage",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/openshift-storage.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"openshift-storage",title:"Data storage"},sidebar:"docs",previous:{title:"Command Line Interface",permalink:"/docs/openshift-commands"},next:{title:"Delete objects (advanced)",permalink:"/docs/openshift-delete-objects"}},g={},m=[{value:"Create the Persistent Storage",id:"create-the-persistent-storage",level:3},{value:"Connect the Existing Persistent Storage",id:"connect-the-existing-persistent-storage",level:3},{value:"Expand existing Persistent Storage",id:"expand-existing-persistent-storage",level:3},{value:"Use the dynamic storage",id:"use-the-dynamic-storage",level:3},{value:"Use the ephemeral storage",id:"use-the-ephemeral-storage",level:3}],y={toc:m};function d(e){var t=e.components,a=(0,r.A)(e,i);return(0,o.yg)("wrapper",(0,n.A)({},y,a,{components:t,mdxType:"MDXLayout"}),(0,o.yg)("p",null,"Different storages can be used when running services on the DSRI:"),(0,o.yg)("p",null,"\ud83e\udd8b ",(0,o.yg)("strong",{parentName:"p"},"Ephemeral storage"),": storage is bound to the pod, data will be lost when the pod is deleted (but this deployment does not require to request the creation of a persistent storage, and is faster to test code)."),(0,o.yg)("p",null,"\u26a1 ",(0,o.yg)("strong",{parentName:"p"},"Dynamic storage"),": automatically create a persistent storage when starting an application. Can also be created in the OpenShift web UI, using the ",(0,o.yg)("inlineCode",{parentName:"p"},"dynamic-maprfs")," Storage Class."),(0,o.yg)("p",null,"\ud83d\uddc4\ufe0f ",(0,o.yg)("strong",{parentName:"p"},"Persistent storage"),": You can use a persistent storage volume to store data. Please see the Create the Persistent Storage section. You can do this yourself. Please keep in mind that there are no backups made of data on DSRI. "),(0,o.yg)("admonition",{title:"Storage per project",type:"caution"},(0,o.yg)("p",{parentName:"admonition"},"A storage (aka. Persistent Volume Claim) is only accessible in the project where it has been created.")),(0,o.yg)("h3",{id:"create-the-persistent-storage"},"Create the Persistent Storage"),(0,o.yg)("ol",null,(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Switch to the ",(0,o.yg)("strong",{parentName:"p"},"Administrator")," view")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Go to the ",(0,o.yg)("strong",{parentName:"p"},"Project")," panel ")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Select your project")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Expand the ",(0,o.yg)("strong",{parentName:"p"},"Storage")," panel then go to the ",(0,o.yg)("strong",{parentName:"p"},"Persistent Volume Claim")," panel")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Click the button call ",(0,o.yg)("strong",{parentName:"p"},"Create Persistent Volume Claim")),(0,o.yg)("p",{parentName:"li"},"then you will redirect the wizard of Create Persistent Volume Claim ")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Provide the unique ",(0,o.yg)("strong",{parentName:"p"},"Persistent Volume Claim Name")," start with ",(0,o.yg)("inlineCode",{parentName:"p"}," pvc-")," "),(0,o.yg)("p",{parentName:"li"},"example: ",(0,o.yg)("inlineCode",{parentName:"p"}," pvc-filebrowser"))),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Select the ",(0,o.yg)("strong",{parentName:"p"},"Access Mode")," ",(0,o.yg)("strong",{parentName:"p"},(0,o.yg)("inlineCode",{parentName:"strong"}," RWX")),"and ",(0,o.yg)("strong",{parentName:"p"},"Storage Size")),(0,o.yg)("table",{parentName:"li"},(0,o.yg)("thead",{parentName:"table"},(0,o.yg)("tr",{parentName:"thead"},(0,o.yg)("th",{parentName:"tr",align:"left"},"Access Mode"),(0,o.yg)("th",{parentName:"tr",align:"left"},"CLI abbreviation"),(0,o.yg)("th",{parentName:"tr",align:"left"},"Description"))),(0,o.yg)("tbody",{parentName:"table"},(0,o.yg)("tr",{parentName:"tbody"},(0,o.yg)("td",{parentName:"tr",align:"left"},"ReadWriteOnce"),(0,o.yg)("td",{parentName:"tr",align:"left"},(0,o.yg)("inlineCode",{parentName:"td"},"RWO")),(0,o.yg)("td",{parentName:"tr",align:"left"},"The volume can be mounted as read-write by a single node.")),(0,o.yg)("tr",{parentName:"tbody"},(0,o.yg)("td",{parentName:"tr",align:"left"},"ReadOnlyMany"),(0,o.yg)("td",{parentName:"tr",align:"left"},(0,o.yg)("inlineCode",{parentName:"td"},"ROX")),(0,o.yg)("td",{parentName:"tr",align:"left"},"The volume can be mounted as read-only by many nodes.")),(0,o.yg)("tr",{parentName:"tbody"},(0,o.yg)("td",{parentName:"tr",align:"left"},"ReadWriteMany"),(0,o.yg)("td",{parentName:"tr",align:"left"},(0,o.yg)("inlineCode",{parentName:"td"},"RWX")),(0,o.yg)("td",{parentName:"tr",align:"left"},"The volume can be mounted as read-write by many nodes."))))),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Click ",(0,o.yg)("strong",{parentName:"p"},"Create")))),(0,o.yg)("img",{src:"/img/screenshot_pvc_storage.png",alt:"Create Persistent Storage",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("img",{src:"/img/screenshot_pvc_storage_create.png",alt:"Create Persistent Storage",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("admonition",{type:"info"},(0,o.yg)("p",{parentName:"admonition"},"The DSRI using the ",(0,o.yg)("a",{parentName:"p",href:"https://www.openshift.com/products/container-storage/"},(0,o.yg)("strong",{parentName:"a"},"Openshift Container Stroage"))," (",(0,o.yg)("inlineCode",{parentName:"p"}," OCS"),") which is based on ",(0,o.yg)("a",{parentName:"p",href:"https://ceph.io/ceph-storage/"},(0,o.yg)("strong",{parentName:"a"},"CEPH"))," offers ",(0,o.yg)("inlineCode",{parentName:"p"},"ReadWriteOnce")," access mode. "),(0,o.yg)("ul",{parentName:"admonition"},(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("inlineCode",{parentName:"li"},"ReadWriteOnce")," (",(0,o.yg)("a",{parentName:"li",href:"https://docs.openshift.com/container-platform/4.6/storage/understanding-persistent-storage.html"},(0,o.yg)("strong",{parentName:"a"},"RWO")),") volumes cannot be mounted on multiple nodes. Use the ",(0,o.yg)("inlineCode",{parentName:"li"},"ReadWriteMany")," (",(0,o.yg)("a",{parentName:"li",href:"https://docs.openshift.com/container-platform/4.6/storage/understanding-persistent-storage.html"},(0,o.yg)("strong",{parentName:"a"},"RWX")),") access mode when possible. If a node fails, the system does not allow the attached RWO volume to be mounted on a new node because it is already assigned to the failed node. If you encounter a multi-attach error message as a result, force delete the pod on a shut down or crashed node. "))),(0,o.yg)("p",null,"Static persistent volumes provides a sustainable persistent storage over time for applications that need to run regular Docker images (which usually use the ",(0,o.yg)("inlineCode",{parentName:"p"},"root")," user)."),(0,o.yg)("admonition",{type:"info"},(0,o.yg)("p",{parentName:"admonition"},"Some Applications such as ",(0,o.yg)("strong",{parentName:"p"},"Jupyter")," template automatically creates a persistent storage")),(0,o.yg)("h3",{id:"connect-the-existing-persistent-storage"},"Connect the Existing Persistent Storage"),(0,o.yg)("p",null,"On the ",(0,o.yg)("strong",{parentName:"p"},"Topology")," page select your application,"),(0,o.yg)("ol",null,(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Click ",(0,o.yg)("strong",{parentName:"p"},"Action")," on your application")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Select the ",(0,o.yg)("strong",{parentName:"p"},"Add Storage")," option from the dropdown list.")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Select the ",(0,o.yg)("strong",{parentName:"p"},"Use Existing Claim")," option from the Add Storage wizard and Select the Claim")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Add the ",(0,o.yg)("strong",{parentName:"p"},"Mount Path"))),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Save"))),(0,o.yg)("img",{src:"/img/screenshot_existing_storage.png",alt:"Add Existing Persistent Storage",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("img",{src:"/img/screenshot_add_storage.png",alt:"Add Existing Persistent Storage",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("admonition",{type:"info"},(0,o.yg)("p",{parentName:"admonition"},"You can try above method if you want to connect ",(0,o.yg)("strong",{parentName:"p"},"more applications to the same storage"))),(0,o.yg)("h3",{id:"expand-existing-persistent-storage"},"Expand existing Persistent Storage"),(0,o.yg)("ol",null,(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Switch to the ",(0,o.yg)("strong",{parentName:"p"},"Administrator")," view")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Go to the ",(0,o.yg)("strong",{parentName:"p"},"Project")," panel ")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Select your project")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Expand the ",(0,o.yg)("strong",{parentName:"p"},"Storage")," panel then go to the ",(0,o.yg)("strong",{parentName:"p"},"Persistent Volume Claim")," panel")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Click on the three dots (\u22ee) next to the ",(0,o.yg)("strong",{parentName:"p"},"Persistent Volume Claim")," you want to expand.")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Click on ",(0,o.yg)("strong",{parentName:"p"},"Expand PVC")," in the menu.")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Enter the size you want to expand your PVC with.")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Hit ",(0,o.yg)("strong",{parentName:"p"},"Expand"),". It can take upto 2 minutes before your PVC is expanded."))),(0,o.yg)("h3",{id:"use-the-dynamic-storage"},"Use the dynamic storage"),(0,o.yg)("p",null,"Dynamic ",(0,o.yg)("strong",{parentName:"p"},"persistent")," volumes can be created automatically by an application template."),(0,o.yg)("p",null,"Dynamic storage can also be created manually, go to ",(0,o.yg)("strong",{parentName:"p"},"Storage")," on the left sidebar in a project:"),(0,o.yg)("ol",null,(0,o.yg)("li",{parentName:"ol"},"Click ",(0,o.yg)("strong",{parentName:"li"},"Create Storage")," top right of the Storage page."),(0,o.yg)("li",{parentName:"ol"},"Storage class: ",(0,o.yg)("strong",{parentName:"li"},"ceph-fs")),(0,o.yg)("li",{parentName:"ol"},"Access Mode:",(0,o.yg)("ul",{parentName:"li"},(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("strong",{parentName:"li"},"Single User (RWO)"),": only the user who created this volume can read/write to this volume."),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("strong",{parentName:"li"},"Shared Access (RWX)"),": all users with access to the projects can read/write this volume."),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("strong",{parentName:"li"},"Read Only (ROX)"),": all users with access to the projects can read this volume.")))),(0,o.yg)("h3",{id:"use-the-ephemeral-storage"},"Use the ephemeral storage"),(0,o.yg)("admonition",{title:"Disabled",type:"warning"},(0,o.yg)("p",{parentName:"admonition"},"We currently disabled this solution by default, as it was confusing for users and would lead to data loss.")),(0,o.yg)("p",null,"When creating a pod, OpenShift will by default use ephemeral storage. It creates a volumes bind to the pod. So the volume will be deleted."),(0,o.yg)("p",null,"It is recommended to use dynamic provisioning for a more sustainable storage solution. But ephemeral storage can be sufficient for testing."))}d.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[4896],{5680:(e,t,a)=>{a.d(t,{xA:()=>g,yg:()=>d});var n=a(6540);function r(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function o(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function i(e){for(var t=1;t=0||(r[a]=e[a]);return r}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(r[a]=e[a])}return r}var p=n.createContext({}),l=function(e){var t=n.useContext(p),a=t;return e&&(a="function"==typeof e?e(t):i(i({},t),e)),a},g=function(e){var t=l(e.components);return n.createElement(p.Provider,{value:t},e.children)},m={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},y=n.forwardRef((function(e,t){var a=e.components,r=e.mdxType,o=e.originalType,p=e.parentName,g=s(e,["components","mdxType","originalType","parentName"]),y=l(a),d=r,c=y["".concat(p,".").concat(d)]||y[d]||m[d]||o;return a?n.createElement(c,i(i({ref:t},g),{},{components:a})):n.createElement(c,i({ref:t},g))}));function d(e,t){var a=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var o=a.length,i=new Array(o);i[0]=y;var s={};for(var p in t)hasOwnProperty.call(t,p)&&(s[p]=t[p]);s.originalType=e,s.mdxType="string"==typeof e?e:r,i[1]=s;for(var l=2;l{a.r(t),a.d(t,{assets:()=>g,contentTitle:()=>p,default:()=>d,frontMatter:()=>s,metadata:()=>l,toc:()=>m});var n=a(9668),r=a(1367),o=(a(6540),a(5680)),i=["components"],s={id:"openshift-storage",title:"Data storage"},p=void 0,l={unversionedId:"openshift-storage",id:"openshift-storage",title:"Data storage",description:"Different storages can be used when running services on the DSRI:",source:"@site/docs/openshift-storage.md",sourceDirName:".",slug:"/openshift-storage",permalink:"/docs/openshift-storage",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/openshift-storage.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"openshift-storage",title:"Data storage"},sidebar:"docs",previous:{title:"Command Line Interface",permalink:"/docs/openshift-commands"},next:{title:"Delete objects (advanced)",permalink:"/docs/openshift-delete-objects"}},g={},m=[{value:"Create the Persistent Storage",id:"create-the-persistent-storage",level:3},{value:"Connect the Existing Persistent Storage",id:"connect-the-existing-persistent-storage",level:3},{value:"Expand existing Persistent Storage",id:"expand-existing-persistent-storage",level:3},{value:"Use the dynamic storage",id:"use-the-dynamic-storage",level:3},{value:"Use the ephemeral storage",id:"use-the-ephemeral-storage",level:3}],y={toc:m};function d(e){var t=e.components,a=(0,r.A)(e,i);return(0,o.yg)("wrapper",(0,n.A)({},y,a,{components:t,mdxType:"MDXLayout"}),(0,o.yg)("p",null,"Different storages can be used when running services on the DSRI:"),(0,o.yg)("p",null,"\ud83e\udd8b ",(0,o.yg)("strong",{parentName:"p"},"Ephemeral storage"),": storage is bound to the pod, data will be lost when the pod is deleted (but this deployment does not require to request the creation of a persistent storage, and is faster to test code)."),(0,o.yg)("p",null,"\u26a1 ",(0,o.yg)("strong",{parentName:"p"},"Dynamic storage"),": automatically create a persistent storage when starting an application. Can also be created in the OpenShift web UI, using the ",(0,o.yg)("inlineCode",{parentName:"p"},"dynamic-maprfs")," Storage Class."),(0,o.yg)("p",null,"\ud83d\uddc4\ufe0f ",(0,o.yg)("strong",{parentName:"p"},"Persistent storage"),": You can use a persistent storage volume to store data. Please see the Create the Persistent Storage section. You can do this yourself. Please keep in mind that there are no backups made of data on DSRI. "),(0,o.yg)("admonition",{title:"Storage per project",type:"caution"},(0,o.yg)("p",{parentName:"admonition"},"A storage (aka. Persistent Volume Claim) is only accessible in the project where it has been created.")),(0,o.yg)("h3",{id:"create-the-persistent-storage"},"Create the Persistent Storage"),(0,o.yg)("ol",null,(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Switch to the ",(0,o.yg)("strong",{parentName:"p"},"Administrator")," view")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Go to the ",(0,o.yg)("strong",{parentName:"p"},"Project")," panel ")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Select your project")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Expand the ",(0,o.yg)("strong",{parentName:"p"},"Storage")," panel then go to the ",(0,o.yg)("strong",{parentName:"p"},"Persistent Volume Claim")," panel")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Click the button call ",(0,o.yg)("strong",{parentName:"p"},"Create Persistent Volume Claim")),(0,o.yg)("p",{parentName:"li"},"then you will redirect the wizard of Create Persistent Volume Claim ")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Provide the unique ",(0,o.yg)("strong",{parentName:"p"},"Persistent Volume Claim Name")," start with ",(0,o.yg)("inlineCode",{parentName:"p"}," pvc-")," "),(0,o.yg)("p",{parentName:"li"},"example: ",(0,o.yg)("inlineCode",{parentName:"p"}," pvc-filebrowser"))),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Select the ",(0,o.yg)("strong",{parentName:"p"},"Access Mode")," ",(0,o.yg)("strong",{parentName:"p"},(0,o.yg)("inlineCode",{parentName:"strong"}," RWX")),"and ",(0,o.yg)("strong",{parentName:"p"},"Storage Size")),(0,o.yg)("table",{parentName:"li"},(0,o.yg)("thead",{parentName:"table"},(0,o.yg)("tr",{parentName:"thead"},(0,o.yg)("th",{parentName:"tr",align:"left"},"Access Mode"),(0,o.yg)("th",{parentName:"tr",align:"left"},"CLI abbreviation"),(0,o.yg)("th",{parentName:"tr",align:"left"},"Description"))),(0,o.yg)("tbody",{parentName:"table"},(0,o.yg)("tr",{parentName:"tbody"},(0,o.yg)("td",{parentName:"tr",align:"left"},"ReadWriteOnce"),(0,o.yg)("td",{parentName:"tr",align:"left"},(0,o.yg)("inlineCode",{parentName:"td"},"RWO")),(0,o.yg)("td",{parentName:"tr",align:"left"},"The volume can be mounted as read-write by a single node.")),(0,o.yg)("tr",{parentName:"tbody"},(0,o.yg)("td",{parentName:"tr",align:"left"},"ReadOnlyMany"),(0,o.yg)("td",{parentName:"tr",align:"left"},(0,o.yg)("inlineCode",{parentName:"td"},"ROX")),(0,o.yg)("td",{parentName:"tr",align:"left"},"The volume can be mounted as read-only by many nodes.")),(0,o.yg)("tr",{parentName:"tbody"},(0,o.yg)("td",{parentName:"tr",align:"left"},"ReadWriteMany"),(0,o.yg)("td",{parentName:"tr",align:"left"},(0,o.yg)("inlineCode",{parentName:"td"},"RWX")),(0,o.yg)("td",{parentName:"tr",align:"left"},"The volume can be mounted as read-write by many nodes."))))),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Click ",(0,o.yg)("strong",{parentName:"p"},"Create")))),(0,o.yg)("img",{src:"/img/screenshot_pvc_storage.png",alt:"Create Persistent Storage",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("img",{src:"/img/screenshot_pvc_storage_create.png",alt:"Create Persistent Storage",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("admonition",{type:"info"},(0,o.yg)("p",{parentName:"admonition"},"The DSRI using the ",(0,o.yg)("a",{parentName:"p",href:"https://www.openshift.com/products/container-storage/"},(0,o.yg)("strong",{parentName:"a"},"Openshift Container Stroage"))," (",(0,o.yg)("inlineCode",{parentName:"p"}," OCS"),") which is based on ",(0,o.yg)("a",{parentName:"p",href:"https://ceph.io/ceph-storage/"},(0,o.yg)("strong",{parentName:"a"},"CEPH"))," offers ",(0,o.yg)("inlineCode",{parentName:"p"},"ReadWriteOnce")," access mode. "),(0,o.yg)("ul",{parentName:"admonition"},(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("inlineCode",{parentName:"li"},"ReadWriteOnce")," (",(0,o.yg)("a",{parentName:"li",href:"https://docs.openshift.com/container-platform/4.6/storage/understanding-persistent-storage.html"},(0,o.yg)("strong",{parentName:"a"},"RWO")),") volumes cannot be mounted on multiple nodes. Use the ",(0,o.yg)("inlineCode",{parentName:"li"},"ReadWriteMany")," (",(0,o.yg)("a",{parentName:"li",href:"https://docs.openshift.com/container-platform/4.6/storage/understanding-persistent-storage.html"},(0,o.yg)("strong",{parentName:"a"},"RWX")),") access mode when possible. If a node fails, the system does not allow the attached RWO volume to be mounted on a new node because it is already assigned to the failed node. If you encounter a multi-attach error message as a result, force delete the pod on a shut down or crashed node. "))),(0,o.yg)("p",null,"Static persistent volumes provides a sustainable persistent storage over time for applications that need to run regular Docker images (which usually use the ",(0,o.yg)("inlineCode",{parentName:"p"},"root")," user)."),(0,o.yg)("admonition",{type:"info"},(0,o.yg)("p",{parentName:"admonition"},"Some Applications such as ",(0,o.yg)("strong",{parentName:"p"},"Jupyter")," template automatically creates a persistent storage")),(0,o.yg)("h3",{id:"connect-the-existing-persistent-storage"},"Connect the Existing Persistent Storage"),(0,o.yg)("p",null,"On the ",(0,o.yg)("strong",{parentName:"p"},"Topology")," page select your application,"),(0,o.yg)("ol",null,(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Click ",(0,o.yg)("strong",{parentName:"p"},"Action")," on your application")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Select the ",(0,o.yg)("strong",{parentName:"p"},"Add Storage")," option from the dropdown list.")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Select the ",(0,o.yg)("strong",{parentName:"p"},"Use Existing Claim")," option from the Add Storage wizard and Select the Claim")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Add the ",(0,o.yg)("strong",{parentName:"p"},"Mount Path"))),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Save"))),(0,o.yg)("img",{src:"/img/screenshot_existing_storage.png",alt:"Add Existing Persistent Storage",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("img",{src:"/img/screenshot_add_storage.png",alt:"Add Existing Persistent Storage",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("admonition",{type:"info"},(0,o.yg)("p",{parentName:"admonition"},"You can try above method if you want to connect ",(0,o.yg)("strong",{parentName:"p"},"more applications to the same storage"))),(0,o.yg)("h3",{id:"expand-existing-persistent-storage"},"Expand existing Persistent Storage"),(0,o.yg)("ol",null,(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Switch to the ",(0,o.yg)("strong",{parentName:"p"},"Administrator")," view")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Go to the ",(0,o.yg)("strong",{parentName:"p"},"Project")," panel ")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Select your project")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Expand the ",(0,o.yg)("strong",{parentName:"p"},"Storage")," panel then go to the ",(0,o.yg)("strong",{parentName:"p"},"Persistent Volume Claim")," panel")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Click on the three dots (\u22ee) next to the ",(0,o.yg)("strong",{parentName:"p"},"Persistent Volume Claim")," you want to expand.")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Click on ",(0,o.yg)("strong",{parentName:"p"},"Expand PVC")," in the menu.")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Enter the size you want to expand your PVC with.")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Hit ",(0,o.yg)("strong",{parentName:"p"},"Expand"),". It can take upto 2 minutes before your PVC is expanded."))),(0,o.yg)("h3",{id:"use-the-dynamic-storage"},"Use the dynamic storage"),(0,o.yg)("p",null,"Dynamic ",(0,o.yg)("strong",{parentName:"p"},"persistent")," volumes can be created automatically by an application template."),(0,o.yg)("p",null,"Dynamic storage can also be created manually, go to ",(0,o.yg)("strong",{parentName:"p"},"Storage")," on the left sidebar in a project:"),(0,o.yg)("ol",null,(0,o.yg)("li",{parentName:"ol"},"Click ",(0,o.yg)("strong",{parentName:"li"},"Create Storage")," top right of the Storage page."),(0,o.yg)("li",{parentName:"ol"},"Storage class: ",(0,o.yg)("strong",{parentName:"li"},"ceph-fs")),(0,o.yg)("li",{parentName:"ol"},"Access Mode:",(0,o.yg)("ul",{parentName:"li"},(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("strong",{parentName:"li"},"Single User (RWO)"),": only the user who created this volume can read/write to this volume."),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("strong",{parentName:"li"},"Shared Access (RWX)"),": all users with access to the projects can read/write this volume."),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("strong",{parentName:"li"},"Read Only (ROX)"),": all users with access to the projects can read this volume.")))),(0,o.yg)("h3",{id:"use-the-ephemeral-storage"},"Use the ephemeral storage"),(0,o.yg)("admonition",{title:"Disabled",type:"warning"},(0,o.yg)("p",{parentName:"admonition"},"We currently disabled this solution by default, as it was confusing for users and would lead to data loss.")),(0,o.yg)("p",null,"When creating a pod, OpenShift will by default use ephemeral storage. It creates a volumes bind to the pod. So the volume will be deleted."),(0,o.yg)("p",null,"It is recommended to use dynamic provisioning for a more sustainable storage solution. But ephemeral storage can be sufficient for testing."))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/2406662c.d71b2976.js b/assets/js/2406662c.69f37c83.js similarity index 99% rename from assets/js/2406662c.d71b2976.js rename to assets/js/2406662c.69f37c83.js index 0155d44e7..2ac69ac15 100644 --- a/assets/js/2406662c.d71b2976.js +++ b/assets/js/2406662c.69f37c83.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[9030],{5680:(e,t,r)=>{r.d(t,{xA:()=>c,yg:()=>y});var o=r(6540);function a(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function n(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,o)}return r}function i(e){for(var t=1;t=0||(a[r]=e[r]);return a}(e,t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);for(o=0;o=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(a[r]=e[r])}return a}var p=o.createContext({}),l=function(e){var t=o.useContext(p),r=t;return e&&(r="function"==typeof e?e(t):i(i({},t),e)),r},c=function(e){var t=l(e.components);return o.createElement(p.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.createElement(o.Fragment,{},t)}},u=o.forwardRef((function(e,t){var r=e.components,a=e.mdxType,n=e.originalType,p=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),u=l(r),y=a,m=u["".concat(p,".").concat(y)]||u[y]||d[y]||n;return r?o.createElement(m,i(i({ref:t},c),{},{components:r})):o.createElement(m,i({ref:t},c))}));function y(e,t){var r=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var n=r.length,i=new Array(n);i[0]=u;var s={};for(var p in t)hasOwnProperty.call(t,p)&&(s[p]=t[p]);s.originalType=e,s.mdxType="string"==typeof e?e:a,i[1]=s;for(var l=2;l{r.r(t),r.d(t,{assets:()=>c,contentTitle:()=>p,default:()=>y,frontMatter:()=>s,metadata:()=>l,toc:()=>d});var o=r(9668),a=r(1367),n=(r(6540),r(5680)),i=["components"],s={id:"prepare-project-for-dsri",title:"Prepare your project"},p=void 0,l={unversionedId:"prepare-project-for-dsri",id:"prepare-project-for-dsri",title:"Prepare your project",description:"Code in a git repository",source:"@site/docs/prepare-project-for-dsri.md",sourceDirName:".",slug:"/prepare-project-for-dsri",permalink:"/docs/prepare-project-for-dsri",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/prepare-project-for-dsri.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"prepare-project-for-dsri",title:"Prepare your project"},sidebar:"docs",previous:{title:"Install the client",permalink:"/docs/openshift-install"},next:{title:"Upload data",permalink:"/docs/openshift-load-data"}},c={},d=[{value:"Code in a git repository",id:"code-in-a-git-repository",level:2},{value:"Get your data ready",id:"get-your-data-ready",level:2},{value:"Data is on your local machine",id:"data-is-on-your-local-machine",level:3},{value:"Data is on a server",id:"data-is-on-a-server",level:3},{value:"Request access to internal UM servers",id:"request-access-to-internal-um-servers",level:3}],u={toc:d};function y(e){var t=e.components,r=(0,a.A)(e,i);return(0,n.yg)("wrapper",(0,o.A)({},u,r,{components:t,mdxType:"MDXLayout"}),(0,n.yg)("h2",{id:"code-in-a-git-repository"},"Code in a git repository"),(0,n.yg)("p",null,"Using ",(0,n.yg)("inlineCode",{parentName:"p"},"git")," is mandatory to deploy your code on the DSRI. Store your code in a git repository to keep track of changes, and make it easier to share and re-use your code outside of your computer."),(0,n.yg)("admonition",{title:"Platform recommendations",type:"info"},(0,n.yg)("p",{parentName:"admonition"},"We recommend those platforms depending on your use-case:"),(0,n.yg)("ul",{parentName:"admonition"},(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("a",{parentName:"li",href:"https://github.com"},"GitHub")," for public repositories"),(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("a",{parentName:"li",href:"https://gitlab.maastrichtuniversity.nl"},"GitLab hosted at Maastricht University")," for private repositories"))),(0,n.yg)("blockquote",null,(0,n.yg)("p",{parentName:"blockquote"},"Any other git platform, such as BitBucket or gitlab.com, is fine too.")),(0,n.yg)("h2",{id:"get-your-data-ready"},"Get your data ready"),(0,n.yg)("p",null,"If your project is using a large amount of data that cannot be pushed to a git repository, you will need to use a persistent storage to store your data on the DSRI. See the ",(0,n.yg)("a",{parentName:"p",href:"/docs/openshift-storage"},"Storage on the DSRI")," documentation for more details about creating a persistent storage."),(0,n.yg)("p",null,"Here are the options to upload your data to the DSRI storage:"),(0,n.yg)("h3",{id:"data-is-on-your-local-machine"},"Data is on your local machine"),(0,n.yg)("p",null,"If the data is stored on a local machine, such as your computer:"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"Drag and drop files from your computer to the VisualStudio Code or JupyterLab web UI, if applicable."),(0,n.yg)("li",{parentName:"ul"},"Otherwise, use the ",(0,n.yg)("inlineCode",{parentName:"li"},"oc cp")," command to copy data to your application pod. See the ",(0,n.yg)("a",{parentName:"li",href:"/docs/openshift-load-data"},"Load data")," documentation page for more information.")),(0,n.yg)("admonition",{title:"Upload to persistent storage",type:"caution"},(0,n.yg)("p",{parentName:"admonition"},"Make sure you ",(0,n.yg)("strong",{parentName:"p"},"upload the data to a folder mounted on a persistent storage")," in the pod to avoid losing your data if the pod restarts.")),(0,n.yg)("h3",{id:"data-is-on-a-server"},"Data is on a server"),(0,n.yg)("p",null,"Same as for your laptop, you will need to install and use the ",(0,n.yg)("inlineCode",{parentName:"p"},"oc cp")," command to copy data to your application pod. See the ",(0,n.yg)("a",{parentName:"p",href:"/docs/openshift-load-data"},"Load data")," documentation page for more information."),(0,n.yg)("h3",{id:"request-access-to-internal-um-servers"},"Request access to internal UM servers"),(0,n.yg)("p",null,"In certain cases, UM servers are not accessible by default from the DSRI. This is even the case for servers that are normally publicly accessible. To be able to access these UM servers from the DSRI, we need to put in the request to open the connection. "),(0,n.yg)("p",null,"Please let us know either the servername and port you like to access, or the URL (e.g. um-vm0057.unimaas.nl on port 443 or ",(0,n.yg)("a",{parentName:"p",href:"https://gitlab.maastrichtuniversity.nl"},"https://gitlab.maastrichtuniversity.nl"),"). You can reach out to us either by mail or by Slack."),(0,n.yg)("p",null,"The procedure is described in the diagram below:"),(0,n.yg)("img",{src:"/img/request-access-um-servers.svg",alt:"Access procedure UM servers",style:{maxWidth:"100%",maxHeight:"100%"}}))}y.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[9030],{5680:(e,t,r)=>{r.d(t,{xA:()=>c,yg:()=>y});var o=r(6540);function a(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function n(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,o)}return r}function i(e){for(var t=1;t=0||(a[r]=e[r]);return a}(e,t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);for(o=0;o=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(a[r]=e[r])}return a}var p=o.createContext({}),l=function(e){var t=o.useContext(p),r=t;return e&&(r="function"==typeof e?e(t):i(i({},t),e)),r},c=function(e){var t=l(e.components);return o.createElement(p.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.createElement(o.Fragment,{},t)}},u=o.forwardRef((function(e,t){var r=e.components,a=e.mdxType,n=e.originalType,p=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),u=l(r),y=a,m=u["".concat(p,".").concat(y)]||u[y]||d[y]||n;return r?o.createElement(m,i(i({ref:t},c),{},{components:r})):o.createElement(m,i({ref:t},c))}));function y(e,t){var r=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var n=r.length,i=new Array(n);i[0]=u;var s={};for(var p in t)hasOwnProperty.call(t,p)&&(s[p]=t[p]);s.originalType=e,s.mdxType="string"==typeof e?e:a,i[1]=s;for(var l=2;l{r.r(t),r.d(t,{assets:()=>c,contentTitle:()=>p,default:()=>y,frontMatter:()=>s,metadata:()=>l,toc:()=>d});var o=r(9668),a=r(1367),n=(r(6540),r(5680)),i=["components"],s={id:"prepare-project-for-dsri",title:"Prepare your project"},p=void 0,l={unversionedId:"prepare-project-for-dsri",id:"prepare-project-for-dsri",title:"Prepare your project",description:"Code in a git repository",source:"@site/docs/prepare-project-for-dsri.md",sourceDirName:".",slug:"/prepare-project-for-dsri",permalink:"/docs/prepare-project-for-dsri",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/prepare-project-for-dsri.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"prepare-project-for-dsri",title:"Prepare your project"},sidebar:"docs",previous:{title:"Install the client",permalink:"/docs/openshift-install"},next:{title:"Upload data",permalink:"/docs/openshift-load-data"}},c={},d=[{value:"Code in a git repository",id:"code-in-a-git-repository",level:2},{value:"Get your data ready",id:"get-your-data-ready",level:2},{value:"Data is on your local machine",id:"data-is-on-your-local-machine",level:3},{value:"Data is on a server",id:"data-is-on-a-server",level:3},{value:"Request access to internal UM servers",id:"request-access-to-internal-um-servers",level:3}],u={toc:d};function y(e){var t=e.components,r=(0,a.A)(e,i);return(0,n.yg)("wrapper",(0,o.A)({},u,r,{components:t,mdxType:"MDXLayout"}),(0,n.yg)("h2",{id:"code-in-a-git-repository"},"Code in a git repository"),(0,n.yg)("p",null,"Using ",(0,n.yg)("inlineCode",{parentName:"p"},"git")," is mandatory to deploy your code on the DSRI. Store your code in a git repository to keep track of changes, and make it easier to share and re-use your code outside of your computer."),(0,n.yg)("admonition",{title:"Platform recommendations",type:"info"},(0,n.yg)("p",{parentName:"admonition"},"We recommend those platforms depending on your use-case:"),(0,n.yg)("ul",{parentName:"admonition"},(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("a",{parentName:"li",href:"https://github.com"},"GitHub")," for public repositories"),(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("a",{parentName:"li",href:"https://gitlab.maastrichtuniversity.nl"},"GitLab hosted at Maastricht University")," for private repositories"))),(0,n.yg)("blockquote",null,(0,n.yg)("p",{parentName:"blockquote"},"Any other git platform, such as BitBucket or gitlab.com, is fine too.")),(0,n.yg)("h2",{id:"get-your-data-ready"},"Get your data ready"),(0,n.yg)("p",null,"If your project is using a large amount of data that cannot be pushed to a git repository, you will need to use a persistent storage to store your data on the DSRI. See the ",(0,n.yg)("a",{parentName:"p",href:"/docs/openshift-storage"},"Storage on the DSRI")," documentation for more details about creating a persistent storage."),(0,n.yg)("p",null,"Here are the options to upload your data to the DSRI storage:"),(0,n.yg)("h3",{id:"data-is-on-your-local-machine"},"Data is on your local machine"),(0,n.yg)("p",null,"If the data is stored on a local machine, such as your computer:"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"Drag and drop files from your computer to the VisualStudio Code or JupyterLab web UI, if applicable."),(0,n.yg)("li",{parentName:"ul"},"Otherwise, use the ",(0,n.yg)("inlineCode",{parentName:"li"},"oc cp")," command to copy data to your application pod. See the ",(0,n.yg)("a",{parentName:"li",href:"/docs/openshift-load-data"},"Load data")," documentation page for more information.")),(0,n.yg)("admonition",{title:"Upload to persistent storage",type:"caution"},(0,n.yg)("p",{parentName:"admonition"},"Make sure you ",(0,n.yg)("strong",{parentName:"p"},"upload the data to a folder mounted on a persistent storage")," in the pod to avoid losing your data if the pod restarts.")),(0,n.yg)("h3",{id:"data-is-on-a-server"},"Data is on a server"),(0,n.yg)("p",null,"Same as for your laptop, you will need to install and use the ",(0,n.yg)("inlineCode",{parentName:"p"},"oc cp")," command to copy data to your application pod. See the ",(0,n.yg)("a",{parentName:"p",href:"/docs/openshift-load-data"},"Load data")," documentation page for more information."),(0,n.yg)("h3",{id:"request-access-to-internal-um-servers"},"Request access to internal UM servers"),(0,n.yg)("p",null,"In certain cases, UM servers are not accessible by default from the DSRI. This is even the case for servers that are normally publicly accessible. To be able to access these UM servers from the DSRI, we need to put in the request to open the connection. "),(0,n.yg)("p",null,"Please let us know either the servername and port you like to access, or the URL (e.g. um-vm0057.unimaas.nl on port 443 or ",(0,n.yg)("a",{parentName:"p",href:"https://gitlab.maastrichtuniversity.nl"},"https://gitlab.maastrichtuniversity.nl"),"). You can reach out to us either by mail or by Slack."),(0,n.yg)("p",null,"The procedure is described in the diagram below:"),(0,n.yg)("img",{src:"/img/request-access-um-servers.svg",alt:"Access procedure UM servers",style:{maxWidth:"100%",maxHeight:"100%"}}))}y.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/2b1adfae.e3ccd5ad.js b/assets/js/2b1adfae.1a8b5388.js similarity index 99% rename from assets/js/2b1adfae.e3ccd5ad.js rename to assets/js/2b1adfae.1a8b5388.js index bc1a9ac2f..43ae1d3f2 100644 --- a/assets/js/2b1adfae.e3ccd5ad.js +++ b/assets/js/2b1adfae.1a8b5388.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[8793],{5680:(e,t,o)=>{o.d(t,{xA:()=>u,yg:()=>h});var r=o(6540);function a(e,t,o){return t in e?Object.defineProperty(e,t,{value:o,enumerable:!0,configurable:!0,writable:!0}):e[t]=o,e}function n(e,t){var o=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),o.push.apply(o,r)}return o}function s(e){for(var t=1;t=0||(a[o]=e[o]);return a}(e,t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,o)&&(a[o]=e[o])}return a}var l=r.createContext({}),p=function(e){var t=r.useContext(l),o=t;return e&&(o="function"==typeof e?e(t):s(s({},t),e)),o},u=function(e){var t=p(e.components);return r.createElement(l.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},c=r.forwardRef((function(e,t){var o=e.components,a=e.mdxType,n=e.originalType,l=e.parentName,u=i(e,["components","mdxType","originalType","parentName"]),c=p(o),h=a,y=c["".concat(l,".").concat(h)]||c[h]||d[h]||n;return o?r.createElement(y,s(s({ref:t},u),{},{components:o})):r.createElement(y,s({ref:t},u))}));function h(e,t){var o=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var n=o.length,s=new Array(n);s[0]=c;var i={};for(var l in t)hasOwnProperty.call(t,l)&&(i[l]=t[l]);i.originalType=e,i.mdxType="string"==typeof e?e:a,s[1]=i;for(var p=2;p{o.r(t),o.d(t,{assets:()=>u,contentTitle:()=>l,default:()=>h,frontMatter:()=>i,metadata:()=>p,toc:()=>d});var r=o(9668),a=o(1367),n=(o(6540),o(5680)),s=["components"],i={id:"increase-process-speed",title:"Increase your processes speed"},l=void 0,p={unversionedId:"increase-process-speed",id:"increase-process-speed",title:"Increase your processes speed",description:"DSRI provides a lot of computing resources, but there are a few things to know if you want to increase the speed of your processes.",source:"@site/docs/increase-process-speed.md",sourceDirName:".",slug:"/increase-process-speed",permalink:"/docs/increase-process-speed",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/increase-process-speed.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"increase-process-speed",title:"Increase your processes speed"},sidebar:"docs",previous:{title:"Enabling VPN access in WSL2",permalink:"/docs/enabling-vpn-wsl"},next:{title:"PyTorch Profiling",permalink:"/docs/profile-pytorch-code"}},u={},d=[{value:"The good",id:"the-good",level:2},{value:"The bad",id:"the-bad",level:2},{value:"The solution",id:"the-solution",level:2}],c={toc:d};function h(e){var t=e.components,o=(0,a.A)(e,s);return(0,n.yg)("wrapper",(0,r.A)({},c,o,{components:t,mdxType:"MDXLayout"}),(0,n.yg)("p",null,"DSRI provides a lot of computing resources, but there are a few things to know if you want to increase the speed of your processes."),(0,n.yg)("h2",{id:"the-good"},"The good"),(0,n.yg)("p",null,"With the DSRI you get access to a workspace with more memory and cores than your laptop (around 200G memory, and 64 cores on the DSRI, against around 16G memory and 8 cores on your laptop)"),(0,n.yg)("p",null,"Those additional resources ",(0,n.yg)("strong",{parentName:"p"},"might")," help to make your workload run faster, but not automatically! It will run faster "),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"If your code can make use of the really large amount of RAM to load more of the data to process in memory. But if your workload does not require dozens of GB of memory, and your laptop does not face out of memory issues, or crash, when you run your workload, then you probably already have enough memory on your laptop, and will not gain a significant boost from the increased memory."),(0,n.yg)("li",{parentName:"ul"},"If you can run your workload in parallel, or enable the libraries you use to use the available cores. This will highly depend on the libraries you use. Do they support running their processes in parallel? Do you need to explicitly enable parallelism on a specific number of cores?")),(0,n.yg)("p",null,"Proper parallelism is not achieved easily, it needs to be manually implemented within the library processes. "),(0,n.yg)("p",null,'For example, Python has a "Global Interpreter Lock" (aka. GIL) that limit threads parallelism by design, so when you are doing some work on a spreadsheet with ',(0,n.yg)("inlineCode",{parentName:"p"},"pandas"),", you are only going to use 1 thread (which is nice, because it makes the conceptualization and understanding of algorithms easier, but it also makes it harder to write truly efficient libraries)"),(0,n.yg)("p",null,"You will need to use complementary libraries if you want to use more threads while processing data with ",(0,n.yg)("inlineCode",{parentName:"p"},"pandas"),". There are multiple ways and libraries to achieve this, but the easiest, if you want to check it yourself with pandas, is to use ",(0,n.yg)("a",{parentName:"p",href:"https://github.com/nalepae/pandarallel"},"pandarallel"),". You could also implement the parallelism yourself with ",(0,n.yg)("inlineCode",{parentName:"p"},"concurrent.futures")),(0,n.yg)("h2",{id:"the-bad"},"The bad"),(0,n.yg)("p",null,"Until now everything seems good, more memory, more cores... So, what's the catch? It can only get better, no?"),(0,n.yg)("p",null,"Application and workspace running on the DSRI uses a persistent volume to avoid losing data when the application is restarted. On most workspaces this persistent volume is mounted on the workspace working directory."),(0,n.yg)("p",null,"This persistent volume is not hosted directly on the same node as your application, it's hosted on the cluster in a distributed fashion (remember you can attach this persistent volume to different applications, which might be hosted on different nodes themselves)"),(0,n.yg)("p",null,"And distributed storage means: slower read and write times! "),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("p",{parentName:"li"},"In your laptop the data is in a hard drive disc sitting at 2 cm of the CPU and memory.")),(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("p",{parentName:"li"},"In the DSRI your workspace might be on node 4, when the persistent volume is on node 8. In this case the data will need to go through the network"))),(0,n.yg)("p",null,"So if you write a script to just load data, do no computing, and write back the data to the persistent volume, it will probably be much faster on your laptop than on the DSRI!"),(0,n.yg)("h2",{id:"the-solution"},"The solution"),(0,n.yg)("p",null,'Only 1 folder (and its subfolders) usually mounted on the persistent volume. The rest is "ephemeral storage", which is the data bound to the application you started, this means the data will be stored on the same node as your workspace. '),(0,n.yg)("p",null,"Which might result in faster read/write speed! But also means the data will be lost if the workspace is restarted (which does not happen everyday, but can happen without notice)"),(0,n.yg)("p",null,"A solution could be to:"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"Keep your code and important data as backup in the persistent volume (the workspace working dir usually)"),(0,n.yg)("li",{parentName:"ul"},"Copy the data your process needs to load in a folder outside of the persistent volume (on the ephemeral storage)"),(0,n.yg)("li",{parentName:"ul"},"Read/write data mostly from this folder on the ephemeral storage, avoid using the data in the persistent volume folder as much as possible"),(0,n.yg)("li",{parentName:"ul"},"Copy the important result files or temporary files you don't want to lose from the folder on the ephemeral storage to the folder on the persistent storage")),(0,n.yg)("p",null,"Let us know how it works for you on the Slack ",(0,n.yg)("strong",{parentName:"p"},"#general")," channel, and if you have suggestions to improve the workspaces."))}h.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[8793],{5680:(e,t,o)=>{o.d(t,{xA:()=>u,yg:()=>h});var r=o(6540);function a(e,t,o){return t in e?Object.defineProperty(e,t,{value:o,enumerable:!0,configurable:!0,writable:!0}):e[t]=o,e}function n(e,t){var o=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),o.push.apply(o,r)}return o}function s(e){for(var t=1;t=0||(a[o]=e[o]);return a}(e,t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,o)&&(a[o]=e[o])}return a}var l=r.createContext({}),p=function(e){var t=r.useContext(l),o=t;return e&&(o="function"==typeof e?e(t):s(s({},t),e)),o},u=function(e){var t=p(e.components);return r.createElement(l.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},c=r.forwardRef((function(e,t){var o=e.components,a=e.mdxType,n=e.originalType,l=e.parentName,u=i(e,["components","mdxType","originalType","parentName"]),c=p(o),h=a,y=c["".concat(l,".").concat(h)]||c[h]||d[h]||n;return o?r.createElement(y,s(s({ref:t},u),{},{components:o})):r.createElement(y,s({ref:t},u))}));function h(e,t){var o=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var n=o.length,s=new Array(n);s[0]=c;var i={};for(var l in t)hasOwnProperty.call(t,l)&&(i[l]=t[l]);i.originalType=e,i.mdxType="string"==typeof e?e:a,s[1]=i;for(var p=2;p{o.r(t),o.d(t,{assets:()=>u,contentTitle:()=>l,default:()=>h,frontMatter:()=>i,metadata:()=>p,toc:()=>d});var r=o(9668),a=o(1367),n=(o(6540),o(5680)),s=["components"],i={id:"increase-process-speed",title:"Increase your processes speed"},l=void 0,p={unversionedId:"increase-process-speed",id:"increase-process-speed",title:"Increase your processes speed",description:"DSRI provides a lot of computing resources, but there are a few things to know if you want to increase the speed of your processes.",source:"@site/docs/increase-process-speed.md",sourceDirName:".",slug:"/increase-process-speed",permalink:"/docs/increase-process-speed",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/increase-process-speed.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"increase-process-speed",title:"Increase your processes speed"},sidebar:"docs",previous:{title:"Enabling VPN access in WSL2",permalink:"/docs/enabling-vpn-wsl"},next:{title:"PyTorch Profiling",permalink:"/docs/profile-pytorch-code"}},u={},d=[{value:"The good",id:"the-good",level:2},{value:"The bad",id:"the-bad",level:2},{value:"The solution",id:"the-solution",level:2}],c={toc:d};function h(e){var t=e.components,o=(0,a.A)(e,s);return(0,n.yg)("wrapper",(0,r.A)({},c,o,{components:t,mdxType:"MDXLayout"}),(0,n.yg)("p",null,"DSRI provides a lot of computing resources, but there are a few things to know if you want to increase the speed of your processes."),(0,n.yg)("h2",{id:"the-good"},"The good"),(0,n.yg)("p",null,"With the DSRI you get access to a workspace with more memory and cores than your laptop (around 200G memory, and 64 cores on the DSRI, against around 16G memory and 8 cores on your laptop)"),(0,n.yg)("p",null,"Those additional resources ",(0,n.yg)("strong",{parentName:"p"},"might")," help to make your workload run faster, but not automatically! It will run faster "),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"If your code can make use of the really large amount of RAM to load more of the data to process in memory. But if your workload does not require dozens of GB of memory, and your laptop does not face out of memory issues, or crash, when you run your workload, then you probably already have enough memory on your laptop, and will not gain a significant boost from the increased memory."),(0,n.yg)("li",{parentName:"ul"},"If you can run your workload in parallel, or enable the libraries you use to use the available cores. This will highly depend on the libraries you use. Do they support running their processes in parallel? Do you need to explicitly enable parallelism on a specific number of cores?")),(0,n.yg)("p",null,"Proper parallelism is not achieved easily, it needs to be manually implemented within the library processes. "),(0,n.yg)("p",null,'For example, Python has a "Global Interpreter Lock" (aka. GIL) that limit threads parallelism by design, so when you are doing some work on a spreadsheet with ',(0,n.yg)("inlineCode",{parentName:"p"},"pandas"),", you are only going to use 1 thread (which is nice, because it makes the conceptualization and understanding of algorithms easier, but it also makes it harder to write truly efficient libraries)"),(0,n.yg)("p",null,"You will need to use complementary libraries if you want to use more threads while processing data with ",(0,n.yg)("inlineCode",{parentName:"p"},"pandas"),". There are multiple ways and libraries to achieve this, but the easiest, if you want to check it yourself with pandas, is to use ",(0,n.yg)("a",{parentName:"p",href:"https://github.com/nalepae/pandarallel"},"pandarallel"),". You could also implement the parallelism yourself with ",(0,n.yg)("inlineCode",{parentName:"p"},"concurrent.futures")),(0,n.yg)("h2",{id:"the-bad"},"The bad"),(0,n.yg)("p",null,"Until now everything seems good, more memory, more cores... So, what's the catch? It can only get better, no?"),(0,n.yg)("p",null,"Application and workspace running on the DSRI uses a persistent volume to avoid losing data when the application is restarted. On most workspaces this persistent volume is mounted on the workspace working directory."),(0,n.yg)("p",null,"This persistent volume is not hosted directly on the same node as your application, it's hosted on the cluster in a distributed fashion (remember you can attach this persistent volume to different applications, which might be hosted on different nodes themselves)"),(0,n.yg)("p",null,"And distributed storage means: slower read and write times! "),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("p",{parentName:"li"},"In your laptop the data is in a hard drive disc sitting at 2 cm of the CPU and memory.")),(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("p",{parentName:"li"},"In the DSRI your workspace might be on node 4, when the persistent volume is on node 8. In this case the data will need to go through the network"))),(0,n.yg)("p",null,"So if you write a script to just load data, do no computing, and write back the data to the persistent volume, it will probably be much faster on your laptop than on the DSRI!"),(0,n.yg)("h2",{id:"the-solution"},"The solution"),(0,n.yg)("p",null,'Only 1 folder (and its subfolders) usually mounted on the persistent volume. The rest is "ephemeral storage", which is the data bound to the application you started, this means the data will be stored on the same node as your workspace. '),(0,n.yg)("p",null,"Which might result in faster read/write speed! But also means the data will be lost if the workspace is restarted (which does not happen everyday, but can happen without notice)"),(0,n.yg)("p",null,"A solution could be to:"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"Keep your code and important data as backup in the persistent volume (the workspace working dir usually)"),(0,n.yg)("li",{parentName:"ul"},"Copy the data your process needs to load in a folder outside of the persistent volume (on the ephemeral storage)"),(0,n.yg)("li",{parentName:"ul"},"Read/write data mostly from this folder on the ephemeral storage, avoid using the data in the persistent volume folder as much as possible"),(0,n.yg)("li",{parentName:"ul"},"Copy the important result files or temporary files you don't want to lose from the folder on the ephemeral storage to the folder on the persistent storage")),(0,n.yg)("p",null,"Let us know how it works for you on the Slack ",(0,n.yg)("strong",{parentName:"p"},"#general")," channel, and if you have suggestions to improve the workspaces."))}h.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/2b7d82ba.ddd044b8.js b/assets/js/2b7d82ba.8e55b696.js similarity index 99% rename from assets/js/2b7d82ba.ddd044b8.js rename to assets/js/2b7d82ba.8e55b696.js index 6d8563ea9..c48a29f28 100644 --- a/assets/js/2b7d82ba.ddd044b8.js +++ b/assets/js/2b7d82ba.8e55b696.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[5447],{5680:(e,n,t)=>{t.d(n,{xA:()=>p,yg:()=>g});var o=t(6540);function r(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function a(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);n&&(o=o.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,o)}return t}function i(e){for(var n=1;n=0||(r[t]=e[t]);return r}(e,n);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(o=0;o=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(r[t]=e[t])}return r}var l=o.createContext({}),u=function(e){var n=o.useContext(l),t=n;return e&&(t="function"==typeof e?e(n):i(i({},n),e)),t},p=function(e){var n=u(e.components);return o.createElement(l.Provider,{value:n},e.children)},c={inlineCode:"code",wrapper:function(e){var n=e.children;return o.createElement(o.Fragment,{},n)}},h=o.forwardRef((function(e,n){var t=e.components,r=e.mdxType,a=e.originalType,l=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),h=u(t),g=r,y=h["".concat(l,".").concat(g)]||h[g]||c[g]||a;return t?o.createElement(y,i(i({ref:n},p),{},{components:t})):o.createElement(y,i({ref:n},p))}));function g(e,n){var t=arguments,r=n&&n.mdxType;if("string"==typeof e||r){var a=t.length,i=new Array(a);i[0]=h;var s={};for(var l in n)hasOwnProperty.call(n,l)&&(s[l]=n[l]);s.originalType=e,s.mdxType="string"==typeof e?e:r,i[1]=s;for(var u=2;u{t.r(n),t.d(n,{assets:()=>p,contentTitle:()=>l,default:()=>g,frontMatter:()=>s,metadata:()=>u,toc:()=>c});var o=t(9668),r=t(1367),a=(t(6540),t(5680)),i=["components"],s={id:"workflows-github-actions",title:"Deploy GitHub Runners"},l=void 0,u={unversionedId:"workflows-github-actions",id:"workflows-github-actions",title:"Deploy GitHub Runners",description:"Deploy a GitHub Actions runner to run workflows simple to define using YAML, and hosted in your GitHub repository on the DSRI. This allows you to run larger workloads than on GitHub-hosted runners, which are limited to 7G RAM, 1 CPU and 6h per job.",source:"@site/docs/workflows-github-actions.md",sourceDirName:".",slug:"/workflows-github-actions",permalink:"/docs/workflows-github-actions",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/workflows-github-actions.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"workflows-github-actions",title:"Deploy GitHub Runners"},sidebar:"docs",previous:{title:"Introduction to workflows",permalink:"/docs/workflows-introduction"},next:{title:"Deploy Airflow",permalink:"/docs/workflows-airflow"}},p={},c=[{value:"Install the chart",id:"install-the-chart",level:2},{value:"Deploy a Runner",id:"deploy-a-runner",level:2},{value:"For an organization",id:"for-an-organization",level:3},{value:"For a repository",id:"for-a-repository",level:3},{value:"Define Actions to run on DSRI",id:"define-actions-to-run-on-dsri",level:2},{value:"Uninstall the runner",id:"uninstall-the-runner",level:2},{value:"Deploy using GitHub Actions workflows",id:"deploy-using-github-actions-workflows",level:2},{value:"See also",id:"see-also",level:2}],h={toc:c};function g(e){var n=e.components,t=(0,r.A)(e,i);return(0,a.yg)("wrapper",(0,o.A)({},h,t,{components:n,mdxType:"MDXLayout"}),(0,a.yg)("p",null,"Deploy a GitHub Actions runner to run workflows simple to define using YAML, and hosted in your GitHub repository on the DSRI. This allows you to run larger workloads than on GitHub-hosted runners, which are limited to 7G RAM, 1 CPU and 6h per job."),(0,a.yg)("p",null,"Here are some of the advantage of GitHub Actions:"),(0,a.yg)("ul",null,(0,a.yg)("li",{parentName:"ul"},"A step can be any Bash command, or a reusable Action from the ",(0,a.yg)("a",{parentName:"li",href:"https://github.com/marketplace/"},"GitHub Marketplace"),", which can be easily define from a Docker container, and share with your collaborators"),(0,a.yg)("li",{parentName:"ul"},"Parallelization can easily be added manually or dynamically to up to 255 jobs"),(0,a.yg)("li",{parentName:"ul"},"It provides a good logging system directly available in your repository on GitHub"),(0,a.yg)("li",{parentName:"ul"},"Define triggers (on code push, cron job, manual request), and secrets (such as passwords) easily")),(0,a.yg)("p",null,"For more information about GitHub Actions workflows, go to ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/features/actions"},"https://github.com/features/actions")),(0,a.yg)("h2",{id:"install-the-chart"},"Install the chart"),(0,a.yg)("p",null,"You will need to have Helm installed on your computer to deploy a GitHub Actions Runner, see the ",(0,a.yg)("a",{parentName:"p",href:"/docs/helm"},"Helm docs")," for more details."),(0,a.yg)("p",null,"Install the Helm chart to be able to deploy the GitHub Actions Runner on the DSRI:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},"helm repo add openshift-actions-runner https://redhat-actions.github.io/openshift-actions-runner-chart\nhelm repo update\n")),(0,a.yg)("p",null,"Then create a GitHub Personal Access Token as per the instructions in the ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/redhat-actions/openshift-actions-runner#pat-guidelines"},"runner image README"),"."),(0,a.yg)("p",null,(0,a.yg)("strong",{parentName:"p"},"tl;dr:")," go to your Settings on GitHub: ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/settings/tokens"},"https://github.com/settings/tokens"),", click the button to create a new token, give it a meaningful name (e.g. ",(0,a.yg)("inlineCode",{parentName:"p"},"DSRI Runner my-project"),"), and check the following permissions:"),(0,a.yg)("p",null,"\u2705\ufe0f ",(0,a.yg)("inlineCode",{parentName:"p"},"repo")," (maybe also ",(0,a.yg)("inlineCode",{parentName:"p"},"workflow"),"?)"),(0,a.yg)("p",null,"\u2705\ufe0f ",(0,a.yg)("inlineCode",{parentName:"p"},"admin:org")," if the Runner is for an organization"),(0,a.yg)("h2",{id:"deploy-a-runner"},"Deploy a Runner"),(0,a.yg)("p",null,"Before deploying the runner, make sure you are in the project where you want to deploy it:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},"oc project my-project\n")),(0,a.yg)("h3",{id:"for-an-organization"},"For an organization"),(0,a.yg)("p",null,"Deploy a runner available for all repositories of an organization (you can fine tune the access via GitHub Settings)"),(0,a.yg)("ol",null,(0,a.yg)("li",{parentName:"ol"},"Provide the token previously created, and the organization name")),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},'export GITHUB_PAT="TOKEN"\nexport GITHUB_OWNER=My-Org\n')),(0,a.yg)("ol",{start:2},(0,a.yg)("li",{parentName:"ol"},"Deploy the runner for the organization:")),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},'helm install actions-runner openshift-actions-runner/actions-runner \\\n --set-string githubPat=$GITHUB_PAT \\\n --set-string githubOwner=$GITHUB_OWNER \\\n --set runnerLabels="{ dsri, $GITHUB_OWNER }" \\\n --set replicas=3 \\\n --set serviceAccountName=anyuid \\\n --set memoryRequest="512Mi" \\\n --set memoryLimit="100Gi" \\\n --set cpuRequest="100m" \\\n --set cpuLimit="64"\n')),(0,a.yg)("p",null,"You can also change the default runner image:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"}," --set runnerImage=ghcr.io/vemonet/github-actions-conda-runner \\\n --set runnerTag=latest\n")),(0,a.yg)("blockquote",null,(0,a.yg)("p",{parentName:"blockquote"},"Checkout ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/redhat-actions/openshift-actions-runner-chart/blob/main/values.yaml"},"all available parameters here"))),(0,a.yg)("ol",{start:3},(0,a.yg)("li",{parentName:"ol"},"Check the deployment:")),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},"helm get manifest actions-runner | kubectl get -f -\n")),(0,a.yg)("p",null,"Go to your organization Settings page on GitHub, then go to the ",(0,a.yg)("strong",{parentName:"p"},"Actions")," tab, and scroll to the bottom. In the list of active runners you should see the runners you just deployed. "),(0,a.yg)("h3",{id:"for-a-repository"},"For a repository"),(0,a.yg)("p",null,"You can also deploy a runner for a specific repository:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},'export GITHUB_PAT="TOKEN"\n# For an org runner, this is the org.\n# For a repo runner, this is the repo owner (org or user).\nexport GITHUB_OWNER=vemonet\n# For an org runner, omit this argument. \n# For a repo runner, the repo name.\nexport GITHUB_REPO=shapes-of-you\n')),(0,a.yg)("p",null,"Deploy the runner:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},'helm install actions-runner openshift-actions-runner/actions-runner \\\n --set-string githubPat=$GITHUB_PAT \\\n --set-string githubOwner=$GITHUB_OWNER \\\n --set-string githubRepository=$GITHUB_REPO \\\n --set runnerLabels="{ dsri, anything-helpful }"\n')),(0,a.yg)("h2",{id:"define-actions-to-run-on-dsri"},"Define Actions to run on DSRI"),(0,a.yg)("p",null,"You can now set GitHub Action workflows, in the ",(0,a.yg)("inlineCode",{parentName:"p"},".github/workflows")," folder, to be run on this runner (the repository needs to be under the organization, or user you added the workflow to). The job will be sent to run on the DSRI:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-yaml"},'jobs:\n your-job:\n runs-on: ["self-hosted", "dsri", "my-org" ]\n steps: ...\n')),(0,a.yg)("h2",{id:"uninstall-the-runner"},"Uninstall the runner"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},"helm uninstall actions-runner\n")),(0,a.yg)("h2",{id:"deploy-using-github-actions-workflows"},"Deploy using GitHub Actions workflows"),(0,a.yg)("admonition",{title:"Experimental",type:"warning"},(0,a.yg)("p",{parentName:"admonition"},"Experimental: this deployment workflow is still experimental, let us know on Slack if you are interested in using it.")),(0,a.yg)("p",null,"Alternatively you can also build and deploy your application using a GitHub Actions workflow."),(0,a.yg)("p",null,"You will need to connect to the UM VPN in your workflow by defining 2 secrets for ",(0,a.yg)("inlineCode",{parentName:"p"},"VPN_USER")," and ",(0,a.yg)("inlineCode",{parentName:"p"},"VPN_PASSWORD"),", this is done by this step:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-yaml"},"- name: Connect to the VPN\n run: |\n sudo apt-get install -y openconnect network-manager-openconnect\n echo '${{ secrets.VPN_PASSWORD }}' | sudo openconnect --passwd-on-stdin --no-xmlpost --non-inter --background --authgroup 01-Employees --user ${{ secrets.VPN_USER }} vpn.maastrichtuniversity.nl\n sleep 10\n")),(0,a.yg)("admonition",{title:"RedHat documentation",type:"info"},(0,a.yg)("p",{parentName:"admonition"},"RedHat provides the following instructions and template to deploy an application on OpenShift")),(0,a.yg)("p",null,"The OpenShift Starter workflow will:"),(0,a.yg)("ul",null,(0,a.yg)("li",{parentName:"ul"},"Checkout your repository"),(0,a.yg)("li",{parentName:"ul"},"Perform a Docker build"),(0,a.yg)("li",{parentName:"ul"},"Push the built image to an image registry"),(0,a.yg)("li",{parentName:"ul"},"Log in to your OpenShift cluster"),(0,a.yg)("li",{parentName:"ul"},"Create an OpenShift app from the image and expose it to the internet.")),(0,a.yg)("p",null,"Before you begin:"),(0,a.yg)("ul",null,(0,a.yg)("li",{parentName:"ul"},"Have write access to a container image registry such as quay.io or Dockerhub."),(0,a.yg)("li",{parentName:"ul"},"Have access to an OpenShift cluster.",(0,a.yg)("ul",{parentName:"li"},(0,a.yg)("li",{parentName:"ul"},"For instructions to get started with OpenShift see ",(0,a.yg)("a",{parentName:"li",href:"https://www.openshift.com/try"},"https://www.openshift.com/try")))),(0,a.yg)("li",{parentName:"ul"},"The project you wish to add this workflow to should have a Dockerfile.",(0,a.yg)("ul",{parentName:"li"},(0,a.yg)("li",{parentName:"ul"},"If you don't have a Dockerfile at the repository root, see the buildah-build step."),(0,a.yg)("li",{parentName:"ul"},"Builds from scratch are also available, but require more configuration.")))),(0,a.yg)("p",null,"To get the workflow running:"),(0,a.yg)("ol",null,(0,a.yg)("li",{parentName:"ol"},"Add this workflow to your repository."),(0,a.yg)("li",{parentName:"ol"},"Edit the top-level 'env' section, which contains a list of environment variables that must be configured."),(0,a.yg)("li",{parentName:"ol"},"Create the secrets referenced in the 'env' section under your repository Settings."),(0,a.yg)("li",{parentName:"ol"},"Edit the 'branches' in the 'on' section to trigger the workflow on a push to your branch."),(0,a.yg)("li",{parentName:"ol"},"Commit and push your changes.")),(0,a.yg)("p",null,"For a more sophisticated example, see ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/redhat-actions/spring-petclinic/blob/main/.github/workflows/petclinic-sample.yaml"},"https://github.com/redhat-actions/spring-petclinic/blob/main/.github/workflows/petclinic-sample.yaml"),"\nAlso see our GitHub organization, ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/redhat-actions/"},"https://github.com/redhat-actions/")),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-yaml"},'name: Deploy to OpenShift\n\n# \u2b07\ufe0f Modify the fields marked with \u2b07\ufe0f to fit your project, and create any secrets that are referenced.\n# https://docs.github.com/en/free-pro-team@latest/actions/reference/encrypted-secrets\nenv:\n # \u2b07\ufe0f EDIT with your registry and registry path.\n REGISTRY: ghcr.io/maastrichtu-ids\n # \u2b07\ufe0f EDIT with your registry username.\n REGISTRY_USER: \n REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}\n\n # \u2b07\ufe0f EDIT to log into your OpenShift cluster and set up the context.\n # See https://github.com/redhat-actions/oc-login#readme for how to retrieve these values.\n OPENSHIFT_SERVER: ${{ secrets.OPENSHIFT_SERVER }}\n OPENSHIFT_TOKEN: ${{ secrets.OPENSHIFT_TOKEN }}\n\n # \u2b07\ufe0f EDIT with the port your application should be accessible on.\n APP_PORT: 8080\n\n # \u2b07\ufe0f EDIT if you wish to set the kube context\'s namespace after login. Leave blank to use the default namespace.\n OPENSHIFT_NAMESPACE: ""\n\n # If you wish to manually provide the APP_NAME and TAG, set them here, otherwise they will be auto-detected.\n APP_NAME: "my-app"\n TAG: ""\n\non:\n # https://docs.github.com/en/free-pro-team@latest/actions/reference/events-that-trigger-workflows\n push:\n # Edit to the branch(es) you want to build and deploy on each push.\n branches: [ main ]\n\njobs:\n openshift-ci-cd:\n name: Build and deploy to OpenShift\n runs-on: ubuntu-20.04\n\n steps:\n - uses: actions/checkout@v2\n\n - name: Determine app name\n if: env.APP_NAME == \'\'\n run: |\n echo "APP_NAME=$(basename $PWD)" | tee -a $GITHUB_ENV\n\n - name: Determine tag\n if: env.TAG == \'\'\n run: |\n echo "TAG=${GITHUB_SHA::7}" | tee -a $GITHUB_ENV\n\n # https://github.com/redhat-actions/buildah-build#readme\n - name: Build from Dockerfile\n uses: redhat-actions/buildah-build@v1\n with:\n image: ${{ env.APP_NAME }}\n tag: ${{ env.TAG }}\n # If you don\'t have a dockerfile, see:\n # https://github.com/redhat-actions/buildah-build#building-from-scratch\n # Otherwise, point this to your Dockerfile relative to the repository root.\n dockerfiles: |\n ./Dockerfile\n\n # https://github.com/redhat-actions/push-to-registry#readme\n - name: Push to registry\n id: push-to-registry\n uses: redhat-actions/push-to-registry@v1\n with:\n image: ${{ env.APP_NAME }}\n tag: ${{ env.TAG }}\n registry: ${{ env.REGISTRY }}\n username: ${{ env.REGISTRY_USER }}\n password: ${{ env.REGISTRY_PASSWORD }}\n\n # The path the image was pushed to is now stored in ${{ steps.push-to-registry.outputs.registry-path }}\n\n - name: Connect to the VPN\n run: |\n sudo apt-get install -y openconnect network-manager-openconnect\n echo \'${{ secrets.VPN_PASSWORD }}\' | sudo openconnect --passwd-on-stdin --no-xmlpost --non-inter --background --authgroup 01-Employees --user ${{ secrets.VPN_USER }} vpn.maastrichtuniversity.nl\n sleep 10\n\n # oc-login works on all platforms, but oc must be installed first.\n # The GitHub Ubuntu runner already includes oc.\n # https://github.com/redhat-actions/oc-login#readme\n - name: Log in to OpenShift\n uses: redhat-actions/oc-login@v1\n with:\n openshift_server_url: ${{ env.OPENSHIFT_SERVER }}\n openshift_token: ${{ env.OPENSHIFT_TOKEN }}\n insecure_skip_tls_verify: true\n namespace: ${{ env.OPENSHIFT_NAMESPACE }}\n\n # This step should create a deployment, service, and route to run your app and expose it to the internet.\n # Feel free to replace this with \'oc apply\', \'helm install\', or however you like to deploy your app.\n - name: Create and expose app\n run: |\n export IMAGE="${{ steps.push-to-registry.outputs.registry-path }}"\n export PORT=${{ env.APP_PORT }}\n\n export SELECTOR="app=${{ env.APP_NAME }}"\n echo "SELECTOR=$SELECTOR" >> $GITHUB_ENV\n\n set -x\n # Take down any old deployment\n oc delete all --selector="$SELECTOR"\n oc new-app --name $APP_NAME --docker-image="$IMAGE"\n\n # Make sure the app port is exposed\n oc patch svc $APP_NAME -p "{ \\"spec\\": { \\"ports\\": [{ \\"name\\": \\"$PORT-tcp\\", \\"port\\": $PORT }] } }"\n oc expose service $APP_NAME --port=$PORT\n\n oc get all --selector="$SELECTOR"\n set +x\n\n export ROUTE="$(oc get route $APP_NAME -o jsonpath=\'{.spec.host}\')"\n echo "$APP_NAME is exposed at $ROUTE"\n echo "ROUTE=$ROUTE" >> $GITHUB_ENV\n\n - name: View application route\n run: |\n [[ -n ${{ env.ROUTE }} ]] || (echo "Determining application route failed in previous step"; exit 1)\n echo "======================== Your application is available at: ========================"\n echo ${{ env.ROUTE }}\n echo "==================================================================================="\n echo\n echo "Your app can be taken down with: \\"oc delete all --selector=\'${{ env.SELECTOR }}\'\\""\n')),(0,a.yg)("h2",{id:"see-also"},"See also"),(0,a.yg)("ul",null,(0,a.yg)("li",{parentName:"ul"},(0,a.yg)("p",{parentName:"li"},"GitHub runner chart repository: ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/redhat-actions/openshift-actions-runner-chart"},"https://github.com/redhat-actions/openshift-actions-runner-chart"))),(0,a.yg)("li",{parentName:"ul"},(0,a.yg)("p",{parentName:"li"},"Image for the runner: ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/redhat-actions/openshift-actions-runner"},"https://github.com/redhat-actions/openshift-actions-runner"))),(0,a.yg)("li",{parentName:"ul"},(0,a.yg)("p",{parentName:"li"},"An action to automatically deploy a runner on a cluster (require to run openconnect to VPN first): ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/redhat-actions/openshift-actions-runner"},"https://github.com/redhat-actions/openshift-actions-runner")))))}g.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[5447],{5680:(e,n,t)=>{t.d(n,{xA:()=>p,yg:()=>g});var o=t(6540);function r(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function a(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);n&&(o=o.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,o)}return t}function i(e){for(var n=1;n=0||(r[t]=e[t]);return r}(e,n);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(o=0;o=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(r[t]=e[t])}return r}var l=o.createContext({}),u=function(e){var n=o.useContext(l),t=n;return e&&(t="function"==typeof e?e(n):i(i({},n),e)),t},p=function(e){var n=u(e.components);return o.createElement(l.Provider,{value:n},e.children)},c={inlineCode:"code",wrapper:function(e){var n=e.children;return o.createElement(o.Fragment,{},n)}},h=o.forwardRef((function(e,n){var t=e.components,r=e.mdxType,a=e.originalType,l=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),h=u(t),g=r,y=h["".concat(l,".").concat(g)]||h[g]||c[g]||a;return t?o.createElement(y,i(i({ref:n},p),{},{components:t})):o.createElement(y,i({ref:n},p))}));function g(e,n){var t=arguments,r=n&&n.mdxType;if("string"==typeof e||r){var a=t.length,i=new Array(a);i[0]=h;var s={};for(var l in n)hasOwnProperty.call(n,l)&&(s[l]=n[l]);s.originalType=e,s.mdxType="string"==typeof e?e:r,i[1]=s;for(var u=2;u{t.r(n),t.d(n,{assets:()=>p,contentTitle:()=>l,default:()=>g,frontMatter:()=>s,metadata:()=>u,toc:()=>c});var o=t(9668),r=t(1367),a=(t(6540),t(5680)),i=["components"],s={id:"workflows-github-actions",title:"Deploy GitHub Runners"},l=void 0,u={unversionedId:"workflows-github-actions",id:"workflows-github-actions",title:"Deploy GitHub Runners",description:"Deploy a GitHub Actions runner to run workflows simple to define using YAML, and hosted in your GitHub repository on the DSRI. This allows you to run larger workloads than on GitHub-hosted runners, which are limited to 7G RAM, 1 CPU and 6h per job.",source:"@site/docs/workflows-github-actions.md",sourceDirName:".",slug:"/workflows-github-actions",permalink:"/docs/workflows-github-actions",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/workflows-github-actions.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"workflows-github-actions",title:"Deploy GitHub Runners"},sidebar:"docs",previous:{title:"Introduction to workflows",permalink:"/docs/workflows-introduction"},next:{title:"Deploy Airflow",permalink:"/docs/workflows-airflow"}},p={},c=[{value:"Install the chart",id:"install-the-chart",level:2},{value:"Deploy a Runner",id:"deploy-a-runner",level:2},{value:"For an organization",id:"for-an-organization",level:3},{value:"For a repository",id:"for-a-repository",level:3},{value:"Define Actions to run on DSRI",id:"define-actions-to-run-on-dsri",level:2},{value:"Uninstall the runner",id:"uninstall-the-runner",level:2},{value:"Deploy using GitHub Actions workflows",id:"deploy-using-github-actions-workflows",level:2},{value:"See also",id:"see-also",level:2}],h={toc:c};function g(e){var n=e.components,t=(0,r.A)(e,i);return(0,a.yg)("wrapper",(0,o.A)({},h,t,{components:n,mdxType:"MDXLayout"}),(0,a.yg)("p",null,"Deploy a GitHub Actions runner to run workflows simple to define using YAML, and hosted in your GitHub repository on the DSRI. This allows you to run larger workloads than on GitHub-hosted runners, which are limited to 7G RAM, 1 CPU and 6h per job."),(0,a.yg)("p",null,"Here are some of the advantage of GitHub Actions:"),(0,a.yg)("ul",null,(0,a.yg)("li",{parentName:"ul"},"A step can be any Bash command, or a reusable Action from the ",(0,a.yg)("a",{parentName:"li",href:"https://github.com/marketplace/"},"GitHub Marketplace"),", which can be easily define from a Docker container, and share with your collaborators"),(0,a.yg)("li",{parentName:"ul"},"Parallelization can easily be added manually or dynamically to up to 255 jobs"),(0,a.yg)("li",{parentName:"ul"},"It provides a good logging system directly available in your repository on GitHub"),(0,a.yg)("li",{parentName:"ul"},"Define triggers (on code push, cron job, manual request), and secrets (such as passwords) easily")),(0,a.yg)("p",null,"For more information about GitHub Actions workflows, go to ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/features/actions"},"https://github.com/features/actions")),(0,a.yg)("h2",{id:"install-the-chart"},"Install the chart"),(0,a.yg)("p",null,"You will need to have Helm installed on your computer to deploy a GitHub Actions Runner, see the ",(0,a.yg)("a",{parentName:"p",href:"/docs/helm"},"Helm docs")," for more details."),(0,a.yg)("p",null,"Install the Helm chart to be able to deploy the GitHub Actions Runner on the DSRI:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},"helm repo add openshift-actions-runner https://redhat-actions.github.io/openshift-actions-runner-chart\nhelm repo update\n")),(0,a.yg)("p",null,"Then create a GitHub Personal Access Token as per the instructions in the ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/redhat-actions/openshift-actions-runner#pat-guidelines"},"runner image README"),"."),(0,a.yg)("p",null,(0,a.yg)("strong",{parentName:"p"},"tl;dr:")," go to your Settings on GitHub: ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/settings/tokens"},"https://github.com/settings/tokens"),", click the button to create a new token, give it a meaningful name (e.g. ",(0,a.yg)("inlineCode",{parentName:"p"},"DSRI Runner my-project"),"), and check the following permissions:"),(0,a.yg)("p",null,"\u2705\ufe0f ",(0,a.yg)("inlineCode",{parentName:"p"},"repo")," (maybe also ",(0,a.yg)("inlineCode",{parentName:"p"},"workflow"),"?)"),(0,a.yg)("p",null,"\u2705\ufe0f ",(0,a.yg)("inlineCode",{parentName:"p"},"admin:org")," if the Runner is for an organization"),(0,a.yg)("h2",{id:"deploy-a-runner"},"Deploy a Runner"),(0,a.yg)("p",null,"Before deploying the runner, make sure you are in the project where you want to deploy it:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},"oc project my-project\n")),(0,a.yg)("h3",{id:"for-an-organization"},"For an organization"),(0,a.yg)("p",null,"Deploy a runner available for all repositories of an organization (you can fine tune the access via GitHub Settings)"),(0,a.yg)("ol",null,(0,a.yg)("li",{parentName:"ol"},"Provide the token previously created, and the organization name")),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},'export GITHUB_PAT="TOKEN"\nexport GITHUB_OWNER=My-Org\n')),(0,a.yg)("ol",{start:2},(0,a.yg)("li",{parentName:"ol"},"Deploy the runner for the organization:")),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},'helm install actions-runner openshift-actions-runner/actions-runner \\\n --set-string githubPat=$GITHUB_PAT \\\n --set-string githubOwner=$GITHUB_OWNER \\\n --set runnerLabels="{ dsri, $GITHUB_OWNER }" \\\n --set replicas=3 \\\n --set serviceAccountName=anyuid \\\n --set memoryRequest="512Mi" \\\n --set memoryLimit="100Gi" \\\n --set cpuRequest="100m" \\\n --set cpuLimit="64"\n')),(0,a.yg)("p",null,"You can also change the default runner image:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"}," --set runnerImage=ghcr.io/vemonet/github-actions-conda-runner \\\n --set runnerTag=latest\n")),(0,a.yg)("blockquote",null,(0,a.yg)("p",{parentName:"blockquote"},"Checkout ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/redhat-actions/openshift-actions-runner-chart/blob/main/values.yaml"},"all available parameters here"))),(0,a.yg)("ol",{start:3},(0,a.yg)("li",{parentName:"ol"},"Check the deployment:")),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},"helm get manifest actions-runner | kubectl get -f -\n")),(0,a.yg)("p",null,"Go to your organization Settings page on GitHub, then go to the ",(0,a.yg)("strong",{parentName:"p"},"Actions")," tab, and scroll to the bottom. In the list of active runners you should see the runners you just deployed. "),(0,a.yg)("h3",{id:"for-a-repository"},"For a repository"),(0,a.yg)("p",null,"You can also deploy a runner for a specific repository:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},'export GITHUB_PAT="TOKEN"\n# For an org runner, this is the org.\n# For a repo runner, this is the repo owner (org or user).\nexport GITHUB_OWNER=vemonet\n# For an org runner, omit this argument. \n# For a repo runner, the repo name.\nexport GITHUB_REPO=shapes-of-you\n')),(0,a.yg)("p",null,"Deploy the runner:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},'helm install actions-runner openshift-actions-runner/actions-runner \\\n --set-string githubPat=$GITHUB_PAT \\\n --set-string githubOwner=$GITHUB_OWNER \\\n --set-string githubRepository=$GITHUB_REPO \\\n --set runnerLabels="{ dsri, anything-helpful }"\n')),(0,a.yg)("h2",{id:"define-actions-to-run-on-dsri"},"Define Actions to run on DSRI"),(0,a.yg)("p",null,"You can now set GitHub Action workflows, in the ",(0,a.yg)("inlineCode",{parentName:"p"},".github/workflows")," folder, to be run on this runner (the repository needs to be under the organization, or user you added the workflow to). The job will be sent to run on the DSRI:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-yaml"},'jobs:\n your-job:\n runs-on: ["self-hosted", "dsri", "my-org" ]\n steps: ...\n')),(0,a.yg)("h2",{id:"uninstall-the-runner"},"Uninstall the runner"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},"helm uninstall actions-runner\n")),(0,a.yg)("h2",{id:"deploy-using-github-actions-workflows"},"Deploy using GitHub Actions workflows"),(0,a.yg)("admonition",{title:"Experimental",type:"warning"},(0,a.yg)("p",{parentName:"admonition"},"Experimental: this deployment workflow is still experimental, let us know on Slack if you are interested in using it.")),(0,a.yg)("p",null,"Alternatively you can also build and deploy your application using a GitHub Actions workflow."),(0,a.yg)("p",null,"You will need to connect to the UM VPN in your workflow by defining 2 secrets for ",(0,a.yg)("inlineCode",{parentName:"p"},"VPN_USER")," and ",(0,a.yg)("inlineCode",{parentName:"p"},"VPN_PASSWORD"),", this is done by this step:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-yaml"},"- name: Connect to the VPN\n run: |\n sudo apt-get install -y openconnect network-manager-openconnect\n echo '${{ secrets.VPN_PASSWORD }}' | sudo openconnect --passwd-on-stdin --no-xmlpost --non-inter --background --authgroup 01-Employees --user ${{ secrets.VPN_USER }} vpn.maastrichtuniversity.nl\n sleep 10\n")),(0,a.yg)("admonition",{title:"RedHat documentation",type:"info"},(0,a.yg)("p",{parentName:"admonition"},"RedHat provides the following instructions and template to deploy an application on OpenShift")),(0,a.yg)("p",null,"The OpenShift Starter workflow will:"),(0,a.yg)("ul",null,(0,a.yg)("li",{parentName:"ul"},"Checkout your repository"),(0,a.yg)("li",{parentName:"ul"},"Perform a Docker build"),(0,a.yg)("li",{parentName:"ul"},"Push the built image to an image registry"),(0,a.yg)("li",{parentName:"ul"},"Log in to your OpenShift cluster"),(0,a.yg)("li",{parentName:"ul"},"Create an OpenShift app from the image and expose it to the internet.")),(0,a.yg)("p",null,"Before you begin:"),(0,a.yg)("ul",null,(0,a.yg)("li",{parentName:"ul"},"Have write access to a container image registry such as quay.io or Dockerhub."),(0,a.yg)("li",{parentName:"ul"},"Have access to an OpenShift cluster.",(0,a.yg)("ul",{parentName:"li"},(0,a.yg)("li",{parentName:"ul"},"For instructions to get started with OpenShift see ",(0,a.yg)("a",{parentName:"li",href:"https://www.openshift.com/try"},"https://www.openshift.com/try")))),(0,a.yg)("li",{parentName:"ul"},"The project you wish to add this workflow to should have a Dockerfile.",(0,a.yg)("ul",{parentName:"li"},(0,a.yg)("li",{parentName:"ul"},"If you don't have a Dockerfile at the repository root, see the buildah-build step."),(0,a.yg)("li",{parentName:"ul"},"Builds from scratch are also available, but require more configuration.")))),(0,a.yg)("p",null,"To get the workflow running:"),(0,a.yg)("ol",null,(0,a.yg)("li",{parentName:"ol"},"Add this workflow to your repository."),(0,a.yg)("li",{parentName:"ol"},"Edit the top-level 'env' section, which contains a list of environment variables that must be configured."),(0,a.yg)("li",{parentName:"ol"},"Create the secrets referenced in the 'env' section under your repository Settings."),(0,a.yg)("li",{parentName:"ol"},"Edit the 'branches' in the 'on' section to trigger the workflow on a push to your branch."),(0,a.yg)("li",{parentName:"ol"},"Commit and push your changes.")),(0,a.yg)("p",null,"For a more sophisticated example, see ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/redhat-actions/spring-petclinic/blob/main/.github/workflows/petclinic-sample.yaml"},"https://github.com/redhat-actions/spring-petclinic/blob/main/.github/workflows/petclinic-sample.yaml"),"\nAlso see our GitHub organization, ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/redhat-actions/"},"https://github.com/redhat-actions/")),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-yaml"},'name: Deploy to OpenShift\n\n# \u2b07\ufe0f Modify the fields marked with \u2b07\ufe0f to fit your project, and create any secrets that are referenced.\n# https://docs.github.com/en/free-pro-team@latest/actions/reference/encrypted-secrets\nenv:\n # \u2b07\ufe0f EDIT with your registry and registry path.\n REGISTRY: ghcr.io/maastrichtu-ids\n # \u2b07\ufe0f EDIT with your registry username.\n REGISTRY_USER: \n REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}\n\n # \u2b07\ufe0f EDIT to log into your OpenShift cluster and set up the context.\n # See https://github.com/redhat-actions/oc-login#readme for how to retrieve these values.\n OPENSHIFT_SERVER: ${{ secrets.OPENSHIFT_SERVER }}\n OPENSHIFT_TOKEN: ${{ secrets.OPENSHIFT_TOKEN }}\n\n # \u2b07\ufe0f EDIT with the port your application should be accessible on.\n APP_PORT: 8080\n\n # \u2b07\ufe0f EDIT if you wish to set the kube context\'s namespace after login. Leave blank to use the default namespace.\n OPENSHIFT_NAMESPACE: ""\n\n # If you wish to manually provide the APP_NAME and TAG, set them here, otherwise they will be auto-detected.\n APP_NAME: "my-app"\n TAG: ""\n\non:\n # https://docs.github.com/en/free-pro-team@latest/actions/reference/events-that-trigger-workflows\n push:\n # Edit to the branch(es) you want to build and deploy on each push.\n branches: [ main ]\n\njobs:\n openshift-ci-cd:\n name: Build and deploy to OpenShift\n runs-on: ubuntu-20.04\n\n steps:\n - uses: actions/checkout@v2\n\n - name: Determine app name\n if: env.APP_NAME == \'\'\n run: |\n echo "APP_NAME=$(basename $PWD)" | tee -a $GITHUB_ENV\n\n - name: Determine tag\n if: env.TAG == \'\'\n run: |\n echo "TAG=${GITHUB_SHA::7}" | tee -a $GITHUB_ENV\n\n # https://github.com/redhat-actions/buildah-build#readme\n - name: Build from Dockerfile\n uses: redhat-actions/buildah-build@v1\n with:\n image: ${{ env.APP_NAME }}\n tag: ${{ env.TAG }}\n # If you don\'t have a dockerfile, see:\n # https://github.com/redhat-actions/buildah-build#building-from-scratch\n # Otherwise, point this to your Dockerfile relative to the repository root.\n dockerfiles: |\n ./Dockerfile\n\n # https://github.com/redhat-actions/push-to-registry#readme\n - name: Push to registry\n id: push-to-registry\n uses: redhat-actions/push-to-registry@v1\n with:\n image: ${{ env.APP_NAME }}\n tag: ${{ env.TAG }}\n registry: ${{ env.REGISTRY }}\n username: ${{ env.REGISTRY_USER }}\n password: ${{ env.REGISTRY_PASSWORD }}\n\n # The path the image was pushed to is now stored in ${{ steps.push-to-registry.outputs.registry-path }}\n\n - name: Connect to the VPN\n run: |\n sudo apt-get install -y openconnect network-manager-openconnect\n echo \'${{ secrets.VPN_PASSWORD }}\' | sudo openconnect --passwd-on-stdin --no-xmlpost --non-inter --background --authgroup 01-Employees --user ${{ secrets.VPN_USER }} vpn.maastrichtuniversity.nl\n sleep 10\n\n # oc-login works on all platforms, but oc must be installed first.\n # The GitHub Ubuntu runner already includes oc.\n # https://github.com/redhat-actions/oc-login#readme\n - name: Log in to OpenShift\n uses: redhat-actions/oc-login@v1\n with:\n openshift_server_url: ${{ env.OPENSHIFT_SERVER }}\n openshift_token: ${{ env.OPENSHIFT_TOKEN }}\n insecure_skip_tls_verify: true\n namespace: ${{ env.OPENSHIFT_NAMESPACE }}\n\n # This step should create a deployment, service, and route to run your app and expose it to the internet.\n # Feel free to replace this with \'oc apply\', \'helm install\', or however you like to deploy your app.\n - name: Create and expose app\n run: |\n export IMAGE="${{ steps.push-to-registry.outputs.registry-path }}"\n export PORT=${{ env.APP_PORT }}\n\n export SELECTOR="app=${{ env.APP_NAME }}"\n echo "SELECTOR=$SELECTOR" >> $GITHUB_ENV\n\n set -x\n # Take down any old deployment\n oc delete all --selector="$SELECTOR"\n oc new-app --name $APP_NAME --docker-image="$IMAGE"\n\n # Make sure the app port is exposed\n oc patch svc $APP_NAME -p "{ \\"spec\\": { \\"ports\\": [{ \\"name\\": \\"$PORT-tcp\\", \\"port\\": $PORT }] } }"\n oc expose service $APP_NAME --port=$PORT\n\n oc get all --selector="$SELECTOR"\n set +x\n\n export ROUTE="$(oc get route $APP_NAME -o jsonpath=\'{.spec.host}\')"\n echo "$APP_NAME is exposed at $ROUTE"\n echo "ROUTE=$ROUTE" >> $GITHUB_ENV\n\n - name: View application route\n run: |\n [[ -n ${{ env.ROUTE }} ]] || (echo "Determining application route failed in previous step"; exit 1)\n echo "======================== Your application is available at: ========================"\n echo ${{ env.ROUTE }}\n echo "==================================================================================="\n echo\n echo "Your app can be taken down with: \\"oc delete all --selector=\'${{ env.SELECTOR }}\'\\""\n')),(0,a.yg)("h2",{id:"see-also"},"See also"),(0,a.yg)("ul",null,(0,a.yg)("li",{parentName:"ul"},(0,a.yg)("p",{parentName:"li"},"GitHub runner chart repository: ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/redhat-actions/openshift-actions-runner-chart"},"https://github.com/redhat-actions/openshift-actions-runner-chart"))),(0,a.yg)("li",{parentName:"ul"},(0,a.yg)("p",{parentName:"li"},"Image for the runner: ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/redhat-actions/openshift-actions-runner"},"https://github.com/redhat-actions/openshift-actions-runner"))),(0,a.yg)("li",{parentName:"ul"},(0,a.yg)("p",{parentName:"li"},"An action to automatically deploy a runner on a cluster (require to run openconnect to VPN first): ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/redhat-actions/openshift-actions-runner"},"https://github.com/redhat-actions/openshift-actions-runner")))))}g.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/2e2e5152.acae7be6.js b/assets/js/2e2e5152.eced3416.js similarity index 99% rename from assets/js/2e2e5152.acae7be6.js rename to assets/js/2e2e5152.eced3416.js index 957a08960..fb713e505 100644 --- a/assets/js/2e2e5152.acae7be6.js +++ b/assets/js/2e2e5152.eced3416.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[2412],{5680:(e,t,n)=>{n.d(t,{xA:()=>g,yg:()=>m});var a=n(6540);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function r(e){for(var t=1;t=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var l=a.createContext({}),p=function(e){var t=a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},g=function(e){var t=p(e.components);return a.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},c=a.forwardRef((function(e,t){var n=e.components,i=e.mdxType,o=e.originalType,l=e.parentName,g=s(e,["components","mdxType","originalType","parentName"]),c=p(n),m=i,d=c["".concat(l,".").concat(m)]||c[m]||u[m]||o;return n?a.createElement(d,r(r({ref:t},g),{},{components:n})):a.createElement(d,r({ref:t},g))}));function m(e,t){var n=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var o=n.length,r=new Array(o);r[0]=c;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s.mdxType="string"==typeof e?e:i,r[1]=s;for(var p=2;p{n.r(t),n.d(t,{assets:()=>g,contentTitle:()=>l,default:()=>m,frontMatter:()=>s,metadata:()=>p,toc:()=>u});var a=n(9668),i=n(1367),o=(n(6540),n(5680)),r=["components"],s={id:"guide-known-issues",title:"Known Issues"},l=void 0,p={unversionedId:"guide-known-issues",id:"guide-known-issues",title:"Known Issues",description:"Cannot access your data in the persistent folder",source:"@site/docs/guide-known-issues.md",sourceDirName:".",slug:"/guide-known-issues",permalink:"/docs/guide-known-issues",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/guide-known-issues.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"guide-known-issues",title:"Known Issues"},sidebar:"docs",previous:{title:"Parallelization using Dask",permalink:"/docs/dask-tutorial"},next:{title:"Publish a Docker image",permalink:"/docs/guide-publish-image"}},g={},u=[{value:"Cannot access your data in the persistent folder",id:"cannot-access-your-data-in-the-persistent-folder",level:2},{value:"Large volumes",id:"large-volumes",level:2},{value:"DockerHub pull limitations",id:"dockerhub-pull-limitations",level:2},{value:"How to run function within a container ''in the background'",id:"how-to-run-function-within-a-container-in-the-background",level:2},{value:"Git authentication issue",id:"git-authentication-issue",level:2},{value:"Windows:",id:"windows",level:5},{value:"Mac:",id:"mac",level:5},{value:"Filebrowser 403 forbidden",id:"filebrowser-403-forbidden",level:2}],c={toc:u};function m(e){var t=e.components,n=(0,i.A)(e,r);return(0,o.yg)("wrapper",(0,a.A)({},c,n,{components:t,mdxType:"MDXLayout"}),(0,o.yg)("h2",{id:"cannot-access-your-data-in-the-persistent-folder"},"Cannot access your data in the persistent folder"),(0,o.yg)("p",null,"Sometimes you cannot access anymore the data you put in the persistent folder of your container. It can be due to a node going down, if the persistent volume your pod is connected to is on this node, then it cannot access it anymore."),(0,o.yg)("p",null,"You can easily fix this issue by restarting the pod of your application, it will make it properly connect to resources on nodes that are up."),(0,o.yg)("p",null,"To restart the pod, go in topology, click on your application, go to the details tab, and decrease the pod count to 0, then put it back up to 1."),(0,o.yg)("h2",{id:"large-volumes"},"Large volumes"),(0,o.yg)("admonition",{title:"Pod or Deployment will not start",type:"warning"},(0,o.yg)("p",{parentName:"admonition"},"You could run into a following message in the ",(0,o.yg)("strong",{parentName:"p"},"Events")," tab that looks similar to this"),(0,o.yg)("pre",{parentName:"admonition"},(0,o.yg)("code",{parentName:"pre"},"Error: kubelet may be retrying requests that are timing out in CRI-O due to system load. Currently at stage container volume configuration: context deadline exceeded: error reserving ctr name\n"))),(0,o.yg)("p",null,"The issue above will occur if you are using a ",(0,o.yg)("strong",{parentName:"p"},"large persistent volume"),". It can be resolved by adding the following to your Deployment(Config):"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre"},"spec:\n template:\n metadata:\n annotations:\n io.kubernetes.cri-o.TrySkipVolumeSELinuxLabel: 'true'\n spec:\n runtimeClassName: selinux\n")),(0,o.yg)("p",null,"Take note of the ",(0,o.yg)("strong",{parentName:"p"},"indentation")," and the place in the file!"),(0,o.yg)("p",null,"An example of this can be found here:"),(0,o.yg)("img",{class:"screenshot",src:"/img/screenshot_large_volume_issue.png",alt:"Storage",style:{zoom:"100%",maxHeight:"500px",maxWidth:"500px"}}),(0,o.yg)("h2",{id:"dockerhub-pull-limitations"},"DockerHub pull limitations"),(0,o.yg)("admonition",{title:"Spot the issue",type:"warning"},(0,o.yg)("p",{parentName:"admonition"},"If the ",(0,o.yg)("strong",{parentName:"p"},"Events")," tab show this error:"),(0,o.yg)("pre",{parentName:"admonition"},(0,o.yg)("code",{parentName:"pre"},"--\x3e Scaling filebrowser-case-1 to 1\nerror: update acceptor rejected my-app-1: pods for rc 'my-project/my-app-1' took longer than 600 seconds to become available\n")),(0,o.yg)("p",{parentName:"admonition"},"Then check for the application ImageStream in ",(0,o.yg)("strong",{parentName:"p"},"Build")," > ",(0,o.yg)("strong",{parentName:"p"},"Images"),", and you might see this for your application image:"),(0,o.yg)("pre",{parentName:"admonition"},(0,o.yg)("code",{parentName:"pre"},"Internal error occurred: toomanyrequests: You have reached your pull rate limit.\nYou may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit.\n"))),(0,o.yg)("p",null,"You can solve this by creating a secret to login to DockerHub in your project:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"oc create secret docker-registry dockerhub-login --docker-server=docker.io --docker-username=dockerhub_username --docker-password=dockerhub_password --docker-email=example@mail.com\n")),(0,o.yg)("p",null,"Linking the login secret to the default service account:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"oc secrets link default dockerhub-login --for=pull\n")),(0,o.yg)("admonition",{type:"tip"},(0,o.yg)("p",{parentName:"admonition"},"Login to DockerHub should raise the limitations")),(0,o.yg)("p",null,"To definitely solve this issue you can publish the DockerHub image to the ",(0,o.yg)("a",{parentName:"p",href:"https://docs.github.com/en/packages/guides/about-github-container-registry"},"GitHub Container Registry"),"."),(0,o.yg)("p",null,"Follow those instructions on your laptop:"),(0,o.yg)("ol",null,(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},(0,o.yg)("a",{parentName:"p",href:"https://maastrichtu-ids.github.io/dsri-documentation/docs/guide-publish-image#login-to-github-container-registry"},"Login to the GitHub Container Registry")," with ",(0,o.yg)("inlineCode",{parentName:"p"},"docker login"),".")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Pull the docker image from"),(0,o.yg)("pre",{parentName:"li"},(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"docker pull myorg/myimage:latest\n")),(0,o.yg)("p",{parentName:"li"},(0,o.yg)("a",{parentName:"p",href:"mailto:git@github.com"},"git@github.com"),":MaastrichtU-IDS/",(0,o.yg)("a",{parentName:"p",href:"mailto:dsri-documentation.gitgit@github.com"},"dsri-documentation.gitgit@github.com"),":MaastrichtU-IDS/",(0,o.yg)("a",{parentName:"p",href:"mailto:dsri-documentation.gitgit@github.com"},"dsri-documentation.gitgit@github.com"),":MaastrichtU-IDS/dsri-documentation.git")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Change its tag"),(0,o.yg)("pre",{parentName:"li"},(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"docker tag myorg/myimage:latest ghcr.io/maastrichtu-ids/myimage:latest\n"))),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Push it back to the GitHub Container Registry:"),(0,o.yg)("pre",{parentName:"li"},(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"docker push ghcr.io/maastrichtu-ids/myimage:latest\n")))),(0,o.yg)("admonition",{title:"Image created automatically",type:"tip"},(0,o.yg)("p",{parentName:"admonition"},"If the image does not exist, GitHub will create automatically when you push it for the first time! You can then head to your ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/orgs/MaastrichtU-IDS/packages"},"organization ",(0,o.yg)("strong",{parentName:"a"},"Packages")," tab")," to see the package.")),(0,o.yg)("admonition",{title:"Make it public",type:"info"},(0,o.yg)("p",{parentName:"admonition"},"By default new images are set as ",(0,o.yg)("inlineCode",{parentName:"p"},"Private"),", go to your ",(0,o.yg)("strong",{parentName:"p"},"Package Settings"),", and click ",(0,o.yg)("strong",{parentName:"p"},"Change Visibility")," to set it as ",(0,o.yg)("inlineCode",{parentName:"p"},"Public"),", this avoids the need to login to pull the image.")),(0,o.yg)("p",null,"You can update the image if you want access to the latest version, you can set a GitHub Actions workflow to do so."),(0,o.yg)("p",null,"Finally you will need to update your DSRI deployment, or template, to use the newly created image on ",(0,o.yg)("inlineCode",{parentName:"p"},"ghcr.io"),", and redeploy the application with the new template."),(0,o.yg)("hr",null),(0,o.yg)("h2",{id:"how-to-run-function-within-a-container-in-the-background"},"How to run function within a container ''in the background'"),(0,o.yg)("admonition",{title:"Spot the issue",type:"warning"},(0,o.yg)("p",{parentName:"admonition"},"If the ",(0,o.yg)("strong",{parentName:"p"},"Events")," tab show this error:"),(0,o.yg)("pre",{parentName:"admonition"},(0,o.yg)("code",{parentName:"pre"},"--\x3e cd /usr/local/src/work2/aerius-sample-sequencing/CD4K4ANXX\n\nTrinity --seqType fq --max_memory 100G --CPU 64 --samples_file samples.txt --output /usr/local/src/work2/Trinity_output_zip_090221\nerror: The function starts but at some points just exits without warnings or errors to Windows folder\n")),(0,o.yg)("pre",{parentName:"admonition"},(0,o.yg)("code",{parentName:"pre"},"DSRI in the container's terminal keep running fine but never finishes. At some point a red label ''disconnected'' appears and the terminal stops and the analysis never continues.\n"))),(0,o.yg)("p",null,"Those two issues are due to the process running attach to the terminal"),(0,o.yg)("p",null,'Should be able to easily run it using the "Bash way": add ',(0,o.yg)("inlineCode",{parentName:"p"},"nohup")," at the beginning and ",(0,o.yg)("inlineCode",{parentName:"p"},"&")," at the end\nIt will run in the back and all output that should have gone to the terminal will go to a file ",(0,o.yg)("inlineCode",{parentName:"p"},"nohup.out")," in the repo"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre"},"nohup Trinity --seqType fq --max_memory 100G --CPU 64 --samples_file samples.txt --output /usr/local/src/work2/Trinity_output_zip_090221 &\n")),(0,o.yg)("p",null,"To check if it is still running:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre"},"ps aux | grep Trinity\n")),(0,o.yg)("p",null,'Be careful make sure the terminal uses bash and not shell ("sh")'),(0,o.yg)("p",null,"To use bash just type bash in the terminal:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre"},"bash\n")),(0,o.yg)("h2",{id:"git-authentication-issue"},"Git authentication issue"),(0,o.yg)("admonition",{type:"warning"},(0,o.yg)("p",{parentName:"admonition"},"\u26a0\ufe0f remote: HTTP Basic: Access denied fatal: Authentication failed for")),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},"It happen every time when we forced to change the Windows password."),(0,o.yg)("img",{class:"screenshot",src:"/img/authentication-issue.png",style:{zoom:"100%",maxHeight:"500px",maxWidth:"500px"}}),(0,o.yg)("img",{class:"screenshot",src:"/img/git-authentication-issue.png",style:{zoom:"100%",maxHeight:"500px",maxWidth:"500px"}})),(0,o.yg)("ol",null,(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Apply command from powershell ",(0,o.yg)("strong",{parentName:"p"},"(run as administrator)")),(0,o.yg)("p",{parentName:"li"},(0,o.yg)("strong",{parentName:"p"},(0,o.yg)("inlineCode",{parentName:"strong"},"git config --system --unset credential.helper")))),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"And then remove ",(0,o.yg)("strong",{parentName:"p"},"gitconfig")," file from ",(0,o.yg)("strong",{parentName:"p"},"C:\\Program Files\\Git\\mingw64/etc/"),' location (Note: this path will be different in MAC like "/Users/username")')),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"After that use git command like ",(0,o.yg)("strong",{parentName:"p"},(0,o.yg)("inlineCode",{parentName:"strong"},"git pull")," or ",(0,o.yg)("inlineCode",{parentName:"strong"},"git push")),", it asked me for username and password. applying valid username and password and git command working."))),(0,o.yg)("h5",{id:"windows"},"Windows:"),(0,o.yg)("ol",null,(0,o.yg)("li",{parentName:"ol"},"Go to Windows ",(0,o.yg)("strong",{parentName:"li"},"Credential Manager"),". This is done in a EN-US Windows by pressing the Windows Key and typing 'credential'. In other localized Windows variants you need to use the localized term.")),(0,o.yg)("img",{class:"screenshot",src:"/img/windows-credentials.png",alt:"Windows Credentials",style:{zoom:"100%",maxHeight:"500px",maxWidth:"500px"}}),(0,o.yg)("p",null," ",(0,o.yg)("em",{parentName:"p"},"alternatively")," you can use the shortcut ",(0,o.yg)("inlineCode",{parentName:"p"},"control /name Microsoft.CredentialManager")," in the run dialog (WIN+R)"),(0,o.yg)("ol",{start:2},(0,o.yg)("li",{parentName:"ol"},"Edit the git entry under Windows Credentials, replacing old password with the new one.")),(0,o.yg)("h5",{id:"mac"},"Mac:"),(0,o.yg)("ol",null,(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},'cmd+space and type "KeyChain Access",')),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},'You should find a key with the name like "gitlab.*.com Access Key for user". You can order by date modified to find it more easily.'))),(0,o.yg)("img",{class:"screenshot",src:"/img/Mac-git-autentication.png",alt:"Mac GIT Autentication",style:{zoom:"100%",maxHeight:"500px",maxWidth:"500px"}}),(0,o.yg)("ol",{start:3},(0,o.yg)("li",{parentName:"ol"},"Right click and delete.")),(0,o.yg)("hr",null),(0,o.yg)("h2",{id:"filebrowser-403-forbidden"},"Filebrowser 403 forbidden"),(0,o.yg)("admonition",{title:"Spot the issue",type:"warning"},(0,o.yg)("p",{parentName:"admonition"},"If you get 403 forbidden issue while try to upload folders / files or creating new folder / file"),(0,o.yg)("pre",{parentName:"admonition"},(0,o.yg)("code",{parentName:"pre"},"403 forbidden\n"))),(0,o.yg)("img",{class:"screenshot",src:"/img/forbidden-issue.png",alt:"Forbidden Issue",style:{zoom:"100%",maxHeight:"500px",maxWidth:"500px"}}),(0,o.yg)("p",null,"Above issue will occur if you are not using the ",(0,o.yg)("strong",{parentName:"p"},"persistent storage"),"."),(0,o.yg)("p",null,"A persistent storage can be created by the DSRI team for a persistent storage of the data. ",(0,o.yg)("a",{parentName:"p",href:"http://localhost:3000/dsri-documentation/help"},"Contact the DSRI team")," to request a persistent storage."),(0,o.yg)("p",null,"You can find the persistent storage name as below"),(0,o.yg)("img",{class:"screenshot",src:"/img/persistent_storage.png",alt:"Storage",style:{zoom:"100%",maxHeight:"500px",maxWidth:"500px"}}))}m.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[2412],{5680:(e,t,n)=>{n.d(t,{xA:()=>g,yg:()=>m});var a=n(6540);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function r(e){for(var t=1;t=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var l=a.createContext({}),p=function(e){var t=a.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},g=function(e){var t=p(e.components);return a.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},c=a.forwardRef((function(e,t){var n=e.components,i=e.mdxType,o=e.originalType,l=e.parentName,g=s(e,["components","mdxType","originalType","parentName"]),c=p(n),m=i,d=c["".concat(l,".").concat(m)]||c[m]||u[m]||o;return n?a.createElement(d,r(r({ref:t},g),{},{components:n})):a.createElement(d,r({ref:t},g))}));function m(e,t){var n=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var o=n.length,r=new Array(o);r[0]=c;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s.mdxType="string"==typeof e?e:i,r[1]=s;for(var p=2;p{n.r(t),n.d(t,{assets:()=>g,contentTitle:()=>l,default:()=>m,frontMatter:()=>s,metadata:()=>p,toc:()=>u});var a=n(9668),i=n(1367),o=(n(6540),n(5680)),r=["components"],s={id:"guide-known-issues",title:"Known Issues"},l=void 0,p={unversionedId:"guide-known-issues",id:"guide-known-issues",title:"Known Issues",description:"Cannot access your data in the persistent folder",source:"@site/docs/guide-known-issues.md",sourceDirName:".",slug:"/guide-known-issues",permalink:"/docs/guide-known-issues",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/guide-known-issues.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"guide-known-issues",title:"Known Issues"},sidebar:"docs",previous:{title:"Parallelization using Dask",permalink:"/docs/dask-tutorial"},next:{title:"Publish a Docker image",permalink:"/docs/guide-publish-image"}},g={},u=[{value:"Cannot access your data in the persistent folder",id:"cannot-access-your-data-in-the-persistent-folder",level:2},{value:"Large volumes",id:"large-volumes",level:2},{value:"DockerHub pull limitations",id:"dockerhub-pull-limitations",level:2},{value:"How to run function within a container ''in the background'",id:"how-to-run-function-within-a-container-in-the-background",level:2},{value:"Git authentication issue",id:"git-authentication-issue",level:2},{value:"Windows:",id:"windows",level:5},{value:"Mac:",id:"mac",level:5},{value:"Filebrowser 403 forbidden",id:"filebrowser-403-forbidden",level:2}],c={toc:u};function m(e){var t=e.components,n=(0,i.A)(e,r);return(0,o.yg)("wrapper",(0,a.A)({},c,n,{components:t,mdxType:"MDXLayout"}),(0,o.yg)("h2",{id:"cannot-access-your-data-in-the-persistent-folder"},"Cannot access your data in the persistent folder"),(0,o.yg)("p",null,"Sometimes you cannot access anymore the data you put in the persistent folder of your container. It can be due to a node going down, if the persistent volume your pod is connected to is on this node, then it cannot access it anymore."),(0,o.yg)("p",null,"You can easily fix this issue by restarting the pod of your application, it will make it properly connect to resources on nodes that are up."),(0,o.yg)("p",null,"To restart the pod, go in topology, click on your application, go to the details tab, and decrease the pod count to 0, then put it back up to 1."),(0,o.yg)("h2",{id:"large-volumes"},"Large volumes"),(0,o.yg)("admonition",{title:"Pod or Deployment will not start",type:"warning"},(0,o.yg)("p",{parentName:"admonition"},"You could run into a following message in the ",(0,o.yg)("strong",{parentName:"p"},"Events")," tab that looks similar to this"),(0,o.yg)("pre",{parentName:"admonition"},(0,o.yg)("code",{parentName:"pre"},"Error: kubelet may be retrying requests that are timing out in CRI-O due to system load. Currently at stage container volume configuration: context deadline exceeded: error reserving ctr name\n"))),(0,o.yg)("p",null,"The issue above will occur if you are using a ",(0,o.yg)("strong",{parentName:"p"},"large persistent volume"),". It can be resolved by adding the following to your Deployment(Config):"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre"},"spec:\n template:\n metadata:\n annotations:\n io.kubernetes.cri-o.TrySkipVolumeSELinuxLabel: 'true'\n spec:\n runtimeClassName: selinux\n")),(0,o.yg)("p",null,"Take note of the ",(0,o.yg)("strong",{parentName:"p"},"indentation")," and the place in the file!"),(0,o.yg)("p",null,"An example of this can be found here:"),(0,o.yg)("img",{class:"screenshot",src:"/img/screenshot_large_volume_issue.png",alt:"Storage",style:{zoom:"100%",maxHeight:"500px",maxWidth:"500px"}}),(0,o.yg)("h2",{id:"dockerhub-pull-limitations"},"DockerHub pull limitations"),(0,o.yg)("admonition",{title:"Spot the issue",type:"warning"},(0,o.yg)("p",{parentName:"admonition"},"If the ",(0,o.yg)("strong",{parentName:"p"},"Events")," tab show this error:"),(0,o.yg)("pre",{parentName:"admonition"},(0,o.yg)("code",{parentName:"pre"},"--\x3e Scaling filebrowser-case-1 to 1\nerror: update acceptor rejected my-app-1: pods for rc 'my-project/my-app-1' took longer than 600 seconds to become available\n")),(0,o.yg)("p",{parentName:"admonition"},"Then check for the application ImageStream in ",(0,o.yg)("strong",{parentName:"p"},"Build")," > ",(0,o.yg)("strong",{parentName:"p"},"Images"),", and you might see this for your application image:"),(0,o.yg)("pre",{parentName:"admonition"},(0,o.yg)("code",{parentName:"pre"},"Internal error occurred: toomanyrequests: You have reached your pull rate limit.\nYou may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit.\n"))),(0,o.yg)("p",null,"You can solve this by creating a secret to login to DockerHub in your project:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"oc create secret docker-registry dockerhub-login --docker-server=docker.io --docker-username=dockerhub_username --docker-password=dockerhub_password --docker-email=example@mail.com\n")),(0,o.yg)("p",null,"Linking the login secret to the default service account:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"oc secrets link default dockerhub-login --for=pull\n")),(0,o.yg)("admonition",{type:"tip"},(0,o.yg)("p",{parentName:"admonition"},"Login to DockerHub should raise the limitations")),(0,o.yg)("p",null,"To definitely solve this issue you can publish the DockerHub image to the ",(0,o.yg)("a",{parentName:"p",href:"https://docs.github.com/en/packages/guides/about-github-container-registry"},"GitHub Container Registry"),"."),(0,o.yg)("p",null,"Follow those instructions on your laptop:"),(0,o.yg)("ol",null,(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},(0,o.yg)("a",{parentName:"p",href:"https://maastrichtu-ids.github.io/dsri-documentation/docs/guide-publish-image#login-to-github-container-registry"},"Login to the GitHub Container Registry")," with ",(0,o.yg)("inlineCode",{parentName:"p"},"docker login"),".")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Pull the docker image from"),(0,o.yg)("pre",{parentName:"li"},(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"docker pull myorg/myimage:latest\n")),(0,o.yg)("p",{parentName:"li"},(0,o.yg)("a",{parentName:"p",href:"mailto:git@github.com"},"git@github.com"),":MaastrichtU-IDS/",(0,o.yg)("a",{parentName:"p",href:"mailto:dsri-documentation.gitgit@github.com"},"dsri-documentation.gitgit@github.com"),":MaastrichtU-IDS/",(0,o.yg)("a",{parentName:"p",href:"mailto:dsri-documentation.gitgit@github.com"},"dsri-documentation.gitgit@github.com"),":MaastrichtU-IDS/dsri-documentation.git")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Change its tag"),(0,o.yg)("pre",{parentName:"li"},(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"docker tag myorg/myimage:latest ghcr.io/maastrichtu-ids/myimage:latest\n"))),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Push it back to the GitHub Container Registry:"),(0,o.yg)("pre",{parentName:"li"},(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"docker push ghcr.io/maastrichtu-ids/myimage:latest\n")))),(0,o.yg)("admonition",{title:"Image created automatically",type:"tip"},(0,o.yg)("p",{parentName:"admonition"},"If the image does not exist, GitHub will create automatically when you push it for the first time! You can then head to your ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/orgs/MaastrichtU-IDS/packages"},"organization ",(0,o.yg)("strong",{parentName:"a"},"Packages")," tab")," to see the package.")),(0,o.yg)("admonition",{title:"Make it public",type:"info"},(0,o.yg)("p",{parentName:"admonition"},"By default new images are set as ",(0,o.yg)("inlineCode",{parentName:"p"},"Private"),", go to your ",(0,o.yg)("strong",{parentName:"p"},"Package Settings"),", and click ",(0,o.yg)("strong",{parentName:"p"},"Change Visibility")," to set it as ",(0,o.yg)("inlineCode",{parentName:"p"},"Public"),", this avoids the need to login to pull the image.")),(0,o.yg)("p",null,"You can update the image if you want access to the latest version, you can set a GitHub Actions workflow to do so."),(0,o.yg)("p",null,"Finally you will need to update your DSRI deployment, or template, to use the newly created image on ",(0,o.yg)("inlineCode",{parentName:"p"},"ghcr.io"),", and redeploy the application with the new template."),(0,o.yg)("hr",null),(0,o.yg)("h2",{id:"how-to-run-function-within-a-container-in-the-background"},"How to run function within a container ''in the background'"),(0,o.yg)("admonition",{title:"Spot the issue",type:"warning"},(0,o.yg)("p",{parentName:"admonition"},"If the ",(0,o.yg)("strong",{parentName:"p"},"Events")," tab show this error:"),(0,o.yg)("pre",{parentName:"admonition"},(0,o.yg)("code",{parentName:"pre"},"--\x3e cd /usr/local/src/work2/aerius-sample-sequencing/CD4K4ANXX\n\nTrinity --seqType fq --max_memory 100G --CPU 64 --samples_file samples.txt --output /usr/local/src/work2/Trinity_output_zip_090221\nerror: The function starts but at some points just exits without warnings or errors to Windows folder\n")),(0,o.yg)("pre",{parentName:"admonition"},(0,o.yg)("code",{parentName:"pre"},"DSRI in the container's terminal keep running fine but never finishes. At some point a red label ''disconnected'' appears and the terminal stops and the analysis never continues.\n"))),(0,o.yg)("p",null,"Those two issues are due to the process running attach to the terminal"),(0,o.yg)("p",null,'Should be able to easily run it using the "Bash way": add ',(0,o.yg)("inlineCode",{parentName:"p"},"nohup")," at the beginning and ",(0,o.yg)("inlineCode",{parentName:"p"},"&")," at the end\nIt will run in the back and all output that should have gone to the terminal will go to a file ",(0,o.yg)("inlineCode",{parentName:"p"},"nohup.out")," in the repo"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre"},"nohup Trinity --seqType fq --max_memory 100G --CPU 64 --samples_file samples.txt --output /usr/local/src/work2/Trinity_output_zip_090221 &\n")),(0,o.yg)("p",null,"To check if it is still running:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre"},"ps aux | grep Trinity\n")),(0,o.yg)("p",null,'Be careful make sure the terminal uses bash and not shell ("sh")'),(0,o.yg)("p",null,"To use bash just type bash in the terminal:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre"},"bash\n")),(0,o.yg)("h2",{id:"git-authentication-issue"},"Git authentication issue"),(0,o.yg)("admonition",{type:"warning"},(0,o.yg)("p",{parentName:"admonition"},"\u26a0\ufe0f remote: HTTP Basic: Access denied fatal: Authentication failed for")),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},"It happen every time when we forced to change the Windows password."),(0,o.yg)("img",{class:"screenshot",src:"/img/authentication-issue.png",style:{zoom:"100%",maxHeight:"500px",maxWidth:"500px"}}),(0,o.yg)("img",{class:"screenshot",src:"/img/git-authentication-issue.png",style:{zoom:"100%",maxHeight:"500px",maxWidth:"500px"}})),(0,o.yg)("ol",null,(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Apply command from powershell ",(0,o.yg)("strong",{parentName:"p"},"(run as administrator)")),(0,o.yg)("p",{parentName:"li"},(0,o.yg)("strong",{parentName:"p"},(0,o.yg)("inlineCode",{parentName:"strong"},"git config --system --unset credential.helper")))),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"And then remove ",(0,o.yg)("strong",{parentName:"p"},"gitconfig")," file from ",(0,o.yg)("strong",{parentName:"p"},"C:\\Program Files\\Git\\mingw64/etc/"),' location (Note: this path will be different in MAC like "/Users/username")')),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"After that use git command like ",(0,o.yg)("strong",{parentName:"p"},(0,o.yg)("inlineCode",{parentName:"strong"},"git pull")," or ",(0,o.yg)("inlineCode",{parentName:"strong"},"git push")),", it asked me for username and password. applying valid username and password and git command working."))),(0,o.yg)("h5",{id:"windows"},"Windows:"),(0,o.yg)("ol",null,(0,o.yg)("li",{parentName:"ol"},"Go to Windows ",(0,o.yg)("strong",{parentName:"li"},"Credential Manager"),". This is done in a EN-US Windows by pressing the Windows Key and typing 'credential'. In other localized Windows variants you need to use the localized term.")),(0,o.yg)("img",{class:"screenshot",src:"/img/windows-credentials.png",alt:"Windows Credentials",style:{zoom:"100%",maxHeight:"500px",maxWidth:"500px"}}),(0,o.yg)("p",null," ",(0,o.yg)("em",{parentName:"p"},"alternatively")," you can use the shortcut ",(0,o.yg)("inlineCode",{parentName:"p"},"control /name Microsoft.CredentialManager")," in the run dialog (WIN+R)"),(0,o.yg)("ol",{start:2},(0,o.yg)("li",{parentName:"ol"},"Edit the git entry under Windows Credentials, replacing old password with the new one.")),(0,o.yg)("h5",{id:"mac"},"Mac:"),(0,o.yg)("ol",null,(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},'cmd+space and type "KeyChain Access",')),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},'You should find a key with the name like "gitlab.*.com Access Key for user". You can order by date modified to find it more easily.'))),(0,o.yg)("img",{class:"screenshot",src:"/img/Mac-git-autentication.png",alt:"Mac GIT Autentication",style:{zoom:"100%",maxHeight:"500px",maxWidth:"500px"}}),(0,o.yg)("ol",{start:3},(0,o.yg)("li",{parentName:"ol"},"Right click and delete.")),(0,o.yg)("hr",null),(0,o.yg)("h2",{id:"filebrowser-403-forbidden"},"Filebrowser 403 forbidden"),(0,o.yg)("admonition",{title:"Spot the issue",type:"warning"},(0,o.yg)("p",{parentName:"admonition"},"If you get 403 forbidden issue while try to upload folders / files or creating new folder / file"),(0,o.yg)("pre",{parentName:"admonition"},(0,o.yg)("code",{parentName:"pre"},"403 forbidden\n"))),(0,o.yg)("img",{class:"screenshot",src:"/img/forbidden-issue.png",alt:"Forbidden Issue",style:{zoom:"100%",maxHeight:"500px",maxWidth:"500px"}}),(0,o.yg)("p",null,"Above issue will occur if you are not using the ",(0,o.yg)("strong",{parentName:"p"},"persistent storage"),"."),(0,o.yg)("p",null,"A persistent storage can be created by the DSRI team for a persistent storage of the data. ",(0,o.yg)("a",{parentName:"p",href:"http://localhost:3000/dsri-documentation/help"},"Contact the DSRI team")," to request a persistent storage."),(0,o.yg)("p",null,"You can find the persistent storage name as below"),(0,o.yg)("img",{class:"screenshot",src:"/img/persistent_storage.png",alt:"Storage",style:{zoom:"100%",maxHeight:"500px",maxWidth:"500px"}}))}m.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/36f9137d.e38080c2.js b/assets/js/36f9137d.67ec300e.js similarity index 99% rename from assets/js/36f9137d.e38080c2.js rename to assets/js/36f9137d.67ec300e.js index 8aa728851..35dd1692c 100644 --- a/assets/js/36f9137d.e38080c2.js +++ b/assets/js/36f9137d.67ec300e.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[5210],{5680:(e,t,a)=>{a.d(t,{xA:()=>u,yg:()=>m});var n=a(6540);function r(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function i(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function o(e){for(var t=1;t=0||(r[a]=e[a]);return r}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(r[a]=e[a])}return r}var p=n.createContext({}),s=function(e){var t=n.useContext(p),a=t;return e&&(a="function"==typeof e?e(t):o(o({},t),e)),a},u=function(e){var t=s(e.components);return n.createElement(p.Provider,{value:t},e.children)},y={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},g=n.forwardRef((function(e,t){var a=e.components,r=e.mdxType,i=e.originalType,p=e.parentName,u=l(e,["components","mdxType","originalType","parentName"]),g=s(a),m=r,c=g["".concat(p,".").concat(m)]||g[m]||y[m]||i;return a?n.createElement(c,o(o({ref:t},u),{},{components:a})):n.createElement(c,o({ref:t},u))}));function m(e,t){var a=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=a.length,o=new Array(i);o[0]=g;var l={};for(var p in t)hasOwnProperty.call(t,p)&&(l[p]=t[p]);l.originalType=e,l.mdxType="string"==typeof e?e:r,o[1]=l;for(var s=2;s{a.r(t),a.d(t,{assets:()=>u,contentTitle:()=>p,default:()=>m,frontMatter:()=>l,metadata:()=>s,toc:()=>y});var n=a(9668),r=a(1367),i=(a(6540),a(5680)),o=["components"],l={id:"deploy-jupyter",title:"Jupyter Notebooks"},p=void 0,s={unversionedId:"deploy-jupyter",id:"deploy-jupyter",title:"Jupyter Notebooks",description:"\ud83e\ude90 Start JupyterLab",source:"@site/docs/deploy-jupyter.md",sourceDirName:".",slug:"/deploy-jupyter",permalink:"/docs/deploy-jupyter",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/deploy-jupyter.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"deploy-jupyter",title:"Jupyter Notebooks"},sidebar:"docs",previous:{title:"Install from Operators",permalink:"/docs/operators"},next:{title:"RStudio",permalink:"/docs/deploy-rstudio"}},u={},y=[{value:"\ud83e\ude90 Start JupyterLab",id:"-start-jupyterlab",level:2},{value:"\ud83d\udce6\ufe0f Manage dependencies with Conda",id:"\ufe0f-manage-dependencies-with-conda",level:2},{value:"\ud83d\udc19 Use git in JupyterLab",id:"-use-git-in-jupyterlab",level:2},{value:"\ud83d\udc36 Example",id:"-example",level:3}],g={toc:y};function m(e){var t=e.components,a=(0,r.A)(e,o);return(0,i.yg)("wrapper",(0,n.A)({},g,a,{components:t,mdxType:"MDXLayout"}),(0,i.yg)("h2",{id:"-start-jupyterlab"},"\ud83e\ude90 Start JupyterLab"),(0,i.yg)("p",null,"Start a JupyterLab container based on the ",(0,i.yg)("a",{parentName:"p",href:"https://github.com/jupyter/docker-stacks"},"official Jupyter docker stacks")," (debian), with ",(0,i.yg)("inlineCode",{parentName:"p"},"sudo")," privileges to install anything you need (e.g. pip or apt packages)"),(0,i.yg)("p",null,"You can start a container using the ",(0,i.yg)("strong",{parentName:"p"},"JupyterLab")," template in the ",(0,i.yg)("a",{parentName:"p",href:"https://console-openshift-console.apps.dsri2.unimaas.nl/catalog"},"Catalog web UI")," (make sure the ",(0,i.yg)("strong",{parentName:"p"},"Templates")," checkbox is checked)"),(0,i.yg)("p",null,"When instantiating the template you can provide a few parameters, such as:"),(0,i.yg)("ul",null,(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("strong",{parentName:"li"},"Password")," to access the notebook"),(0,i.yg)("li",{parentName:"ul"},"Optionally you can provide a ",(0,i.yg)("strong",{parentName:"li"},"git repository")," to be automatically cloned in the JupyterLab (if there is a ",(0,i.yg)("inlineCode",{parentName:"li"},"requirements.txt")," packages will be automatically installed with ",(0,i.yg)("inlineCode",{parentName:"li"},"pip"),")"),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("strong",{parentName:"li"},"Docker image")," to use for the notebook (see below for more details on customizing the docker image) "),(0,i.yg)("li",{parentName:"ul"},"Your ",(0,i.yg)("strong",{parentName:"li"},"git username and email")," to automatically configure git")),(0,i.yg)("p",null,"The DSRI will automatically create a persistent volume to store data you will put in the ",(0,i.yg)("inlineCode",{parentName:"p"},"/home/jovyan/work")," folder (the folder used by the notebook interface). You can find the persistent volumes in the DSRI web UI, go to the ",(0,i.yg)("strong",{parentName:"p"},"Administrator")," view > ",(0,i.yg)("strong",{parentName:"p"},"Storage")," > ",(0,i.yg)("strong",{parentName:"p"},"Persistent Volume Claims"),"."),(0,i.yg)("img",{src:"/img/screenshot-deploy-jupyter.png",alt:"Deploy Jupyter",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("p",null,"With this template you can use any image based on the official Jupyter docker stack: ",(0,i.yg)("a",{parentName:"p",href:"https://github.com/jupyter/docker-stacks"},"https://github.com/jupyter/docker-stacks")),(0,i.yg)("ul",null,(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("inlineCode",{parentName:"li"},"ghcr.io/maastrichtu-ids/jupyterlab:latest"),": custom image for data science on the DSRI, with additional kernels (Java), conda integration, VisualStudio Code, and autocomplete for Python"),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("inlineCode",{parentName:"li"},"ghcr.io/maastrichtu-ids/jupyterlab:knowledge-graph"),": custom image for working with knowledge graph on the DSRI, with SPARQL kernel and OpenRefine"),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("inlineCode",{parentName:"li"},"jupyter/scipy-notebook"),": some packages for science are preinstalled "),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("inlineCode",{parentName:"li"},"jupyter/datascience-notebook"),": with Julia kernel"),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("inlineCode",{parentName:"li"},"jupyter/tensorflow-notebook"),": with tensorflow package pre-installed"),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("inlineCode",{parentName:"li"},"jupyter/r-notebook"),": to work with R"),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("inlineCode",{parentName:"li"},"jupyter/pyspark-notebook"),": if you want to connect to a Spark cluster"),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("inlineCode",{parentName:"li"},"jupyter/all-spark-notebook"),": if you want to run Spark locally in the notebook")),(0,i.yg)("p",null,"You can also build your own image, we recommend to use this repository as example to extend a JupyterLab image: ",(0,i.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/jupyterlab"},"https://github.com/MaastrichtU-IDS/jupyterlab")),(0,i.yg)("h2",{id:"\ufe0f-manage-dependencies-with-conda"},"\ud83d\udce6\ufe0f Manage dependencies with Conda"),(0,i.yg)("p",null,"With the ",(0,i.yg)("inlineCode",{parentName:"p"},"ghcr.io/maastrichtu-ids/jupyterlab:latest")," image, you can easily start notebooks from the JupyterLab Launcher page using installed conda environments, at the condition ",(0,i.yg)("inlineCode",{parentName:"p"},"nb_conda_kernels")," and ",(0,i.yg)("inlineCode",{parentName:"p"},"ipykernel")," are installed in those environments."),(0,i.yg)("ul",null,(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("p",{parentName:"li"},"You can pass a Git repository URL which contains an ",(0,i.yg)("inlineCode",{parentName:"p"},"environment.yml")," file in the root folder when starting JupyterLab, the conda environment will automatically be installed at the start of your container, and available in the JupyterLab Launcher page. You can use this repository as example: ",(0,i.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-demo"},"https://github.com/MaastrichtU-IDS/dsri-demo"))),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("p",{parentName:"li"},"Or you can install it directly in a running JupyterLab (we use ",(0,i.yg)("inlineCode",{parentName:"p"},"mamba")," which is like ",(0,i.yg)("inlineCode",{parentName:"p"},"conda")," but faster):"),(0,i.yg)("pre",{parentName:"li"},(0,i.yg)("code",{parentName:"pre",className:"language-bash"},"mamba env create -f environment.yml\n")),(0,i.yg)("p",{parentName:"li"},"You'll need to wait for 1 or 2 minutes before the new conda environment becomes available on the JupyterLab Launcher page."))),(0,i.yg)("p",null,"You can easily install an environment with a different version of Python if you need it. Here is an example of an ",(0,i.yg)("inlineCode",{parentName:"p"},"environment.yml")," file to create an environment with Python 3.9, install the minimal dependencies required to easily starts notebooks in this environment with ",(0,i.yg)("inlineCode",{parentName:"p"},"conda"),", and install a ",(0,i.yg)("inlineCode",{parentName:"p"},"pip")," package:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},"name: custom-env\nchannels:\n - defaults\n - conda-forge\n - anaconda\ndependencies:\n - python=3.9\n - ipykernel \n - nb_conda_kernels\n - pip\n - pip:\n - matplotlib\n")),(0,i.yg)("p",null,"\u26a0\ufe0f You cannot use ",(0,i.yg)("inlineCode",{parentName:"p"},"conda activate")," in a Docker container, so you will need to either open a notebook using the kernel for your conda env, or use ",(0,i.yg)("inlineCode",{parentName:"p"},"conda run")," to run scripts in the new environment:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-bash"},"conda run -n custom-env python --version\n")),(0,i.yg)("h2",{id:"-use-git-in-jupyterlab"},"\ud83d\udc19 Use git in JupyterLab"),(0,i.yg)("p",null,"You can always use ",(0,i.yg)("inlineCode",{parentName:"p"},"git")," from the terminal."),(0,i.yg)("admonition",{title:"Configure username",type:"caution"},(0,i.yg)("p",{parentName:"admonition"},"Before pushing back to GitHub or GitLab, you will need to ",(0,i.yg)("strong",{parentName:"p"},"configure you username and email")," in VSCode terminal:"),(0,i.yg)("pre",{parentName:"admonition"},(0,i.yg)("code",{parentName:"pre",className:"language-bash"},'git config --global user.name "Jean Dupont"\ngit config --global user.email jeandupont@gmail.com\n'))),(0,i.yg)("admonition",{title:"Save your password",type:"info"},(0,i.yg)("p",{parentName:"admonition"},"You can run this command to ask git to save your password for 15min:"),(0,i.yg)("pre",{parentName:"admonition"},(0,i.yg)("code",{parentName:"pre",className:"language-bash"},"git config credential.helper cache\n")),(0,i.yg)("p",{parentName:"admonition"},"Or store the password in a plain text file:"),(0,i.yg)("pre",{parentName:"admonition"},(0,i.yg)("code",{parentName:"pre",className:"language-bash"},"git config --global credential.helper 'store --file ~/.git-credentials'\n"))),(0,i.yg)("admonition",{title:"Git tip",type:"tip"},(0,i.yg)("p",{parentName:"admonition"},"We recommend to use SSH instead of HTTPS connection when possible, checkout ",(0,i.yg)("a",{parentName:"p",href:"https://docs.github.com/en/free-pro-team@latest/github/authenticating-to-github/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent"},"here")," how to generate SSH keys and use them with your GitHub account.")),(0,i.yg)("p",null,"You can also enable and use the ",(0,i.yg)("a",{parentName:"p",href:"https://github.com/jupyterlab/jupyterlab-git"},"JupyterLab Git extension")," to clone and manage your ",(0,i.yg)("inlineCode",{parentName:"p"},"git")," repositories."),(0,i.yg)("p",null,"It will prompt you for a username and password if the repository is private."),(0,i.yg)("img",{src:"https://raw.githubusercontent.com/jupyterlab/jupyterlab-git/master/docs/figs/preview.gif",alt:"JupyterLab Git extension",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("h3",{id:"-example"},"\ud83d\udc36 Example"),(0,i.yg)("p",null,"Initialize repository"),(0,i.yg)("img",{src:"/img/sample_git_page.png",alt:"Initialize repo",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("p",null,"Include git details in DSRI project setup"),(0,i.yg)("img",{src:"/img/sample_git_details_jupyter.png",alt:"git details",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("p",null,"Verify automatic deployment"),(0,i.yg)("img",{src:"/img/sample_workspace.png",alt:"workspace",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("img",{src:"/img/sample_requirements.png",alt:"requirements",style:{maxWidth:"100%",maxHeight:"100%"}}))}m.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[5210],{5680:(e,t,a)=>{a.d(t,{xA:()=>u,yg:()=>m});var n=a(6540);function r(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function i(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function o(e){for(var t=1;t=0||(r[a]=e[a]);return r}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(r[a]=e[a])}return r}var p=n.createContext({}),s=function(e){var t=n.useContext(p),a=t;return e&&(a="function"==typeof e?e(t):o(o({},t),e)),a},u=function(e){var t=s(e.components);return n.createElement(p.Provider,{value:t},e.children)},y={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},g=n.forwardRef((function(e,t){var a=e.components,r=e.mdxType,i=e.originalType,p=e.parentName,u=l(e,["components","mdxType","originalType","parentName"]),g=s(a),m=r,c=g["".concat(p,".").concat(m)]||g[m]||y[m]||i;return a?n.createElement(c,o(o({ref:t},u),{},{components:a})):n.createElement(c,o({ref:t},u))}));function m(e,t){var a=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=a.length,o=new Array(i);o[0]=g;var l={};for(var p in t)hasOwnProperty.call(t,p)&&(l[p]=t[p]);l.originalType=e,l.mdxType="string"==typeof e?e:r,o[1]=l;for(var s=2;s{a.r(t),a.d(t,{assets:()=>u,contentTitle:()=>p,default:()=>m,frontMatter:()=>l,metadata:()=>s,toc:()=>y});var n=a(9668),r=a(1367),i=(a(6540),a(5680)),o=["components"],l={id:"deploy-jupyter",title:"Jupyter Notebooks"},p=void 0,s={unversionedId:"deploy-jupyter",id:"deploy-jupyter",title:"Jupyter Notebooks",description:"\ud83e\ude90 Start JupyterLab",source:"@site/docs/deploy-jupyter.md",sourceDirName:".",slug:"/deploy-jupyter",permalink:"/docs/deploy-jupyter",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/deploy-jupyter.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"deploy-jupyter",title:"Jupyter Notebooks"},sidebar:"docs",previous:{title:"Install from Operators",permalink:"/docs/operators"},next:{title:"RStudio",permalink:"/docs/deploy-rstudio"}},u={},y=[{value:"\ud83e\ude90 Start JupyterLab",id:"-start-jupyterlab",level:2},{value:"\ud83d\udce6\ufe0f Manage dependencies with Conda",id:"\ufe0f-manage-dependencies-with-conda",level:2},{value:"\ud83d\udc19 Use git in JupyterLab",id:"-use-git-in-jupyterlab",level:2},{value:"\ud83d\udc36 Example",id:"-example",level:3}],g={toc:y};function m(e){var t=e.components,a=(0,r.A)(e,o);return(0,i.yg)("wrapper",(0,n.A)({},g,a,{components:t,mdxType:"MDXLayout"}),(0,i.yg)("h2",{id:"-start-jupyterlab"},"\ud83e\ude90 Start JupyterLab"),(0,i.yg)("p",null,"Start a JupyterLab container based on the ",(0,i.yg)("a",{parentName:"p",href:"https://github.com/jupyter/docker-stacks"},"official Jupyter docker stacks")," (debian), with ",(0,i.yg)("inlineCode",{parentName:"p"},"sudo")," privileges to install anything you need (e.g. pip or apt packages)"),(0,i.yg)("p",null,"You can start a container using the ",(0,i.yg)("strong",{parentName:"p"},"JupyterLab")," template in the ",(0,i.yg)("a",{parentName:"p",href:"https://console-openshift-console.apps.dsri2.unimaas.nl/catalog"},"Catalog web UI")," (make sure the ",(0,i.yg)("strong",{parentName:"p"},"Templates")," checkbox is checked)"),(0,i.yg)("p",null,"When instantiating the template you can provide a few parameters, such as:"),(0,i.yg)("ul",null,(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("strong",{parentName:"li"},"Password")," to access the notebook"),(0,i.yg)("li",{parentName:"ul"},"Optionally you can provide a ",(0,i.yg)("strong",{parentName:"li"},"git repository")," to be automatically cloned in the JupyterLab (if there is a ",(0,i.yg)("inlineCode",{parentName:"li"},"requirements.txt")," packages will be automatically installed with ",(0,i.yg)("inlineCode",{parentName:"li"},"pip"),")"),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("strong",{parentName:"li"},"Docker image")," to use for the notebook (see below for more details on customizing the docker image) "),(0,i.yg)("li",{parentName:"ul"},"Your ",(0,i.yg)("strong",{parentName:"li"},"git username and email")," to automatically configure git")),(0,i.yg)("p",null,"The DSRI will automatically create a persistent volume to store data you will put in the ",(0,i.yg)("inlineCode",{parentName:"p"},"/home/jovyan/work")," folder (the folder used by the notebook interface). You can find the persistent volumes in the DSRI web UI, go to the ",(0,i.yg)("strong",{parentName:"p"},"Administrator")," view > ",(0,i.yg)("strong",{parentName:"p"},"Storage")," > ",(0,i.yg)("strong",{parentName:"p"},"Persistent Volume Claims"),"."),(0,i.yg)("img",{src:"/img/screenshot-deploy-jupyter.png",alt:"Deploy Jupyter",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("p",null,"With this template you can use any image based on the official Jupyter docker stack: ",(0,i.yg)("a",{parentName:"p",href:"https://github.com/jupyter/docker-stacks"},"https://github.com/jupyter/docker-stacks")),(0,i.yg)("ul",null,(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("inlineCode",{parentName:"li"},"ghcr.io/maastrichtu-ids/jupyterlab:latest"),": custom image for data science on the DSRI, with additional kernels (Java), conda integration, VisualStudio Code, and autocomplete for Python"),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("inlineCode",{parentName:"li"},"ghcr.io/maastrichtu-ids/jupyterlab:knowledge-graph"),": custom image for working with knowledge graph on the DSRI, with SPARQL kernel and OpenRefine"),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("inlineCode",{parentName:"li"},"jupyter/scipy-notebook"),": some packages for science are preinstalled "),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("inlineCode",{parentName:"li"},"jupyter/datascience-notebook"),": with Julia kernel"),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("inlineCode",{parentName:"li"},"jupyter/tensorflow-notebook"),": with tensorflow package pre-installed"),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("inlineCode",{parentName:"li"},"jupyter/r-notebook"),": to work with R"),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("inlineCode",{parentName:"li"},"jupyter/pyspark-notebook"),": if you want to connect to a Spark cluster"),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("inlineCode",{parentName:"li"},"jupyter/all-spark-notebook"),": if you want to run Spark locally in the notebook")),(0,i.yg)("p",null,"You can also build your own image, we recommend to use this repository as example to extend a JupyterLab image: ",(0,i.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/jupyterlab"},"https://github.com/MaastrichtU-IDS/jupyterlab")),(0,i.yg)("h2",{id:"\ufe0f-manage-dependencies-with-conda"},"\ud83d\udce6\ufe0f Manage dependencies with Conda"),(0,i.yg)("p",null,"With the ",(0,i.yg)("inlineCode",{parentName:"p"},"ghcr.io/maastrichtu-ids/jupyterlab:latest")," image, you can easily start notebooks from the JupyterLab Launcher page using installed conda environments, at the condition ",(0,i.yg)("inlineCode",{parentName:"p"},"nb_conda_kernels")," and ",(0,i.yg)("inlineCode",{parentName:"p"},"ipykernel")," are installed in those environments."),(0,i.yg)("ul",null,(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("p",{parentName:"li"},"You can pass a Git repository URL which contains an ",(0,i.yg)("inlineCode",{parentName:"p"},"environment.yml")," file in the root folder when starting JupyterLab, the conda environment will automatically be installed at the start of your container, and available in the JupyterLab Launcher page. You can use this repository as example: ",(0,i.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-demo"},"https://github.com/MaastrichtU-IDS/dsri-demo"))),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("p",{parentName:"li"},"Or you can install it directly in a running JupyterLab (we use ",(0,i.yg)("inlineCode",{parentName:"p"},"mamba")," which is like ",(0,i.yg)("inlineCode",{parentName:"p"},"conda")," but faster):"),(0,i.yg)("pre",{parentName:"li"},(0,i.yg)("code",{parentName:"pre",className:"language-bash"},"mamba env create -f environment.yml\n")),(0,i.yg)("p",{parentName:"li"},"You'll need to wait for 1 or 2 minutes before the new conda environment becomes available on the JupyterLab Launcher page."))),(0,i.yg)("p",null,"You can easily install an environment with a different version of Python if you need it. Here is an example of an ",(0,i.yg)("inlineCode",{parentName:"p"},"environment.yml")," file to create an environment with Python 3.9, install the minimal dependencies required to easily starts notebooks in this environment with ",(0,i.yg)("inlineCode",{parentName:"p"},"conda"),", and install a ",(0,i.yg)("inlineCode",{parentName:"p"},"pip")," package:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},"name: custom-env\nchannels:\n - defaults\n - conda-forge\n - anaconda\ndependencies:\n - python=3.9\n - ipykernel \n - nb_conda_kernels\n - pip\n - pip:\n - matplotlib\n")),(0,i.yg)("p",null,"\u26a0\ufe0f You cannot use ",(0,i.yg)("inlineCode",{parentName:"p"},"conda activate")," in a Docker container, so you will need to either open a notebook using the kernel for your conda env, or use ",(0,i.yg)("inlineCode",{parentName:"p"},"conda run")," to run scripts in the new environment:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-bash"},"conda run -n custom-env python --version\n")),(0,i.yg)("h2",{id:"-use-git-in-jupyterlab"},"\ud83d\udc19 Use git in JupyterLab"),(0,i.yg)("p",null,"You can always use ",(0,i.yg)("inlineCode",{parentName:"p"},"git")," from the terminal."),(0,i.yg)("admonition",{title:"Configure username",type:"caution"},(0,i.yg)("p",{parentName:"admonition"},"Before pushing back to GitHub or GitLab, you will need to ",(0,i.yg)("strong",{parentName:"p"},"configure you username and email")," in VSCode terminal:"),(0,i.yg)("pre",{parentName:"admonition"},(0,i.yg)("code",{parentName:"pre",className:"language-bash"},'git config --global user.name "Jean Dupont"\ngit config --global user.email jeandupont@gmail.com\n'))),(0,i.yg)("admonition",{title:"Save your password",type:"info"},(0,i.yg)("p",{parentName:"admonition"},"You can run this command to ask git to save your password for 15min:"),(0,i.yg)("pre",{parentName:"admonition"},(0,i.yg)("code",{parentName:"pre",className:"language-bash"},"git config credential.helper cache\n")),(0,i.yg)("p",{parentName:"admonition"},"Or store the password in a plain text file:"),(0,i.yg)("pre",{parentName:"admonition"},(0,i.yg)("code",{parentName:"pre",className:"language-bash"},"git config --global credential.helper 'store --file ~/.git-credentials'\n"))),(0,i.yg)("admonition",{title:"Git tip",type:"tip"},(0,i.yg)("p",{parentName:"admonition"},"We recommend to use SSH instead of HTTPS connection when possible, checkout ",(0,i.yg)("a",{parentName:"p",href:"https://docs.github.com/en/free-pro-team@latest/github/authenticating-to-github/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent"},"here")," how to generate SSH keys and use them with your GitHub account.")),(0,i.yg)("p",null,"You can also enable and use the ",(0,i.yg)("a",{parentName:"p",href:"https://github.com/jupyterlab/jupyterlab-git"},"JupyterLab Git extension")," to clone and manage your ",(0,i.yg)("inlineCode",{parentName:"p"},"git")," repositories."),(0,i.yg)("p",null,"It will prompt you for a username and password if the repository is private."),(0,i.yg)("img",{src:"https://raw.githubusercontent.com/jupyterlab/jupyterlab-git/master/docs/figs/preview.gif",alt:"JupyterLab Git extension",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("h3",{id:"-example"},"\ud83d\udc36 Example"),(0,i.yg)("p",null,"Initialize repository"),(0,i.yg)("img",{src:"/img/sample_git_page.png",alt:"Initialize repo",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("p",null,"Include git details in DSRI project setup"),(0,i.yg)("img",{src:"/img/sample_git_details_jupyter.png",alt:"git details",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("p",null,"Verify automatic deployment"),(0,i.yg)("img",{src:"/img/sample_workspace.png",alt:"workspace",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("img",{src:"/img/sample_requirements.png",alt:"requirements",style:{maxWidth:"100%",maxHeight:"100%"}}))}m.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/389e2b0f.bd7f82de.js b/assets/js/389e2b0f.605dea03.js similarity index 99% rename from assets/js/389e2b0f.bd7f82de.js rename to assets/js/389e2b0f.605dea03.js index 3367ced30..ce1c0e764 100644 --- a/assets/js/389e2b0f.bd7f82de.js +++ b/assets/js/389e2b0f.605dea03.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[610],{5680:(e,t,a)=>{a.d(t,{xA:()=>c,yg:()=>g});var n=a(6540);function o(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function r(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function l(e){for(var t=1;t=0||(o[a]=e[a]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(o[a]=e[a])}return o}var i=n.createContext({}),d=function(e){var t=n.useContext(i),a=t;return e&&(a="function"==typeof e?e(t):l(l({},t),e)),a},c=function(e){var t=d(e.components);return n.createElement(i.Provider,{value:t},e.children)},s={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},m=n.forwardRef((function(e,t){var a=e.components,o=e.mdxType,r=e.originalType,i=e.parentName,c=p(e,["components","mdxType","originalType","parentName"]),m=d(a),g=o,y=m["".concat(i,".").concat(g)]||m[g]||s[g]||r;return a?n.createElement(y,l(l({ref:t},c),{},{components:a})):n.createElement(y,l({ref:t},c))}));function g(e,t){var a=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=a.length,l=new Array(r);l[0]=m;var p={};for(var i in t)hasOwnProperty.call(t,i)&&(p[i]=t[i]);p.originalType=e,p.mdxType="string"==typeof e?e:o,l[1]=p;for(var d=2;d{a.r(t),a.d(t,{assets:()=>c,contentTitle:()=>i,default:()=>g,frontMatter:()=>p,metadata:()=>d,toc:()=>s});var n=a(9668),o=a(1367),r=(a(6540),a(5680)),l=["components"],p={id:"openshift-commands",title:"Command Line Interface"},i=void 0,d={unversionedId:"openshift-commands",id:"openshift-commands",title:"Command Line Interface",description:"Overview",source:"@site/docs/openshift-commands.md",sourceDirName:".",slug:"/openshift-commands",permalink:"/docs/openshift-commands",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/openshift-commands.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"openshift-commands",title:"Command Line Interface"},sidebar:"docs",previous:{title:"Login to Docker registries",permalink:"/docs/login-docker-registry"},next:{title:"Data storage",permalink:"/docs/openshift-storage"}},c={},s=[{value:"Overview",id:"overview",level:2},{value:"Projects",id:"projects",level:2},{value:"List projects",id:"list-projects",level:3},{value:"Connect to project",id:"connect-to-project",level:3},{value:"ImageStreams",id:"imagestreams",level:2},{value:"Pods",id:"pods",level:2},{value:"Create pod from YAML",id:"create-pod-from-yaml",level:3},{value:"List pods",id:"list-pods",level:3},{value:"Get specific pod",id:"get-specific-pod",level:3},{value:"Remote Shell connection",id:"remote-shell-connection",level:3},{value:"Execute command in pod",id:"execute-command-in-pod",level:3},{value:"Delete pod",id:"delete-pod",level:3},{value:"Get pod logs",id:"get-pod-logs",level:3},{value:"Create app from template",id:"create-app-from-template",level:2},{value:"Copy files",id:"copy-files",level:3}],m={toc:s};function g(e){var t=e.components,a=(0,o.A)(e,l);return(0,r.yg)("wrapper",(0,n.A)({},m,a,{components:t,mdxType:"MDXLayout"}),(0,r.yg)("h2",{id:"overview"},"Overview"),(0,r.yg)("p",null,"Here is an overview of common ",(0,r.yg)("inlineCode",{parentName:"p"},"oc")," commands:"),(0,r.yg)("table",null,(0,r.yg)("thead",{parentName:"table"},(0,r.yg)("tr",{parentName:"thead"},(0,r.yg)("th",{parentName:"tr",align:null},(0,r.yg)("strong",{parentName:"th"},"Command")),(0,r.yg)("th",{parentName:"tr",align:null},(0,r.yg)("strong",{parentName:"th"},"Description")))),(0,r.yg)("tbody",{parentName:"table"},(0,r.yg)("tr",{parentName:"tbody"},(0,r.yg)("td",{parentName:"tr",align:null},(0,r.yg)("inlineCode",{parentName:"td"},"oc login --token=")),(0,r.yg)("td",{parentName:"tr",align:null},"Login to the DSRI OpenShift cluster in your terminal")),(0,r.yg)("tr",{parentName:"tbody"},(0,r.yg)("td",{parentName:"tr",align:null},(0,r.yg)("inlineCode",{parentName:"td"},"oc get projects")),(0,r.yg)("td",{parentName:"tr",align:null},"List all available projects")),(0,r.yg)("tr",{parentName:"tbody"},(0,r.yg)("td",{parentName:"tr",align:null},(0,r.yg)("inlineCode",{parentName:"td"},"oc project ")),(0,r.yg)("td",{parentName:"tr",align:null},"Switch to project")),(0,r.yg)("tr",{parentName:"tbody"},(0,r.yg)("td",{parentName:"tr",align:null},(0,r.yg)("inlineCode",{parentName:"td"},"oc get pods ")),(0,r.yg)("td",{parentName:"tr",align:null},"Get running pods (a pod can run one or multiple containers for your application)")),(0,r.yg)("tr",{parentName:"tbody"},(0,r.yg)("td",{parentName:"tr",align:null},(0,r.yg)("inlineCode",{parentName:"td"},"oc rsh ")),(0,r.yg)("td",{parentName:"tr",align:null},"Remote terminal connexion to a pod (Shell/Bash)")),(0,r.yg)("tr",{parentName:"tbody"},(0,r.yg)("td",{parentName:"tr",align:null},(0,r.yg)("inlineCode",{parentName:"td"},"oc cp ")),(0,r.yg)("td",{parentName:"tr",align:null},"Copy files from host to container or vice versa, e.g. from host: ",(0,r.yg)("inlineCode",{parentName:"td"},"oc cp :")," or from to host: ",(0,r.yg)("inlineCode",{parentName:"td"},"oc cp : "))),(0,r.yg)("tr",{parentName:"tbody"},(0,r.yg)("td",{parentName:"tr",align:null},(0,r.yg)("inlineCode",{parentName:"td"},"oc rsync ")),(0,r.yg)("td",{parentName:"tr",align:null},"Similar to rsync command on Linux to synchronize directories between container and host or the other way around")),(0,r.yg)("tr",{parentName:"tbody"},(0,r.yg)("td",{parentName:"tr",align:null},(0,r.yg)("inlineCode",{parentName:"td"},"oc exec ")),(0,r.yg)("td",{parentName:"tr",align:null},"Execute command in pods")),(0,r.yg)("tr",{parentName:"tbody"},(0,r.yg)("td",{parentName:"tr",align:null},(0,r.yg)("inlineCode",{parentName:"td"},"oc delete pod ")),(0,r.yg)("td",{parentName:"tr",align:null},"Delete pod")))),(0,r.yg)("h2",{id:"projects"},"Projects"),(0,r.yg)("h3",{id:"list-projects"},"List projects"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"oc projects\n")),(0,r.yg)("h3",{id:"connect-to-project"},"Connect to project"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"oc project my-project\n")),(0,r.yg)("hr",null),(0,r.yg)("h2",{id:"imagestreams"},"ImageStreams"),(0,r.yg)("p",null,"To update an ImageStream in your project to pull the latest update from the external repository (e.g. from ghcr.io or DockerHub):"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc import-image \n")),(0,r.yg)("hr",null),(0,r.yg)("h2",{id:"pods"},"Pods"),(0,r.yg)("h3",{id:"create-pod-from-yaml"},"Create pod from YAML"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"oc create -f my-pod.yaml\n")),(0,r.yg)("blockquote",null,(0,r.yg)("p",{parentName:"blockquote"},"E.g. ",(0,r.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/d2s-core/blob/master/argo/d2s-pod-virtuoso.yaml"},"d2s-pod-virtuoso.yaml"),".")),(0,r.yg)("h3",{id:"list-pods"},"List pods"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"oc get pod\n")),(0,r.yg)("p",null,"List running pods:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc get pods --field-selector=status.phase=Running\n")),(0,r.yg)("h3",{id:"get-specific-pod"},"Get specific pod"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"oc get pod | grep \n")),(0,r.yg)("p",null,"Using selector with Apache Flink as example, and showing only the pod id without header:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc get pod --selector app=flink --selector component=jobmanager --no-headers -o=custom-columns=NAME:.metadata.name\n")),(0,r.yg)("h3",{id:"remote-shell-connection"},"Remote Shell connection"),(0,r.yg)("p",null,"Connect to a pod with ",(0,r.yg)("a",{parentName:"p",href:"https://devhints.io/bash"},"Bash"),"."),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"oc rsh \n")),(0,r.yg)("h3",{id:"execute-command-in-pod"},"Execute command in pod"),(0,r.yg)("p",null,"Example creating a folder:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"oc exec -- mkdir -p /mnt/workspace/resources\n")),(0,r.yg)("h3",{id:"delete-pod"},"Delete pod"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"oc delete pod \n")),(0,r.yg)("admonition",{title:"Force pod deletion",type:"caution"},(0,r.yg)("p",{parentName:"admonition"},"If the pod is not properly deleted, you can force its deletion:"),(0,r.yg)("pre",{parentName:"admonition"},(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc delete pod --force --grace-period=0 \n"))),(0,r.yg)("h3",{id:"get-pod-logs"},"Get pod logs"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"oc logs -f \n")),(0,r.yg)("admonition",{title:"Debug a pod",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"Get more details on how to ",(0,r.yg)("a",{parentName:"p",href:"/docs/guide-monitoring"},"debug a pod"),".")),(0,r.yg)("h2",{id:"create-app-from-template"},"Create app from template"),(0,r.yg)("p",null,"Create app from template using the CLI and providing parameters as arguments:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc new-app my-template -p APPLICATION_NAME=my-app -p ADMIN_PASSWORD=mypassword\n")),(0,r.yg)("p",null,"Example for the Semantic Web course notebooks:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc new-app template-jupyterstack-notebook -p APPLICATION_NAME=swcourseName -p NOTEBOOK_PASSWORD=PASSWORD\n\noc delete all --selector template=template-jupyterstack-notebook\n")),(0,r.yg)("h3",{id:"copy-files"},"Copy files"),(0,r.yg)("p",null,"See the ",(0,r.yg)("a",{parentName:"p",href:"https://maastrichtu-ids.github.io/dsri-documentation/docs/openshift-load-data"},"Load data")," page."))}g.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[610],{5680:(e,t,a)=>{a.d(t,{xA:()=>c,yg:()=>g});var n=a(6540);function o(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function r(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function l(e){for(var t=1;t=0||(o[a]=e[a]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(o[a]=e[a])}return o}var i=n.createContext({}),d=function(e){var t=n.useContext(i),a=t;return e&&(a="function"==typeof e?e(t):l(l({},t),e)),a},c=function(e){var t=d(e.components);return n.createElement(i.Provider,{value:t},e.children)},s={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},m=n.forwardRef((function(e,t){var a=e.components,o=e.mdxType,r=e.originalType,i=e.parentName,c=p(e,["components","mdxType","originalType","parentName"]),m=d(a),g=o,y=m["".concat(i,".").concat(g)]||m[g]||s[g]||r;return a?n.createElement(y,l(l({ref:t},c),{},{components:a})):n.createElement(y,l({ref:t},c))}));function g(e,t){var a=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=a.length,l=new Array(r);l[0]=m;var p={};for(var i in t)hasOwnProperty.call(t,i)&&(p[i]=t[i]);p.originalType=e,p.mdxType="string"==typeof e?e:o,l[1]=p;for(var d=2;d{a.r(t),a.d(t,{assets:()=>c,contentTitle:()=>i,default:()=>g,frontMatter:()=>p,metadata:()=>d,toc:()=>s});var n=a(9668),o=a(1367),r=(a(6540),a(5680)),l=["components"],p={id:"openshift-commands",title:"Command Line Interface"},i=void 0,d={unversionedId:"openshift-commands",id:"openshift-commands",title:"Command Line Interface",description:"Overview",source:"@site/docs/openshift-commands.md",sourceDirName:".",slug:"/openshift-commands",permalink:"/docs/openshift-commands",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/openshift-commands.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"openshift-commands",title:"Command Line Interface"},sidebar:"docs",previous:{title:"Login to Docker registries",permalink:"/docs/login-docker-registry"},next:{title:"Data storage",permalink:"/docs/openshift-storage"}},c={},s=[{value:"Overview",id:"overview",level:2},{value:"Projects",id:"projects",level:2},{value:"List projects",id:"list-projects",level:3},{value:"Connect to project",id:"connect-to-project",level:3},{value:"ImageStreams",id:"imagestreams",level:2},{value:"Pods",id:"pods",level:2},{value:"Create pod from YAML",id:"create-pod-from-yaml",level:3},{value:"List pods",id:"list-pods",level:3},{value:"Get specific pod",id:"get-specific-pod",level:3},{value:"Remote Shell connection",id:"remote-shell-connection",level:3},{value:"Execute command in pod",id:"execute-command-in-pod",level:3},{value:"Delete pod",id:"delete-pod",level:3},{value:"Get pod logs",id:"get-pod-logs",level:3},{value:"Create app from template",id:"create-app-from-template",level:2},{value:"Copy files",id:"copy-files",level:3}],m={toc:s};function g(e){var t=e.components,a=(0,o.A)(e,l);return(0,r.yg)("wrapper",(0,n.A)({},m,a,{components:t,mdxType:"MDXLayout"}),(0,r.yg)("h2",{id:"overview"},"Overview"),(0,r.yg)("p",null,"Here is an overview of common ",(0,r.yg)("inlineCode",{parentName:"p"},"oc")," commands:"),(0,r.yg)("table",null,(0,r.yg)("thead",{parentName:"table"},(0,r.yg)("tr",{parentName:"thead"},(0,r.yg)("th",{parentName:"tr",align:null},(0,r.yg)("strong",{parentName:"th"},"Command")),(0,r.yg)("th",{parentName:"tr",align:null},(0,r.yg)("strong",{parentName:"th"},"Description")))),(0,r.yg)("tbody",{parentName:"table"},(0,r.yg)("tr",{parentName:"tbody"},(0,r.yg)("td",{parentName:"tr",align:null},(0,r.yg)("inlineCode",{parentName:"td"},"oc login --token=")),(0,r.yg)("td",{parentName:"tr",align:null},"Login to the DSRI OpenShift cluster in your terminal")),(0,r.yg)("tr",{parentName:"tbody"},(0,r.yg)("td",{parentName:"tr",align:null},(0,r.yg)("inlineCode",{parentName:"td"},"oc get projects")),(0,r.yg)("td",{parentName:"tr",align:null},"List all available projects")),(0,r.yg)("tr",{parentName:"tbody"},(0,r.yg)("td",{parentName:"tr",align:null},(0,r.yg)("inlineCode",{parentName:"td"},"oc project ")),(0,r.yg)("td",{parentName:"tr",align:null},"Switch to project")),(0,r.yg)("tr",{parentName:"tbody"},(0,r.yg)("td",{parentName:"tr",align:null},(0,r.yg)("inlineCode",{parentName:"td"},"oc get pods ")),(0,r.yg)("td",{parentName:"tr",align:null},"Get running pods (a pod can run one or multiple containers for your application)")),(0,r.yg)("tr",{parentName:"tbody"},(0,r.yg)("td",{parentName:"tr",align:null},(0,r.yg)("inlineCode",{parentName:"td"},"oc rsh ")),(0,r.yg)("td",{parentName:"tr",align:null},"Remote terminal connexion to a pod (Shell/Bash)")),(0,r.yg)("tr",{parentName:"tbody"},(0,r.yg)("td",{parentName:"tr",align:null},(0,r.yg)("inlineCode",{parentName:"td"},"oc cp ")),(0,r.yg)("td",{parentName:"tr",align:null},"Copy files from host to container or vice versa, e.g. from host: ",(0,r.yg)("inlineCode",{parentName:"td"},"oc cp :")," or from to host: ",(0,r.yg)("inlineCode",{parentName:"td"},"oc cp : "))),(0,r.yg)("tr",{parentName:"tbody"},(0,r.yg)("td",{parentName:"tr",align:null},(0,r.yg)("inlineCode",{parentName:"td"},"oc rsync ")),(0,r.yg)("td",{parentName:"tr",align:null},"Similar to rsync command on Linux to synchronize directories between container and host or the other way around")),(0,r.yg)("tr",{parentName:"tbody"},(0,r.yg)("td",{parentName:"tr",align:null},(0,r.yg)("inlineCode",{parentName:"td"},"oc exec ")),(0,r.yg)("td",{parentName:"tr",align:null},"Execute command in pods")),(0,r.yg)("tr",{parentName:"tbody"},(0,r.yg)("td",{parentName:"tr",align:null},(0,r.yg)("inlineCode",{parentName:"td"},"oc delete pod ")),(0,r.yg)("td",{parentName:"tr",align:null},"Delete pod")))),(0,r.yg)("h2",{id:"projects"},"Projects"),(0,r.yg)("h3",{id:"list-projects"},"List projects"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"oc projects\n")),(0,r.yg)("h3",{id:"connect-to-project"},"Connect to project"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"oc project my-project\n")),(0,r.yg)("hr",null),(0,r.yg)("h2",{id:"imagestreams"},"ImageStreams"),(0,r.yg)("p",null,"To update an ImageStream in your project to pull the latest update from the external repository (e.g. from ghcr.io or DockerHub):"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc import-image \n")),(0,r.yg)("hr",null),(0,r.yg)("h2",{id:"pods"},"Pods"),(0,r.yg)("h3",{id:"create-pod-from-yaml"},"Create pod from YAML"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"oc create -f my-pod.yaml\n")),(0,r.yg)("blockquote",null,(0,r.yg)("p",{parentName:"blockquote"},"E.g. ",(0,r.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/d2s-core/blob/master/argo/d2s-pod-virtuoso.yaml"},"d2s-pod-virtuoso.yaml"),".")),(0,r.yg)("h3",{id:"list-pods"},"List pods"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"oc get pod\n")),(0,r.yg)("p",null,"List running pods:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc get pods --field-selector=status.phase=Running\n")),(0,r.yg)("h3",{id:"get-specific-pod"},"Get specific pod"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"oc get pod | grep \n")),(0,r.yg)("p",null,"Using selector with Apache Flink as example, and showing only the pod id without header:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc get pod --selector app=flink --selector component=jobmanager --no-headers -o=custom-columns=NAME:.metadata.name\n")),(0,r.yg)("h3",{id:"remote-shell-connection"},"Remote Shell connection"),(0,r.yg)("p",null,"Connect to a pod with ",(0,r.yg)("a",{parentName:"p",href:"https://devhints.io/bash"},"Bash"),"."),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"oc rsh \n")),(0,r.yg)("h3",{id:"execute-command-in-pod"},"Execute command in pod"),(0,r.yg)("p",null,"Example creating a folder:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"oc exec -- mkdir -p /mnt/workspace/resources\n")),(0,r.yg)("h3",{id:"delete-pod"},"Delete pod"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"oc delete pod \n")),(0,r.yg)("admonition",{title:"Force pod deletion",type:"caution"},(0,r.yg)("p",{parentName:"admonition"},"If the pod is not properly deleted, you can force its deletion:"),(0,r.yg)("pre",{parentName:"admonition"},(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc delete pod --force --grace-period=0 \n"))),(0,r.yg)("h3",{id:"get-pod-logs"},"Get pod logs"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"oc logs -f \n")),(0,r.yg)("admonition",{title:"Debug a pod",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"Get more details on how to ",(0,r.yg)("a",{parentName:"p",href:"/docs/guide-monitoring"},"debug a pod"),".")),(0,r.yg)("h2",{id:"create-app-from-template"},"Create app from template"),(0,r.yg)("p",null,"Create app from template using the CLI and providing parameters as arguments:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc new-app my-template -p APPLICATION_NAME=my-app -p ADMIN_PASSWORD=mypassword\n")),(0,r.yg)("p",null,"Example for the Semantic Web course notebooks:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc new-app template-jupyterstack-notebook -p APPLICATION_NAME=swcourseName -p NOTEBOOK_PASSWORD=PASSWORD\n\noc delete all --selector template=template-jupyterstack-notebook\n")),(0,r.yg)("h3",{id:"copy-files"},"Copy files"),(0,r.yg)("p",null,"See the ",(0,r.yg)("a",{parentName:"p",href:"https://maastrichtu-ids.github.io/dsri-documentation/docs/openshift-load-data"},"Load data")," page."))}g.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/391576c4.f8426e12.js b/assets/js/391576c4.c238dfde.js similarity index 98% rename from assets/js/391576c4.f8426e12.js rename to assets/js/391576c4.c238dfde.js index 760ed6131..8b7b5e8dc 100644 --- a/assets/js/391576c4.f8426e12.js +++ b/assets/js/391576c4.c238dfde.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[2567],{5680:(e,n,t)=>{t.d(n,{xA:()=>p,yg:()=>f});var r=t(6540);function o(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function l(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);n&&(r=r.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,r)}return t}function a(e){for(var n=1;n=0||(o[t]=e[t]);return o}(e,n);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(o[t]=e[t])}return o}var i=r.createContext({}),c=function(e){var n=r.useContext(i),t=n;return e&&(t="function"==typeof e?e(n):a(a({},n),e)),t},p=function(e){var n=c(e.components);return r.createElement(i.Provider,{value:n},e.children)},u={inlineCode:"code",wrapper:function(e){var n=e.children;return r.createElement(r.Fragment,{},n)}},d=r.forwardRef((function(e,n){var t=e.components,o=e.mdxType,l=e.originalType,i=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),d=c(t),f=o,y=d["".concat(i,".").concat(f)]||d[f]||u[f]||l;return t?r.createElement(y,a(a({ref:n},p),{},{components:t})):r.createElement(y,a({ref:n},p))}));function f(e,n){var t=arguments,o=n&&n.mdxType;if("string"==typeof e||o){var l=t.length,a=new Array(l);a[0]=d;var s={};for(var i in n)hasOwnProperty.call(n,i)&&(s[i]=n[i]);s.originalType=e,s.mdxType="string"==typeof e?e:o,a[1]=s;for(var c=2;c{t.r(n),t.d(n,{assets:()=>p,contentTitle:()=>i,default:()=>f,frontMatter:()=>s,metadata:()=>c,toc:()=>u});var r=t(9668),o=t(1367),l=(t(6540),t(5680)),a=["components"],s={id:"enabling-vpn-wsl",title:"Enabling VPN access in WSL2"},i=void 0,c={unversionedId:"enabling-vpn-wsl",id:"enabling-vpn-wsl",title:"Enabling VPN access in WSL2",description:"Follow these steps in the WSL2 environment:",source:"@site/docs/enabling-vpn-wsl.md",sourceDirName:".",slug:"/enabling-vpn-wsl",permalink:"/docs/enabling-vpn-wsl",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/enabling-vpn-wsl.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"enabling-vpn-wsl",title:"Enabling VPN access in WSL2"},sidebar:"docs",previous:{title:"Contribute",permalink:"/docs/contribute"},next:{title:"Increase your processes speed",permalink:"/docs/increase-process-speed"}},p={},u=[{value:"Follow these steps in the WSL2 environment:",id:"follow-these-steps-in-the-wsl2-environment",level:2}],d={toc:u};function f(e){var n=e.components,t=(0,o.A)(e,a);return(0,l.yg)("wrapper",(0,r.A)({},d,t,{components:n,mdxType:"MDXLayout"}),(0,l.yg)("h2",{id:"follow-these-steps-in-the-wsl2-environment"},"Follow these steps in the WSL2 environment:"),(0,l.yg)("p",null,"Create a file in /etc/wsl.conf:"),(0,l.yg)("p",null,"[network]"),(0,l.yg)("p",null,"generateResolvConf = false"),(0,l.yg)("p",null,"This makes sure that WSL2 does not generate it's own resolv.conf anymore."),(0,l.yg)("p",null,"Edit the file /etc/resolv.conf and add the appropiate nameservers:"),(0,l.yg)("p",null,"nameserver 137.120.1.1"),(0,l.yg)("p",null,"nameserver 137.120.1.5"),(0,l.yg)("p",null,"nameserver 8.8.8.8 # OR OF YOUR CHOOSING"),(0,l.yg)("p",null,"search unimaas.nl"),(0,l.yg)("p",null,"These are all the steps you should take in WSL2. Now you should do the following step after you connected to the VPN.\nYou can run this command in Powershell:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-Powershell"},'Get-NetAdapter | Where-Object {$_.InterfaceDescription -Match "Cisco AnyConnect"} | Set-NetIPInterface -InterfaceMetric 6000\n\n')),(0,l.yg)("p",null,"you should now be able to verify that WSL2 has connectivity:"),(0,l.yg)("p",null,"ping google.com -c 4"))}f.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[2567],{5680:(e,n,t)=>{t.d(n,{xA:()=>p,yg:()=>f});var r=t(6540);function o(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function l(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);n&&(r=r.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,r)}return t}function a(e){for(var n=1;n=0||(o[t]=e[t]);return o}(e,n);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(o[t]=e[t])}return o}var i=r.createContext({}),c=function(e){var n=r.useContext(i),t=n;return e&&(t="function"==typeof e?e(n):a(a({},n),e)),t},p=function(e){var n=c(e.components);return r.createElement(i.Provider,{value:n},e.children)},u={inlineCode:"code",wrapper:function(e){var n=e.children;return r.createElement(r.Fragment,{},n)}},d=r.forwardRef((function(e,n){var t=e.components,o=e.mdxType,l=e.originalType,i=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),d=c(t),f=o,y=d["".concat(i,".").concat(f)]||d[f]||u[f]||l;return t?r.createElement(y,a(a({ref:n},p),{},{components:t})):r.createElement(y,a({ref:n},p))}));function f(e,n){var t=arguments,o=n&&n.mdxType;if("string"==typeof e||o){var l=t.length,a=new Array(l);a[0]=d;var s={};for(var i in n)hasOwnProperty.call(n,i)&&(s[i]=n[i]);s.originalType=e,s.mdxType="string"==typeof e?e:o,a[1]=s;for(var c=2;c{t.r(n),t.d(n,{assets:()=>p,contentTitle:()=>i,default:()=>f,frontMatter:()=>s,metadata:()=>c,toc:()=>u});var r=t(9668),o=t(1367),l=(t(6540),t(5680)),a=["components"],s={id:"enabling-vpn-wsl",title:"Enabling VPN access in WSL2"},i=void 0,c={unversionedId:"enabling-vpn-wsl",id:"enabling-vpn-wsl",title:"Enabling VPN access in WSL2",description:"Follow these steps in the WSL2 environment:",source:"@site/docs/enabling-vpn-wsl.md",sourceDirName:".",slug:"/enabling-vpn-wsl",permalink:"/docs/enabling-vpn-wsl",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/enabling-vpn-wsl.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"enabling-vpn-wsl",title:"Enabling VPN access in WSL2"},sidebar:"docs",previous:{title:"Contribute",permalink:"/docs/contribute"},next:{title:"Increase your processes speed",permalink:"/docs/increase-process-speed"}},p={},u=[{value:"Follow these steps in the WSL2 environment:",id:"follow-these-steps-in-the-wsl2-environment",level:2}],d={toc:u};function f(e){var n=e.components,t=(0,o.A)(e,a);return(0,l.yg)("wrapper",(0,r.A)({},d,t,{components:n,mdxType:"MDXLayout"}),(0,l.yg)("h2",{id:"follow-these-steps-in-the-wsl2-environment"},"Follow these steps in the WSL2 environment:"),(0,l.yg)("p",null,"Create a file in /etc/wsl.conf:"),(0,l.yg)("p",null,"[network]"),(0,l.yg)("p",null,"generateResolvConf = false"),(0,l.yg)("p",null,"This makes sure that WSL2 does not generate it's own resolv.conf anymore."),(0,l.yg)("p",null,"Edit the file /etc/resolv.conf and add the appropiate nameservers:"),(0,l.yg)("p",null,"nameserver 137.120.1.1"),(0,l.yg)("p",null,"nameserver 137.120.1.5"),(0,l.yg)("p",null,"nameserver 8.8.8.8 # OR OF YOUR CHOOSING"),(0,l.yg)("p",null,"search unimaas.nl"),(0,l.yg)("p",null,"These are all the steps you should take in WSL2. Now you should do the following step after you connected to the VPN.\nYou can run this command in Powershell:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-Powershell"},'Get-NetAdapter | Where-Object {$_.InterfaceDescription -Match "Cisco AnyConnect"} | Set-NetIPInterface -InterfaceMetric 6000\n\n')),(0,l.yg)("p",null,"you should now be able to verify that WSL2 has connectivity:"),(0,l.yg)("p",null,"ping google.com -c 4"))}f.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/3a93ea81.2cd6a8e7.js b/assets/js/3a93ea81.569426c0.js similarity index 99% rename from assets/js/3a93ea81.2cd6a8e7.js rename to assets/js/3a93ea81.569426c0.js index dd6fd422d..14b93d760 100644 --- a/assets/js/3a93ea81.2cd6a8e7.js +++ b/assets/js/3a93ea81.569426c0.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[3684],{5680:(e,t,a)=>{a.d(t,{xA:()=>u,yg:()=>d});var n=a(6540);function r(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function l(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function o(e){for(var t=1;t=0||(r[a]=e[a]);return r}(e,t);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(r[a]=e[a])}return r}var i=n.createContext({}),s=function(e){var t=n.useContext(i),a=t;return e&&(a="function"==typeof e?e(t):o(o({},t),e)),a},u=function(e){var t=s(e.components);return n.createElement(i.Provider,{value:t},e.children)},c={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},y=n.forwardRef((function(e,t){var a=e.components,r=e.mdxType,l=e.originalType,i=e.parentName,u=p(e,["components","mdxType","originalType","parentName"]),y=s(a),d=r,m=y["".concat(i,".").concat(d)]||y[d]||c[d]||l;return a?n.createElement(m,o(o({ref:t},u),{},{components:a})):n.createElement(m,o({ref:t},u))}));function d(e,t){var a=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var l=a.length,o=new Array(l);o[0]=y;var p={};for(var i in t)hasOwnProperty.call(t,i)&&(p[i]=t[i]);p.originalType=e,p.mdxType="string"==typeof e?e:r,o[1]=p;for(var s=2;s{a.r(t),a.d(t,{assets:()=>u,contentTitle:()=>i,default:()=>d,frontMatter:()=>p,metadata:()=>s,toc:()=>c});var n=a(9668),r=a(1367),l=(a(6540),a(5680)),o=["components"],p={id:"jupyterhub-spark",title:"JupyterHub with Spark"},i=void 0,s={unversionedId:"jupyterhub-spark",id:"jupyterhub-spark",title:"JupyterHub with Spark",description:"JupyterHub is ideal to enable multiple users easily start predefined workspaces in the same project. The complimentary Apache Spark cluster can be used from the workspaces to perform distributed processing.",source:"@site/docs/jupyterhub-spark.md",sourceDirName:".",slug:"/jupyterhub-spark",permalink:"/docs/jupyterhub-spark",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/jupyterhub-spark.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"jupyterhub-spark",title:"JupyterHub with Spark"}},u={},c=[{value:"\ud83e\uddca Install kfctl",id:"-install-kfctl",level:2},{value:"\ud83e\ude90 Deploy JupyterHub and Spark",id:"-deploy-jupyterhub-and-spark",level:2},{value:"\u2728 Use the Spark cluster",id:"-use-the-spark-cluster",level:2},{value:"Match the version",id:"match-the-version",level:3},{value:"Spark UI",id:"spark-ui",level:3},{value:"New Spark cluster",id:"new-spark-cluster",level:3},{value:"\ud83d\uddd1\ufe0f Delete the deployment",id:"\ufe0f-delete-the-deployment",level:2}],y={toc:c};function d(e){var t=e.components,a=(0,r.A)(e,o);return(0,l.yg)("wrapper",(0,n.A)({},y,a,{components:t,mdxType:"MDXLayout"}),(0,l.yg)("p",null,"JupyterHub is ideal to enable multiple users easily start predefined workspaces in the same project. The complimentary Apache Spark cluster can be used from the workspaces to perform distributed processing."),(0,l.yg)("h2",{id:"-install-kfctl"},"\ud83e\uddca Install kfctl"),(0,l.yg)("p",null,"You will need to have the usual ",(0,l.yg)("inlineCode",{parentName:"p"},"oc")," tool installed, and to install ",(0,l.yg)("inlineCode",{parentName:"p"},"kfctl")," on your machine, a tool to deploy Kubeflow applications, download the ",(0,l.yg)("a",{parentName:"p",href:"https://github.com/kubeflow/kfctl/releases"},"latest version for your OS \ud83d\udce5\ufe0f")," "),(0,l.yg)("p",null,"You can then install it by downloading the binary and putting it in your path, for example on Linux:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-bash"},"wget https://github.com/kubeflow/kfctl/releases/download/v1.2.0/kfctl_v1.2.0-0-gbc038f9_linux.tar.gz\ntar -xzf kfctl_v1.2.0-0-gbc038f9_linux.tar.gz\nsudo mv kfctl /usr/local/bin/\n")),(0,l.yg)("p",null,"Clone the repository with the DSRI custom images and deployments for the OpenDataHub platform, and go to the ",(0,l.yg)("inlineCode",{parentName:"p"},"kfdef")," folder:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-bash"},"git clone https://github.com/MaastrichtU-IDS/odh-manifests\ncd odh-manifests/kfdef\n")),(0,l.yg)("h2",{id:"-deploy-jupyterhub-and-spark"},"\ud83e\ude90 Deploy JupyterHub and Spark"),(0,l.yg)("admonition",{title:"Go the the kfdef folder",type:"info"},(0,l.yg)("p",{parentName:"admonition"},"All scripts need to be run from the ",(0,l.yg)("inlineCode",{parentName:"p"},"kfdef")," folder \ud83d\udcc2")),(0,l.yg)("p",null,"You can deploy JupyterHub with 2 different authentications system, use the file corresponding to your choice:"),(0,l.yg)("ul",null,(0,l.yg)("li",{parentName:"ul"},(0,l.yg)("p",{parentName:"li"},"For the default DSRI authentication use ",(0,l.yg)("inlineCode",{parentName:"p"},"kfctl_openshift_dsri.yaml"))),(0,l.yg)("li",{parentName:"ul"},(0,l.yg)("p",{parentName:"li"},"For GitHub authentication use ",(0,l.yg)("inlineCode",{parentName:"p"},"kfctl_openshift_github.yaml")),(0,l.yg)("ul",{parentName:"li"},(0,l.yg)("li",{parentName:"ul"},(0,l.yg)("p",{parentName:"li"},"You need to create a new GitHub OAuth app: ",(0,l.yg)("a",{parentName:"p",href:"https://github.com/settings/developers"},"https://github.com/settings/developers"))),(0,l.yg)("li",{parentName:"ul"},(0,l.yg)("p",{parentName:"li"},"And provide the GitHub client ID and secret through environment variable before running the start script:"),(0,l.yg)("pre",{parentName:"li"},(0,l.yg)("code",{parentName:"pre",className:"language-bash"},"export GITHUB_CLIENT_ID=YOUR_CLIENT_ID\nexport GITHUB_CLIENT_SECRET=YOUR_CLIENT_SECRET\n")))))),(0,l.yg)("p",null,"First you will need to change the ",(0,l.yg)("inlineCode",{parentName:"p"},"namespace:")," in the file you want to deploy, to provide the project where you want to start JupyterHub (currently ",(0,l.yg)("inlineCode",{parentName:"p"},"opendatahub-ids"),"), then you can deploy JupyterHub and Spark with ",(0,l.yg)("inlineCode",{parentName:"p"},"kfctl"),":"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-bash"},"./start_odh.sh kfctl_openshift_dsri.yaml\n")),(0,l.yg)("p",null,"\ud83d\uddc4\ufe0f Persistent volumes are automatically created for each instance started in JupyterHub to insure persistence of the data even JupyterHub is stopped. You can find the persistent volumes in the DSRI web UI, go to the ",(0,l.yg)("strong",{parentName:"p"},"Administrator")," view > ",(0,l.yg)("strong",{parentName:"p"},"Storage")," > ",(0,l.yg)("strong",{parentName:"p"},"Persistent Volume Claims"),"."),(0,l.yg)("p",null,"\u26a1\ufe0f A Spark cluster with 3 workers is automatically created with the service name ",(0,l.yg)("inlineCode",{parentName:"p"},"spark-cluster"),", you can use the URL of the master node to access it from your workspace: ",(0,l.yg)("inlineCode",{parentName:"p"},"spark://spark-cluster:7077")),(0,l.yg)("h2",{id:"-use-the-spark-cluster"},"\u2728 Use the Spark cluster"),(0,l.yg)("admonition",{title:"Matching Spark versions",type:"caution"},(0,l.yg)("p",{parentName:"admonition"},"Make sure all the Spark versions are matching, the current default version is ",(0,l.yg)("inlineCode",{parentName:"p"},"3.0.1"))),(0,l.yg)("p",null,"You can test the Spark cluster connection with PySpark:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-python"},"from pyspark.sql import SparkSession, SQLContext\nimport os\nimport socket\n# Create a Spark session\nspark_cluster_url = \"spark://spark-cluster:7077\"\nspark = SparkSession.builder.master(spark_cluster_url).getOrCreate()\nsc = spark.sparkContext\n\n# Test your Spark connection\nspark.range(5, numPartitions=5).rdd.map(lambda x: socket.gethostname()).distinct().collect()\n# Or try:\n#x = ['spark', 'rdd', 'example', 'sample', 'example']\nx = [1, 2, 3, 4, 5]\ny = sc.parallelize(x)\ny.collect()\n# Or try:\ndata = [1, 2, 3, 4, 5]\ndistData = sc.parallelize(data)\ndistData.reduce(lambda a, b: a + b)\n")),(0,l.yg)("h3",{id:"match-the-version"},"Match the version"),(0,l.yg)("p",null,"Make sure all the Spark versions are matching, the current default version is ",(0,l.yg)("inlineCode",{parentName:"p"},"3.0.1"),":"),(0,l.yg)("ul",null,(0,l.yg)("li",{parentName:"ul"},"Go to the Spark UI to verify the version of the Spark cluster"),(0,l.yg)("li",{parentName:"ul"},"Run ",(0,l.yg)("inlineCode",{parentName:"li"},"spark-shell --version")," to verify the version of the Spark binary installed in the workspace"),(0,l.yg)("li",{parentName:"ul"},"Run ",(0,l.yg)("inlineCode",{parentName:"li"},"pip list | grep pyspark")," to verify the version of the PySpark library")),(0,l.yg)("p",null,"Check the ",(0,l.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/jupyterlab/blob/main/Dockerfile#L14"},"JupyterLab workspace ",(0,l.yg)("inlineCode",{parentName:"a"},"Dockerfile"))," to change the version of Spark installed in the workspace, and see how you can download and install a new version of the Spark binary."),(0,l.yg)("p",null,"If you need to change the Python, Java or PySpark version in the workspace you can create a ",(0,l.yg)("inlineCode",{parentName:"p"},"environment.yml")," file, for example for ",(0,l.yg)("inlineCode",{parentName:"p"},"2.4.5"),":"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-yaml"},"name: spark\nchannels:\n - defaults\n - conda-forge\n - anaconda\ndependencies:\n - python=3.7\n - openjdk=8\n - ipykernel \n - nb_conda_kernels\n - pip\n - pip:\n - pyspark==2.4.5\n")),(0,l.yg)("p",null,"Create the environment with ",(0,l.yg)("inlineCode",{parentName:"p"},"conda"),":"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-bash"},"mamba env create -f environment.yml\n")),(0,l.yg)("h3",{id:"spark-ui"},"Spark UI"),(0,l.yg)("p",null,"You can also create a route to access the Spark UI and monitor the activity on the Spark cluster:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-bash"},"oc expose svc/spark-cluster-ui\n")),(0,l.yg)("p",null,"Get the Spark UI URL:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-bash"},"oc get route --selector radanalytics.io/service=ui --no-headers -o=custom-columns=HOST:.spec.host\n")),(0,l.yg)("h3",{id:"new-spark-cluster"},"New Spark cluster"),(0,l.yg)("p",null,"You can create a new Spark cluster, for example here using Spark ",(0,l.yg)("inlineCode",{parentName:"p"},"3.0.1")," with the installed Spark Operator:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-bash"},"cat <{a.d(t,{xA:()=>u,yg:()=>d});var n=a(6540);function r(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function l(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function o(e){for(var t=1;t=0||(r[a]=e[a]);return r}(e,t);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(r[a]=e[a])}return r}var i=n.createContext({}),s=function(e){var t=n.useContext(i),a=t;return e&&(a="function"==typeof e?e(t):o(o({},t),e)),a},u=function(e){var t=s(e.components);return n.createElement(i.Provider,{value:t},e.children)},c={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},y=n.forwardRef((function(e,t){var a=e.components,r=e.mdxType,l=e.originalType,i=e.parentName,u=p(e,["components","mdxType","originalType","parentName"]),y=s(a),d=r,m=y["".concat(i,".").concat(d)]||y[d]||c[d]||l;return a?n.createElement(m,o(o({ref:t},u),{},{components:a})):n.createElement(m,o({ref:t},u))}));function d(e,t){var a=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var l=a.length,o=new Array(l);o[0]=y;var p={};for(var i in t)hasOwnProperty.call(t,i)&&(p[i]=t[i]);p.originalType=e,p.mdxType="string"==typeof e?e:r,o[1]=p;for(var s=2;s{a.r(t),a.d(t,{assets:()=>u,contentTitle:()=>i,default:()=>d,frontMatter:()=>p,metadata:()=>s,toc:()=>c});var n=a(9668),r=a(1367),l=(a(6540),a(5680)),o=["components"],p={id:"jupyterhub-spark",title:"JupyterHub with Spark"},i=void 0,s={unversionedId:"jupyterhub-spark",id:"jupyterhub-spark",title:"JupyterHub with Spark",description:"JupyterHub is ideal to enable multiple users easily start predefined workspaces in the same project. The complimentary Apache Spark cluster can be used from the workspaces to perform distributed processing.",source:"@site/docs/jupyterhub-spark.md",sourceDirName:".",slug:"/jupyterhub-spark",permalink:"/docs/jupyterhub-spark",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/jupyterhub-spark.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"jupyterhub-spark",title:"JupyterHub with Spark"}},u={},c=[{value:"\ud83e\uddca Install kfctl",id:"-install-kfctl",level:2},{value:"\ud83e\ude90 Deploy JupyterHub and Spark",id:"-deploy-jupyterhub-and-spark",level:2},{value:"\u2728 Use the Spark cluster",id:"-use-the-spark-cluster",level:2},{value:"Match the version",id:"match-the-version",level:3},{value:"Spark UI",id:"spark-ui",level:3},{value:"New Spark cluster",id:"new-spark-cluster",level:3},{value:"\ud83d\uddd1\ufe0f Delete the deployment",id:"\ufe0f-delete-the-deployment",level:2}],y={toc:c};function d(e){var t=e.components,a=(0,r.A)(e,o);return(0,l.yg)("wrapper",(0,n.A)({},y,a,{components:t,mdxType:"MDXLayout"}),(0,l.yg)("p",null,"JupyterHub is ideal to enable multiple users easily start predefined workspaces in the same project. The complimentary Apache Spark cluster can be used from the workspaces to perform distributed processing."),(0,l.yg)("h2",{id:"-install-kfctl"},"\ud83e\uddca Install kfctl"),(0,l.yg)("p",null,"You will need to have the usual ",(0,l.yg)("inlineCode",{parentName:"p"},"oc")," tool installed, and to install ",(0,l.yg)("inlineCode",{parentName:"p"},"kfctl")," on your machine, a tool to deploy Kubeflow applications, download the ",(0,l.yg)("a",{parentName:"p",href:"https://github.com/kubeflow/kfctl/releases"},"latest version for your OS \ud83d\udce5\ufe0f")," "),(0,l.yg)("p",null,"You can then install it by downloading the binary and putting it in your path, for example on Linux:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-bash"},"wget https://github.com/kubeflow/kfctl/releases/download/v1.2.0/kfctl_v1.2.0-0-gbc038f9_linux.tar.gz\ntar -xzf kfctl_v1.2.0-0-gbc038f9_linux.tar.gz\nsudo mv kfctl /usr/local/bin/\n")),(0,l.yg)("p",null,"Clone the repository with the DSRI custom images and deployments for the OpenDataHub platform, and go to the ",(0,l.yg)("inlineCode",{parentName:"p"},"kfdef")," folder:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-bash"},"git clone https://github.com/MaastrichtU-IDS/odh-manifests\ncd odh-manifests/kfdef\n")),(0,l.yg)("h2",{id:"-deploy-jupyterhub-and-spark"},"\ud83e\ude90 Deploy JupyterHub and Spark"),(0,l.yg)("admonition",{title:"Go the the kfdef folder",type:"info"},(0,l.yg)("p",{parentName:"admonition"},"All scripts need to be run from the ",(0,l.yg)("inlineCode",{parentName:"p"},"kfdef")," folder \ud83d\udcc2")),(0,l.yg)("p",null,"You can deploy JupyterHub with 2 different authentications system, use the file corresponding to your choice:"),(0,l.yg)("ul",null,(0,l.yg)("li",{parentName:"ul"},(0,l.yg)("p",{parentName:"li"},"For the default DSRI authentication use ",(0,l.yg)("inlineCode",{parentName:"p"},"kfctl_openshift_dsri.yaml"))),(0,l.yg)("li",{parentName:"ul"},(0,l.yg)("p",{parentName:"li"},"For GitHub authentication use ",(0,l.yg)("inlineCode",{parentName:"p"},"kfctl_openshift_github.yaml")),(0,l.yg)("ul",{parentName:"li"},(0,l.yg)("li",{parentName:"ul"},(0,l.yg)("p",{parentName:"li"},"You need to create a new GitHub OAuth app: ",(0,l.yg)("a",{parentName:"p",href:"https://github.com/settings/developers"},"https://github.com/settings/developers"))),(0,l.yg)("li",{parentName:"ul"},(0,l.yg)("p",{parentName:"li"},"And provide the GitHub client ID and secret through environment variable before running the start script:"),(0,l.yg)("pre",{parentName:"li"},(0,l.yg)("code",{parentName:"pre",className:"language-bash"},"export GITHUB_CLIENT_ID=YOUR_CLIENT_ID\nexport GITHUB_CLIENT_SECRET=YOUR_CLIENT_SECRET\n")))))),(0,l.yg)("p",null,"First you will need to change the ",(0,l.yg)("inlineCode",{parentName:"p"},"namespace:")," in the file you want to deploy, to provide the project where you want to start JupyterHub (currently ",(0,l.yg)("inlineCode",{parentName:"p"},"opendatahub-ids"),"), then you can deploy JupyterHub and Spark with ",(0,l.yg)("inlineCode",{parentName:"p"},"kfctl"),":"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-bash"},"./start_odh.sh kfctl_openshift_dsri.yaml\n")),(0,l.yg)("p",null,"\ud83d\uddc4\ufe0f Persistent volumes are automatically created for each instance started in JupyterHub to insure persistence of the data even JupyterHub is stopped. You can find the persistent volumes in the DSRI web UI, go to the ",(0,l.yg)("strong",{parentName:"p"},"Administrator")," view > ",(0,l.yg)("strong",{parentName:"p"},"Storage")," > ",(0,l.yg)("strong",{parentName:"p"},"Persistent Volume Claims"),"."),(0,l.yg)("p",null,"\u26a1\ufe0f A Spark cluster with 3 workers is automatically created with the service name ",(0,l.yg)("inlineCode",{parentName:"p"},"spark-cluster"),", you can use the URL of the master node to access it from your workspace: ",(0,l.yg)("inlineCode",{parentName:"p"},"spark://spark-cluster:7077")),(0,l.yg)("h2",{id:"-use-the-spark-cluster"},"\u2728 Use the Spark cluster"),(0,l.yg)("admonition",{title:"Matching Spark versions",type:"caution"},(0,l.yg)("p",{parentName:"admonition"},"Make sure all the Spark versions are matching, the current default version is ",(0,l.yg)("inlineCode",{parentName:"p"},"3.0.1"))),(0,l.yg)("p",null,"You can test the Spark cluster connection with PySpark:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-python"},"from pyspark.sql import SparkSession, SQLContext\nimport os\nimport socket\n# Create a Spark session\nspark_cluster_url = \"spark://spark-cluster:7077\"\nspark = SparkSession.builder.master(spark_cluster_url).getOrCreate()\nsc = spark.sparkContext\n\n# Test your Spark connection\nspark.range(5, numPartitions=5).rdd.map(lambda x: socket.gethostname()).distinct().collect()\n# Or try:\n#x = ['spark', 'rdd', 'example', 'sample', 'example']\nx = [1, 2, 3, 4, 5]\ny = sc.parallelize(x)\ny.collect()\n# Or try:\ndata = [1, 2, 3, 4, 5]\ndistData = sc.parallelize(data)\ndistData.reduce(lambda a, b: a + b)\n")),(0,l.yg)("h3",{id:"match-the-version"},"Match the version"),(0,l.yg)("p",null,"Make sure all the Spark versions are matching, the current default version is ",(0,l.yg)("inlineCode",{parentName:"p"},"3.0.1"),":"),(0,l.yg)("ul",null,(0,l.yg)("li",{parentName:"ul"},"Go to the Spark UI to verify the version of the Spark cluster"),(0,l.yg)("li",{parentName:"ul"},"Run ",(0,l.yg)("inlineCode",{parentName:"li"},"spark-shell --version")," to verify the version of the Spark binary installed in the workspace"),(0,l.yg)("li",{parentName:"ul"},"Run ",(0,l.yg)("inlineCode",{parentName:"li"},"pip list | grep pyspark")," to verify the version of the PySpark library")),(0,l.yg)("p",null,"Check the ",(0,l.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/jupyterlab/blob/main/Dockerfile#L14"},"JupyterLab workspace ",(0,l.yg)("inlineCode",{parentName:"a"},"Dockerfile"))," to change the version of Spark installed in the workspace, and see how you can download and install a new version of the Spark binary."),(0,l.yg)("p",null,"If you need to change the Python, Java or PySpark version in the workspace you can create a ",(0,l.yg)("inlineCode",{parentName:"p"},"environment.yml")," file, for example for ",(0,l.yg)("inlineCode",{parentName:"p"},"2.4.5"),":"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-yaml"},"name: spark\nchannels:\n - defaults\n - conda-forge\n - anaconda\ndependencies:\n - python=3.7\n - openjdk=8\n - ipykernel \n - nb_conda_kernels\n - pip\n - pip:\n - pyspark==2.4.5\n")),(0,l.yg)("p",null,"Create the environment with ",(0,l.yg)("inlineCode",{parentName:"p"},"conda"),":"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-bash"},"mamba env create -f environment.yml\n")),(0,l.yg)("h3",{id:"spark-ui"},"Spark UI"),(0,l.yg)("p",null,"You can also create a route to access the Spark UI and monitor the activity on the Spark cluster:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-bash"},"oc expose svc/spark-cluster-ui\n")),(0,l.yg)("p",null,"Get the Spark UI URL:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-bash"},"oc get route --selector radanalytics.io/service=ui --no-headers -o=custom-columns=HOST:.spec.host\n")),(0,l.yg)("h3",{id:"new-spark-cluster"},"New Spark cluster"),(0,l.yg)("p",null,"You can create a new Spark cluster, for example here using Spark ",(0,l.yg)("inlineCode",{parentName:"p"},"3.0.1")," with the installed Spark Operator:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-bash"},"cat <{a.d(t,{xA:()=>g,yg:()=>y});var i=a(6540);function o(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function r(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);t&&(i=i.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,i)}return a}function n(e){for(var t=1;t=0||(o[a]=e[a]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(i=0;i=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(o[a]=e[a])}return o}var u=i.createContext({}),s=function(e){var t=i.useContext(u),a=t;return e&&(a="function"==typeof e?e(t):n(n({},t),e)),a},g=function(e){var t=s(e.components);return i.createElement(u.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return i.createElement(i.Fragment,{},t)}},c=i.forwardRef((function(e,t){var a=e.components,o=e.mdxType,r=e.originalType,u=e.parentName,g=l(e,["components","mdxType","originalType","parentName"]),c=s(a),y=o,m=c["".concat(u,".").concat(y)]||c[y]||p[y]||r;return a?i.createElement(m,n(n({ref:t},g),{},{components:a})):i.createElement(m,n({ref:t},g))}));function y(e,t){var a=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=a.length,n=new Array(r);n[0]=c;var l={};for(var u in t)hasOwnProperty.call(t,u)&&(l[u]=t[u]);l.originalType=e,l.mdxType="string"==typeof e?e:o,n[1]=l;for(var s=2;s{a.r(t),a.d(t,{assets:()=>g,contentTitle:()=>u,default:()=>y,frontMatter:()=>l,metadata:()=>s,toc:()=>p});var i=a(9668),o=a(1367),r=(a(6540),a(5680)),n=["components"],l={id:"guide-publish-image",title:"Publish a Docker image"},u=void 0,s={unversionedId:"guide-publish-image",id:"guide-publish-image",title:"Publish a Docker image",description:"\u26a0\ufe0f DockerHub imposes strict pull limitations for clusters like the DSRI (using DockerHub might result in failing to pull your images on the DSRI).",source:"@site/docs/guide-publish-image.md",sourceDirName:".",slug:"/guide-publish-image",permalink:"/docs/guide-publish-image",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/guide-publish-image.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"guide-publish-image",title:"Publish a Docker image"},sidebar:"docs",previous:{title:"Known Issues",permalink:"/docs/guide-known-issues"},next:{title:"Install UM VPN",permalink:"/docs/guide-vpn"}},g={},p=[{value:"Login to Container Registries \ud83d\udd11",id:"login-to-container-registries-",level:2},{value:"Login to GitHub Container Registry",id:"login-to-github-container-registry",level:3},{value:"Login to quay.io",id:"login-to-quayio",level:3},{value:"Login to DockerHub",id:"login-to-dockerhub",level:3},{value:"Publish your image \ud83d\udce2",id:"publish-your-image-",level:2},{value:"Publish to GitHub Container Registry",id:"publish-to-github-container-registry",level:3},{value:"Publish to Quay.io",id:"publish-to-quayio",level:3},{value:"Publish to DockerHub",id:"publish-to-dockerhub",level:3},{value:"Use automated workflows",id:"use-automated-workflows",level:3}],c={toc:p};function y(e){var t=e.components,a=(0,o.A)(e,n);return(0,r.yg)("wrapper",(0,i.A)({},c,a,{components:t,mdxType:"MDXLayout"}),(0,r.yg)("admonition",{title:"DockerHub pull rates limitations",type:"warning"},(0,r.yg)("p",{parentName:"admonition"},"\u26a0\ufe0f ",(0,r.yg)("strong",{parentName:"p"},"DockerHub imposes strict pull limitations for clusters")," like the DSRI (using DockerHub might result in failing to pull your images on the DSRI). "),(0,r.yg)("p",{parentName:"admonition"},"We highly recommend to ",(0,r.yg)("strong",{parentName:"p"},"use the ",(0,r.yg)("a",{parentName:"strong",href:"https://docs.github.com/en/free-pro-team@latest/packages/getting-started-with-github-container-registry/about-github-container-registry"},"GitHub Container Registry")," or ",(0,r.yg)("a",{parentName:"strong",href:"https://quay.io/"},"RedHat quay.io Container Registry")," to publish public Docker images"),".")),(0,r.yg)("blockquote",null,(0,r.yg)("p",{parentName:"blockquote"},"You can also login to DockerHub using a Secret in OpenShift to increase the pull rates limitations from 100 to 200 every 6 hours (this will mitigate the issue, but not solve it completely if you do not have a paid account on DockerHub):"),(0,r.yg)("pre",{parentName:"blockquote"},(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc create secret docker-registry docker-hub-secret --docker-server=docker.io --docker-username=your-dockerhub-username --docker-password=your-dockerhub-password --docker-email=your-dockerhub-email\n"))),(0,r.yg)("h2",{id:"login-to-container-registries-"},"Login to Container Registries \ud83d\udd11"),(0,r.yg)("h3",{id:"login-to-github-container-registry"},"Login to GitHub Container Registry"),(0,r.yg)("p",null,"Use your existing ",(0,r.yg)("a",{parentName:"p",href:"https://github.com"},"GitHub")," account if you have one:"),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},"Create a ",(0,r.yg)("strong",{parentName:"li"},"Personal Access Token")," for GitHub packages at ",(0,r.yg)("strong",{parentName:"li"},(0,r.yg)("a",{parentName:"strong",href:"https://github.com/settings/tokens/new"},"https://github.com/settings/tokens/new"))),(0,r.yg)("li",{parentName:"ol"},"Provide a meaningful description for the token, and enable the following scopes when creating the token:",(0,r.yg)("ul",{parentName:"li"},(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("inlineCode",{parentName:"li"},"write:packages"),": publish container images to GitHub Container Registry"),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("inlineCode",{parentName:"li"},"delete:packages"),": delete specified versions of private or public container images from GitHub Container Registry"))),(0,r.yg)("li",{parentName:"ol"},"You might want to store this token in a safe place, as you will not be able to retrieve it later on github.com (you can still delete it, and create a new token easily if you lose your token)"),(0,r.yg)("li",{parentName:"ol"},"\ud83d\udc68\u200d\ud83d\udcbb Log in to the GitHub Container Registry in your terminal (change ",(0,r.yg)("inlineCode",{parentName:"li"},"USERNAME")," and ",(0,r.yg)("inlineCode",{parentName:"li"},"ACCESS_TOKEN")," to yours):")),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},'echo "ACCESS_TOKEN" | docker login ghcr.io -u USERNAME --password-stdin\n')),(0,r.yg)("p",null,"On Windows use this command:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},'docker login ghcr.io -u USERNAME -p "ACCESS_TOKEN"\n')),(0,r.yg)("blockquote",null,(0,r.yg)("p",{parentName:"blockquote"},"See the ",(0,r.yg)("a",{parentName:"p",href:"https://docs.github.com/en/free-pro-team@latest/packages/using-github-packages-with-your-projects-ecosystem/configuring-docker-for-use-with-github-packages"},"official GitHub documentation"),".")),(0,r.yg)("h3",{id:"login-to-quayio"},"Login to quay.io"),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},"Create an account at ",(0,r.yg)("a",{parentName:"li",href:"https://quay.io"},"https://quay.io")," "),(0,r.yg)("li",{parentName:"ol"},"Login in your terminal (you will be asked for username and password)")),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"docker login quay.io\n")),(0,r.yg)("h3",{id:"login-to-dockerhub"},"Login to DockerHub"),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"Get a ",(0,r.yg)("a",{parentName:"p",href:"https://hub.docker.com/"},"DockerHub")," account at ",(0,r.yg)("a",{parentName:"p",href:"https://hub.docker.com"},"https://hub.docker.com")," (you most probably already have one if you installed Docker Desktop)")),(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"\ud83d\udc69\u200d\ud83d\udcbb Run in your terminal:"))),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"docker login\n")),(0,r.yg)("ol",{start:3},(0,r.yg)("li",{parentName:"ol"},"Provide your DockerHub username and password.")),(0,r.yg)("h2",{id:"publish-your-image-"},"Publish your image \ud83d\udce2"),(0,r.yg)("p",null,"Once you built a Docker image, and you logged in to a Container Registry, you might want to publish the image to pull and re-use it easily later."),(0,r.yg)("h3",{id:"publish-to-github-container-registry"},"Publish to GitHub Container Registry"),(0,r.yg)("admonition",{title:"Free for public images",type:"tip"},(0,r.yg)("p",{parentName:"admonition"},"The ",(0,r.yg)("a",{parentName:"p",href:"https://docs.github.com/en/free-pro-team@latest/packages/getting-started-with-github-container-registry"},"GitHub Container Registry")," is still in beta but will be free for public images when fully released. It enables you to store your Docker images at the same place you keep your code! \ud83d\udce6")),(0,r.yg)("p",null,"Publish to your user Container Registry on GitHub:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"docker build -t ghcr.io/github-username/my-image:latest .\ndocker push ghcr.io/github-username/my-image:latest\n")),(0,r.yg)("p",null,"For example, to the ",(0,r.yg)("a",{parentName:"p",href:"https://github.com/orgs/MaastrichtU-IDS/packages"},"MaastrichtU-IDS organization Container Registry on GitHub"),":"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"docker build -t ghcr.io/maastrichtu-ids/jupyterlab:latest .\ndocker push ghcr.io/maastrichtu-ids/jupyterlab:latest\n")),(0,r.yg)("admonition",{title:"Created automatically",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"If the image does not exist, GitHub Container Registry will create it automatically and set it as ",(0,r.yg)("strong",{parentName:"p"},"Private")," by default. You can easily change it to ",(0,r.yg)("strong",{parentName:"p"},"Public")," in the image settings on github.com.")),(0,r.yg)("h3",{id:"publish-to-quayio"},"Publish to Quay.io"),(0,r.yg)("admonition",{title:"Free for public images",type:"tip"},(0,r.yg)("p",{parentName:"admonition"},"Quay.io is free for public images and does not restrict images pulls.")),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"Create the image on ",(0,r.yg)("a",{parentName:"p",href:"https://quay.io/"},"quay.io"))),(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"Build and push to ",(0,r.yg)("a",{parentName:"p",href:"https://quay.io/"},"quay.io")))),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"docker build -t ghcr.io/quay-username/my-image:latest .\ndocker push quay.io/quay-username/my-image:latest\n")),(0,r.yg)("h3",{id:"publish-to-dockerhub"},"Publish to DockerHub"),(0,r.yg)("admonition",{title:"DockerHub pull rates limitations",type:"warning"},(0,r.yg)("p",{parentName:"admonition"},"\u26a0\ufe0f ",(0,r.yg)("strong",{parentName:"p"},"DockerHub imposes strict pull limitations for clusters")," like the DSRI (using DockerHub might result in failing to pull your images on the DSRI). "),(0,r.yg)("p",{parentName:"admonition"},"We highly recommend to ",(0,r.yg)("strong",{parentName:"p"},"use the ",(0,r.yg)("a",{parentName:"strong",href:"https://docs.github.com/en/free-pro-team@latest/packages/getting-started-with-github-container-registry/about-github-container-registry"},"GitHub Container Registry")," or ",(0,r.yg)("a",{parentName:"strong",href:"https://quay.io/"},"RedHat quay.io Container Registry")," to publish public Docker images"),".")),(0,r.yg)("admonition",{title:"Logged in",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"If you are login with your DockerHub user on the DSRI, it should allow you to pull DockerHub images in your project (see above).")),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},"Create the repository on ",(0,r.yg)("a",{parentName:"li",href:"https://hub.docker.com/"},"DockerHub")," (attached to your user or an ",(0,r.yg)("a",{parentName:"li",href:"https://hub.docker.com/orgs/umids/repositories"},"organization"),")"),(0,r.yg)("li",{parentName:"ol"},"Build and push the image:")),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"docker build -t dockerhub-username/jupyterlab:latest .\ndocker push dockerhub-username/jupyterlab:latest\n")),(0,r.yg)("p",null,"You can also change the name (aka. tag) of an existing image:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"docker build -t my-jupyterlab .\ndocker tag my-jupyterlab ghcr.io/github-username/jupyterlab:latest\n")),(0,r.yg)("h3",{id:"use-automated-workflows"},"Use automated workflows"),(0,r.yg)("p",null,"You can automate the building and publication of Docker images using GitHub Actions workflows \ud83d\udd04"),(0,r.yg)("admonition",{title:"Use a working workflow as example",type:"tip"},(0,r.yg)("p",{parentName:"admonition"},"\ud83d\udc40 Check the ",(0,r.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/get-started-with-docker/blob/main/.github/workflows/publish-docker.yml"},".github/workflows/publish-docker.yml file")," to see an example of a workflow to publish an image to the GitHub Container Registry.")),(0,r.yg)("p",null,"\ud83d\udc69\u200d\ud83d\udcbb You only need to change the ",(0,r.yg)("inlineCode",{parentName:"p"},"IMAGE_NAME"),", and use it in your GitHub repository to publish a Docker image for your application automatically! It will build from a ",(0,r.yg)("inlineCode",{parentName:"p"},"Dockerfile")," at the root of the repository."),(0,r.yg)("admonition",{title:"Workflow triggers",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"The workflow can be easily configured to:"),(0,r.yg)("ul",{parentName:"admonition"},(0,r.yg)("li",{parentName:"ul"},"publish a new image to the ",(0,r.yg)("inlineCode",{parentName:"li"},"latest")," tag at each push to the main branch"),(0,r.yg)("li",{parentName:"ul"},"publish an image to a new tag if a release is pushed on GitHub (using the git tag)",(0,r.yg)("ul",{parentName:"li"},(0,r.yg)("li",{parentName:"ul"},"e.g. ",(0,r.yg)("inlineCode",{parentName:"li"},"v0.0.1")," published as image ",(0,r.yg)("inlineCode",{parentName:"li"},"0.0.1")))))))}y.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[4792],{5680:(e,t,a)=>{a.d(t,{xA:()=>g,yg:()=>y});var i=a(6540);function o(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function r(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);t&&(i=i.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,i)}return a}function n(e){for(var t=1;t=0||(o[a]=e[a]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(i=0;i=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(o[a]=e[a])}return o}var u=i.createContext({}),s=function(e){var t=i.useContext(u),a=t;return e&&(a="function"==typeof e?e(t):n(n({},t),e)),a},g=function(e){var t=s(e.components);return i.createElement(u.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return i.createElement(i.Fragment,{},t)}},c=i.forwardRef((function(e,t){var a=e.components,o=e.mdxType,r=e.originalType,u=e.parentName,g=l(e,["components","mdxType","originalType","parentName"]),c=s(a),y=o,m=c["".concat(u,".").concat(y)]||c[y]||p[y]||r;return a?i.createElement(m,n(n({ref:t},g),{},{components:a})):i.createElement(m,n({ref:t},g))}));function y(e,t){var a=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=a.length,n=new Array(r);n[0]=c;var l={};for(var u in t)hasOwnProperty.call(t,u)&&(l[u]=t[u]);l.originalType=e,l.mdxType="string"==typeof e?e:o,n[1]=l;for(var s=2;s{a.r(t),a.d(t,{assets:()=>g,contentTitle:()=>u,default:()=>y,frontMatter:()=>l,metadata:()=>s,toc:()=>p});var i=a(9668),o=a(1367),r=(a(6540),a(5680)),n=["components"],l={id:"guide-publish-image",title:"Publish a Docker image"},u=void 0,s={unversionedId:"guide-publish-image",id:"guide-publish-image",title:"Publish a Docker image",description:"\u26a0\ufe0f DockerHub imposes strict pull limitations for clusters like the DSRI (using DockerHub might result in failing to pull your images on the DSRI).",source:"@site/docs/guide-publish-image.md",sourceDirName:".",slug:"/guide-publish-image",permalink:"/docs/guide-publish-image",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/guide-publish-image.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"guide-publish-image",title:"Publish a Docker image"},sidebar:"docs",previous:{title:"Known Issues",permalink:"/docs/guide-known-issues"},next:{title:"Install UM VPN",permalink:"/docs/guide-vpn"}},g={},p=[{value:"Login to Container Registries \ud83d\udd11",id:"login-to-container-registries-",level:2},{value:"Login to GitHub Container Registry",id:"login-to-github-container-registry",level:3},{value:"Login to quay.io",id:"login-to-quayio",level:3},{value:"Login to DockerHub",id:"login-to-dockerhub",level:3},{value:"Publish your image \ud83d\udce2",id:"publish-your-image-",level:2},{value:"Publish to GitHub Container Registry",id:"publish-to-github-container-registry",level:3},{value:"Publish to Quay.io",id:"publish-to-quayio",level:3},{value:"Publish to DockerHub",id:"publish-to-dockerhub",level:3},{value:"Use automated workflows",id:"use-automated-workflows",level:3}],c={toc:p};function y(e){var t=e.components,a=(0,o.A)(e,n);return(0,r.yg)("wrapper",(0,i.A)({},c,a,{components:t,mdxType:"MDXLayout"}),(0,r.yg)("admonition",{title:"DockerHub pull rates limitations",type:"warning"},(0,r.yg)("p",{parentName:"admonition"},"\u26a0\ufe0f ",(0,r.yg)("strong",{parentName:"p"},"DockerHub imposes strict pull limitations for clusters")," like the DSRI (using DockerHub might result in failing to pull your images on the DSRI). "),(0,r.yg)("p",{parentName:"admonition"},"We highly recommend to ",(0,r.yg)("strong",{parentName:"p"},"use the ",(0,r.yg)("a",{parentName:"strong",href:"https://docs.github.com/en/free-pro-team@latest/packages/getting-started-with-github-container-registry/about-github-container-registry"},"GitHub Container Registry")," or ",(0,r.yg)("a",{parentName:"strong",href:"https://quay.io/"},"RedHat quay.io Container Registry")," to publish public Docker images"),".")),(0,r.yg)("blockquote",null,(0,r.yg)("p",{parentName:"blockquote"},"You can also login to DockerHub using a Secret in OpenShift to increase the pull rates limitations from 100 to 200 every 6 hours (this will mitigate the issue, but not solve it completely if you do not have a paid account on DockerHub):"),(0,r.yg)("pre",{parentName:"blockquote"},(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc create secret docker-registry docker-hub-secret --docker-server=docker.io --docker-username=your-dockerhub-username --docker-password=your-dockerhub-password --docker-email=your-dockerhub-email\n"))),(0,r.yg)("h2",{id:"login-to-container-registries-"},"Login to Container Registries \ud83d\udd11"),(0,r.yg)("h3",{id:"login-to-github-container-registry"},"Login to GitHub Container Registry"),(0,r.yg)("p",null,"Use your existing ",(0,r.yg)("a",{parentName:"p",href:"https://github.com"},"GitHub")," account if you have one:"),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},"Create a ",(0,r.yg)("strong",{parentName:"li"},"Personal Access Token")," for GitHub packages at ",(0,r.yg)("strong",{parentName:"li"},(0,r.yg)("a",{parentName:"strong",href:"https://github.com/settings/tokens/new"},"https://github.com/settings/tokens/new"))),(0,r.yg)("li",{parentName:"ol"},"Provide a meaningful description for the token, and enable the following scopes when creating the token:",(0,r.yg)("ul",{parentName:"li"},(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("inlineCode",{parentName:"li"},"write:packages"),": publish container images to GitHub Container Registry"),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("inlineCode",{parentName:"li"},"delete:packages"),": delete specified versions of private or public container images from GitHub Container Registry"))),(0,r.yg)("li",{parentName:"ol"},"You might want to store this token in a safe place, as you will not be able to retrieve it later on github.com (you can still delete it, and create a new token easily if you lose your token)"),(0,r.yg)("li",{parentName:"ol"},"\ud83d\udc68\u200d\ud83d\udcbb Log in to the GitHub Container Registry in your terminal (change ",(0,r.yg)("inlineCode",{parentName:"li"},"USERNAME")," and ",(0,r.yg)("inlineCode",{parentName:"li"},"ACCESS_TOKEN")," to yours):")),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},'echo "ACCESS_TOKEN" | docker login ghcr.io -u USERNAME --password-stdin\n')),(0,r.yg)("p",null,"On Windows use this command:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},'docker login ghcr.io -u USERNAME -p "ACCESS_TOKEN"\n')),(0,r.yg)("blockquote",null,(0,r.yg)("p",{parentName:"blockquote"},"See the ",(0,r.yg)("a",{parentName:"p",href:"https://docs.github.com/en/free-pro-team@latest/packages/using-github-packages-with-your-projects-ecosystem/configuring-docker-for-use-with-github-packages"},"official GitHub documentation"),".")),(0,r.yg)("h3",{id:"login-to-quayio"},"Login to quay.io"),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},"Create an account at ",(0,r.yg)("a",{parentName:"li",href:"https://quay.io"},"https://quay.io")," "),(0,r.yg)("li",{parentName:"ol"},"Login in your terminal (you will be asked for username and password)")),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"docker login quay.io\n")),(0,r.yg)("h3",{id:"login-to-dockerhub"},"Login to DockerHub"),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"Get a ",(0,r.yg)("a",{parentName:"p",href:"https://hub.docker.com/"},"DockerHub")," account at ",(0,r.yg)("a",{parentName:"p",href:"https://hub.docker.com"},"https://hub.docker.com")," (you most probably already have one if you installed Docker Desktop)")),(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"\ud83d\udc69\u200d\ud83d\udcbb Run in your terminal:"))),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"docker login\n")),(0,r.yg)("ol",{start:3},(0,r.yg)("li",{parentName:"ol"},"Provide your DockerHub username and password.")),(0,r.yg)("h2",{id:"publish-your-image-"},"Publish your image \ud83d\udce2"),(0,r.yg)("p",null,"Once you built a Docker image, and you logged in to a Container Registry, you might want to publish the image to pull and re-use it easily later."),(0,r.yg)("h3",{id:"publish-to-github-container-registry"},"Publish to GitHub Container Registry"),(0,r.yg)("admonition",{title:"Free for public images",type:"tip"},(0,r.yg)("p",{parentName:"admonition"},"The ",(0,r.yg)("a",{parentName:"p",href:"https://docs.github.com/en/free-pro-team@latest/packages/getting-started-with-github-container-registry"},"GitHub Container Registry")," is still in beta but will be free for public images when fully released. It enables you to store your Docker images at the same place you keep your code! \ud83d\udce6")),(0,r.yg)("p",null,"Publish to your user Container Registry on GitHub:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"docker build -t ghcr.io/github-username/my-image:latest .\ndocker push ghcr.io/github-username/my-image:latest\n")),(0,r.yg)("p",null,"For example, to the ",(0,r.yg)("a",{parentName:"p",href:"https://github.com/orgs/MaastrichtU-IDS/packages"},"MaastrichtU-IDS organization Container Registry on GitHub"),":"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"docker build -t ghcr.io/maastrichtu-ids/jupyterlab:latest .\ndocker push ghcr.io/maastrichtu-ids/jupyterlab:latest\n")),(0,r.yg)("admonition",{title:"Created automatically",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"If the image does not exist, GitHub Container Registry will create it automatically and set it as ",(0,r.yg)("strong",{parentName:"p"},"Private")," by default. You can easily change it to ",(0,r.yg)("strong",{parentName:"p"},"Public")," in the image settings on github.com.")),(0,r.yg)("h3",{id:"publish-to-quayio"},"Publish to Quay.io"),(0,r.yg)("admonition",{title:"Free for public images",type:"tip"},(0,r.yg)("p",{parentName:"admonition"},"Quay.io is free for public images and does not restrict images pulls.")),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"Create the image on ",(0,r.yg)("a",{parentName:"p",href:"https://quay.io/"},"quay.io"))),(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"Build and push to ",(0,r.yg)("a",{parentName:"p",href:"https://quay.io/"},"quay.io")))),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"docker build -t ghcr.io/quay-username/my-image:latest .\ndocker push quay.io/quay-username/my-image:latest\n")),(0,r.yg)("h3",{id:"publish-to-dockerhub"},"Publish to DockerHub"),(0,r.yg)("admonition",{title:"DockerHub pull rates limitations",type:"warning"},(0,r.yg)("p",{parentName:"admonition"},"\u26a0\ufe0f ",(0,r.yg)("strong",{parentName:"p"},"DockerHub imposes strict pull limitations for clusters")," like the DSRI (using DockerHub might result in failing to pull your images on the DSRI). "),(0,r.yg)("p",{parentName:"admonition"},"We highly recommend to ",(0,r.yg)("strong",{parentName:"p"},"use the ",(0,r.yg)("a",{parentName:"strong",href:"https://docs.github.com/en/free-pro-team@latest/packages/getting-started-with-github-container-registry/about-github-container-registry"},"GitHub Container Registry")," or ",(0,r.yg)("a",{parentName:"strong",href:"https://quay.io/"},"RedHat quay.io Container Registry")," to publish public Docker images"),".")),(0,r.yg)("admonition",{title:"Logged in",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"If you are login with your DockerHub user on the DSRI, it should allow you to pull DockerHub images in your project (see above).")),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},"Create the repository on ",(0,r.yg)("a",{parentName:"li",href:"https://hub.docker.com/"},"DockerHub")," (attached to your user or an ",(0,r.yg)("a",{parentName:"li",href:"https://hub.docker.com/orgs/umids/repositories"},"organization"),")"),(0,r.yg)("li",{parentName:"ol"},"Build and push the image:")),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"docker build -t dockerhub-username/jupyterlab:latest .\ndocker push dockerhub-username/jupyterlab:latest\n")),(0,r.yg)("p",null,"You can also change the name (aka. tag) of an existing image:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"docker build -t my-jupyterlab .\ndocker tag my-jupyterlab ghcr.io/github-username/jupyterlab:latest\n")),(0,r.yg)("h3",{id:"use-automated-workflows"},"Use automated workflows"),(0,r.yg)("p",null,"You can automate the building and publication of Docker images using GitHub Actions workflows \ud83d\udd04"),(0,r.yg)("admonition",{title:"Use a working workflow as example",type:"tip"},(0,r.yg)("p",{parentName:"admonition"},"\ud83d\udc40 Check the ",(0,r.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/get-started-with-docker/blob/main/.github/workflows/publish-docker.yml"},".github/workflows/publish-docker.yml file")," to see an example of a workflow to publish an image to the GitHub Container Registry.")),(0,r.yg)("p",null,"\ud83d\udc69\u200d\ud83d\udcbb You only need to change the ",(0,r.yg)("inlineCode",{parentName:"p"},"IMAGE_NAME"),", and use it in your GitHub repository to publish a Docker image for your application automatically! It will build from a ",(0,r.yg)("inlineCode",{parentName:"p"},"Dockerfile")," at the root of the repository."),(0,r.yg)("admonition",{title:"Workflow triggers",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"The workflow can be easily configured to:"),(0,r.yg)("ul",{parentName:"admonition"},(0,r.yg)("li",{parentName:"ul"},"publish a new image to the ",(0,r.yg)("inlineCode",{parentName:"li"},"latest")," tag at each push to the main branch"),(0,r.yg)("li",{parentName:"ul"},"publish an image to a new tag if a release is pushed on GitHub (using the git tag)",(0,r.yg)("ul",{parentName:"li"},(0,r.yg)("li",{parentName:"ul"},"e.g. ",(0,r.yg)("inlineCode",{parentName:"li"},"v0.0.1")," published as image ",(0,r.yg)("inlineCode",{parentName:"li"},"0.0.1")))))))}y.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/3f4d8b80.10fb0d58.js b/assets/js/3f4d8b80.10fb0d58.js new file mode 100644 index 000000000..519aa191d --- /dev/null +++ b/assets/js/3f4d8b80.10fb0d58.js @@ -0,0 +1 @@ +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[4190],{5680:(e,t,n)=>{n.d(t,{xA:()=>p,yg:()=>c});var a=n(6540);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function r(e){for(var t=1;t=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var s=a.createContext({}),u=function(e){var t=a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},p=function(e){var t=u(e.components);return a.createElement(s.Provider,{value:t},e.children)},h={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},y=a.forwardRef((function(e,t){var n=e.components,i=e.mdxType,o=e.originalType,s=e.parentName,p=l(e,["components","mdxType","originalType","parentName"]),y=u(n),c=i,g=y["".concat(s,".").concat(c)]||y[c]||h[c]||o;return n?a.createElement(g,r(r({ref:t},p),{},{components:n})):a.createElement(g,r({ref:t},p))}));function c(e,t){var n=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var o=n.length,r=new Array(o);r[0]=y;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:i,r[1]=l;for(var u=2;u{n.r(t),n.d(t,{assets:()=>p,contentTitle:()=>s,default:()=>c,frontMatter:()=>l,metadata:()=>u,toc:()=>h});var a=n(9668),i=n(1367),o=(n(6540),n(5680)),r=["components"],l={id:"deploy-jupyterhub",title:"JupyterHub"},s=void 0,u={unversionedId:"deploy-jupyterhub",id:"deploy-jupyterhub",title:"JupyterHub",description:"JupyterHub is ideal to enable multiple users easily start predefined workspaces in the same project.",source:"@site/docs/deploy-jupyterhub.md",sourceDirName:".",slug:"/deploy-jupyterhub",permalink:"/docs/deploy-jupyterhub",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/deploy-jupyterhub.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"deploy-jupyterhub",title:"JupyterHub"},sidebar:"docs",previous:{title:"Matlab",permalink:"/docs/deploy-matlab"},next:{title:"Deploy Dask Cluster",permalink:"/docs/dask-cluster"}},p={},h=[{value:"Downloading and adjusting the config-basic.yaml",id:"downloading-and-adjusting-the-config-basicyaml",level:2},{value:"Setting user's default persistent volume size",id:"setting-users-default-persistent-volume-size",level:3},{value:"Persistent volumes",id:"persistent-volumes",level:4},{value:"Configuring an authentication method",id:"configuring-an-authentication-method",level:3},{value:"Dummy authentication",id:"dummy-authentication",level:4},{value:"allow_users / admin_users authentication",id:"allow_users--admin_users-authentication",level:4},{value:"GitHub OAuth authentication",id:"github-oauth-authentication",level:4},{value:"Configure the notebook image",id:"configure-the-notebook-image",level:3},{value:"Extensive customization options in config-extensive.yaml",id:"extensive-customization-options-in-config-extensiveyaml",level:2},{value:"Deploying JupyterHub using the DSRI website \ud83e\ude90",id:"deploying-jupyterhub-using-the-dsri-website-",level:2},{value:"Installing the JupyterHub Helm Chart repository",id:"installing-the-jupyterhub-helm-chart-repository",level:3},{value:"Installing the JupyterHub Helm Chart",id:"installing-the-jupyterhub-helm-chart",level:3},{value:"Creating a secured route",id:"creating-a-secured-route",level:3},{value:"Upgrading the config-basic.yaml",id:"upgrading-the-config-basicyaml",level:3},{value:"Deploying JupyterHub using the Command Line Interface (CLI) \ud83e\ude90",id:"deploying-jupyterhub-using-the-command-line-interface-cli-",level:2},{value:"Installing the JupyterHub Helm Chart repository",id:"installing-the-jupyterhub-helm-chart-repository-1",level:3},{value:"Installing the JupyterHub Helm Chart",id:"installing-the-jupyterhub-helm-chart-1",level:3},{value:"Creating a secured route",id:"creating-a-secured-route-1",level:3},{value:"Upgrading the config-basic.yaml",id:"upgrading-the-config-basicyaml-1",level:3}],y={toc:h};function c(e){var t=e.components,n=(0,i.A)(e,r);return(0,o.yg)("wrapper",(0,a.A)({},y,n,{components:t,mdxType:"MDXLayout"}),(0,o.yg)("p",null,"JupyterHub is ideal to enable multiple users easily start predefined workspaces in the same project. "),(0,o.yg)("admonition",{title:"Experimental \ud83e\uddea ",type:"info"},(0,o.yg)("p",{parentName:"admonition"},"Deploying JupyterHub is still experimental, and it can be a bit tricky to configure. Feel free to ",(0,o.yg)("a",{parentName:"p",href:"https://servicedesk.icts.maastrichtuniversity.nl/tas/public/ssp/content/serviceflow?unid=1ffa93e9ecd94d938ad46e3cb24c2392"},"submit a ticket")," to ask for help.")),(0,o.yg)("h2",{id:"downloading-and-adjusting-the-config-basicyaml"},"Downloading and adjusting the config-basic.yaml"),(0,o.yg)("admonition",{title:"Before you begin download the config-basic.yaml",type:"warning"},(0,o.yg)("p",{parentName:"admonition"},"Download the preconfigured ",(0,o.yg)("inlineCode",{parentName:"p"},"config-basic.yaml")," from our ",(0,o.yg)("a",{parentName:"p",href:"https://raw.githubusercontent.com/MaastrichtU-IDS/dsri-documentation/refs/heads/master/applications/jupyterhub/config-basic.yaml"},"GitHub repository"),".\nThe default config that is provided by JupyterHub will not work. ")),(0,o.yg)("h3",{id:"setting-users-default-persistent-volume-size"},"Setting user's default persistent volume size"),(0,o.yg)("h4",{id:"persistent-volumes"},"Persistent volumes"),(0,o.yg)("p",null,"Persistent volumes are automatically created for each user and instance started in JupyterHub to ensure persistence of the data even if JupyterHub is stopped. You can find the persistent volumes in the DSRI web UI, go to the ",(0,o.yg)("strong",{parentName:"p"},"Administrator")," view > ",(0,o.yg)("strong",{parentName:"p"},"Storage")," > ",(0,o.yg)("strong",{parentName:"p"},"Persistent Volume Claims"),"."),(0,o.yg)("p",null,"It is possible to change the default size of a persistent volume claim for a user in the ",(0,o.yg)("inlineCode",{parentName:"p"},"config-basic.yaml"),". In our ",(0,o.yg)("inlineCode",{parentName:"p"},"config-basic.yaml")," the default value is ",(0,o.yg)("inlineCode",{parentName:"p"},"2Gi"),". However if you think that your users will need more storage space you can change this default size in the ",(0,o.yg)("inlineCode",{parentName:"p"},"config-basic.yaml"),". "),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"singleuser:\n # ...\n storage:\n capacity: 2Gi\n")),(0,o.yg)("h3",{id:"configuring-an-authentication-method"},"Configuring an authentication method"),(0,o.yg)("p",null,"At the moment we support three different authentication methods. One for testing purposes (dummy authentication), one for people who are working alone in a JupyterHub instance or with one or two collaborators (allowed_users / admin_users authentication), and one for allowing groups of people to collaborate in the same JupyterHub instance (GitHub OAuth). By default the dummy authentication is set in the ",(0,o.yg)("inlineCode",{parentName:"p"},"config-basic.yaml"),". ",(0,o.yg)("strong",{parentName:"p"},"Note that this is only for testing purposes!!!")," However, with very few changes to the ",(0,o.yg)("inlineCode",{parentName:"p"},"config-basic.yaml")," you can set up the other authentication methods. For reference see ",(0,o.yg)("a",{parentName:"p",href:"https://z2jh.jupyter.org/en/stable/administrator/authentication.html"},"the zero2jupyterhub documentation about authentication methods")),(0,o.yg)("h4",{id:"dummy-authentication"},"Dummy authentication"),(0,o.yg)("p",null,"This authentication method is set by default and is only there so that you can easily test your JupyterHub instance without the need of setting up proper authentication. The catch with this method is that whatever username/password combination you fill in, you will get access! In other words this is ",(0,o.yg)("strong",{parentName:"p"},"completely not safe to use in use cases other than testing!"),"\nIn the ",(0,o.yg)("inlineCode",{parentName:"p"},"config-basic.yaml")," you see -besides the commented out other authentication methods- the following block of text:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"hub:\n # ...\n config:\n JupyterHub:\n admin_access: true\n authenticator_class: dummy\n")),(0,o.yg)("p",null,"Some parts are intentionally left out here, shown as dots ",(0,o.yg)("inlineCode",{parentName:"p"},"# ...")," for better representation. If you are first setting up your JupyterHub instance you can leave this as is. Upon going to your instance via the URL you will get prompted with a login screen:"),(0,o.yg)("img",{src:"/img/jupyterhub-dummy-login.png",alt:"",style:{maxWidth:"75%",maxHeight:"75%"}}),(0,o.yg)("h1",{id:""}),(0,o.yg)("p",null,"Fill in any usernamer and password combination you would like and the user account will be made. Note that this user account literally is made and has its own pod in the project. It also has a persistent volume and all the other properties of any other user account created. However, you can use whatever password you want to access this account. In other words do not use this user actively and definitely do not store any (sensitive) data in this user account!"),(0,o.yg)("h4",{id:"allow_users--admin_users-authentication"},"allow_users / admin_users authentication"),(0,o.yg)("p",null,"If you will be working on your own in your JupyterHub instance it will be easiest to use the allow_users / admin_users authentication method. This method allows you to specify a user and admin account with a shared password. ",(0,o.yg)("strong",{parentName:"p"},"It is important that you keep this password a secret and safe! If people will get their hands on this they can acces your JupyterHub instance and login as an admin, which can lead to hefty consequences.")," "),(0,o.yg)("p",null,"If you want to make use of this config uncomment the following block of text and comment out the previous block of text seen at the ",(0,o.yg)("inlineCode",{parentName:"p"},"Dummy authentication")," section above:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"hub:\n # ...\n config:\n Authenticator:\n admin_users:\n - admin\n allowed_users:\n - user1\n DummyAuthenticator:\n password: a-shared-secret-password\n JupyterHub:\n authenticator_class: dummy\n")),(0,o.yg)("p",null,"Note that this password is in plaintext in your ",(0,o.yg)("inlineCode",{parentName:"p"},"config-basic.yaml"),". ",(0,o.yg)("strong",{parentName:"p"},"Do not use a password you use for other accounts, this is never a good idea and is surely not a good idea in this case!")," Unfortunately it is not possible to set passwords in JupyterHub using secrets in the DSRI at the moment. If you need to share your JupyterHub instance with others we recommend you to use the GitHub OAuth authentication method described below. "),(0,o.yg)("h4",{id:"github-oauth-authentication"},"GitHub OAuth authentication"),(0,o.yg)("p",null,"This authentication method is the most secure option we provide at the moment. The major caveat is that you and the people you want to collaborate with need a GitHub account. Moreover, you will need to create an organization and team within that organization, or have access to an organization and team. To set up an organization and team, please refer to GitHub's ",(0,o.yg)("a",{parentName:"p",href:"https://docs.github.com/en/organizations"},"documentation"),". You grant the people authorization to log in into the JupyterHub instance with their GitHub account by adding them to a team in an organization in GitHub."),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"hub:\n # ...\n config:\n GitHubOAuthenticator:\n client_id: your-client-id\n client_secret: your-client-secret\n oauth_callback_url: https://-.apps.dsri2.unimaas.nl/hub/oauth_callback\n JupyterHub:\n authenticator_class: github\n")),(0,o.yg)("p",null,"For creating an OAuth app in GitHub please refer to GitHub's ",(0,o.yg)("a",{parentName:"p",href:"https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/creating-an-oauth-app"},"documentation"),". The GitHub OAuth app will provide the client ID and client secret. Fill in the ",(0,o.yg)("inlineCode",{parentName:"p"},"")," and ",(0,o.yg)("inlineCode",{parentName:"p"},"")," at the ",(0,o.yg)("inlineCode",{parentName:"p"},"oauth_callback_url")," section. To set up a route to get your ",(0,o.yg)("inlineCode",{parentName:"p"},"")," see the following section: ",(0,o.yg)("a",{parentName:"p",href:"https://dsri.maastrichtuniversity.nl/docs/deploy-jupyterhub#creating-a-secured-route"},"Creating a secured route using the DSRI website"),", or ",(0,o.yg)("a",{parentName:"p",href:"https://dsri.maastrichtuniversity.nl/docs/deploy-jupyterhub#creating-a-secured-route-1"},"Creating a secured route using the CLI"),". Note that you can change the ",(0,o.yg)("inlineCode",{parentName:"p"},"")," at a later moment by upgrading the ",(0,o.yg)("inlineCode",{parentName:"p"},"config-basic.yaml"),". "),(0,o.yg)("h3",{id:"configure-the-notebook-image"},"Configure the notebook image"),(0,o.yg)("p",null,"The default notebook image used in the ",(0,o.yg)("inlineCode",{parentName:"p"},"config-basic.yaml")," is the ",(0,o.yg)("inlineCode",{parentName:"p"},"k8s-singleuser-sample")," image, version ",(0,o.yg)("inlineCode",{parentName:"p"},"3.3.8"),"."),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre"},"singleuser:\n #...\n image:\n name: quay.io/jupyterhub/k8s-singleuser-sample\n pullPolicy: null\n pullSecrets: []\n tag: 3.3.8\n")),(0,o.yg)("p",null,"It is, however, possible to change this notebook image to configure the JupyterHub instance to start up user pods with a customized image, which could have various things installed such as different kernels, packages and/or extensions.\nFor more information about different images provided by Jupyter, please refer to their ",(0,o.yg)("a",{parentName:"p",href:"https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html"},"documentation"),"."),(0,o.yg)("p",null,"For example you can change the notebook image to their Tensorflow image, which comes with Tensorflow pre-installed!"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre"},"singleuser\n #...\n image:\n name: quay.io/jupyter/tensorflow-notebook\n pullPolicy: null\n pullSecrets: []\n tag: 87b37b4fd818\n")),(0,o.yg)("p",null,"Note that we chose the latest tag at the time of writing: ",(0,o.yg)("inlineCode",{parentName:"p"},"87b37b4fd818"),". Change this tag accordingly if a more recent release is available! You can find their releases via their Quay.io repository: ",(0,o.yg)("a",{parentName:"p",href:"https://quay.io/organization/jupyter"},"https://quay.io/organization/jupyter"),"."),(0,o.yg)("h1",{id:"-1"}),(0,o.yg)("h2",{id:"extensive-customization-options-in-config-extensiveyaml"},"Extensive customization options in config-extensive.yaml"),(0,o.yg)("p",null,"It is possible to customize your JupyterHub instance further. For example we created a ",(0,o.yg)("inlineCode",{parentName:"p"},"config-extensive.yaml")," file which has some customization options configured."),(0,o.yg)("admonition",{title:"Download the config-extensive.yaml",type:"info"},(0,o.yg)("p",{parentName:"admonition"},"Download the preconfigured ",(0,o.yg)("inlineCode",{parentName:"p"},"config-extensive.yaml")," from our ",(0,o.yg)("a",{parentName:"p",href:"https://raw.githubusercontent.com/MaastrichtU-IDS/dsri-documentation/refs/heads/master/applications/jupyterhub/config-extensive.yaml"},"GitHub repository"),". ")),(0,o.yg)("p",null,"In this configuration we disabled culling so that user pods will not be scaled down after a time of inactivity. This can easily be activated again changing ",(0,o.yg)("inlineCode",{parentName:"p"},"cull: > enabled: false")," to ",(0,o.yg)("inlineCode",{parentName:"p"},"true"),"."),(0,o.yg)("p",null,"Next, we extended the startup command to install a Python package ",(0,o.yg)("inlineCode",{parentName:"p"},"papersize")," upon user pod creation. Additionally, we pull a public GitHub repository, if it does not exist already, to the ",(0,o.yg)("inlineCode",{parentName:"p"},"/home/jovyan/materials")," directory. This directory will automatically be made if it does not exist already. "),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre"},"singleuser:\n #...\n # Remove the git clone part or fill in a valid URL, otherwise the user-pod will not start!\n cmd:\n - /bin/bash\n - '-c'\n - >-\n pip install papersize && [ \"$(ls -A /home/jovyan/materials 2>/dev/null)\" ]\n || git clone https://github.com/EbookFoundation/free-programming-books.git\n /home/jovyan/materials && exec jupyterhub-singleuser\n")),(0,o.yg)("p",null,"Finally, we chose to include two additional notebook images. Besides the minimal image, we include the data science and Tensorflow images. These notebook images come with other kernels, packages and extensions installed. For more information about different images provided by Jupyter, please refer to their ",(0,o.yg)("a",{parentName:"p",href:"https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html"},"documentation"),"."),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre"},'image:\n name: quay.io/jupyter/minimal-notebook\n pullPolicy: null\n pullSecrets: []\n tag: 87b37b4fd818\n profileList:\n - display_name: "Minimal environment"\n description: "To avoid too much bells and whistles: Python."\n default: true\n - display_name: "Datascience environment"\n description: "If you want the additional bells and whistles: Python, R, and Julia."\n kubespawner_override:\n image: quay.io/jupyter/datascience-notebook:87b37b4fd818\n - display_name: "Tensorflow environment"\n description: "Here you have Tensorflow installed!"\n kubespawner_override:\n image: quay.io/jupyter/tensorflow-notebook:87b37b4fd818\n')),(0,o.yg)("p",null,"Note that we chose the latest tag at the time of writing: ",(0,o.yg)("inlineCode",{parentName:"p"},"87b37b4fd818"),". Change this tag accordingly if a more recent release is available! You can find their releases via their Quay.io repository: ",(0,o.yg)("a",{parentName:"p",href:"https://quay.io/organization/jupyter"},"https://quay.io/organization/jupyter"),"."),(0,o.yg)("p",null,"Upon the first creation of the user pod, in other words when the user logs in for the first time. They will see a menu where they can choose their preconfigured notebook by choice. "),(0,o.yg)("img",{src:"/img/jupyterhub-notebookchoice-login.png",alt:"",style:{maxWidth:"75%",maxHeight:"75%"}}),(0,o.yg)("h1",{id:"-2"}),(0,o.yg)("h2",{id:"deploying-jupyterhub-using-the-dsri-website-"},"Deploying JupyterHub using the DSRI website \ud83e\ude90"),(0,o.yg)("admonition",{title:"Before you begin download the config-basic.yaml",type:"warning"},(0,o.yg)("p",{parentName:"admonition"},"Download the preconfigured ",(0,o.yg)("inlineCode",{parentName:"p"},"config-basic.yaml")," from our ",(0,o.yg)("a",{parentName:"p",href:"https://raw.githubusercontent.com/MaastrichtU-IDS/dsri-documentation/refs/heads/master/applications/jupyterhub/config-basic.yaml"},"GitHub repository"),".\nThe default config that is provided by JupyterHub will not work. ")),(0,o.yg)("h3",{id:"installing-the-jupyterhub-helm-chart-repository"},"Installing the JupyterHub Helm Chart repository"),(0,o.yg)("p",null,"After you have created a project you can start with installing the JupyterHub Helm Chart. If you do not have access to DSRI or created a project yet, and you need to find out how, please refer to our ",(0,o.yg)("a",{parentName:"p",href:"https://dsri.maastrichtuniversity.nl/docs/"},"documentation.")),(0,o.yg)("admonition",{title:"Helm Chart already available",type:"info"},(0,o.yg)("p",{parentName:"admonition"},"The Helm Chart should be already made available for everyone to use on the DSRI platform. There will be no need to install the repository yourself.")),(0,o.yg)("h1",{id:"-3"}),(0,o.yg)("p",null,"In ",(0,o.yg)("inlineCode",{parentName:"p"},"Developer")," mode in your project, go to ",(0,o.yg)("inlineCode",{parentName:"p"},"Helm")," in the sidepanel (1). Next, click on ",(0,o.yg)("inlineCode",{parentName:"p"},"Create")," and choose ",(0,o.yg)("inlineCode",{parentName:"p"},"Repository")," (2)."),(0,o.yg)("img",{src:"/img/jupyterhub-helm-chart-repo-add-one.png",alt:"",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("h1",{id:"-4"}),(0,o.yg)("p",null,"Then fill in the ",(0,o.yg)("inlineCode",{parentName:"p"},"Name"),", ",(0,o.yg)("inlineCode",{parentName:"p"},"Display Name"),", give it a ",(0,o.yg)("inlineCode",{parentName:"p"},"Description")," and fill in the ",(0,o.yg)("inlineCode",{parentName:"p"},"URL"),": ",(0,o.yg)("a",{parentName:"p",href:"https://hub.jupyter.org/helm-chart/"},"https://hub.jupyter.org/helm-chart/"),". "),(0,o.yg)("h1",{id:"-5"}),(0,o.yg)("p",null,"Next, click ",(0,o.yg)("inlineCode",{parentName:"p"},"Create"),". "),(0,o.yg)("img",{src:"/img/jupyterhub-helm-chart-repo-add-two.png",alt:"",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("h1",{id:"-6"}),(0,o.yg)("h3",{id:"installing-the-jupyterhub-helm-chart"},"Installing the JupyterHub Helm Chart"),(0,o.yg)("admonition",{type:"info"},(0,o.yg)("p",{parentName:"admonition"},"At the moment the latest -and only- Helm Chart version which is supported by DSRI is version 3.3.8. Newer versions will not work, and older versions are not tested and/or configured!")),(0,o.yg)("h1",{id:"-7"}),(0,o.yg)("p",null,"In ",(0,o.yg)("inlineCode",{parentName:"p"},"Developer")," mode in your project, go to ",(0,o.yg)("inlineCode",{parentName:"p"},"Helm")," in the sidepanel (1). Next, click on ",(0,o.yg)("inlineCode",{parentName:"p"},"Create")," and choose ",(0,o.yg)("inlineCode",{parentName:"p"},"Helm Release")," (2)"),(0,o.yg)("img",{src:"/img/jupyterhub-helm-chart-install-one.png",alt:"",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("h1",{id:"-8"}),(0,o.yg)("p",null,"Search for ",(0,o.yg)("inlineCode",{parentName:"p"},"jupyterhub")," (or the name you gave the repository if you added the repository yourself), and choose the ",(0,o.yg)("inlineCode",{parentName:"p"},"JupyterHub")," Helm Chart (1)."),(0,o.yg)("img",{src:"/img/jupyterhub-helm-chart-install-two.png",alt:"",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("h1",{id:"-9"}),(0,o.yg)("p",null,"Click ",(0,o.yg)("inlineCode",{parentName:"p"},"Create"),"."),(0,o.yg)("img",{src:"/img/jupyterhub-helm-chart-install-three.png",alt:"",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("h1",{id:"-10"}),(0,o.yg)("p",null,"Click the ",(0,o.yg)("inlineCode",{parentName:"p"},"Chart version")," drop down menu (1)."),(0,o.yg)("img",{src:"/img/jupyterhub-helm-chart-install-four.png",alt:"",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("h1",{id:"-11"}),(0,o.yg)("p",null,"And choose the right Chart version: ",(0,o.yg)("inlineCode",{parentName:"p"},"3.3.8")," (1). Note that this is an important step, as we only support version 3.3.8 at the moment. Newer versions do not work yet and older versions we did not configure and/or test! "),(0,o.yg)("img",{src:"/img/jupyterhub-helm-chart-install-five.png",alt:"",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("h1",{id:"-12"}),(0,o.yg)("p",null,"Now, change the config with the content of the ",(0,o.yg)("inlineCode",{parentName:"p"},"config-basic.yaml")," you have downloaded from our ",(0,o.yg)("a",{parentName:"p",href:"https://raw.githubusercontent.com/MaastrichtU-IDS/dsri-documentation/refs/heads/master/applications/jupyterhub/config-basic.yaml"},"GitHub repository"),".\nCopy the content of the ",(0,o.yg)("inlineCode",{parentName:"p"},"config-basic.yaml")," and paste it in the highlighted box to replace the old with the new config. Click ",(0,o.yg)("inlineCode",{parentName:"p"},"Create")," to install the JupyterHub Helm Chart."),(0,o.yg)("img",{src:"/img/jupyterhub-helm-chart-install-six.png",alt:"",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("h1",{id:"-13"}),(0,o.yg)("h3",{id:"creating-a-secured-route"},"Creating a secured route"),(0,o.yg)("p",null,"Create a secured route, with TLS edge termination."),(0,o.yg)("p",null,"In ",(0,o.yg)("inlineCode",{parentName:"p"},"Developer")," mode in your project, go to ",(0,o.yg)("inlineCode",{parentName:"p"},"Project")," in the sidepanel (1). Next, click on ",(0,o.yg)("inlineCode",{parentName:"p"},"Route")," (2)."),(0,o.yg)("img",{src:"/img/jupyterhub-helm-chart-route-one.png",alt:"",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("h1",{id:"-14"}),(0,o.yg)("p",null,"Next, click ",(0,o.yg)("inlineCode",{parentName:"p"},"Create"),"."),(0,o.yg)("img",{src:"/img/jupyterhub-helm-chart-route-two.png",alt:"",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("h1",{id:"-15"}),(0,o.yg)("p",null,"Fill in the ",(0,o.yg)("inlineCode",{parentName:"p"},"Name")," (1), choose the ",(0,o.yg)("inlineCode",{parentName:"p"},"Service"),": ",(0,o.yg)("inlineCode",{parentName:"p"},"proxy-public")," (2), choose the ",(0,o.yg)("inlineCode",{parentName:"p"},"Target Port"),": ",(0,o.yg)("inlineCode",{parentName:"p"},"80 -> http (TCP)")," (3), tick the box ",(0,o.yg)("inlineCode",{parentName:"p"},"Secure Route")," (4), and finally choose ",(0,o.yg)("inlineCode",{parentName:"p"},"TLS Termination"),": ",(0,o.yg)("inlineCode",{parentName:"p"},"Edge")," (5). Next, click ",(0,o.yg)("inlineCode",{parentName:"p"},"Create"),", to create the route."),(0,o.yg)("img",{src:"/img/jupyterhub-helm-chart-route-three.png",alt:"",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("h3",{id:"upgrading-the-config-basicyaml"},"Upgrading the config-basic.yaml"),(0,o.yg)("p",null,"You can upgrade your ",(0,o.yg)("inlineCode",{parentName:"p"},"config-basic.yaml")," easily in the DSRI web UI if you would like to change certain settings, such as user's default persistent volume claims, authentication methods, and many more things. Note that in some cases users who created an account with an old authentication method will still have access via that method, make sure you set up your preferred authentication method before allowing users to authenticate and use the JupyterHub instance."),(0,o.yg)("h1",{id:"-16"}),(0,o.yg)("p",null,"In ",(0,o.yg)("inlineCode",{parentName:"p"},"Developer")," mode in your project, go to ",(0,o.yg)("inlineCode",{parentName:"p"},"Helm")," in the sidepanel (1). Next, click on your Helm Chart Release (2)."),(0,o.yg)("img",{src:"/img/jupyterhub-helm-chart-upgrade-chart-one.png",alt:"",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("h1",{id:"-17"}),(0,o.yg)("p",null,"Now, click the ",(0,o.yg)("inlineCode",{parentName:"p"},"Actions")," drop down menu, and choose ",(0,o.yg)("inlineCode",{parentName:"p"},"Upgrade")," (1)."),(0,o.yg)("img",{src:"/img/jupyterhub-helm-chart-upgrade-chart-two.png",alt:"",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("h1",{id:"-18"}),(0,o.yg)("p",null,"In the box -highlighted in the picutre below- you can make changes to the config-basic.yaml. After you have made your changes, click ",(0,o.yg)("inlineCode",{parentName:"p"},"Upgrade")," and your upgraded JupyterHub Helm Chart Release will automatically be deployed."),(0,o.yg)("img",{src:"/img/jupyterhub-helm-chart-upgrade-chart-three.png",alt:"",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("h1",{id:"-19"}),(0,o.yg)("admonition",{title:"Configure JupyterHub",type:"success"},(0,o.yg)("p",{parentName:"admonition"},"Feel free to ",(0,o.yg)("a",{parentName:"p",href:"https://servicedesk.icts.maastrichtuniversity.nl/tas/public/ssp/content/serviceflow?unid=1ffa93e9ecd94d938ad46e3cb24c2392"},"submit a ticket")," to ask for help configuring your JupyterHub.")),(0,o.yg)("h1",{id:"-20"}),(0,o.yg)("h2",{id:"deploying-jupyterhub-using-the-command-line-interface-cli-"},"Deploying JupyterHub using the Command Line Interface (CLI) \ud83e\ude90"),(0,o.yg)("admonition",{title:"Before you begin download the config-basic.yaml",type:"warning"},(0,o.yg)("p",{parentName:"admonition"},"Download the preconfigured ",(0,o.yg)("inlineCode",{parentName:"p"},"config-basic.yaml")," from our ",(0,o.yg)("a",{parentName:"p",href:"https://raw.githubusercontent.com/MaastrichtU-IDS/dsri-documentation/refs/heads/master/applications/jupyterhub/config-basic.yaml"},"GitHub repository"),".\nThe default config that is provided by JupyterHub will not work. ")),(0,o.yg)("h3",{id:"installing-the-jupyterhub-helm-chart-repository-1"},"Installing the JupyterHub Helm Chart repository"),(0,o.yg)("p",null,"After you have created a project you can start with installing the JupyterHub Helm Chart. If you do not have access to DSRI or created a project yet, and you need to find out how, please refer to our ",(0,o.yg)("a",{parentName:"p",href:"https://dsri.maastrichtuniversity.nl/docs/"},"documentation.")),(0,o.yg)("admonition",{title:"Helm Chart already available",type:"info"},(0,o.yg)("p",{parentName:"admonition"},"The Helm Chart should be already made available for everyone to use on the DSRI platform. There will be no need to install the repository yourself.")),(0,o.yg)("h1",{id:"-21"}),(0,o.yg)("p",null,"Add the JupyterHub Helm Chart repository:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"helm repo add jupyterhub https://hub.jupyter.org/helm-chart/\nhelm repo update\n")),(0,o.yg)("h3",{id:"installing-the-jupyterhub-helm-chart-1"},"Installing the JupyterHub Helm Chart"),(0,o.yg)("admonition",{type:"info"},(0,o.yg)("p",{parentName:"admonition"},"At the moment the latest -and only- Helm Chart version which is supported by DSRI is version 3.3.8. Newer versions will not work, and older versions are not tested and/or configured!")),(0,o.yg)("h1",{id:"-22"}),(0,o.yg)("p",null,"Make sure you use the right ",(0,o.yg)("inlineCode",{parentName:"p"},"config-basic.yaml")," downloaded from our ",(0,o.yg)("a",{parentName:"p",href:"https://raw.githubusercontent.com/MaastrichtU-IDS/dsri-documentation/refs/heads/master/applications/jupyterhub/config-basic.yaml"},"GitHub repository"),"."),(0,o.yg)("p",null,"Install the Helm Chart using the following command:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"helm upgrade --cleanup-on-fail \\\n --install jupyterhub jupyterhub/jupyterhub \\\n --version=3.3.8 \\\n --namespace= \\\n --values config-basic.yaml\n")),(0,o.yg)("p",null,(0,o.yg)("inlineCode",{parentName:"p"},"")," is the name of your project."),(0,o.yg)("h3",{id:"creating-a-secured-route-1"},"Creating a secured route"),(0,o.yg)("p",null,"Create a secured route, with TLS edge termination:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"oc create route edge --namespace --service=proxy-public --port=http\n")),(0,o.yg)("p",null,(0,o.yg)("inlineCode",{parentName:"p"},"")," is the name of your project.\n",(0,o.yg)("inlineCode",{parentName:"p"},"")," is the name of the route. "),(0,o.yg)("h3",{id:"upgrading-the-config-basicyaml-1"},"Upgrading the config-basic.yaml"),(0,o.yg)("p",null,"Run the following command with your new config-basic.yaml:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"helm upgrade --cleanup-on-fail \\\n --install jupyterhub jupyterhub/jupyterhub \\\n --version=3.3.8 \\\n --namespace= \\\n --values config-basic.yaml\n")),(0,o.yg)("p",null,(0,o.yg)("inlineCode",{parentName:"p"},"")," is the name of your project."),(0,o.yg)("h1",{id:"-23"}),(0,o.yg)("admonition",{title:"Configure JupyterHub",type:"success"},(0,o.yg)("p",{parentName:"admonition"},"Feel free to ",(0,o.yg)("a",{parentName:"p",href:"https://servicedesk.icts.maastrichtuniversity.nl/tas/public/ssp/content/serviceflow?unid=1ffa93e9ecd94d938ad46e3cb24c2392"},"submit a ticket")," to ask for help configuring your JupyterHub.")))}c.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/3f4d8b80.405a42ab.js b/assets/js/3f4d8b80.405a42ab.js deleted file mode 100644 index 678624c61..000000000 --- a/assets/js/3f4d8b80.405a42ab.js +++ /dev/null @@ -1 +0,0 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[4190],{5680:(e,t,n)=>{n.d(t,{xA:()=>p,yg:()=>d});var a=n(6540);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function r(e){for(var t=1;t=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var u=a.createContext({}),s=function(e){var t=a.useContext(u),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},p=function(e){var t=s(e.components);return a.createElement(u.Provider,{value:t},e.children)},h={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},y=a.forwardRef((function(e,t){var n=e.components,i=e.mdxType,o=e.originalType,u=e.parentName,p=l(e,["components","mdxType","originalType","parentName"]),y=s(n),d=i,g=y["".concat(u,".").concat(d)]||y[d]||h[d]||o;return n?a.createElement(g,r(r({ref:t},p),{},{components:n})):a.createElement(g,r({ref:t},p))}));function d(e,t){var n=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var o=n.length,r=new Array(o);r[0]=y;var l={};for(var u in t)hasOwnProperty.call(t,u)&&(l[u]=t[u]);l.originalType=e,l.mdxType="string"==typeof e?e:i,r[1]=l;for(var s=2;s{n.r(t),n.d(t,{assets:()=>p,contentTitle:()=>u,default:()=>d,frontMatter:()=>l,metadata:()=>s,toc:()=>h});var a=n(9668),i=n(1367),o=(n(6540),n(5680)),r=["components"],l={id:"deploy-jupyterhub",title:"JupyterHub"},u=void 0,s={unversionedId:"deploy-jupyterhub",id:"deploy-jupyterhub",title:"JupyterHub",description:"JupyterHub is ideal to enable multiple users easily start predefined workspaces in the same project.",source:"@site/docs/deploy-jupyterhub.md",sourceDirName:".",slug:"/deploy-jupyterhub",permalink:"/docs/deploy-jupyterhub",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/deploy-jupyterhub.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"deploy-jupyterhub",title:"JupyterHub"},sidebar:"docs",previous:{title:"Matlab",permalink:"/docs/deploy-matlab"},next:{title:"Deploy Dask Cluster",permalink:"/docs/dask-cluster"}},p={},h=[{value:"Downloading and adjusting the config.yaml",id:"downloading-and-adjusting-the-configyaml",level:2},{value:"Setting user's default persistent volume size",id:"setting-users-default-persistent-volume-size",level:3},{value:"Persistent volumes",id:"persistent-volumes",level:4},{value:"Configuring an authentication method",id:"configuring-an-authentication-method",level:3},{value:"Dummy authentication",id:"dummy-authentication",level:4},{value:"allow_users / admin_users authentication",id:"allow_users--admin_users-authentication",level:4},{value:"GitHub OAuth authentication",id:"github-oauth-authentication",level:4},{value:"Deploying JupyterHub using the DSRI website \ud83e\ude90",id:"deploying-jupyterhub-using-the-dsri-website-",level:2},{value:"Installing the JupyterHub Helm Chart repository",id:"installing-the-jupyterhub-helm-chart-repository",level:3},{value:"Installing the JupyterHub Helm Chart",id:"installing-the-jupyterhub-helm-chart",level:3},{value:"Creating a secured route",id:"creating-a-secured-route",level:3},{value:"Upgrading the config.yaml",id:"upgrading-the-configyaml",level:3},{value:"Deploying JupyterHub using the Command Line Interface (CLI) \ud83e\ude90",id:"deploying-jupyterhub-using-the-command-line-interface-cli-",level:2},{value:"Installing the JupyterHub Helm Chart repository",id:"installing-the-jupyterhub-helm-chart-repository-1",level:3},{value:"Installing the JupyterHub Helm Chart",id:"installing-the-jupyterhub-helm-chart-1",level:3},{value:"Creating a secured route",id:"creating-a-secured-route-1",level:3},{value:"Upgrading the config.yaml",id:"upgrading-the-configyaml-1",level:3}],y={toc:h};function d(e){var t=e.components,n=(0,i.A)(e,r);return(0,o.yg)("wrapper",(0,a.A)({},y,n,{components:t,mdxType:"MDXLayout"}),(0,o.yg)("p",null,"JupyterHub is ideal to enable multiple users easily start predefined workspaces in the same project. "),(0,o.yg)("admonition",{title:"Experimental \ud83e\uddea ",type:"info"},(0,o.yg)("p",{parentName:"admonition"},"Deploying JupyterHub is still experimental, and it can be a bit tricky to configure. Feel free to ",(0,o.yg)("a",{parentName:"p",href:"https://servicedesk.icts.maastrichtuniversity.nl/tas/public/ssp/content/serviceflow?unid=1ffa93e9ecd94d938ad46e3cb24c2392"},"submit a ticket")," to ask for help.")),(0,o.yg)("h2",{id:"downloading-and-adjusting-the-configyaml"},"Downloading and adjusting the config.yaml"),(0,o.yg)("admonition",{title:"Before you begin download the config.yaml",type:"warning"},(0,o.yg)("p",{parentName:"admonition"},"Download the preconfigured ",(0,o.yg)("inlineCode",{parentName:"p"},"config.yaml")," from our ",(0,o.yg)("a",{parentName:"p",href:"https://raw.githubusercontent.com/MaastrichtU-IDS/dsri-documentation/refs/heads/master/applications/jupyterhub/config.yaml"},"GitHub repository"),".\nThe default config that is provided by JupyterHub will not work. ")),(0,o.yg)("h1",{id:""}),(0,o.yg)("h3",{id:"setting-users-default-persistent-volume-size"},"Setting user's default persistent volume size"),(0,o.yg)("h4",{id:"persistent-volumes"},"Persistent volumes"),(0,o.yg)("p",null,"Persistent volumes are automatically created for each user and instance started in JupyterHub to ensure persistence of the data even if JupyterHub is stopped. You can find the persistent volumes in the DSRI web UI, go to the ",(0,o.yg)("strong",{parentName:"p"},"Administrator")," view > ",(0,o.yg)("strong",{parentName:"p"},"Storage")," > ",(0,o.yg)("strong",{parentName:"p"},"Persistent Volume Claims"),"."),(0,o.yg)("p",null,"It is possible to change the default size of a persistent volume claim for a user in the ",(0,o.yg)("inlineCode",{parentName:"p"},"config.yaml"),". In our ",(0,o.yg)("inlineCode",{parentName:"p"},"config.yaml")," the default value is ",(0,o.yg)("inlineCode",{parentName:"p"},"2Gi"),". However if you think that your users will need more storage space you can change this default size in the ",(0,o.yg)("inlineCode",{parentName:"p"},"config.yaml"),". "),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"singleuser:\n # ...\n storage:\n capacity: 2Gi\n")),(0,o.yg)("h3",{id:"configuring-an-authentication-method"},"Configuring an authentication method"),(0,o.yg)("p",null,"At the moment we support three different authentication methods. One for testing purposes (dummy authentication), one for people who are working alone in a JupyterHub instance or with one or two collaborators (allowed_users / admin_users authentication), and one for allowing groups of people to collaborate in the same JupyterHub instance (GitHub OAuth). By default the dummy authentication is set in the ",(0,o.yg)("inlineCode",{parentName:"p"},"config.yaml"),". ",(0,o.yg)("strong",{parentName:"p"},"Note that this is only for testing purposes!!!")," However, with very few changes to the ",(0,o.yg)("inlineCode",{parentName:"p"},"config.yaml")," you can set up the other authentication methods. For reference see ",(0,o.yg)("a",{parentName:"p",href:"https://z2jh.jupyter.org/en/stable/administrator/authentication.html"},"the zero2jupyterhub documentation about authentication methods")),(0,o.yg)("h4",{id:"dummy-authentication"},"Dummy authentication"),(0,o.yg)("p",null,"This authentication method is set by default and is only there so that you can easily test your JupyterHub instance without the need of setting up proper authentication. The catch with this method is that whatever username/password combination you fill in, you will get access! In other words this is ",(0,o.yg)("strong",{parentName:"p"},"completely not safe to use in use cases other than testing!"),"\nIn the ",(0,o.yg)("inlineCode",{parentName:"p"},"config.yaml")," you see -besides the commented out other authentication methods- the following block of text:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"hub:\n # ...\n config:\n JupyterHub:\n admin_access: true\n authenticator_class: dummy\n")),(0,o.yg)("p",null,"Some parts are intentionally left out here, shown as dots ",(0,o.yg)("inlineCode",{parentName:"p"},"# ...")," for better representation. If you are first setting up your JupyterHub instance you can leave this as is. Upon going to your instance via the URL you will get prompted with a login screen:"),(0,o.yg)("img",{src:"/img/jupyterhub-dummy-login.png",alt:"",style:{maxWidth:"75%",maxHeight:"75%"}}),(0,o.yg)("h1",{id:"-1"}),(0,o.yg)("p",null,"Fill in any usernamer and password combination you would like and the user account will be made. Note that this user account literally is made and has its own pod in the project. It also has a persistent volume and all the other properties of any other user account created. However, you can use whatever password you want to access this account. In other words do not use this user actively and definitely do not store any (sensitive) data in this user account!"),(0,o.yg)("h4",{id:"allow_users--admin_users-authentication"},"allow_users / admin_users authentication"),(0,o.yg)("p",null,"If you will be working on your own in your JupyterHub instance it will be easiest to use the allow_users / admin_users authentication method. This method allows you to specify a user and admin account with a shared password. ",(0,o.yg)("strong",{parentName:"p"},"It is important that you keep this password a secret and safe! If people will get their hands on this they can acces your JupyterHub instance and login as an admin, which can lead to hefty consequences.")," "),(0,o.yg)("p",null,"If you want to make use of this config uncomment the following block of text and comment out the previous block of text seen at the ",(0,o.yg)("inlineCode",{parentName:"p"},"Dummy authentication")," section above:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"hub:\n # ...\n config:\n Authenticator:\n admin_users:\n - admin\n allowed_users:\n - user1\n DummyAuthenticator:\n password: a-shared-secret-password\n JupyterHub:\n authenticator_class: dummy\n")),(0,o.yg)("p",null,"Note that this password is in plaintext in your ",(0,o.yg)("inlineCode",{parentName:"p"},"config.yaml"),". ",(0,o.yg)("strong",{parentName:"p"},"Do not use a password you use for other accounts, this is never a good idea and is surely not a good idea in this case!")," Unfortunately it is not possible to set passwords in JupyterHub using secrets in the DSRI at the moment. If you need to share your JupyterHub instance with others we recommend you to use the GitHub OAuth authentication method described below. "),(0,o.yg)("h4",{id:"github-oauth-authentication"},"GitHub OAuth authentication"),(0,o.yg)("p",null,"This authentication method is the most secure option we provide at the moment. The major caveat is that you and the people you want to collaborate with need a GitHub account. Moreover, you will need to create an organization and team within that organization, or have access to an organization and team. To set up an organization and team, please refer to GitHub's ",(0,o.yg)("a",{parentName:"p",href:"https://docs.github.com/en/organizations"},"documentation"),". You grant the people authorization to log in into the JupyterHub instance with their GitHub account by adding them to a team in an organization in GitHub."),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"hub:\n # ...\n config:\n GitHubOAuthenticator:\n client_id: your-client-id\n client_secret: your-client-secret\n oauth_callback_url: https://-.apps.dsri2.unimaas.nl/hub/oauth_callback\n JupyterHub:\n authenticator_class: github\n")),(0,o.yg)("p",null,"For creating an OAuth app in GitHub please refer to GitHub's ",(0,o.yg)("a",{parentName:"p",href:"https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/creating-an-oauth-app"},"documentation"),". The GitHub OAuth app will provide the client ID and client secret. Fill in the ",(0,o.yg)("inlineCode",{parentName:"p"},"")," and ",(0,o.yg)("inlineCode",{parentName:"p"},"")," at the ",(0,o.yg)("inlineCode",{parentName:"p"},"oauth_callback_url")," section. To set up a route to get your ",(0,o.yg)("inlineCode",{parentName:"p"},"")," see the following section: ",(0,o.yg)("a",{parentName:"p",href:"https://dsri.maastrichtuniversity.nl/docs/deploy-jupyterhub#creating-a-secured-route"},"Creating a secured route using the DSRI website"),", or ",(0,o.yg)("a",{parentName:"p",href:"https://dsri.maastrichtuniversity.nl/docs/deploy-jupyterhub#creating-a-secured-route-1"},"Creating a secured route using the CLI"),". Note that you can change the ",(0,o.yg)("inlineCode",{parentName:"p"},"")," at a later moment by upgrading the ",(0,o.yg)("inlineCode",{parentName:"p"},"config.yaml"),". "),(0,o.yg)("h1",{id:"-2"}),(0,o.yg)("h2",{id:"deploying-jupyterhub-using-the-dsri-website-"},"Deploying JupyterHub using the DSRI website \ud83e\ude90"),(0,o.yg)("admonition",{title:"Before you begin download the config.yaml",type:"warning"},(0,o.yg)("p",{parentName:"admonition"},"Download the preconfigured ",(0,o.yg)("inlineCode",{parentName:"p"},"config.yaml")," from our ",(0,o.yg)("a",{parentName:"p",href:"https://raw.githubusercontent.com/MaastrichtU-IDS/dsri-documentation/refs/heads/master/applications/jupyterhub/config.yaml"},"GitHub repository"),".\nThe default config that is provided by JupyterHub will not work. ")),(0,o.yg)("h3",{id:"installing-the-jupyterhub-helm-chart-repository"},"Installing the JupyterHub Helm Chart repository"),(0,o.yg)("p",null,"After you have created a project you can start with installing the JupyterHub Helm Chart. If you do not have access to DSRI or created a project yet, and you need to find out how, please refer to our ",(0,o.yg)("a",{parentName:"p",href:"https://dsri.maastrichtuniversity.nl/docs/"},"documentation.")),(0,o.yg)("admonition",{title:"Helm Chart already available",type:"info"},(0,o.yg)("p",{parentName:"admonition"},"The Helm Chart should be already made available for everyone to use on the DSRI platform. There will be no need to install the repository yourself.")),(0,o.yg)("h1",{id:"-3"}),(0,o.yg)("p",null,"In ",(0,o.yg)("inlineCode",{parentName:"p"},"Developer")," mode in your project, go to ",(0,o.yg)("inlineCode",{parentName:"p"},"Helm")," in the sidepanel (1). Next, click on ",(0,o.yg)("inlineCode",{parentName:"p"},"Create")," and choose ",(0,o.yg)("inlineCode",{parentName:"p"},"Repository")," (2)."),(0,o.yg)("img",{src:"/img/jupyterhub-helm-chart-repo-add-one.png",alt:"",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("h1",{id:"-4"}),(0,o.yg)("p",null,"Then fill in the ",(0,o.yg)("inlineCode",{parentName:"p"},"Name"),", ",(0,o.yg)("inlineCode",{parentName:"p"},"Display Name"),", give it a ",(0,o.yg)("inlineCode",{parentName:"p"},"Description")," and fill in the ",(0,o.yg)("inlineCode",{parentName:"p"},"URL"),": ",(0,o.yg)("a",{parentName:"p",href:"https://hub.jupyter.org/helm-chart/"},"https://hub.jupyter.org/helm-chart/"),". "),(0,o.yg)("h1",{id:"-5"}),(0,o.yg)("p",null,"Next, click ",(0,o.yg)("inlineCode",{parentName:"p"},"Create"),". "),(0,o.yg)("img",{src:"/img/jupyterhub-helm-chart-repo-add-two.png",alt:"",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("h1",{id:"-6"}),(0,o.yg)("h3",{id:"installing-the-jupyterhub-helm-chart"},"Installing the JupyterHub Helm Chart"),(0,o.yg)("admonition",{type:"info"},(0,o.yg)("p",{parentName:"admonition"},"At the moment the latest -and only- Helm Chart version which is supported by DSRI is version 3.3.8. Newer versions will not work, and older versions are not tested and/or configured!")),(0,o.yg)("h1",{id:"-7"}),(0,o.yg)("p",null,"In ",(0,o.yg)("inlineCode",{parentName:"p"},"Developer")," mode in your project, go to ",(0,o.yg)("inlineCode",{parentName:"p"},"Helm")," in the sidepanel (1). Next, click on ",(0,o.yg)("inlineCode",{parentName:"p"},"Create")," and choose ",(0,o.yg)("inlineCode",{parentName:"p"},"Helm Release")," (2)"),(0,o.yg)("img",{src:"/img/jupyterhub-helm-chart-install-one.png",alt:"",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("h1",{id:"-8"}),(0,o.yg)("p",null,"Search for ",(0,o.yg)("inlineCode",{parentName:"p"},"jupyterhub")," (or the name you gave the repository if you added the repository yourself), and choose the ",(0,o.yg)("inlineCode",{parentName:"p"},"JupyterHub")," Helm Chart (1)."),(0,o.yg)("img",{src:"/img/jupyterhub-helm-chart-install-two.png",alt:"",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("h1",{id:"-9"}),(0,o.yg)("p",null,"Click ",(0,o.yg)("inlineCode",{parentName:"p"},"Create"),"."),(0,o.yg)("img",{src:"/img/jupyterhub-helm-chart-install-three.png",alt:"",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("h1",{id:"-10"}),(0,o.yg)("p",null,"Click the ",(0,o.yg)("inlineCode",{parentName:"p"},"Chart version")," drop down menu (1)."),(0,o.yg)("img",{src:"/img/jupyterhub-helm-chart-install-four.png",alt:"",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("h1",{id:"-11"}),(0,o.yg)("p",null,"And choose the right Chart version: ",(0,o.yg)("inlineCode",{parentName:"p"},"3.3.8")," (1). Note that this is an important step, as we only support version 3.3.8 at the moment. Newer versions do not work yet and older versions we did not configure and/or test! "),(0,o.yg)("img",{src:"/img/jupyterhub-helm-chart-install-five.png",alt:"",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("h1",{id:"-12"}),(0,o.yg)("p",null,"Now, change the config with the content of the ",(0,o.yg)("inlineCode",{parentName:"p"},"config.yaml")," you have downloaded from our ",(0,o.yg)("a",{parentName:"p",href:"https://raw.githubusercontent.com/MaastrichtU-IDS/dsri-documentation/refs/heads/master/applications/jupyterhub/config.yaml"},"GitHub repository"),".\nCopy the content of the ",(0,o.yg)("inlineCode",{parentName:"p"},"config.yaml")," and paste it in the highlighted box to replace the old with the new config. Click ",(0,o.yg)("inlineCode",{parentName:"p"},"Create")," to install the JupyterHub Helm Chart."),(0,o.yg)("img",{src:"/img/jupyterhub-helm-chart-install-six.png",alt:"",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("h1",{id:"-13"}),(0,o.yg)("h3",{id:"creating-a-secured-route"},"Creating a secured route"),(0,o.yg)("p",null,"Create a secured route, with TLS edge termination."),(0,o.yg)("p",null,"In ",(0,o.yg)("inlineCode",{parentName:"p"},"Developer")," mode in your project, go to ",(0,o.yg)("inlineCode",{parentName:"p"},"Project")," in the sidepanel (1). Next, click on ",(0,o.yg)("inlineCode",{parentName:"p"},"Route")," (2)."),(0,o.yg)("img",{src:"/img/jupyterhub-helm-chart-route-one.png",alt:"",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("h1",{id:"-14"}),(0,o.yg)("p",null,"Next, click ",(0,o.yg)("inlineCode",{parentName:"p"},"Create"),"."),(0,o.yg)("img",{src:"/img/jupyterhub-helm-chart-route-two.png",alt:"",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("h1",{id:"-15"}),(0,o.yg)("p",null,"Fill in the ",(0,o.yg)("inlineCode",{parentName:"p"},"Name")," (1), choose the ",(0,o.yg)("inlineCode",{parentName:"p"},"Service"),": ",(0,o.yg)("inlineCode",{parentName:"p"},"proxy-public")," (2), choose the ",(0,o.yg)("inlineCode",{parentName:"p"},"Target Port"),": ",(0,o.yg)("inlineCode",{parentName:"p"},"80 -> http (TCP)")," (3), tick the box ",(0,o.yg)("inlineCode",{parentName:"p"},"Secure Route")," (4), and finally choose ",(0,o.yg)("inlineCode",{parentName:"p"},"TLS Termination"),": ",(0,o.yg)("inlineCode",{parentName:"p"},"Edge")," (5). Next, click ",(0,o.yg)("inlineCode",{parentName:"p"},"Create"),", to create the route."),(0,o.yg)("img",{src:"/img/jupyterhub-helm-chart-route-three.png",alt:"",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("h3",{id:"upgrading-the-configyaml"},"Upgrading the config.yaml"),(0,o.yg)("p",null,"You can upgrade your ",(0,o.yg)("inlineCode",{parentName:"p"},"config.yaml")," easily in the DSRI web UI if you would like to change certain settings, such as user's default persistent volume claims, authentication methods, and many more things. Note that in some cases users who created an account with an old authentication method will still have access via that method, make sure you set up your preferred authentication method before allowing users to authenticate and use the JupyterHub instance."),(0,o.yg)("h1",{id:"-16"}),(0,o.yg)("p",null,"In ",(0,o.yg)("inlineCode",{parentName:"p"},"Developer")," mode in your project, go to ",(0,o.yg)("inlineCode",{parentName:"p"},"Helm")," in the sidepanel (1). Next, click on your Helm Chart Release (2)."),(0,o.yg)("img",{src:"/img/jupyterhub-helm-chart-upgrade-chart-one.png",alt:"",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("h1",{id:"-17"}),(0,o.yg)("p",null,"Now, click the ",(0,o.yg)("inlineCode",{parentName:"p"},"Actions")," drop down menu, and choose ",(0,o.yg)("inlineCode",{parentName:"p"},"Upgrade")," (1)."),(0,o.yg)("img",{src:"/img/jupyterhub-helm-chart-upgrade-chart-two.png",alt:"",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("h1",{id:"-18"}),(0,o.yg)("p",null,"In the box -highlighted in the picutre below- you can make changes to the config.yaml. After you have made your changes, click ",(0,o.yg)("inlineCode",{parentName:"p"},"Upgrade")," and your upgraded JupyterHub Helm Chart Release will automatically be deployed."),(0,o.yg)("img",{src:"/img/jupyterhub-helm-chart-upgrade-chart-three.png",alt:"",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("h1",{id:"-19"}),(0,o.yg)("admonition",{title:"Configure JupyterHub",type:"success"},(0,o.yg)("p",{parentName:"admonition"},"Feel free to ",(0,o.yg)("a",{parentName:"p",href:"https://servicedesk.icts.maastrichtuniversity.nl/tas/public/ssp/content/serviceflow?unid=1ffa93e9ecd94d938ad46e3cb24c2392"},"submit a ticket")," to ask for help configuring your JupyterHub.")),(0,o.yg)("h1",{id:"-20"}),(0,o.yg)("h2",{id:"deploying-jupyterhub-using-the-command-line-interface-cli-"},"Deploying JupyterHub using the Command Line Interface (CLI) \ud83e\ude90"),(0,o.yg)("admonition",{title:"Before you begin download the config.yaml",type:"warning"},(0,o.yg)("p",{parentName:"admonition"},"Download the preconfigured ",(0,o.yg)("inlineCode",{parentName:"p"},"config.yaml")," from our ",(0,o.yg)("a",{parentName:"p",href:"https://raw.githubusercontent.com/MaastrichtU-IDS/dsri-documentation/refs/heads/master/applications/jupyterhub/config.yaml"},"GitHub repository"),".\nThe default config that is provided by JupyterHub will not work. ")),(0,o.yg)("h3",{id:"installing-the-jupyterhub-helm-chart-repository-1"},"Installing the JupyterHub Helm Chart repository"),(0,o.yg)("p",null,"After you have created a project you can start with installing the JupyterHub Helm Chart. If you do not have access to DSRI or created a project yet, and you need to find out how, please refer to our ",(0,o.yg)("a",{parentName:"p",href:"https://dsri.maastrichtuniversity.nl/docs/"},"documentation.")),(0,o.yg)("admonition",{title:"Helm Chart already available",type:"info"},(0,o.yg)("p",{parentName:"admonition"},"The Helm Chart should be already made available for everyone to use on the DSRI platform. There will be no need to install the repository yourself.")),(0,o.yg)("h1",{id:"-21"}),(0,o.yg)("p",null,"Add the JupyterHub Helm Chart repository:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"helm repo add jupyterhub https://hub.jupyter.org/helm-chart/\nhelm repo update\n")),(0,o.yg)("h3",{id:"installing-the-jupyterhub-helm-chart-1"},"Installing the JupyterHub Helm Chart"),(0,o.yg)("admonition",{type:"info"},(0,o.yg)("p",{parentName:"admonition"},"At the moment the latest -and only- Helm Chart version which is supported by DSRI is version 3.3.8. Newer versions will not work, and older versions are not tested and/or configured!")),(0,o.yg)("h1",{id:"-22"}),(0,o.yg)("p",null,"Make sure you use the right ",(0,o.yg)("inlineCode",{parentName:"p"},"config.yaml")," downloaded from our ",(0,o.yg)("a",{parentName:"p",href:"https://raw.githubusercontent.com/MaastrichtU-IDS/dsri-documentation/refs/heads/master/applications/jupyterhub/config.yaml"},"GitHub repository"),"."),(0,o.yg)("p",null,"Install the Helm Chart using the following command:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"helm upgrade --cleanup-on-fail \\\n --install jupyterhub jupyterhub/jupyterhub \\\n --version=3.3.8 \\\n --namespace= \\\n --values config.yaml\n")),(0,o.yg)("p",null,(0,o.yg)("inlineCode",{parentName:"p"},"")," is the name of your project."),(0,o.yg)("h3",{id:"creating-a-secured-route-1"},"Creating a secured route"),(0,o.yg)("p",null,"Create a secured route, with TLS edge termination:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"oc create route edge --namespace --service=proxy-public --port=http\n")),(0,o.yg)("p",null,(0,o.yg)("inlineCode",{parentName:"p"},"")," is the name of your project.\n",(0,o.yg)("inlineCode",{parentName:"p"},"")," is the name of the route. "),(0,o.yg)("h3",{id:"upgrading-the-configyaml-1"},"Upgrading the config.yaml"),(0,o.yg)("p",null,"Run the following command with your new config.yaml:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"helm upgrade --cleanup-on-fail \\\n --install jupyterhub jupyterhub/jupyterhub \\\n --version=3.3.8 \\\n --namespace= \\\n --values config.yaml\n")),(0,o.yg)("p",null,(0,o.yg)("inlineCode",{parentName:"p"},"")," is the name of your project."),(0,o.yg)("h1",{id:"-23"}),(0,o.yg)("admonition",{title:"Configure JupyterHub",type:"success"},(0,o.yg)("p",{parentName:"admonition"},"Feel free to ",(0,o.yg)("a",{parentName:"p",href:"https://servicedesk.icts.maastrichtuniversity.nl/tas/public/ssp/content/serviceflow?unid=1ffa93e9ecd94d938ad46e3cb24c2392"},"submit a ticket")," to ask for help configuring your JupyterHub.")))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/47a4a695.d75cb9e8.js b/assets/js/47a4a695.9461a409.js similarity index 99% rename from assets/js/47a4a695.d75cb9e8.js rename to assets/js/47a4a695.9461a409.js index caacbbda1..1bc2bc991 100644 --- a/assets/js/47a4a695.d75cb9e8.js +++ b/assets/js/47a4a695.9461a409.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[3836],{5680:(e,r,t)=>{t.d(r,{xA:()=>g,yg:()=>y});var o=t(6540);function n(e,r,t){return r in e?Object.defineProperty(e,r,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[r]=t,e}function a(e,r){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);r&&(o=o.filter((function(r){return Object.getOwnPropertyDescriptor(e,r).enumerable}))),t.push.apply(t,o)}return t}function i(e){for(var r=1;r=0||(n[t]=e[t]);return n}(e,r);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(o=0;o=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(n[t]=e[t])}return n}var l=o.createContext({}),s=function(e){var r=o.useContext(l),t=r;return e&&(t="function"==typeof e?e(r):i(i({},r),e)),t},g=function(e){var r=s(e.components);return o.createElement(l.Provider,{value:r},e.children)},p={inlineCode:"code",wrapper:function(e){var r=e.children;return o.createElement(o.Fragment,{},r)}},u=o.forwardRef((function(e,r){var t=e.components,n=e.mdxType,a=e.originalType,l=e.parentName,g=c(e,["components","mdxType","originalType","parentName"]),u=s(t),y=n,m=u["".concat(l,".").concat(y)]||u[y]||p[y]||a;return t?o.createElement(m,i(i({ref:r},g),{},{components:t})):o.createElement(m,i({ref:r},g))}));function y(e,r){var t=arguments,n=r&&r.mdxType;if("string"==typeof e||n){var a=t.length,i=new Array(a);i[0]=u;var c={};for(var l in r)hasOwnProperty.call(r,l)&&(c[l]=r[l]);c.originalType=e,c.mdxType="string"==typeof e?e:n,i[1]=c;for(var s=2;s{t.r(r),t.d(r,{assets:()=>g,contentTitle:()=>l,default:()=>y,frontMatter:()=>c,metadata:()=>s,toc:()=>p});var o=t(9668),n=t(1367),a=(t(6540),t(5680)),i=["components"],c={id:"login-docker-registry",title:"Login to Docker registries"},l=void 0,s={unversionedId:"login-docker-registry",id:"login-docker-registry",title:"Login to Docker registries",description:"Login to an external container registry can be helpful to pull private images, or increase the DockerHub pull limitations.",source:"@site/docs/login-docker-registry.md",sourceDirName:".",slug:"/login-docker-registry",permalink:"/docs/login-docker-registry",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/login-docker-registry.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"login-docker-registry",title:"Login to Docker registries"},sidebar:"docs",previous:{title:"Prepare a workshop",permalink:"/docs/guide-workshop"},next:{title:"Command Line Interface",permalink:"/docs/openshift-commands"}},g={},p=[{value:"UM Container registry",id:"um-container-registry",level:2},{value:"Logging in with Docker CLI",id:"logging-in-with-docker-cli",level:3},{value:"Using a Proxy Cache",id:"using-a-proxy-cache",level:3},{value:"Creating your own project",id:"creating-your-own-project",level:3},{value:"Using your own user",id:"using-your-own-user",level:3},{value:"Using a robot account",id:"using-a-robot-account",level:3},{value:"GitHub Container Registry",id:"github-container-registry",level:2},{value:"DockerHub",id:"dockerhub",level:2}],u={toc:p};function y(e){var r=e.components,t=(0,n.A)(e,i);return(0,a.yg)("wrapper",(0,o.A)({},u,t,{components:r,mdxType:"MDXLayout"}),(0,a.yg)("p",null,"Login to an external container registry can be helpful to pull private images, or increase the DockerHub pull limitations."),(0,a.yg)("p",null,"You will need to create a secret in your project, then link it to the default service account of this project."),(0,a.yg)("p",null,"We detail here the process for ",(0,a.yg)("a",{parentName:"p",href:"https://cr.icts.unimaas.nl"},"UM Container registry"),", ",(0,a.yg)("a",{parentName:"p",href:"https://docs.github.com/en/packages/guides/about-github-container-registry"},"GitHub Container Registry")," and ",(0,a.yg)("a",{parentName:"p",href:"https://hub.docker.com/"},"Docker Hub"),", but the process is similar for any other container registry (e.g. ",(0,a.yg)("a",{parentName:"p",href:"https://quay.io/"},"quay.io"),")"),(0,a.yg)("h2",{id:"um-container-registry"},"UM Container registry"),(0,a.yg)("admonition",{title:"Access",type:"info"},(0,a.yg)("p",{parentName:"admonition"},"You need to be connected to the UM network to access this container registry.")),(0,a.yg)("p",null,"This container registry is available at ",(0,a.yg)("a",{parentName:"p",href:"https://cr.icts.unimaas.nl"},"UM Container registry"),". Here you can login using your ",(0,a.yg)("strong",{parentName:"p"},"UM credentials"),' by clicking on the "Login via OIDC provider"'),(0,a.yg)("img",{class:"screenshot",src:"/img/screenshot_harbor_login_page.png",alt:"Harbor_login_page",style:{zoom:"100%",maxHeight:"500px",maxWidth:"500px"}}),(0,a.yg)("admonition",{title:"Public Projects",type:"info"},(0,a.yg)("p",{parentName:"admonition"},"You don't need to follow the steps below if you are using one of the Public projects. These are available without credentials.")),(0,a.yg)("h3",{id:"logging-in-with-docker-cli"},"Logging in with Docker CLI"),(0,a.yg)("ol",null,(0,a.yg)("li",{parentName:"ol"},"Go to ",(0,a.yg)("a",{parentName:"li",href:"https://cr.icts.unimaas.nl"},"UM Container registry"),", click on your username in the top right corner followed by clicking on ",(0,a.yg)("strong",{parentName:"li"},"User Profile"),". Click on the ",(0,a.yg)("strong",{parentName:"li"},"Copy")," icon."),(0,a.yg)("li",{parentName:"ol"},"Login with your credentials:")),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre"},"docker login cr.icts.unimaas.nl\n\n(Username)\n(Copied in Step 1)\n")),(0,a.yg)("h3",{id:"using-a-proxy-cache"},"Using a Proxy Cache"),(0,a.yg)("ol",null,(0,a.yg)("li",{parentName:"ol"},"Go to ",(0,a.yg)("a",{parentName:"li",href:"https://cr.icts.unimaas.nl"},"UM Container registry"),", look for a project of type ",(0,a.yg)("strong",{parentName:"li"},"Proxy Cache"),". For each of the mayor registries we created a ",(0,a.yg)("strong",{parentName:"li"},"Proxy Cache"),". Remember the project name, for example ",(0,a.yg)("strong",{parentName:"li"},"dockerhub"),"."),(0,a.yg)("li",{parentName:"ol"},"On the DSRI you can deploy an image like in this example:")),(0,a.yg)("img",{class:"screenshot",src:"/img/screenshot_harbor_proxy_cache.png",alt:"Harbor_proxy_cache",style:{zoom:"100%",maxHeight:"500px",maxWidth:"500px"}}),(0,a.yg)("admonition",{title:"Docker CLI",type:"info"},(0,a.yg)("p",{parentName:"admonition"},"The same concept can be applied using the docker CLI"),(0,a.yg)("pre",{parentName:"admonition"},(0,a.yg)("code",{parentName:"pre"},"docker pull cr.icts.unimaas.nl/dockerhub/ubuntu:22.04\n"))),(0,a.yg)("h3",{id:"creating-your-own-project"},"Creating your own project"),(0,a.yg)("ol",null,(0,a.yg)("li",{parentName:"ol"},(0,a.yg)("p",{parentName:"li"},"Go to ",(0,a.yg)("a",{parentName:"p",href:"https://cr.icts.unimaas.nl"},"UM Container registry"),", click on ",(0,a.yg)("strong",{parentName:"p"},"+ NEW PROJECT"),". Fill in the details of project name and Access Level (preferred method is to leave the checkbox unchecked).")),(0,a.yg)("li",{parentName:"ol"},(0,a.yg)("p",{parentName:"li"},"Click OK"))),(0,a.yg)("h3",{id:"using-your-own-user"},"Using your own user"),(0,a.yg)("ol",null,(0,a.yg)("li",{parentName:"ol"},(0,a.yg)("p",{parentName:"li"},"Go to ",(0,a.yg)("a",{parentName:"p",href:"https://cr.icts.unimaas.nl"},"UM Container registry"),", click on your username in the top right corner followed by clicking on ",(0,a.yg)("strong",{parentName:"p"},"User Profile"),". Click on the ",(0,a.yg)("strong",{parentName:"p"},"Copy")," icon.")),(0,a.yg)("li",{parentName:"ol"},(0,a.yg)("p",{parentName:"li"},"Create a secret to login to UM Harbor Container Registry in your project:"))),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre"},"oc create secret docker-registry um-harbor-secret --docker-server=cr.icts.unimaas.nl --docker-username= --docker-password=\n")),(0,a.yg)("ol",{start:3},(0,a.yg)("li",{parentName:"ol"},"Link the login secret to the default service account:")),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre"},"oc secrets link default um-harbor-secret --for=pull\n")),(0,a.yg)("h3",{id:"using-a-robot-account"},"Using a robot account"),(0,a.yg)("ol",null,(0,a.yg)("li",{parentName:"ol"},(0,a.yg)("p",{parentName:"li"},"Go to ",(0,a.yg)("a",{parentName:"p",href:"https://cr.icts.unimaas.nl"},"UM Container registry"),", click on your project if you already created one.")),(0,a.yg)("li",{parentName:"ol"},(0,a.yg)("p",{parentName:"li"},"Click on the tab ",(0,a.yg)("strong",{parentName:"p"},"Robot Accounts"))),(0,a.yg)("li",{parentName:"ol"},(0,a.yg)("p",{parentName:"li"},"Click on ",(0,a.yg)("strong",{parentName:"p"},"New Robot Account"))),(0,a.yg)("li",{parentName:"ol"},(0,a.yg)("p",{parentName:"li"},"Create the Robot account to your liking")),(0,a.yg)("li",{parentName:"ol"},(0,a.yg)("p",{parentName:"li"},"Copy the secret or export it")),(0,a.yg)("li",{parentName:"ol"},(0,a.yg)("p",{parentName:"li"},"Create a secret to login to UM Harbor Container Registry in your project:"))),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre"},"oc create secret docker-registry um-harbor-secret --docker-server=cr.icts.unimaas.nl --docker-username= --docker-password=\n")),(0,a.yg)("ol",{start:7},(0,a.yg)("li",{parentName:"ol"},"Link the login secret to the default service account:")),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre"},"oc secrets link default um-harbor-secret --for=pull\n")),(0,a.yg)("h2",{id:"github-container-registry"},"GitHub Container Registry"),(0,a.yg)("ol",null,(0,a.yg)("li",{parentName:"ol"},(0,a.yg)("p",{parentName:"li"},"Go to ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/settings/tokens"},"GitHub Settings"),", and create a Personal Access Token (PAT) which will be used as password to connect to the GitHub Container Registry")),(0,a.yg)("li",{parentName:"ol"},(0,a.yg)("p",{parentName:"li"},"Create a secret to login to GitHub Container Registry in your project:"))),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},"oc create secret docker-registry github-ghcr-secret --docker-server=ghcr.io --docker-username= --docker-password= --docker-email=\n")),(0,a.yg)("ol",{start:3},(0,a.yg)("li",{parentName:"ol"},"Link the login secret to the default service account:")),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},"oc secrets link default github-ghcr-secret --for=pull\n")),(0,a.yg)("h2",{id:"dockerhub"},"DockerHub"),(0,a.yg)("admonition",{title:"Increase DockerHub limitations",type:"tip"},(0,a.yg)("p",{parentName:"admonition"},"Login with DockerHub also increase the DockerHub limitations to pull images in your project")),(0,a.yg)("ol",null,(0,a.yg)("li",{parentName:"ol"},"Create a secret to login to DockerHub in your project:")),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},"oc create secret docker-registry dockerhub-secret --docker-server=docker.io --docker-username= --docker-password= --docker-email=\n")),(0,a.yg)("ol",{start:2},(0,a.yg)("li",{parentName:"ol"},"Link the login secret to the default service account:")),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},"oc secrets link default dockerhub-secret --for=pull\n")))}y.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[3836],{5680:(e,r,t)=>{t.d(r,{xA:()=>g,yg:()=>y});var o=t(6540);function n(e,r,t){return r in e?Object.defineProperty(e,r,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[r]=t,e}function a(e,r){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);r&&(o=o.filter((function(r){return Object.getOwnPropertyDescriptor(e,r).enumerable}))),t.push.apply(t,o)}return t}function i(e){for(var r=1;r=0||(n[t]=e[t]);return n}(e,r);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(o=0;o=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(n[t]=e[t])}return n}var l=o.createContext({}),s=function(e){var r=o.useContext(l),t=r;return e&&(t="function"==typeof e?e(r):i(i({},r),e)),t},g=function(e){var r=s(e.components);return o.createElement(l.Provider,{value:r},e.children)},p={inlineCode:"code",wrapper:function(e){var r=e.children;return o.createElement(o.Fragment,{},r)}},u=o.forwardRef((function(e,r){var t=e.components,n=e.mdxType,a=e.originalType,l=e.parentName,g=c(e,["components","mdxType","originalType","parentName"]),u=s(t),y=n,m=u["".concat(l,".").concat(y)]||u[y]||p[y]||a;return t?o.createElement(m,i(i({ref:r},g),{},{components:t})):o.createElement(m,i({ref:r},g))}));function y(e,r){var t=arguments,n=r&&r.mdxType;if("string"==typeof e||n){var a=t.length,i=new Array(a);i[0]=u;var c={};for(var l in r)hasOwnProperty.call(r,l)&&(c[l]=r[l]);c.originalType=e,c.mdxType="string"==typeof e?e:n,i[1]=c;for(var s=2;s{t.r(r),t.d(r,{assets:()=>g,contentTitle:()=>l,default:()=>y,frontMatter:()=>c,metadata:()=>s,toc:()=>p});var o=t(9668),n=t(1367),a=(t(6540),t(5680)),i=["components"],c={id:"login-docker-registry",title:"Login to Docker registries"},l=void 0,s={unversionedId:"login-docker-registry",id:"login-docker-registry",title:"Login to Docker registries",description:"Login to an external container registry can be helpful to pull private images, or increase the DockerHub pull limitations.",source:"@site/docs/login-docker-registry.md",sourceDirName:".",slug:"/login-docker-registry",permalink:"/docs/login-docker-registry",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/login-docker-registry.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"login-docker-registry",title:"Login to Docker registries"},sidebar:"docs",previous:{title:"Prepare a workshop",permalink:"/docs/guide-workshop"},next:{title:"Command Line Interface",permalink:"/docs/openshift-commands"}},g={},p=[{value:"UM Container registry",id:"um-container-registry",level:2},{value:"Logging in with Docker CLI",id:"logging-in-with-docker-cli",level:3},{value:"Using a Proxy Cache",id:"using-a-proxy-cache",level:3},{value:"Creating your own project",id:"creating-your-own-project",level:3},{value:"Using your own user",id:"using-your-own-user",level:3},{value:"Using a robot account",id:"using-a-robot-account",level:3},{value:"GitHub Container Registry",id:"github-container-registry",level:2},{value:"DockerHub",id:"dockerhub",level:2}],u={toc:p};function y(e){var r=e.components,t=(0,n.A)(e,i);return(0,a.yg)("wrapper",(0,o.A)({},u,t,{components:r,mdxType:"MDXLayout"}),(0,a.yg)("p",null,"Login to an external container registry can be helpful to pull private images, or increase the DockerHub pull limitations."),(0,a.yg)("p",null,"You will need to create a secret in your project, then link it to the default service account of this project."),(0,a.yg)("p",null,"We detail here the process for ",(0,a.yg)("a",{parentName:"p",href:"https://cr.icts.unimaas.nl"},"UM Container registry"),", ",(0,a.yg)("a",{parentName:"p",href:"https://docs.github.com/en/packages/guides/about-github-container-registry"},"GitHub Container Registry")," and ",(0,a.yg)("a",{parentName:"p",href:"https://hub.docker.com/"},"Docker Hub"),", but the process is similar for any other container registry (e.g. ",(0,a.yg)("a",{parentName:"p",href:"https://quay.io/"},"quay.io"),")"),(0,a.yg)("h2",{id:"um-container-registry"},"UM Container registry"),(0,a.yg)("admonition",{title:"Access",type:"info"},(0,a.yg)("p",{parentName:"admonition"},"You need to be connected to the UM network to access this container registry.")),(0,a.yg)("p",null,"This container registry is available at ",(0,a.yg)("a",{parentName:"p",href:"https://cr.icts.unimaas.nl"},"UM Container registry"),". Here you can login using your ",(0,a.yg)("strong",{parentName:"p"},"UM credentials"),' by clicking on the "Login via OIDC provider"'),(0,a.yg)("img",{class:"screenshot",src:"/img/screenshot_harbor_login_page.png",alt:"Harbor_login_page",style:{zoom:"100%",maxHeight:"500px",maxWidth:"500px"}}),(0,a.yg)("admonition",{title:"Public Projects",type:"info"},(0,a.yg)("p",{parentName:"admonition"},"You don't need to follow the steps below if you are using one of the Public projects. These are available without credentials.")),(0,a.yg)("h3",{id:"logging-in-with-docker-cli"},"Logging in with Docker CLI"),(0,a.yg)("ol",null,(0,a.yg)("li",{parentName:"ol"},"Go to ",(0,a.yg)("a",{parentName:"li",href:"https://cr.icts.unimaas.nl"},"UM Container registry"),", click on your username in the top right corner followed by clicking on ",(0,a.yg)("strong",{parentName:"li"},"User Profile"),". Click on the ",(0,a.yg)("strong",{parentName:"li"},"Copy")," icon."),(0,a.yg)("li",{parentName:"ol"},"Login with your credentials:")),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre"},"docker login cr.icts.unimaas.nl\n\n(Username)\n(Copied in Step 1)\n")),(0,a.yg)("h3",{id:"using-a-proxy-cache"},"Using a Proxy Cache"),(0,a.yg)("ol",null,(0,a.yg)("li",{parentName:"ol"},"Go to ",(0,a.yg)("a",{parentName:"li",href:"https://cr.icts.unimaas.nl"},"UM Container registry"),", look for a project of type ",(0,a.yg)("strong",{parentName:"li"},"Proxy Cache"),". For each of the mayor registries we created a ",(0,a.yg)("strong",{parentName:"li"},"Proxy Cache"),". Remember the project name, for example ",(0,a.yg)("strong",{parentName:"li"},"dockerhub"),"."),(0,a.yg)("li",{parentName:"ol"},"On the DSRI you can deploy an image like in this example:")),(0,a.yg)("img",{class:"screenshot",src:"/img/screenshot_harbor_proxy_cache.png",alt:"Harbor_proxy_cache",style:{zoom:"100%",maxHeight:"500px",maxWidth:"500px"}}),(0,a.yg)("admonition",{title:"Docker CLI",type:"info"},(0,a.yg)("p",{parentName:"admonition"},"The same concept can be applied using the docker CLI"),(0,a.yg)("pre",{parentName:"admonition"},(0,a.yg)("code",{parentName:"pre"},"docker pull cr.icts.unimaas.nl/dockerhub/ubuntu:22.04\n"))),(0,a.yg)("h3",{id:"creating-your-own-project"},"Creating your own project"),(0,a.yg)("ol",null,(0,a.yg)("li",{parentName:"ol"},(0,a.yg)("p",{parentName:"li"},"Go to ",(0,a.yg)("a",{parentName:"p",href:"https://cr.icts.unimaas.nl"},"UM Container registry"),", click on ",(0,a.yg)("strong",{parentName:"p"},"+ NEW PROJECT"),". Fill in the details of project name and Access Level (preferred method is to leave the checkbox unchecked).")),(0,a.yg)("li",{parentName:"ol"},(0,a.yg)("p",{parentName:"li"},"Click OK"))),(0,a.yg)("h3",{id:"using-your-own-user"},"Using your own user"),(0,a.yg)("ol",null,(0,a.yg)("li",{parentName:"ol"},(0,a.yg)("p",{parentName:"li"},"Go to ",(0,a.yg)("a",{parentName:"p",href:"https://cr.icts.unimaas.nl"},"UM Container registry"),", click on your username in the top right corner followed by clicking on ",(0,a.yg)("strong",{parentName:"p"},"User Profile"),". Click on the ",(0,a.yg)("strong",{parentName:"p"},"Copy")," icon.")),(0,a.yg)("li",{parentName:"ol"},(0,a.yg)("p",{parentName:"li"},"Create a secret to login to UM Harbor Container Registry in your project:"))),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre"},"oc create secret docker-registry um-harbor-secret --docker-server=cr.icts.unimaas.nl --docker-username= --docker-password=\n")),(0,a.yg)("ol",{start:3},(0,a.yg)("li",{parentName:"ol"},"Link the login secret to the default service account:")),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre"},"oc secrets link default um-harbor-secret --for=pull\n")),(0,a.yg)("h3",{id:"using-a-robot-account"},"Using a robot account"),(0,a.yg)("ol",null,(0,a.yg)("li",{parentName:"ol"},(0,a.yg)("p",{parentName:"li"},"Go to ",(0,a.yg)("a",{parentName:"p",href:"https://cr.icts.unimaas.nl"},"UM Container registry"),", click on your project if you already created one.")),(0,a.yg)("li",{parentName:"ol"},(0,a.yg)("p",{parentName:"li"},"Click on the tab ",(0,a.yg)("strong",{parentName:"p"},"Robot Accounts"))),(0,a.yg)("li",{parentName:"ol"},(0,a.yg)("p",{parentName:"li"},"Click on ",(0,a.yg)("strong",{parentName:"p"},"New Robot Account"))),(0,a.yg)("li",{parentName:"ol"},(0,a.yg)("p",{parentName:"li"},"Create the Robot account to your liking")),(0,a.yg)("li",{parentName:"ol"},(0,a.yg)("p",{parentName:"li"},"Copy the secret or export it")),(0,a.yg)("li",{parentName:"ol"},(0,a.yg)("p",{parentName:"li"},"Create a secret to login to UM Harbor Container Registry in your project:"))),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre"},"oc create secret docker-registry um-harbor-secret --docker-server=cr.icts.unimaas.nl --docker-username= --docker-password=\n")),(0,a.yg)("ol",{start:7},(0,a.yg)("li",{parentName:"ol"},"Link the login secret to the default service account:")),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre"},"oc secrets link default um-harbor-secret --for=pull\n")),(0,a.yg)("h2",{id:"github-container-registry"},"GitHub Container Registry"),(0,a.yg)("ol",null,(0,a.yg)("li",{parentName:"ol"},(0,a.yg)("p",{parentName:"li"},"Go to ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/settings/tokens"},"GitHub Settings"),", and create a Personal Access Token (PAT) which will be used as password to connect to the GitHub Container Registry")),(0,a.yg)("li",{parentName:"ol"},(0,a.yg)("p",{parentName:"li"},"Create a secret to login to GitHub Container Registry in your project:"))),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},"oc create secret docker-registry github-ghcr-secret --docker-server=ghcr.io --docker-username= --docker-password= --docker-email=\n")),(0,a.yg)("ol",{start:3},(0,a.yg)("li",{parentName:"ol"},"Link the login secret to the default service account:")),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},"oc secrets link default github-ghcr-secret --for=pull\n")),(0,a.yg)("h2",{id:"dockerhub"},"DockerHub"),(0,a.yg)("admonition",{title:"Increase DockerHub limitations",type:"tip"},(0,a.yg)("p",{parentName:"admonition"},"Login with DockerHub also increase the DockerHub limitations to pull images in your project")),(0,a.yg)("ol",null,(0,a.yg)("li",{parentName:"ol"},"Create a secret to login to DockerHub in your project:")),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},"oc create secret docker-registry dockerhub-secret --docker-server=docker.io --docker-username= --docker-password= --docker-email=\n")),(0,a.yg)("ol",{start:2},(0,a.yg)("li",{parentName:"ol"},"Link the login secret to the default service account:")),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},"oc secrets link default dockerhub-secret --for=pull\n")))}y.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/4ba7e5a3.0b2467c5.js b/assets/js/4ba7e5a3.b1143b9f.js similarity index 99% rename from assets/js/4ba7e5a3.0b2467c5.js rename to assets/js/4ba7e5a3.b1143b9f.js index d533b23bf..effcc65bb 100644 --- a/assets/js/4ba7e5a3.0b2467c5.js +++ b/assets/js/4ba7e5a3.b1143b9f.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[9591],{5680:(e,t,n)=>{n.d(t,{xA:()=>u,yg:()=>m});var a=n(6540);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function i(e){for(var t=1;t=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var s=a.createContext({}),c=function(e){var t=a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},u=function(e){var t=c(e.components);return a.createElement(s.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},d=a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,o=e.originalType,s=e.parentName,u=l(e,["components","mdxType","originalType","parentName"]),d=c(n),m=r,g=d["".concat(s,".").concat(m)]||d[m]||p[m]||o;return n?a.createElement(g,i(i({ref:t},u),{},{components:n})):a.createElement(g,i({ref:t},u))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var o=n.length,i=new Array(o);i[0]=d;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:r,i[1]=l;for(var c=2;c{n.r(t),n.d(t,{assets:()=>u,contentTitle:()=>s,default:()=>m,frontMatter:()=>l,metadata:()=>c,toc:()=>p});var a=n(9668),r=n(1367),o=(n(6540),n(5680)),i=["components"],l={id:"contribute",title:"Contribute"},s="Contribute",c={unversionedId:"contribute",id:"contribute",title:"Contribute",description:"Check if there are issues related to your contribution, or post a new issue to discuss improvement to the documentation.",source:"@site/docs/contribute.md",sourceDirName:".",slug:"/contribute",permalink:"/docs/contribute",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/contribute.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"contribute",title:"Contribute"},sidebar:"docs",previous:{title:"Run CWL workflows",permalink:"/docs/workflows-cwl"},next:{title:"Enabling VPN access in WSL2",permalink:"/docs/enabling-vpn-wsl"}},u={},p=[{value:"\u26a1 Quick edit on GitHub",id:"-quick-edit-on-github",level:2},{value:"\ud83c\udfd7\ufe0f Larger changes locally",id:"\ufe0f-larger-changes-locally",level:2},{value:"\ud83d\udd04 Automated deployment",id:"-automated-deployment",level:2},{value:"\ud83d\udcdd Help",id:"-help",level:2},{value:"\ud83d\udd0e Files locations",id:"-files-locations",level:3},{value:"\ud83e\udd84 Markdown tip",id:"-markdown-tip",level:3},{value:"\u2714\ufe0f Pull Request process",id:"\ufe0f-pull-request-process",level:2}],d={toc:p};function m(e){var t=e.components,n=(0,r.A)(e,i);return(0,o.yg)("wrapper",(0,a.A)({},d,n,{components:t,mdxType:"MDXLayout"}),(0,o.yg)("h1",{id:"contribute"},"Contribute"),(0,o.yg)("p",null,"Check if there are ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/issues"},"issues")," related to your contribution, or post a ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/issues/new"},"new issue")," to discuss improvement to the documentation. "),(0,o.yg)("a",{href:"https://github.com/MaastrichtU-IDS/dsri-documentation/issues",target:"_blank",rel:"noopener noreferrer","aria-label":"GitHub issues"},(0,o.yg)("img",{alt:"GitHub issues",src:"https://img.shields.io/github/issues/MaastrichtU-IDS/dsri-documentation?label=dsri-documentation"})),(0,o.yg)("admonition",{title:"Fork this repository",type:"info"},(0,o.yg)("p",{parentName:"admonition"},"Otherwise you will need to first ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/fork"},"fork this repository"),", then send a pull request when your changes have been pushed.")),(0,o.yg)("admonition",{title:"Direct change if permission",type:"note"},(0,o.yg)("p",{parentName:"admonition"},"If you are part of the ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS"},"MaastrichtU-IDS organization on GitHub")," you can directly create a new branch to make your change in the main repository. ")),(0,o.yg)("hr",null),(0,o.yg)("h2",{id:"-quick-edit-on-github"},"\u26a1 Quick edit on GitHub"),(0,o.yg)("p",null,"You can really easily make quick changes directly on the GitHub website by clicking the ",(0,o.yg)("strong",{parentName:"p"},"Edit this page")," button at the bottom left of each documentation page. Or browsing to your forked repository."),(0,o.yg)("p",null,"For example to edit the introduction page you can go to ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/introduction.md"},"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/introduction.md")),(0,o.yg)("hr",null),(0,o.yg)("h2",{id:"\ufe0f-larger-changes-locally"},"\ud83c\udfd7\ufe0f Larger changes locally"),(0,o.yg)("p",null,"To edit the documentation it is easier to clone the repository on your laptop, and use a ",(0,o.yg)("a",{parentName:"p",href:"https://typora.io"},"user-friendly markdown editor"),"."),(0,o.yg)("admonition",{title:"Use a Markdown editor",type:"info"},(0,o.yg)("p",{parentName:"admonition"},"We strongly recommend you to use a markdown editor, such as ",(0,o.yg)("a",{parentName:"p",href:"https://typora.io"},"Typora"),". It makes writing documentation much faster, and more enjoyable.")),(0,o.yg)("ol",null,(0,o.yg)("li",{parentName:"ol"},"Clone the repository on your machine:")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"git clone https://github.com/MaastrichtU-IDS/dsri-documentation.git\ncd dsri-documentation\n")),(0,o.yg)("ol",{start:2},(0,o.yg)("li",{parentName:"ol"},"Create a new branch from the ",(0,o.yg)("inlineCode",{parentName:"li"},"master")," branch \ud83d\udd4a\ufe0f")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"git checkout -b my-branch\n")),(0,o.yg)("ol",{start:3},(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Add your changes in this branch \u2712\ufe0f")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Start the website on ",(0,o.yg)("a",{parentName:"p",href:"http://localhost:3000"},"http://localhost:3000")," to test it:"))),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"cd website\nyarn install\nyarn start\n")),(0,o.yg)("admonition",{title:"Send a pull request",type:"info"},(0,o.yg)("p",{parentName:"admonition"},"Send a pull request to the ",(0,o.yg)("inlineCode",{parentName:"p"},"master")," branch when your changes are done")),(0,o.yg)("admonition",{title:"Development documentation",type:"note"},(0,o.yg)("p",{parentName:"admonition"},"Read more about running the API in development at ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation#run-for-development"},"https://github.com/MaastrichtU-IDS/dsri-documentation#run-for-development"))),(0,o.yg)("hr",null),(0,o.yg)("h2",{id:"-automated-deployment"},"\ud83d\udd04 Automated deployment"),(0,o.yg)("p",null,"The documentation website is automatically updated and redeployed at each change to the ",(0,o.yg)("inlineCode",{parentName:"p"},"main")," branch using a ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/actions"},"GitHub Actions workflow"),"."),(0,o.yg)("p",null,(0,o.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/actions?query=workflow%3A%22Publish+to+GitHub+Pages%22"},(0,o.yg)("img",{parentName:"a",src:"https://github.com/MaastrichtU-IDS/dsri-documentation/workflows/Publish%20to%20GitHub%20Pages/badge.svg",alt:"Publish to GitHub Pages"}))),(0,o.yg)("hr",null),(0,o.yg)("h2",{id:"-help"},"\ud83d\udcdd Help"),(0,o.yg)("p",null,"Most pages of this website are written in Markdown, hence they are really easy to edit, especially when you are using a ",(0,o.yg)("a",{parentName:"p",href:"https://typora.io/"},"convenient markdown editor"),". Only the ",(0,o.yg)("inlineCode",{parentName:"p"},"index.js")," page is written in React JavaScript."),(0,o.yg)("h3",{id:"-files-locations"},"\ud83d\udd0e Files locations"),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},"Main DSRI documentation markdown files in ",(0,o.yg)("inlineCode",{parentName:"li"},"website/docs"),(0,o.yg)("ul",{parentName:"li"},(0,o.yg)("li",{parentName:"ul"},"Left docs menu defined in ",(0,o.yg)("inlineCode",{parentName:"li"},"website/sidebars.json")," "))),(0,o.yg)("li",{parentName:"ul"},"Blog articles as markdown files in ",(0,o.yg)("inlineCode",{parentName:"li"},"website/docs")),(0,o.yg)("li",{parentName:"ul"},"Index and contribute pages in ",(0,o.yg)("inlineCode",{parentName:"li"},"website/src/pages")),(0,o.yg)("li",{parentName:"ul"},"Images in ",(0,o.yg)("inlineCode",{parentName:"li"},"website/src/static/img")),(0,o.yg)("li",{parentName:"ul"},"Website configuration file in ",(0,o.yg)("inlineCode",{parentName:"li"},"website/docusaurus.config.js")," ")),(0,o.yg)("h3",{id:"-markdown-tip"},"\ud83e\udd84 Markdown tip"),(0,o.yg)("admonition",{title:"Colorful boxes",type:"warning"},(0,o.yg)("p",{parentName:"admonition"},"Use the following tags to create colorful boxes in markdown files:")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-markdown"},":::note You can specify an optional title\nGrey box\n:::\n\n:::tip Green box\nThe content and title *can* include markdown.\n:::\n\n:::info Blue box\nUseful information.\n:::\n\n:::caution Be careful!\nYellow box\n:::\n\n:::danger Fire red box\nDanger danger, mayday!\n:::\n")),(0,o.yg)("h2",{id:"\ufe0f-pull-request-process"},"\u2714\ufe0f Pull Request process"),(0,o.yg)("ol",null,(0,o.yg)("li",{parentName:"ol"},"Before sending a pull request make sure the DSRI documentation website still work as expected with the new changes properly integrated:")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre"},"cd website\nyarn install\nyarn start\n")),(0,o.yg)("ol",{start:2},(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/compare"},"Send a pull request")," to the ",(0,o.yg)("inlineCode",{parentName:"li"},"master")," branch."),(0,o.yg)("li",{parentName:"ol"},"Project contributors will review your change as soon as they can!")))}m.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[9591],{5680:(e,t,n)=>{n.d(t,{xA:()=>u,yg:()=>m});var a=n(6540);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function i(e){for(var t=1;t=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var s=a.createContext({}),c=function(e){var t=a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},u=function(e){var t=c(e.components);return a.createElement(s.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},d=a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,o=e.originalType,s=e.parentName,u=l(e,["components","mdxType","originalType","parentName"]),d=c(n),m=r,g=d["".concat(s,".").concat(m)]||d[m]||p[m]||o;return n?a.createElement(g,i(i({ref:t},u),{},{components:n})):a.createElement(g,i({ref:t},u))}));function m(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var o=n.length,i=new Array(o);i[0]=d;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:r,i[1]=l;for(var c=2;c{n.r(t),n.d(t,{assets:()=>u,contentTitle:()=>s,default:()=>m,frontMatter:()=>l,metadata:()=>c,toc:()=>p});var a=n(9668),r=n(1367),o=(n(6540),n(5680)),i=["components"],l={id:"contribute",title:"Contribute"},s="Contribute",c={unversionedId:"contribute",id:"contribute",title:"Contribute",description:"Check if there are issues related to your contribution, or post a new issue to discuss improvement to the documentation.",source:"@site/docs/contribute.md",sourceDirName:".",slug:"/contribute",permalink:"/docs/contribute",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/contribute.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"contribute",title:"Contribute"},sidebar:"docs",previous:{title:"Run CWL workflows",permalink:"/docs/workflows-cwl"},next:{title:"Enabling VPN access in WSL2",permalink:"/docs/enabling-vpn-wsl"}},u={},p=[{value:"\u26a1 Quick edit on GitHub",id:"-quick-edit-on-github",level:2},{value:"\ud83c\udfd7\ufe0f Larger changes locally",id:"\ufe0f-larger-changes-locally",level:2},{value:"\ud83d\udd04 Automated deployment",id:"-automated-deployment",level:2},{value:"\ud83d\udcdd Help",id:"-help",level:2},{value:"\ud83d\udd0e Files locations",id:"-files-locations",level:3},{value:"\ud83e\udd84 Markdown tip",id:"-markdown-tip",level:3},{value:"\u2714\ufe0f Pull Request process",id:"\ufe0f-pull-request-process",level:2}],d={toc:p};function m(e){var t=e.components,n=(0,r.A)(e,i);return(0,o.yg)("wrapper",(0,a.A)({},d,n,{components:t,mdxType:"MDXLayout"}),(0,o.yg)("h1",{id:"contribute"},"Contribute"),(0,o.yg)("p",null,"Check if there are ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/issues"},"issues")," related to your contribution, or post a ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/issues/new"},"new issue")," to discuss improvement to the documentation. "),(0,o.yg)("a",{href:"https://github.com/MaastrichtU-IDS/dsri-documentation/issues",target:"_blank",rel:"noopener noreferrer","aria-label":"GitHub issues"},(0,o.yg)("img",{alt:"GitHub issues",src:"https://img.shields.io/github/issues/MaastrichtU-IDS/dsri-documentation?label=dsri-documentation"})),(0,o.yg)("admonition",{title:"Fork this repository",type:"info"},(0,o.yg)("p",{parentName:"admonition"},"Otherwise you will need to first ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/fork"},"fork this repository"),", then send a pull request when your changes have been pushed.")),(0,o.yg)("admonition",{title:"Direct change if permission",type:"note"},(0,o.yg)("p",{parentName:"admonition"},"If you are part of the ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS"},"MaastrichtU-IDS organization on GitHub")," you can directly create a new branch to make your change in the main repository. ")),(0,o.yg)("hr",null),(0,o.yg)("h2",{id:"-quick-edit-on-github"},"\u26a1 Quick edit on GitHub"),(0,o.yg)("p",null,"You can really easily make quick changes directly on the GitHub website by clicking the ",(0,o.yg)("strong",{parentName:"p"},"Edit this page")," button at the bottom left of each documentation page. Or browsing to your forked repository."),(0,o.yg)("p",null,"For example to edit the introduction page you can go to ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/introduction.md"},"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/introduction.md")),(0,o.yg)("hr",null),(0,o.yg)("h2",{id:"\ufe0f-larger-changes-locally"},"\ud83c\udfd7\ufe0f Larger changes locally"),(0,o.yg)("p",null,"To edit the documentation it is easier to clone the repository on your laptop, and use a ",(0,o.yg)("a",{parentName:"p",href:"https://typora.io"},"user-friendly markdown editor"),"."),(0,o.yg)("admonition",{title:"Use a Markdown editor",type:"info"},(0,o.yg)("p",{parentName:"admonition"},"We strongly recommend you to use a markdown editor, such as ",(0,o.yg)("a",{parentName:"p",href:"https://typora.io"},"Typora"),". It makes writing documentation much faster, and more enjoyable.")),(0,o.yg)("ol",null,(0,o.yg)("li",{parentName:"ol"},"Clone the repository on your machine:")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"git clone https://github.com/MaastrichtU-IDS/dsri-documentation.git\ncd dsri-documentation\n")),(0,o.yg)("ol",{start:2},(0,o.yg)("li",{parentName:"ol"},"Create a new branch from the ",(0,o.yg)("inlineCode",{parentName:"li"},"master")," branch \ud83d\udd4a\ufe0f")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"git checkout -b my-branch\n")),(0,o.yg)("ol",{start:3},(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Add your changes in this branch \u2712\ufe0f")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Start the website on ",(0,o.yg)("a",{parentName:"p",href:"http://localhost:3000"},"http://localhost:3000")," to test it:"))),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"cd website\nyarn install\nyarn start\n")),(0,o.yg)("admonition",{title:"Send a pull request",type:"info"},(0,o.yg)("p",{parentName:"admonition"},"Send a pull request to the ",(0,o.yg)("inlineCode",{parentName:"p"},"master")," branch when your changes are done")),(0,o.yg)("admonition",{title:"Development documentation",type:"note"},(0,o.yg)("p",{parentName:"admonition"},"Read more about running the API in development at ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation#run-for-development"},"https://github.com/MaastrichtU-IDS/dsri-documentation#run-for-development"))),(0,o.yg)("hr",null),(0,o.yg)("h2",{id:"-automated-deployment"},"\ud83d\udd04 Automated deployment"),(0,o.yg)("p",null,"The documentation website is automatically updated and redeployed at each change to the ",(0,o.yg)("inlineCode",{parentName:"p"},"main")," branch using a ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/actions"},"GitHub Actions workflow"),"."),(0,o.yg)("p",null,(0,o.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/actions?query=workflow%3A%22Publish+to+GitHub+Pages%22"},(0,o.yg)("img",{parentName:"a",src:"https://github.com/MaastrichtU-IDS/dsri-documentation/workflows/Publish%20to%20GitHub%20Pages/badge.svg",alt:"Publish to GitHub Pages"}))),(0,o.yg)("hr",null),(0,o.yg)("h2",{id:"-help"},"\ud83d\udcdd Help"),(0,o.yg)("p",null,"Most pages of this website are written in Markdown, hence they are really easy to edit, especially when you are using a ",(0,o.yg)("a",{parentName:"p",href:"https://typora.io/"},"convenient markdown editor"),". Only the ",(0,o.yg)("inlineCode",{parentName:"p"},"index.js")," page is written in React JavaScript."),(0,o.yg)("h3",{id:"-files-locations"},"\ud83d\udd0e Files locations"),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},"Main DSRI documentation markdown files in ",(0,o.yg)("inlineCode",{parentName:"li"},"website/docs"),(0,o.yg)("ul",{parentName:"li"},(0,o.yg)("li",{parentName:"ul"},"Left docs menu defined in ",(0,o.yg)("inlineCode",{parentName:"li"},"website/sidebars.json")," "))),(0,o.yg)("li",{parentName:"ul"},"Blog articles as markdown files in ",(0,o.yg)("inlineCode",{parentName:"li"},"website/docs")),(0,o.yg)("li",{parentName:"ul"},"Index and contribute pages in ",(0,o.yg)("inlineCode",{parentName:"li"},"website/src/pages")),(0,o.yg)("li",{parentName:"ul"},"Images in ",(0,o.yg)("inlineCode",{parentName:"li"},"website/src/static/img")),(0,o.yg)("li",{parentName:"ul"},"Website configuration file in ",(0,o.yg)("inlineCode",{parentName:"li"},"website/docusaurus.config.js")," ")),(0,o.yg)("h3",{id:"-markdown-tip"},"\ud83e\udd84 Markdown tip"),(0,o.yg)("admonition",{title:"Colorful boxes",type:"warning"},(0,o.yg)("p",{parentName:"admonition"},"Use the following tags to create colorful boxes in markdown files:")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-markdown"},":::note You can specify an optional title\nGrey box\n:::\n\n:::tip Green box\nThe content and title *can* include markdown.\n:::\n\n:::info Blue box\nUseful information.\n:::\n\n:::caution Be careful!\nYellow box\n:::\n\n:::danger Fire red box\nDanger danger, mayday!\n:::\n")),(0,o.yg)("h2",{id:"\ufe0f-pull-request-process"},"\u2714\ufe0f Pull Request process"),(0,o.yg)("ol",null,(0,o.yg)("li",{parentName:"ol"},"Before sending a pull request make sure the DSRI documentation website still work as expected with the new changes properly integrated:")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre"},"cd website\nyarn install\nyarn start\n")),(0,o.yg)("ol",{start:2},(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/compare"},"Send a pull request")," to the ",(0,o.yg)("inlineCode",{parentName:"li"},"master")," branch."),(0,o.yg)("li",{parentName:"ol"},"Project contributors will review your change as soon as they can!")))}m.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/521adc3d.ad458d64.js b/assets/js/521adc3d.76dc0d57.js similarity index 99% rename from assets/js/521adc3d.ad458d64.js rename to assets/js/521adc3d.76dc0d57.js index a4b06a9a2..011d7568f 100644 --- a/assets/js/521adc3d.ad458d64.js +++ b/assets/js/521adc3d.76dc0d57.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[7234],{5680:(e,t,a)=>{a.d(t,{xA:()=>c,yg:()=>y});var r=a(6540);function n(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function o(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,r)}return a}function p(e){for(var t=1;t=0||(n[a]=e[a]);return n}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(n[a]=e[a])}return n}var s=r.createContext({}),l=function(e){var t=r.useContext(s),a=t;return e&&(a="function"==typeof e?e(t):p(p({},t),e)),a},c=function(e){var t=l(e.components);return r.createElement(s.Provider,{value:t},e.children)},g={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},m=r.forwardRef((function(e,t){var a=e.components,n=e.mdxType,o=e.originalType,s=e.parentName,c=i(e,["components","mdxType","originalType","parentName"]),m=l(a),y=n,d=m["".concat(s,".").concat(y)]||m[y]||g[y]||o;return a?r.createElement(d,p(p({ref:t},c),{},{components:a})):r.createElement(d,p({ref:t},c))}));function y(e,t){var a=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var o=a.length,p=new Array(o);p[0]=m;var i={};for(var s in t)hasOwnProperty.call(t,s)&&(i[s]=t[s]);i.originalType=e,i.mdxType="string"==typeof e?e:n,p[1]=i;for(var l=2;l{a.r(t),a.d(t,{assets:()=>c,contentTitle:()=>s,default:()=>y,frontMatter:()=>i,metadata:()=>l,toc:()=>g});var r=a(9668),n=a(1367),o=(a(6540),a(5680)),p=["components"],i={id:"project-management",title:"Create a new Project"},s=void 0,l={unversionedId:"project-management",id:"project-management",title:"Create a new Project",description:"Create a project using the web UI",source:"@site/docs/create-dsri-project.md",sourceDirName:".",slug:"/project-management",permalink:"/docs/project-management",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/create-dsri-project.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"project-management",title:"Create a new Project"},sidebar:"docs",previous:{title:"Delete objects (advanced)",permalink:"/docs/openshift-delete-objects"},next:{title:"Libraries for Machine Learning",permalink:"/docs/tools-machine-learning"}},c={},g=[{value:"Create a project using the web UI",id:"create-a-project-using-the-web-ui",level:2},{value:"Create a project using the CLI",id:"create-a-project-using-the-cli",level:2},{value:"Access permissions for developers to your project",id:"access-permissions-for-developers-to-your-project",level:2},{value:"Delete a project using the web UI",id:"delete-a-project-using-the-web-ui",level:2},{value:"Delete a project using the CLI",id:"delete-a-project-using-the-cli",level:2}],m={toc:g};function y(e){var t=e.components,a=(0,n.A)(e,p);return(0,o.yg)("wrapper",(0,r.A)({},m,a,{components:t,mdxType:"MDXLayout"}),(0,o.yg)("h2",{id:"create-a-project-using-the-web-ui"},"Create a project using the web UI"),(0,o.yg)("admonition",{title:"Avoid creating multiple projects",type:"caution"},(0,o.yg)("p",{parentName:"admonition"},"Try to avoid to create multiple projects for nothing please. Be responsible and delete applications you are not using anymore in your project to free resources, instead of creating a new project with a different number at the end."),(0,o.yg)("p",{parentName:"admonition"},"It is also easier to connect your different applications containers and storages when you create them in the same project.")),(0,o.yg)("p",null,"You can create a project using the ",(0,o.yg)("strong",{parentName:"p"},"Developer")," perspective, as follows:"),(0,o.yg)("ol",null,(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Click the ",(0,o.yg)("strong",{parentName:"p"},"Project")," drop-down menu to see a list of all available projects. Select ",(0,o.yg)("strong",{parentName:"p"},"Create Project"),".")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"In the ",(0,o.yg)("strong",{parentName:"p"},"Create Project")," dialog box, enter a unique name in the ",(0,o.yg)("strong",{parentName:"p"},"Name")," field. Use a short and meaningful name for your project as the project identifier is unique across all projects, such as ",(0,o.yg)("inlineCode",{parentName:"p"},"workspace-yourname")," or ",(0,o.yg)("inlineCode",{parentName:"p"},"ml-covid-pathways"))),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Add the ",(0,o.yg)("strong",{parentName:"p"},"Display Name")," ",(0,o.yg)("inlineCode",{parentName:"p"}," DSR Workshop"),"and ",(0,o.yg)("strong",{parentName:"p"},"Description")," ",(0,o.yg)("inlineCode",{parentName:"p"}," DSRI Community Workshop Projects"),"details for the project.")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Click ",(0,o.yg)("strong",{parentName:"p"},"Create"),".")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Use the left navigation panel to navigate to the ",(0,o.yg)("strong",{parentName:"p"},"Project")," view and see the dashboard for your project."))),(0,o.yg)("img",{src:"/img/screenshot_create_project.png",alt:"Create Project",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("ol",{start:6},(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Optional:"),(0,o.yg)("ul",{parentName:"li"},(0,o.yg)("li",{parentName:"ul"},"Use the ",(0,o.yg)("strong",{parentName:"li"},"Project")," drop-down menu at the top of the screen and select ",(0,o.yg)("strong",{parentName:"li"},"all projects")," to list all of the projects in your cluster."),(0,o.yg)("li",{parentName:"ul"},"Use the ",(0,o.yg)("strong",{parentName:"li"},"Details")," tab to see the project details."),(0,o.yg)("li",{parentName:"ul"},"If you have adequate permissions for a project, you can use the ",(0,o.yg)("strong",{parentName:"li"},"Project Access")," tab to provide or revoke ",(0,o.yg)("em",{parentName:"li"},"administrator"),", ",(0,o.yg)("em",{parentName:"li"},"edit"),", and ",(0,o.yg)("em",{parentName:"li"},"view")," privileges for the project.")))),(0,o.yg)("h2",{id:"create-a-project-using-the-cli"},"Create a project using the CLI"),(0,o.yg)("p",null,"You need to be logged in to the DSRI and ",(0,o.yg)("a",{parentName:"p",href:"https://maastrichtu-ids.github.io/dsri-documentation/docs/openshift-install#login-in-the-terminal-with-oc"},"copy the login command"),"."),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("p",{parentName:"li"},"Run"),(0,o.yg)("pre",{parentName:"li"},(0,o.yg)("code",{parentName:"pre",className:"language-bash"},'oc new-project --description="" --display-name=""\n'))),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("p",{parentName:"li"},"Example"),(0,o.yg)("pre",{parentName:"li"},(0,o.yg)("code",{parentName:"pre",className:"language-bash"},'oc new-project dsri-workshop --description="DSRI Workshop" \\\n --display-name="DSRI Community Workshop Projects"\n')))),(0,o.yg)("admonition",{title:"Reuse your project",type:"caution"},(0,o.yg)("p",{parentName:"admonition"},"Only create new projects when it is necessary (for a new project). You can easily ",(0,o.yg)("a",{parentName:"p",href:"https://maastrichtu-ids.github.io/dsri-documentation/docs/project-management#delete-a-project-using-the-web-ui"},"clean up your current project")," instead of creating a new one every time you want to try something.")),(0,o.yg)("h2",{id:"access-permissions-for-developers-to-your-project"},"Access permissions for developers to your project"),(0,o.yg)("p",null,"You can use the ",(0,o.yg)("strong",{parentName:"p"},"Project")," view in the ",(0,o.yg)("strong",{parentName:"p"},"Developer")," perspective to grant or revoke access permissions to your project."),(0,o.yg)("p",null,"To add users to your project and provide ",(0,o.yg)("strong",{parentName:"p"},"Admin"),", ",(0,o.yg)("strong",{parentName:"p"},"Edit"),", or ",(0,o.yg)("strong",{parentName:"p"},"View")," access to them:"),(0,o.yg)("ol",null,(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"In the ",(0,o.yg)("strong",{parentName:"p"},"Developer")," perspective, navigate to the ",(0,o.yg)("strong",{parentName:"p"},"Project")," view.")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"In the ",(0,o.yg)("strong",{parentName:"p"},"Project")," page, select the ",(0,o.yg)("strong",{parentName:"p"},"Project Access")," tab.")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Click ",(0,o.yg)("strong",{parentName:"p"},"Add Access")," to add a new row of permissions to the default ones."),(0,o.yg)("img",{src:"/img/screenshot_project_access.png",alt:"Project Access",style:{maxWidth:"100%",maxHeight:"100%"}}))),(0,o.yg)("ol",{start:4},(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Enter the user name, click the ",(0,o.yg)("strong",{parentName:"p"},"Select a role")," drop-down list, and select an appropriate role.")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Click ",(0,o.yg)("strong",{parentName:"p"},"Save")," to add the new permissions."))),(0,o.yg)("p",null,"You can also use:"),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("p",{parentName:"li"},"The ",(0,o.yg)("strong",{parentName:"p"},"Select a role")," drop-down list, to modify the access permissions of an existing user.")),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("p",{parentName:"li"},"The ",(0,o.yg)("strong",{parentName:"p"},"Remove Access")," icon, to completely remove the access permissions of an existing user to the project."))),(0,o.yg)("admonition",{type:"info"},(0,o.yg)("p",{parentName:"admonition"}," Advanced role-based access control is managed in the ",(0,o.yg)("strong",{parentName:"p"},"Roles")," and ",(0,o.yg)("strong",{parentName:"p"},"Roles Binding")," views in the ",(0,o.yg)("strong",{parentName:"p"},"Administrator")," perspective")),(0,o.yg)("h2",{id:"delete-a-project-using-the-web-ui"},"Delete a project using the web UI"),(0,o.yg)("ol",null,(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Navigate to ",(0,o.yg)("strong",{parentName:"p"},"Home")," \u2192 ",(0,o.yg)("strong",{parentName:"p"},"Projects"),".")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Locate the project that you want to delete from the list of projects.")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"On the far right side of the project listing, select ",(0,o.yg)("strong",{parentName:"p"},"Delete Project")," from the Options menu ",(0,o.yg)("img",{parentName:"p",src:"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABsAAAAjCAIAAADqn+bCAAAACXBIWXMAAA7EAAAOxAGVKw4bAAAA+0lEQVRIie2WMQqEMBBFJ47gUXRBLyBYqbUXULCx9CR2XsAb6AlUEM9kpckW7obdZhwWYWHXX/3i8TPJZEKEUgpOlXFu3JX4V4kmB2qaZhgGKSUiZlkWxzEBC84N9zxv27bdO47Tti0Bs3at4wBgXVca/lJnfN/XPggCGmadIwAsywIAiGhZFk1ydy2EYJKgGCqK4vZUVVU0zKpxnmftp2mi4S/1GhG1N82DMWNNYVmW4zgqpRAxTVMa5t4evlg11nXd9/1eY57nSZIQMKtG13WllLu3bbvrOgJmdUbHwfur8Xniqw6Hh5UYRdGDNowwDA+WvP4UV+JPJ94B1gKUWcTOCT0AAAAASUVORK5CYII=",alt:"kebab"}),".")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"When the ",(0,o.yg)("strong",{parentName:"p"},"Delete Project")," pane opens, enter the name of the project that you want to delete in the field.")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Click ",(0,o.yg)("strong",{parentName:"p"},"Delete"),"."),(0,o.yg)("img",{src:"/img/screenshot_delete_project.png",alt:"Delete Project",style:{maxWidth:"100%",maxHeight:"100%"}}))),(0,o.yg)("h2",{id:"delete-a-project-using-the-cli"},"Delete a project using the CLI"),(0,o.yg)("admonition",{title:"Delete Project",type:"caution"},(0,o.yg)("p",{parentName:"admonition"},"When you delete a project, the server updates the project status to ",(0,o.yg)("strong",{parentName:"p"},"Terminating")," from ",(0,o.yg)("strong",{parentName:"p"},"Active"),". Then, the server clears all content from a project that is in the ",(0,o.yg)("strong",{parentName:"p"},"Terminating")," state before finally removing the project. While a project is in ",(0,o.yg)("strong",{parentName:"p"},"Terminating")," status, you cannot add new content to the project. Projects can be deleted from the CLI or the web console.")),(0,o.yg)("p",null,"You need to be logged in to the DSRI and ",(0,o.yg)("a",{parentName:"p",href:"https://maastrichtu-ids.github.io/dsri-documentation/docs/openshift-install#login-in-the-terminal-with-oc"},"copy the login command"),"."),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("p",{parentName:"li"},"Run"),(0,o.yg)("pre",{parentName:"li"},(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"oc delete project \n")))),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("p",{parentName:"li"},"Example"),(0,o.yg)("pre",{parentName:"li"},(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"oc delete project dsri-workshop\n")))))}y.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[7234],{5680:(e,t,a)=>{a.d(t,{xA:()=>c,yg:()=>y});var r=a(6540);function n(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function o(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,r)}return a}function p(e){for(var t=1;t=0||(n[a]=e[a]);return n}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(n[a]=e[a])}return n}var s=r.createContext({}),l=function(e){var t=r.useContext(s),a=t;return e&&(a="function"==typeof e?e(t):p(p({},t),e)),a},c=function(e){var t=l(e.components);return r.createElement(s.Provider,{value:t},e.children)},g={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},m=r.forwardRef((function(e,t){var a=e.components,n=e.mdxType,o=e.originalType,s=e.parentName,c=i(e,["components","mdxType","originalType","parentName"]),m=l(a),y=n,d=m["".concat(s,".").concat(y)]||m[y]||g[y]||o;return a?r.createElement(d,p(p({ref:t},c),{},{components:a})):r.createElement(d,p({ref:t},c))}));function y(e,t){var a=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var o=a.length,p=new Array(o);p[0]=m;var i={};for(var s in t)hasOwnProperty.call(t,s)&&(i[s]=t[s]);i.originalType=e,i.mdxType="string"==typeof e?e:n,p[1]=i;for(var l=2;l{a.r(t),a.d(t,{assets:()=>c,contentTitle:()=>s,default:()=>y,frontMatter:()=>i,metadata:()=>l,toc:()=>g});var r=a(9668),n=a(1367),o=(a(6540),a(5680)),p=["components"],i={id:"project-management",title:"Create a new Project"},s=void 0,l={unversionedId:"project-management",id:"project-management",title:"Create a new Project",description:"Create a project using the web UI",source:"@site/docs/create-dsri-project.md",sourceDirName:".",slug:"/project-management",permalink:"/docs/project-management",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/create-dsri-project.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"project-management",title:"Create a new Project"},sidebar:"docs",previous:{title:"Delete objects (advanced)",permalink:"/docs/openshift-delete-objects"},next:{title:"Libraries for Machine Learning",permalink:"/docs/tools-machine-learning"}},c={},g=[{value:"Create a project using the web UI",id:"create-a-project-using-the-web-ui",level:2},{value:"Create a project using the CLI",id:"create-a-project-using-the-cli",level:2},{value:"Access permissions for developers to your project",id:"access-permissions-for-developers-to-your-project",level:2},{value:"Delete a project using the web UI",id:"delete-a-project-using-the-web-ui",level:2},{value:"Delete a project using the CLI",id:"delete-a-project-using-the-cli",level:2}],m={toc:g};function y(e){var t=e.components,a=(0,n.A)(e,p);return(0,o.yg)("wrapper",(0,r.A)({},m,a,{components:t,mdxType:"MDXLayout"}),(0,o.yg)("h2",{id:"create-a-project-using-the-web-ui"},"Create a project using the web UI"),(0,o.yg)("admonition",{title:"Avoid creating multiple projects",type:"caution"},(0,o.yg)("p",{parentName:"admonition"},"Try to avoid to create multiple projects for nothing please. Be responsible and delete applications you are not using anymore in your project to free resources, instead of creating a new project with a different number at the end."),(0,o.yg)("p",{parentName:"admonition"},"It is also easier to connect your different applications containers and storages when you create them in the same project.")),(0,o.yg)("p",null,"You can create a project using the ",(0,o.yg)("strong",{parentName:"p"},"Developer")," perspective, as follows:"),(0,o.yg)("ol",null,(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Click the ",(0,o.yg)("strong",{parentName:"p"},"Project")," drop-down menu to see a list of all available projects. Select ",(0,o.yg)("strong",{parentName:"p"},"Create Project"),".")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"In the ",(0,o.yg)("strong",{parentName:"p"},"Create Project")," dialog box, enter a unique name in the ",(0,o.yg)("strong",{parentName:"p"},"Name")," field. Use a short and meaningful name for your project as the project identifier is unique across all projects, such as ",(0,o.yg)("inlineCode",{parentName:"p"},"workspace-yourname")," or ",(0,o.yg)("inlineCode",{parentName:"p"},"ml-covid-pathways"))),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Add the ",(0,o.yg)("strong",{parentName:"p"},"Display Name")," ",(0,o.yg)("inlineCode",{parentName:"p"}," DSR Workshop"),"and ",(0,o.yg)("strong",{parentName:"p"},"Description")," ",(0,o.yg)("inlineCode",{parentName:"p"}," DSRI Community Workshop Projects"),"details for the project.")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Click ",(0,o.yg)("strong",{parentName:"p"},"Create"),".")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Use the left navigation panel to navigate to the ",(0,o.yg)("strong",{parentName:"p"},"Project")," view and see the dashboard for your project."))),(0,o.yg)("img",{src:"/img/screenshot_create_project.png",alt:"Create Project",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("ol",{start:6},(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Optional:"),(0,o.yg)("ul",{parentName:"li"},(0,o.yg)("li",{parentName:"ul"},"Use the ",(0,o.yg)("strong",{parentName:"li"},"Project")," drop-down menu at the top of the screen and select ",(0,o.yg)("strong",{parentName:"li"},"all projects")," to list all of the projects in your cluster."),(0,o.yg)("li",{parentName:"ul"},"Use the ",(0,o.yg)("strong",{parentName:"li"},"Details")," tab to see the project details."),(0,o.yg)("li",{parentName:"ul"},"If you have adequate permissions for a project, you can use the ",(0,o.yg)("strong",{parentName:"li"},"Project Access")," tab to provide or revoke ",(0,o.yg)("em",{parentName:"li"},"administrator"),", ",(0,o.yg)("em",{parentName:"li"},"edit"),", and ",(0,o.yg)("em",{parentName:"li"},"view")," privileges for the project.")))),(0,o.yg)("h2",{id:"create-a-project-using-the-cli"},"Create a project using the CLI"),(0,o.yg)("p",null,"You need to be logged in to the DSRI and ",(0,o.yg)("a",{parentName:"p",href:"https://maastrichtu-ids.github.io/dsri-documentation/docs/openshift-install#login-in-the-terminal-with-oc"},"copy the login command"),"."),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("p",{parentName:"li"},"Run"),(0,o.yg)("pre",{parentName:"li"},(0,o.yg)("code",{parentName:"pre",className:"language-bash"},'oc new-project --description="" --display-name=""\n'))),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("p",{parentName:"li"},"Example"),(0,o.yg)("pre",{parentName:"li"},(0,o.yg)("code",{parentName:"pre",className:"language-bash"},'oc new-project dsri-workshop --description="DSRI Workshop" \\\n --display-name="DSRI Community Workshop Projects"\n')))),(0,o.yg)("admonition",{title:"Reuse your project",type:"caution"},(0,o.yg)("p",{parentName:"admonition"},"Only create new projects when it is necessary (for a new project). You can easily ",(0,o.yg)("a",{parentName:"p",href:"https://maastrichtu-ids.github.io/dsri-documentation/docs/project-management#delete-a-project-using-the-web-ui"},"clean up your current project")," instead of creating a new one every time you want to try something.")),(0,o.yg)("h2",{id:"access-permissions-for-developers-to-your-project"},"Access permissions for developers to your project"),(0,o.yg)("p",null,"You can use the ",(0,o.yg)("strong",{parentName:"p"},"Project")," view in the ",(0,o.yg)("strong",{parentName:"p"},"Developer")," perspective to grant or revoke access permissions to your project."),(0,o.yg)("p",null,"To add users to your project and provide ",(0,o.yg)("strong",{parentName:"p"},"Admin"),", ",(0,o.yg)("strong",{parentName:"p"},"Edit"),", or ",(0,o.yg)("strong",{parentName:"p"},"View")," access to them:"),(0,o.yg)("ol",null,(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"In the ",(0,o.yg)("strong",{parentName:"p"},"Developer")," perspective, navigate to the ",(0,o.yg)("strong",{parentName:"p"},"Project")," view.")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"In the ",(0,o.yg)("strong",{parentName:"p"},"Project")," page, select the ",(0,o.yg)("strong",{parentName:"p"},"Project Access")," tab.")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Click ",(0,o.yg)("strong",{parentName:"p"},"Add Access")," to add a new row of permissions to the default ones."),(0,o.yg)("img",{src:"/img/screenshot_project_access.png",alt:"Project Access",style:{maxWidth:"100%",maxHeight:"100%"}}))),(0,o.yg)("ol",{start:4},(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Enter the user name, click the ",(0,o.yg)("strong",{parentName:"p"},"Select a role")," drop-down list, and select an appropriate role.")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Click ",(0,o.yg)("strong",{parentName:"p"},"Save")," to add the new permissions."))),(0,o.yg)("p",null,"You can also use:"),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("p",{parentName:"li"},"The ",(0,o.yg)("strong",{parentName:"p"},"Select a role")," drop-down list, to modify the access permissions of an existing user.")),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("p",{parentName:"li"},"The ",(0,o.yg)("strong",{parentName:"p"},"Remove Access")," icon, to completely remove the access permissions of an existing user to the project."))),(0,o.yg)("admonition",{type:"info"},(0,o.yg)("p",{parentName:"admonition"}," Advanced role-based access control is managed in the ",(0,o.yg)("strong",{parentName:"p"},"Roles")," and ",(0,o.yg)("strong",{parentName:"p"},"Roles Binding")," views in the ",(0,o.yg)("strong",{parentName:"p"},"Administrator")," perspective")),(0,o.yg)("h2",{id:"delete-a-project-using-the-web-ui"},"Delete a project using the web UI"),(0,o.yg)("ol",null,(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Navigate to ",(0,o.yg)("strong",{parentName:"p"},"Home")," \u2192 ",(0,o.yg)("strong",{parentName:"p"},"Projects"),".")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Locate the project that you want to delete from the list of projects.")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"On the far right side of the project listing, select ",(0,o.yg)("strong",{parentName:"p"},"Delete Project")," from the Options menu ",(0,o.yg)("img",{parentName:"p",src:"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABsAAAAjCAIAAADqn+bCAAAACXBIWXMAAA7EAAAOxAGVKw4bAAAA+0lEQVRIie2WMQqEMBBFJ47gUXRBLyBYqbUXULCx9CR2XsAb6AlUEM9kpckW7obdZhwWYWHXX/3i8TPJZEKEUgpOlXFu3JX4V4kmB2qaZhgGKSUiZlkWxzEBC84N9zxv27bdO47Tti0Bs3at4wBgXVca/lJnfN/XPggCGmadIwAsywIAiGhZFk1ydy2EYJKgGCqK4vZUVVU0zKpxnmftp2mi4S/1GhG1N82DMWNNYVmW4zgqpRAxTVMa5t4evlg11nXd9/1eY57nSZIQMKtG13WllLu3bbvrOgJmdUbHwfur8Xniqw6Hh5UYRdGDNowwDA+WvP4UV+JPJ94B1gKUWcTOCT0AAAAASUVORK5CYII=",alt:"kebab"}),".")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"When the ",(0,o.yg)("strong",{parentName:"p"},"Delete Project")," pane opens, enter the name of the project that you want to delete in the field.")),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("p",{parentName:"li"},"Click ",(0,o.yg)("strong",{parentName:"p"},"Delete"),"."),(0,o.yg)("img",{src:"/img/screenshot_delete_project.png",alt:"Delete Project",style:{maxWidth:"100%",maxHeight:"100%"}}))),(0,o.yg)("h2",{id:"delete-a-project-using-the-cli"},"Delete a project using the CLI"),(0,o.yg)("admonition",{title:"Delete Project",type:"caution"},(0,o.yg)("p",{parentName:"admonition"},"When you delete a project, the server updates the project status to ",(0,o.yg)("strong",{parentName:"p"},"Terminating")," from ",(0,o.yg)("strong",{parentName:"p"},"Active"),". Then, the server clears all content from a project that is in the ",(0,o.yg)("strong",{parentName:"p"},"Terminating")," state before finally removing the project. While a project is in ",(0,o.yg)("strong",{parentName:"p"},"Terminating")," status, you cannot add new content to the project. Projects can be deleted from the CLI or the web console.")),(0,o.yg)("p",null,"You need to be logged in to the DSRI and ",(0,o.yg)("a",{parentName:"p",href:"https://maastrichtu-ids.github.io/dsri-documentation/docs/openshift-install#login-in-the-terminal-with-oc"},"copy the login command"),"."),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("p",{parentName:"li"},"Run"),(0,o.yg)("pre",{parentName:"li"},(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"oc delete project \n")))),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("p",{parentName:"li"},"Example"),(0,o.yg)("pre",{parentName:"li"},(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"oc delete project dsri-workshop\n")))))}y.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/5514662e.d419ee2b.js b/assets/js/5514662e.e5887652.js similarity index 98% rename from assets/js/5514662e.d419ee2b.js rename to assets/js/5514662e.e5887652.js index c2b9172c0..618737fdf 100644 --- a/assets/js/5514662e.d419ee2b.js +++ b/assets/js/5514662e.e5887652.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[2865],{5680:(e,t,a)=>{a.d(t,{xA:()=>s,yg:()=>d});var n=a(6540);function r(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function o(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function c(e){for(var t=1;t=0||(r[a]=e[a]);return r}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(r[a]=e[a])}return r}var l=n.createContext({}),p=function(e){var t=n.useContext(l),a=t;return e&&(a="function"==typeof e?e(t):c(c({},t),e)),a},s=function(e){var t=p(e.components);return n.createElement(l.Provider,{value:t},e.children)},m={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},u=n.forwardRef((function(e,t){var a=e.components,r=e.mdxType,o=e.originalType,l=e.parentName,s=i(e,["components","mdxType","originalType","parentName"]),u=p(a),d=r,g=u["".concat(l,".").concat(d)]||u[d]||m[d]||o;return a?n.createElement(g,c(c({ref:t},s),{},{components:a})):n.createElement(g,c({ref:t},s))}));function d(e,t){var a=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var o=a.length,c=new Array(o);c[0]=u;var i={};for(var l in t)hasOwnProperty.call(t,l)&&(i[l]=t[l]);i.originalType=e,i.mdxType="string"==typeof e?e:r,c[1]=i;for(var p=2;p{a.r(t),a.d(t,{assets:()=>s,contentTitle:()=>l,default:()=>d,frontMatter:()=>i,metadata:()=>p,toc:()=>m});var n=a(9668),r=a(1367),o=(a(6540),a(5680)),c=["components"],i={id:"catalog-data-streaming",title:"Data streaming"},l=void 0,p={unversionedId:"catalog-data-streaming",id:"catalog-data-streaming",title:"Data streaming",description:"Apache Flink",source:"@site/docs/catalog-data-streaming.md",sourceDirName:".",slug:"/catalog-data-streaming",permalink:"/docs/catalog-data-streaming",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/catalog-data-streaming.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"catalog-data-streaming",title:"Data streaming"}},s={},m=[{value:"Apache Flink",id:"apache-flink",level:2}],u={toc:m};function d(e){var t=e.components,a=(0,r.A)(e,c);return(0,o.yg)("wrapper",(0,n.A)({},u,a,{components:t,mdxType:"MDXLayout"}),(0,o.yg)("h2",{id:"apache-flink"},"Apache Flink"),(0,o.yg)("p",null,(0,o.yg)("a",{parentName:"p",href:"https://flink.apache.org/"},"Apache Flink")," enables processing of Data Streams using languages such as Java or Scala ."),(0,o.yg)("admonition",{title:"Root permission required",type:"caution"},(0,o.yg)("p",{parentName:"admonition"},"\ud83d\udd12 You need root containers enabled (aka. anyuid) in your project to start this application.")),(0,o.yg)("p",null,"Create the Apache Flink template in your project using ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/vemonet/flink-on-openshift"},"vemonet/flink-on-openshift")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"oc apply -f https://raw.githubusercontent.com/vemonet/flink-on-openshift/master/template-flink-dsri.yml\n")),(0,o.yg)("p",null,"Use the template to start the cluster from the catalog."),(0,o.yg)("p",null,"Use this command to get the Flink Jobmanager pod id and copy file to the pod."),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"oc get pod --selector app=flink --selector component=jobmanager --no-headers -o=custom-columns=NAME:.metadata.name\n\n# Example creating the workspace folder and copying the RMLStreamer.jar to the pod\noc exec -- mkdir -p /mnt/workspace/resources\noc cp workspace/resources/RMLStreamer.jar :/mnt/\n")),(0,o.yg)("p",null,"Delete the Apache Flink cluster (change the application name):"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"oc delete all,secret,configmaps,serviceaccount,rolebinding --selector app=flink-cluster\n")))}d.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[2865],{5680:(e,t,a)=>{a.d(t,{xA:()=>s,yg:()=>d});var n=a(6540);function r(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function o(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function c(e){for(var t=1;t=0||(r[a]=e[a]);return r}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(r[a]=e[a])}return r}var l=n.createContext({}),p=function(e){var t=n.useContext(l),a=t;return e&&(a="function"==typeof e?e(t):c(c({},t),e)),a},s=function(e){var t=p(e.components);return n.createElement(l.Provider,{value:t},e.children)},m={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},u=n.forwardRef((function(e,t){var a=e.components,r=e.mdxType,o=e.originalType,l=e.parentName,s=i(e,["components","mdxType","originalType","parentName"]),u=p(a),d=r,g=u["".concat(l,".").concat(d)]||u[d]||m[d]||o;return a?n.createElement(g,c(c({ref:t},s),{},{components:a})):n.createElement(g,c({ref:t},s))}));function d(e,t){var a=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var o=a.length,c=new Array(o);c[0]=u;var i={};for(var l in t)hasOwnProperty.call(t,l)&&(i[l]=t[l]);i.originalType=e,i.mdxType="string"==typeof e?e:r,c[1]=i;for(var p=2;p{a.r(t),a.d(t,{assets:()=>s,contentTitle:()=>l,default:()=>d,frontMatter:()=>i,metadata:()=>p,toc:()=>m});var n=a(9668),r=a(1367),o=(a(6540),a(5680)),c=["components"],i={id:"catalog-data-streaming",title:"Data streaming"},l=void 0,p={unversionedId:"catalog-data-streaming",id:"catalog-data-streaming",title:"Data streaming",description:"Apache Flink",source:"@site/docs/catalog-data-streaming.md",sourceDirName:".",slug:"/catalog-data-streaming",permalink:"/docs/catalog-data-streaming",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/catalog-data-streaming.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"catalog-data-streaming",title:"Data streaming"}},s={},m=[{value:"Apache Flink",id:"apache-flink",level:2}],u={toc:m};function d(e){var t=e.components,a=(0,r.A)(e,c);return(0,o.yg)("wrapper",(0,n.A)({},u,a,{components:t,mdxType:"MDXLayout"}),(0,o.yg)("h2",{id:"apache-flink"},"Apache Flink"),(0,o.yg)("p",null,(0,o.yg)("a",{parentName:"p",href:"https://flink.apache.org/"},"Apache Flink")," enables processing of Data Streams using languages such as Java or Scala ."),(0,o.yg)("admonition",{title:"Root permission required",type:"caution"},(0,o.yg)("p",{parentName:"admonition"},"\ud83d\udd12 You need root containers enabled (aka. anyuid) in your project to start this application.")),(0,o.yg)("p",null,"Create the Apache Flink template in your project using ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/vemonet/flink-on-openshift"},"vemonet/flink-on-openshift")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"oc apply -f https://raw.githubusercontent.com/vemonet/flink-on-openshift/master/template-flink-dsri.yml\n")),(0,o.yg)("p",null,"Use the template to start the cluster from the catalog."),(0,o.yg)("p",null,"Use this command to get the Flink Jobmanager pod id and copy file to the pod."),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"oc get pod --selector app=flink --selector component=jobmanager --no-headers -o=custom-columns=NAME:.metadata.name\n\n# Example creating the workspace folder and copying the RMLStreamer.jar to the pod\noc exec -- mkdir -p /mnt/workspace/resources\noc cp workspace/resources/RMLStreamer.jar :/mnt/\n")),(0,o.yg)("p",null,"Delete the Apache Flink cluster (change the application name):"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"oc delete all,secret,configmaps,serviceaccount,rolebinding --selector app=flink-cluster\n")))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/5890eb41.e4043ef2.js b/assets/js/5890eb41.60ab0726.js similarity index 99% rename from assets/js/5890eb41.e4043ef2.js rename to assets/js/5890eb41.60ab0726.js index a846c98e4..1e805c770 100644 --- a/assets/js/5890eb41.e4043ef2.js +++ b/assets/js/5890eb41.60ab0726.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[8603],{5680:(e,t,n)=>{n.d(t,{xA:()=>c,yg:()=>g});var a=n(6540);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function o(e){for(var t=1;t=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var s=a.createContext({}),p=function(e){var t=a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):o(o({},t),e)),n},c=function(e){var t=p(e.components);return a.createElement(s.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},y=a.forwardRef((function(e,t){var n=e.components,i=e.mdxType,r=e.originalType,s=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),y=p(n),g=i,m=y["".concat(s,".").concat(g)]||y[g]||u[g]||r;return n?a.createElement(m,o(o({ref:t},c),{},{components:n})):a.createElement(m,o({ref:t},c))}));function g(e,t){var n=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var r=n.length,o=new Array(r);o[0]=y;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:i,o[1]=l;for(var p=2;p{n.r(t),n.d(t,{assets:()=>c,contentTitle:()=>s,default:()=>g,frontMatter:()=>l,metadata:()=>p,toc:()=>u});var a=n(9668),i=n(1367),r=(n(6540),n(5680)),o=["components"],l={id:"guide-vpn",title:"Install UM VPN"},s=void 0,p={unversionedId:"guide-vpn",id:"guide-vpn",title:"Install UM VPN",description:"Request an account",source:"@site/docs/guide-vpn.md",sourceDirName:".",slug:"/guide-vpn",permalink:"/docs/guide-vpn",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/guide-vpn.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"guide-vpn",title:"Install UM VPN"},sidebar:"docs",previous:{title:"Publish a Docker image",permalink:"/docs/guide-publish-image"},next:{title:"Prepare a workshop",permalink:"/docs/guide-workshop"}},c={},u=[{value:"Request an account",id:"request-an-account",level:2},{value:"Connect to the UM network",id:"connect-to-the-um-network",level:2},{value:"Install the VPN (AnyConnect Secure Mobility Client) on Windows",id:"install-the-vpn-anyconnect-secure-mobility-client-on-windows",level:4},{value:"Log in to the VPN (AnyConnect Secure Mobility Client)",id:"log-in-to-the-vpn-anyconnect-secure-mobility-client",level:4},{value:"Install the VPN (AnyConnect Secure Mobility Client) on Linux",id:"install-the-vpn-anyconnect-secure-mobility-client-on-linux",level:4}],y={toc:u};function g(e){var t=e.components,n=(0,i.A)(e,o);return(0,r.yg)("wrapper",(0,a.A)({},y,n,{components:t,mdxType:"MDXLayout"}),(0,r.yg)("h2",{id:"request-an-account"},"Request an account"),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"You will need to have an account at Maastricht University with an email ending with ",(0,r.yg)("inlineCode",{parentName:"p"},"@maastrichtuniversity.nl")," or ",(0,r.yg)("inlineCode",{parentName:"p"},"@student.maastrichtuniversity.nl"),".")),(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"Request access to the DSRI for your account Please fill this ",(0,r.yg)("a",{parentName:"p",href:"/register"},"form \ud83d\udcec"),". to provide us some information on what you plan to do with the DSRI."))),(0,r.yg)("h2",{id:"connect-to-the-um-network"},"Connect to the UM network"),(0,r.yg)("p",null,"You need to be connected to the UM network to access the DSRI."),(0,r.yg)("ul",null,(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("p",{parentName:"li"},"Connect to ",(0,r.yg)("strong",{parentName:"p"},"UMnet")," or ",(0,r.yg)("strong",{parentName:"p"},"eduroam")," WiFi at Maastricht University")),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("p",{parentName:"li"},"Use the ",(0,r.yg)("strong",{parentName:"p"},"Maastricht University VPN")," at ",(0,r.yg)("strong",{parentName:"p"},(0,r.yg)("a",{parentName:"strong",href:"https://vpn.maastrichtuniversity.nl/"},"vpn.maastrichtuniversity.nl"))),(0,r.yg)("p",{parentName:"li"},"Log in to that using your UM username and password."))),(0,r.yg)("admonition",{title:"Students",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"By default the UM VPN is only available to employees. As a student you can access UM resources from any location via ",(0,r.yg)("a",{parentName:"p",href:"https://athenadesktop.maastrichtuniversity.nl"},"Student Desktop Anywhere"),".\u202fHowever, if VPN access is absolutely necessary you can request access via your course coordinator. ")),(0,r.yg)("ul",null,(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("p",{parentName:"li"},"The ",(0,r.yg)("strong",{parentName:"p"},"prefix of your UM email address")," with the first letter capitalized, e.g. ",(0,r.yg)("inlineCode",{parentName:"p"},"Firstname.Lastname")," or ",(0,r.yg)("inlineCode",{parentName:"p"},"F.Lastname"),"Or your ",(0,r.yg)("strong",{parentName:"p"},"employee number")," at Maastricht University (a.k.a. P number), e.g. ",(0,r.yg)("inlineCode",{parentName:"p"},"P7000000")),(0,r.yg)("img",{src:"/img/vpn-login.png",alt:"VPN Log in View",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("p",{parentName:"li"},"Then You will see below page to download the ",(0,r.yg)("strong",{parentName:"p"},"AnyConnect Secure Mobility Client")),(0,r.yg)("img",{src:"/img/VPN-anyconnect.png",alt:"Download AnyConnect Secure Mobility Client",style:{maxWidth:"100%",maxHeight:"100%"}}))),(0,r.yg)("h4",{id:"install-the-vpn-anyconnect-secure-mobility-client-on-windows"},"Install the VPN (AnyConnect Secure Mobility Client) on Windows"),(0,r.yg)("p",null,"Double click on the ",(0,r.yg)("inlineCode",{parentName:"p"},".exe")," file to install the VPN."),(0,r.yg)("p",null,"You can follow below steps as in pictures."),(0,r.yg)("img",{src:"/img/vpnsetup1.png",alt:"Install VPN",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("img",{src:"/img/vpnsetup2.png",alt:"Install VPN",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("img",{src:"/img/vpnsetup3.png",alt:"Install VPN",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("img",{src:"/img/vpnsetup4.png",alt:"Install VPN",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("h4",{id:"log-in-to-the-vpn-anyconnect-secure-mobility-client"},"Log in to the VPN (AnyConnect Secure Mobility Client)"),(0,r.yg)("p",null,"Once you finish installing you can run the Cisco AnyConnect Secure Mobility Client. "),(0,r.yg)("img",{src:"/img/vpnsetup7.png",alt:"Log in to the VPN",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("p",null,"Then after you will get the bellow wizard and click connect"),(0,r.yg)("img",{src:"/img/vpnsetup5.png",alt:"Log in to the VPN",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("p",null,"Provide your UM username and password. (",(0,r.yg)("strong",{parentName:"p"},"employee number")," at Maastricht University (a.k.a. P number), e.g. ",(0,r.yg)("inlineCode",{parentName:"p"},"P7000000"),")"),(0,r.yg)("img",{src:"/img/vpnsetup6.png",alt:"Log in to the VPN",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("h4",{id:"install-the-vpn-anyconnect-secure-mobility-client-on-linux"},"Install the VPN (AnyConnect Secure Mobility Client) on Linux"),(0,r.yg)("ul",null,(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("p",{parentName:"li"},"Connect to ",(0,r.yg)("strong",{parentName:"p"},"UMnet")," or ",(0,r.yg)("strong",{parentName:"p"},"eduroam")," WiFi at Maastricht University")),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("p",{parentName:"li"},"For ",(0,r.yg)("strong",{parentName:"p"},"Linux"),", use ",(0,r.yg)("inlineCode",{parentName:"p"},"openconnect")," to connect to the UM VPN. You can easily install it on Ubuntu and Debian distributions with ",(0,r.yg)("inlineCode",{parentName:"p"},"apt"),":"),(0,r.yg)("pre",{parentName:"li"},(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"sudo apt install openconnect\nsudo openconnect -u YOUR.USER --authgroup 01-Employees --useragent=AnyConnect vpn.maastrichtuniversity.nl\n")),(0,r.yg)("blockquote",{parentName:"li"},(0,r.yg)("p",{parentName:"blockquote"},"Provide your UM password when prompted."))),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("p",{parentName:"li"},"For ",(0,r.yg)("strong",{parentName:"p"},"students"),":"),(0,r.yg)("ul",{parentName:"li"},(0,r.yg)("li",{parentName:"ul"},"By default the UM VPN is only available to employees. As a student you can access UM resources from any location via ",(0,r.yg)("a",{parentName:"li",href:"https://athenadesktop.maastrichtuniversity.nl"},"Student Desktop Anywhere"),".\u202fHowever, if VPN access is absolutely necessary you can request access via your course coordinator.")))))}g.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[8603],{5680:(e,t,n)=>{n.d(t,{xA:()=>c,yg:()=>g});var a=n(6540);function i(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function o(e){for(var t=1;t=0||(i[n]=e[n]);return i}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(i[n]=e[n])}return i}var s=a.createContext({}),p=function(e){var t=a.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):o(o({},t),e)),n},c=function(e){var t=p(e.components);return a.createElement(s.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},y=a.forwardRef((function(e,t){var n=e.components,i=e.mdxType,r=e.originalType,s=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),y=p(n),g=i,m=y["".concat(s,".").concat(g)]||y[g]||u[g]||r;return n?a.createElement(m,o(o({ref:t},c),{},{components:n})):a.createElement(m,o({ref:t},c))}));function g(e,t){var n=arguments,i=t&&t.mdxType;if("string"==typeof e||i){var r=n.length,o=new Array(r);o[0]=y;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:i,o[1]=l;for(var p=2;p{n.r(t),n.d(t,{assets:()=>c,contentTitle:()=>s,default:()=>g,frontMatter:()=>l,metadata:()=>p,toc:()=>u});var a=n(9668),i=n(1367),r=(n(6540),n(5680)),o=["components"],l={id:"guide-vpn",title:"Install UM VPN"},s=void 0,p={unversionedId:"guide-vpn",id:"guide-vpn",title:"Install UM VPN",description:"Request an account",source:"@site/docs/guide-vpn.md",sourceDirName:".",slug:"/guide-vpn",permalink:"/docs/guide-vpn",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/guide-vpn.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"guide-vpn",title:"Install UM VPN"},sidebar:"docs",previous:{title:"Publish a Docker image",permalink:"/docs/guide-publish-image"},next:{title:"Prepare a workshop",permalink:"/docs/guide-workshop"}},c={},u=[{value:"Request an account",id:"request-an-account",level:2},{value:"Connect to the UM network",id:"connect-to-the-um-network",level:2},{value:"Install the VPN (AnyConnect Secure Mobility Client) on Windows",id:"install-the-vpn-anyconnect-secure-mobility-client-on-windows",level:4},{value:"Log in to the VPN (AnyConnect Secure Mobility Client)",id:"log-in-to-the-vpn-anyconnect-secure-mobility-client",level:4},{value:"Install the VPN (AnyConnect Secure Mobility Client) on Linux",id:"install-the-vpn-anyconnect-secure-mobility-client-on-linux",level:4}],y={toc:u};function g(e){var t=e.components,n=(0,i.A)(e,o);return(0,r.yg)("wrapper",(0,a.A)({},y,n,{components:t,mdxType:"MDXLayout"}),(0,r.yg)("h2",{id:"request-an-account"},"Request an account"),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"You will need to have an account at Maastricht University with an email ending with ",(0,r.yg)("inlineCode",{parentName:"p"},"@maastrichtuniversity.nl")," or ",(0,r.yg)("inlineCode",{parentName:"p"},"@student.maastrichtuniversity.nl"),".")),(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"Request access to the DSRI for your account Please fill this ",(0,r.yg)("a",{parentName:"p",href:"/register"},"form \ud83d\udcec"),". to provide us some information on what you plan to do with the DSRI."))),(0,r.yg)("h2",{id:"connect-to-the-um-network"},"Connect to the UM network"),(0,r.yg)("p",null,"You need to be connected to the UM network to access the DSRI."),(0,r.yg)("ul",null,(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("p",{parentName:"li"},"Connect to ",(0,r.yg)("strong",{parentName:"p"},"UMnet")," or ",(0,r.yg)("strong",{parentName:"p"},"eduroam")," WiFi at Maastricht University")),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("p",{parentName:"li"},"Use the ",(0,r.yg)("strong",{parentName:"p"},"Maastricht University VPN")," at ",(0,r.yg)("strong",{parentName:"p"},(0,r.yg)("a",{parentName:"strong",href:"https://vpn.maastrichtuniversity.nl/"},"vpn.maastrichtuniversity.nl"))),(0,r.yg)("p",{parentName:"li"},"Log in to that using your UM username and password."))),(0,r.yg)("admonition",{title:"Students",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"By default the UM VPN is only available to employees. As a student you can access UM resources from any location via ",(0,r.yg)("a",{parentName:"p",href:"https://athenadesktop.maastrichtuniversity.nl"},"Student Desktop Anywhere"),".\u202fHowever, if VPN access is absolutely necessary you can request access via your course coordinator. ")),(0,r.yg)("ul",null,(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("p",{parentName:"li"},"The ",(0,r.yg)("strong",{parentName:"p"},"prefix of your UM email address")," with the first letter capitalized, e.g. ",(0,r.yg)("inlineCode",{parentName:"p"},"Firstname.Lastname")," or ",(0,r.yg)("inlineCode",{parentName:"p"},"F.Lastname"),"Or your ",(0,r.yg)("strong",{parentName:"p"},"employee number")," at Maastricht University (a.k.a. P number), e.g. ",(0,r.yg)("inlineCode",{parentName:"p"},"P7000000")),(0,r.yg)("img",{src:"/img/vpn-login.png",alt:"VPN Log in View",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("p",{parentName:"li"},"Then You will see below page to download the ",(0,r.yg)("strong",{parentName:"p"},"AnyConnect Secure Mobility Client")),(0,r.yg)("img",{src:"/img/VPN-anyconnect.png",alt:"Download AnyConnect Secure Mobility Client",style:{maxWidth:"100%",maxHeight:"100%"}}))),(0,r.yg)("h4",{id:"install-the-vpn-anyconnect-secure-mobility-client-on-windows"},"Install the VPN (AnyConnect Secure Mobility Client) on Windows"),(0,r.yg)("p",null,"Double click on the ",(0,r.yg)("inlineCode",{parentName:"p"},".exe")," file to install the VPN."),(0,r.yg)("p",null,"You can follow below steps as in pictures."),(0,r.yg)("img",{src:"/img/vpnsetup1.png",alt:"Install VPN",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("img",{src:"/img/vpnsetup2.png",alt:"Install VPN",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("img",{src:"/img/vpnsetup3.png",alt:"Install VPN",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("img",{src:"/img/vpnsetup4.png",alt:"Install VPN",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("h4",{id:"log-in-to-the-vpn-anyconnect-secure-mobility-client"},"Log in to the VPN (AnyConnect Secure Mobility Client)"),(0,r.yg)("p",null,"Once you finish installing you can run the Cisco AnyConnect Secure Mobility Client. "),(0,r.yg)("img",{src:"/img/vpnsetup7.png",alt:"Log in to the VPN",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("p",null,"Then after you will get the bellow wizard and click connect"),(0,r.yg)("img",{src:"/img/vpnsetup5.png",alt:"Log in to the VPN",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("p",null,"Provide your UM username and password. (",(0,r.yg)("strong",{parentName:"p"},"employee number")," at Maastricht University (a.k.a. P number), e.g. ",(0,r.yg)("inlineCode",{parentName:"p"},"P7000000"),")"),(0,r.yg)("img",{src:"/img/vpnsetup6.png",alt:"Log in to the VPN",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("h4",{id:"install-the-vpn-anyconnect-secure-mobility-client-on-linux"},"Install the VPN (AnyConnect Secure Mobility Client) on Linux"),(0,r.yg)("ul",null,(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("p",{parentName:"li"},"Connect to ",(0,r.yg)("strong",{parentName:"p"},"UMnet")," or ",(0,r.yg)("strong",{parentName:"p"},"eduroam")," WiFi at Maastricht University")),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("p",{parentName:"li"},"For ",(0,r.yg)("strong",{parentName:"p"},"Linux"),", use ",(0,r.yg)("inlineCode",{parentName:"p"},"openconnect")," to connect to the UM VPN. You can easily install it on Ubuntu and Debian distributions with ",(0,r.yg)("inlineCode",{parentName:"p"},"apt"),":"),(0,r.yg)("pre",{parentName:"li"},(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"sudo apt install openconnect\nsudo openconnect -u YOUR.USER --authgroup 01-Employees --useragent=AnyConnect vpn.maastrichtuniversity.nl\n")),(0,r.yg)("blockquote",{parentName:"li"},(0,r.yg)("p",{parentName:"blockquote"},"Provide your UM password when prompted."))),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("p",{parentName:"li"},"For ",(0,r.yg)("strong",{parentName:"p"},"students"),":"),(0,r.yg)("ul",{parentName:"li"},(0,r.yg)("li",{parentName:"ul"},"By default the UM VPN is only available to employees. As a student you can access UM resources from any location via ",(0,r.yg)("a",{parentName:"li",href:"https://athenadesktop.maastrichtuniversity.nl"},"Student Desktop Anywhere"),".\u202fHowever, if VPN access is absolutely necessary you can request access via your course coordinator.")))))}g.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/5db33872.3a0ed34c.js b/assets/js/5db33872.94a7a163.js similarity index 99% rename from assets/js/5db33872.3a0ed34c.js rename to assets/js/5db33872.94a7a163.js index 1e339c33a..f67ee0f3f 100644 --- a/assets/js/5db33872.3a0ed34c.js +++ b/assets/js/5db33872.94a7a163.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[2479],{5680:(e,t,n)=>{n.d(t,{xA:()=>c,yg:()=>y});var a=n(6540);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function r(e){for(var t=1;t=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var p=a.createContext({}),s=function(e){var t=a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},c=function(e){var t=s(e.components);return a.createElement(p.Provider,{value:t},e.children)},m={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},g=a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,i=e.originalType,p=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),g=s(n),y=o,d=g["".concat(p,".").concat(y)]||g[y]||m[y]||i;return n?a.createElement(d,r(r({ref:t},c),{},{components:n})):a.createElement(d,r({ref:t},c))}));function y(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var i=n.length,r=new Array(i);r[0]=g;var l={};for(var p in t)hasOwnProperty.call(t,p)&&(l[p]=t[p]);l.originalType=e,l.mdxType="string"==typeof e?e:o,r[1]=l;for(var s=2;s{n.r(t),n.d(t,{assets:()=>c,contentTitle:()=>p,default:()=>y,frontMatter:()=>l,metadata:()=>s,toc:()=>m});var a=n(9668),o=n(1367),i=(n(6540),n(5680)),r=["components"],l={id:"openshift-install",title:"Install the client"},p=void 0,s={unversionedId:"openshift-install",id:"openshift-install",title:"Install the client",description:"Install the OpenShift Command Line Interface (CLI): oc to access the DSRI from your computer's terminal.",source:"@site/docs/openshift-install.md",sourceDirName:".",slug:"/openshift-install",permalink:"/docs/openshift-install",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/openshift-install.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"openshift-install",title:"Install the client"},sidebar:"docs",previous:{title:"Start your workspace",permalink:"/docs/start-workspace"},next:{title:"Prepare your project",permalink:"/docs/prepare-project-for-dsri"}},c={},m=[{value:"Install the oc client",id:"install-the-oc-client",level:2},{value:"On Linux",id:"on-linux",level:3},{value:"On Mac",id:"on-mac",level:3},{value:"On Windows",id:"on-windows",level:3},{value:"Login in the terminal with oc",id:"login-in-the-terminal-with-oc",level:2}],g={toc:m};function y(e){var t=e.components,n=(0,o.A)(e,r);return(0,i.yg)("wrapper",(0,a.A)({},g,n,{components:t,mdxType:"MDXLayout"}),(0,i.yg)("p",null,"Install the ",(0,i.yg)("strong",{parentName:"p"},"OpenShift Command Line Interface (CLI)"),": ",(0,i.yg)("inlineCode",{parentName:"p"},"oc")," to access the DSRI from your computer's terminal. "),(0,i.yg)("p",null,"The ",(0,i.yg)("inlineCode",{parentName:"p"},"oc")," CLI enables to perform operations on your applications deployed on the DSRI, such as:"),(0,i.yg)("ul",null,(0,i.yg)("li",{parentName:"ul"},"Copy large files to or from the DSRI using ",(0,i.yg)("inlineCode",{parentName:"li"},"oc cp")),(0,i.yg)("li",{parentName:"ul"},"Connect to an application terminal using ",(0,i.yg)("inlineCode",{parentName:"li"},"oc rsh")),(0,i.yg)("li",{parentName:"ul"},"Get the applications running in your project with ",(0,i.yg)("inlineCode",{parentName:"li"},"oc get pods"))),(0,i.yg)("h2",{id:"install-the-oc-client"},"Install the ",(0,i.yg)("inlineCode",{parentName:"h2"},"oc")," client"),(0,i.yg)("h3",{id:"on-linux"},"On Linux"),(0,i.yg)("p",null,"Download the ",(0,i.yg)("inlineCode",{parentName:"p"},"oc")," and ",(0,i.yg)("inlineCode",{parentName:"p"},"kubectl")," Command Line Interface clients:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-shell"},"wget https://mirror.openshift.com/pub/openshift-v4/clients/oc/latest/linux/oc.tar.gz && tar xvf oc.tar.gz\nsudo mv oc kubectl /usr/local/bin/\n")),(0,i.yg)("h3",{id:"on-mac"},"On Mac"),(0,i.yg)("p",null,"Use ",(0,i.yg)("a",{parentName:"p",href:"https://brew.sh"},(0,i.yg)("inlineCode",{parentName:"a"},"brew")),":"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-bash"},"brew install openshift-cli\n")),(0,i.yg)("p",null,"Or manually download the program and add it to your path:"),(0,i.yg)("ol",null,(0,i.yg)("li",{parentName:"ol"},(0,i.yg)("p",{parentName:"li"},"Download ",(0,i.yg)("a",{parentName:"p",href:"https://mirror.openshift.com/pub/openshift-v4/clients/oc/latest/macosx/oc.tar.gz"},"https://mirror.openshift.com/pub/openshift-v4/clients/oc/latest/macosx/oc.tar.gz"))),(0,i.yg)("li",{parentName:"ol"},(0,i.yg)("p",{parentName:"li"},"Unzip the archive")),(0,i.yg)("li",{parentName:"ol"},(0,i.yg)("p",{parentName:"li"},"Move the ",(0,i.yg)("inlineCode",{parentName:"p"},"oc")," binary to a directory on your PATH."),(0,i.yg)("p",{parentName:"li"},"To check your ",(0,i.yg)("inlineCode",{parentName:"p"},"PATH"),", open a terminal and execute the following command:"),(0,i.yg)("pre",{parentName:"li"},(0,i.yg)("code",{parentName:"pre",className:"language-bash"},"echo $PATH\n")))),(0,i.yg)("h3",{id:"on-windows"},"On Windows"),(0,i.yg)("ol",null,(0,i.yg)("li",{parentName:"ol"},"Create a folder for OpenShift in Program Files: ",(0,i.yg)("inlineCode",{parentName:"li"},"C:\\Program Files (x86)\\OpenShift")),(0,i.yg)("li",{parentName:"ol"},"Click ",(0,i.yg)("a",{parentName:"li",href:"https://mirror.openshift.com/pub/openshift-v4/clients/oc/latest/windows/oc.zip"},"here")," to download the ",(0,i.yg)("inlineCode",{parentName:"li"},"oc")," tool ",(0,i.yg)("inlineCode",{parentName:"li"},".zip")," file, and move it to ",(0,i.yg)("inlineCode",{parentName:"li"},"C:\\Program Files (x86)\\OpenShift"),"."),(0,i.yg)("li",{parentName:"ol"},"Extract the ",(0,i.yg)("inlineCode",{parentName:"li"},".zip")," file."),(0,i.yg)("li",{parentName:"ol"},"Next set the system ",(0,i.yg)("strong",{parentName:"li"},"PATH")," environment variables for the directory containing the ",(0,i.yg)("inlineCode",{parentName:"li"},"oc.exe")," file, which now resides in your newly created ",(0,i.yg)("strong",{parentName:"li"},"OpenShift")," folder inside of ",(0,i.yg)("inlineCode",{parentName:"li"},"C:\\Program Files (x86)\\OpenShift"),(0,i.yg)("ol",{parentName:"li"},(0,i.yg)("li",{parentName:"ol"},"Open the Control Panel, and click on ",(0,i.yg)("strong",{parentName:"li"},"System")),(0,i.yg)("li",{parentName:"ol"},"Click on ",(0,i.yg)("strong",{parentName:"li"},"Advance system settings")," on the left or open the ",(0,i.yg)("strong",{parentName:"li"},"Advance")," tab of ",(0,i.yg)("em",{parentName:"li"},"System Properties.")," "),(0,i.yg)("li",{parentName:"ol"},"Click the button labeled ",(0,i.yg)("strong",{parentName:"li"},"Environment Variables...")," at the bottom. "),(0,i.yg)("li",{parentName:"ol"},"Look for the option ",(0,i.yg)("strong",{parentName:"li"},"Path")," in either the ",(0,i.yg)("strong",{parentName:"li"},"User variables")," section (for the current user) or the ",(0,i.yg)("strong",{parentName:"li"},"System variables")," section (for all users on the system).")))),(0,i.yg)("img",{class:"screenshot",src:"/img/OC_Path.png",alt:"Set OC Path",style:{zoom:"100%",maxHeight:"500px",maxWidth:"500px"}}),(0,i.yg)("p",null,"This makes it easy to access the ",(0,i.yg)("inlineCode",{parentName:"p"},"oc")," command line interface by simply opening up the ",(0,i.yg)("strong",{parentName:"p"},"PowerShell")," and typing in the ",(0,i.yg)("inlineCode",{parentName:"p"},"oc")," command, e.g.:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-powershell"},"oc version\n")),(0,i.yg)("admonition",{title:"Official documentation",type:"note"},(0,i.yg)("p",{parentName:"admonition"},"See the ",(0,i.yg)("a",{parentName:"p",href:"https://docs.okd.io/latest/cli_reference/openshift_cli/getting-started-cli.html#installing-openshift-cli"},"official documentation to install the client")," if needed.")),(0,i.yg)("h2",{id:"login-in-the-terminal-with-oc"},"Login in the terminal with ",(0,i.yg)("inlineCode",{parentName:"h2"},"oc")),(0,i.yg)("p",null,"To use the ",(0,i.yg)("inlineCode",{parentName:"p"},"oc")," Command Line Interface, you will need to authenticate to the ",(0,i.yg)("a",{parentName:"p",href:"https://console-openshift-console.apps.dsri2.unimaas.nl/console"},"DSRI")," in your terminal:"),(0,i.yg)("admonition",{title:"PASSWORD NOT SUPPORTED",type:"warning"},(0,i.yg)("p",{parentName:"admonition"},"Authentication to the ",(0,i.yg)("inlineCode",{parentName:"p"},"oc")," Command Line Interface using your password is not supported. ")),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-bash"},"oc login --token=\n")),(0,i.yg)("p",null,"The token is provided by the Web UI:"),(0,i.yg)("ol",null,(0,i.yg)("li",{parentName:"ol"},(0,i.yg)("p",{parentName:"li"},"Go to the ",(0,i.yg)("a",{parentName:"p",href:"https://console-openshift-console.apps.dsri2.unimaas.nl/console"},"DSRI web UI"),".")),(0,i.yg)("li",{parentName:"ol"},(0,i.yg)("p",{parentName:"li"},"Click on the ",(0,i.yg)("strong",{parentName:"p"},"Copy Login Command")," button (in the top right of the page)."),(0,i.yg)("img",{src:"/img/screenshot_copy_login.png",alt:"Deploy VSCode",style:{maxWidth:"100%",maxHeight:"100%"}})),(0,i.yg)("li",{parentName:"ol"},(0,i.yg)("p",{parentName:"li"},"Paste the copied command in your terminal, and execute it to login with ",(0,i.yg)("inlineCode",{parentName:"p"},"oc")," \ud83d\udd11"))),(0,i.yg)("admonition",{title:"Login command",type:"info"},(0,i.yg)("p",{parentName:"admonition"},"The command should look like this:"),(0,i.yg)("pre",{parentName:"admonition"},(0,i.yg)("code",{parentName:"pre",className:"language-bash"},"oc login https://api.dsri2.unimaas.nl:6443 --token=$GENERATED_TOKEN\n"))))}y.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[2479],{5680:(e,t,n)=>{n.d(t,{xA:()=>c,yg:()=>y});var a=n(6540);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function r(e){for(var t=1;t=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var p=a.createContext({}),s=function(e){var t=a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},c=function(e){var t=s(e.components);return a.createElement(p.Provider,{value:t},e.children)},m={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},g=a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,i=e.originalType,p=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),g=s(n),y=o,d=g["".concat(p,".").concat(y)]||g[y]||m[y]||i;return n?a.createElement(d,r(r({ref:t},c),{},{components:n})):a.createElement(d,r({ref:t},c))}));function y(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var i=n.length,r=new Array(i);r[0]=g;var l={};for(var p in t)hasOwnProperty.call(t,p)&&(l[p]=t[p]);l.originalType=e,l.mdxType="string"==typeof e?e:o,r[1]=l;for(var s=2;s{n.r(t),n.d(t,{assets:()=>c,contentTitle:()=>p,default:()=>y,frontMatter:()=>l,metadata:()=>s,toc:()=>m});var a=n(9668),o=n(1367),i=(n(6540),n(5680)),r=["components"],l={id:"openshift-install",title:"Install the client"},p=void 0,s={unversionedId:"openshift-install",id:"openshift-install",title:"Install the client",description:"Install the OpenShift Command Line Interface (CLI): oc to access the DSRI from your computer's terminal.",source:"@site/docs/openshift-install.md",sourceDirName:".",slug:"/openshift-install",permalink:"/docs/openshift-install",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/openshift-install.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"openshift-install",title:"Install the client"},sidebar:"docs",previous:{title:"Start your workspace",permalink:"/docs/start-workspace"},next:{title:"Prepare your project",permalink:"/docs/prepare-project-for-dsri"}},c={},m=[{value:"Install the oc client",id:"install-the-oc-client",level:2},{value:"On Linux",id:"on-linux",level:3},{value:"On Mac",id:"on-mac",level:3},{value:"On Windows",id:"on-windows",level:3},{value:"Login in the terminal with oc",id:"login-in-the-terminal-with-oc",level:2}],g={toc:m};function y(e){var t=e.components,n=(0,o.A)(e,r);return(0,i.yg)("wrapper",(0,a.A)({},g,n,{components:t,mdxType:"MDXLayout"}),(0,i.yg)("p",null,"Install the ",(0,i.yg)("strong",{parentName:"p"},"OpenShift Command Line Interface (CLI)"),": ",(0,i.yg)("inlineCode",{parentName:"p"},"oc")," to access the DSRI from your computer's terminal. "),(0,i.yg)("p",null,"The ",(0,i.yg)("inlineCode",{parentName:"p"},"oc")," CLI enables to perform operations on your applications deployed on the DSRI, such as:"),(0,i.yg)("ul",null,(0,i.yg)("li",{parentName:"ul"},"Copy large files to or from the DSRI using ",(0,i.yg)("inlineCode",{parentName:"li"},"oc cp")),(0,i.yg)("li",{parentName:"ul"},"Connect to an application terminal using ",(0,i.yg)("inlineCode",{parentName:"li"},"oc rsh")),(0,i.yg)("li",{parentName:"ul"},"Get the applications running in your project with ",(0,i.yg)("inlineCode",{parentName:"li"},"oc get pods"))),(0,i.yg)("h2",{id:"install-the-oc-client"},"Install the ",(0,i.yg)("inlineCode",{parentName:"h2"},"oc")," client"),(0,i.yg)("h3",{id:"on-linux"},"On Linux"),(0,i.yg)("p",null,"Download the ",(0,i.yg)("inlineCode",{parentName:"p"},"oc")," and ",(0,i.yg)("inlineCode",{parentName:"p"},"kubectl")," Command Line Interface clients:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-shell"},"wget https://mirror.openshift.com/pub/openshift-v4/clients/oc/latest/linux/oc.tar.gz && tar xvf oc.tar.gz\nsudo mv oc kubectl /usr/local/bin/\n")),(0,i.yg)("h3",{id:"on-mac"},"On Mac"),(0,i.yg)("p",null,"Use ",(0,i.yg)("a",{parentName:"p",href:"https://brew.sh"},(0,i.yg)("inlineCode",{parentName:"a"},"brew")),":"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-bash"},"brew install openshift-cli\n")),(0,i.yg)("p",null,"Or manually download the program and add it to your path:"),(0,i.yg)("ol",null,(0,i.yg)("li",{parentName:"ol"},(0,i.yg)("p",{parentName:"li"},"Download ",(0,i.yg)("a",{parentName:"p",href:"https://mirror.openshift.com/pub/openshift-v4/clients/oc/latest/macosx/oc.tar.gz"},"https://mirror.openshift.com/pub/openshift-v4/clients/oc/latest/macosx/oc.tar.gz"))),(0,i.yg)("li",{parentName:"ol"},(0,i.yg)("p",{parentName:"li"},"Unzip the archive")),(0,i.yg)("li",{parentName:"ol"},(0,i.yg)("p",{parentName:"li"},"Move the ",(0,i.yg)("inlineCode",{parentName:"p"},"oc")," binary to a directory on your PATH."),(0,i.yg)("p",{parentName:"li"},"To check your ",(0,i.yg)("inlineCode",{parentName:"p"},"PATH"),", open a terminal and execute the following command:"),(0,i.yg)("pre",{parentName:"li"},(0,i.yg)("code",{parentName:"pre",className:"language-bash"},"echo $PATH\n")))),(0,i.yg)("h3",{id:"on-windows"},"On Windows"),(0,i.yg)("ol",null,(0,i.yg)("li",{parentName:"ol"},"Create a folder for OpenShift in Program Files: ",(0,i.yg)("inlineCode",{parentName:"li"},"C:\\Program Files (x86)\\OpenShift")),(0,i.yg)("li",{parentName:"ol"},"Click ",(0,i.yg)("a",{parentName:"li",href:"https://mirror.openshift.com/pub/openshift-v4/clients/oc/latest/windows/oc.zip"},"here")," to download the ",(0,i.yg)("inlineCode",{parentName:"li"},"oc")," tool ",(0,i.yg)("inlineCode",{parentName:"li"},".zip")," file, and move it to ",(0,i.yg)("inlineCode",{parentName:"li"},"C:\\Program Files (x86)\\OpenShift"),"."),(0,i.yg)("li",{parentName:"ol"},"Extract the ",(0,i.yg)("inlineCode",{parentName:"li"},".zip")," file."),(0,i.yg)("li",{parentName:"ol"},"Next set the system ",(0,i.yg)("strong",{parentName:"li"},"PATH")," environment variables for the directory containing the ",(0,i.yg)("inlineCode",{parentName:"li"},"oc.exe")," file, which now resides in your newly created ",(0,i.yg)("strong",{parentName:"li"},"OpenShift")," folder inside of ",(0,i.yg)("inlineCode",{parentName:"li"},"C:\\Program Files (x86)\\OpenShift"),(0,i.yg)("ol",{parentName:"li"},(0,i.yg)("li",{parentName:"ol"},"Open the Control Panel, and click on ",(0,i.yg)("strong",{parentName:"li"},"System")),(0,i.yg)("li",{parentName:"ol"},"Click on ",(0,i.yg)("strong",{parentName:"li"},"Advance system settings")," on the left or open the ",(0,i.yg)("strong",{parentName:"li"},"Advance")," tab of ",(0,i.yg)("em",{parentName:"li"},"System Properties.")," "),(0,i.yg)("li",{parentName:"ol"},"Click the button labeled ",(0,i.yg)("strong",{parentName:"li"},"Environment Variables...")," at the bottom. "),(0,i.yg)("li",{parentName:"ol"},"Look for the option ",(0,i.yg)("strong",{parentName:"li"},"Path")," in either the ",(0,i.yg)("strong",{parentName:"li"},"User variables")," section (for the current user) or the ",(0,i.yg)("strong",{parentName:"li"},"System variables")," section (for all users on the system).")))),(0,i.yg)("img",{class:"screenshot",src:"/img/OC_Path.png",alt:"Set OC Path",style:{zoom:"100%",maxHeight:"500px",maxWidth:"500px"}}),(0,i.yg)("p",null,"This makes it easy to access the ",(0,i.yg)("inlineCode",{parentName:"p"},"oc")," command line interface by simply opening up the ",(0,i.yg)("strong",{parentName:"p"},"PowerShell")," and typing in the ",(0,i.yg)("inlineCode",{parentName:"p"},"oc")," command, e.g.:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-powershell"},"oc version\n")),(0,i.yg)("admonition",{title:"Official documentation",type:"note"},(0,i.yg)("p",{parentName:"admonition"},"See the ",(0,i.yg)("a",{parentName:"p",href:"https://docs.okd.io/latest/cli_reference/openshift_cli/getting-started-cli.html#installing-openshift-cli"},"official documentation to install the client")," if needed.")),(0,i.yg)("h2",{id:"login-in-the-terminal-with-oc"},"Login in the terminal with ",(0,i.yg)("inlineCode",{parentName:"h2"},"oc")),(0,i.yg)("p",null,"To use the ",(0,i.yg)("inlineCode",{parentName:"p"},"oc")," Command Line Interface, you will need to authenticate to the ",(0,i.yg)("a",{parentName:"p",href:"https://console-openshift-console.apps.dsri2.unimaas.nl/console"},"DSRI")," in your terminal:"),(0,i.yg)("admonition",{title:"PASSWORD NOT SUPPORTED",type:"warning"},(0,i.yg)("p",{parentName:"admonition"},"Authentication to the ",(0,i.yg)("inlineCode",{parentName:"p"},"oc")," Command Line Interface using your password is not supported. ")),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-bash"},"oc login --token=\n")),(0,i.yg)("p",null,"The token is provided by the Web UI:"),(0,i.yg)("ol",null,(0,i.yg)("li",{parentName:"ol"},(0,i.yg)("p",{parentName:"li"},"Go to the ",(0,i.yg)("a",{parentName:"p",href:"https://console-openshift-console.apps.dsri2.unimaas.nl/console"},"DSRI web UI"),".")),(0,i.yg)("li",{parentName:"ol"},(0,i.yg)("p",{parentName:"li"},"Click on the ",(0,i.yg)("strong",{parentName:"p"},"Copy Login Command")," button (in the top right of the page)."),(0,i.yg)("img",{src:"/img/screenshot_copy_login.png",alt:"Deploy VSCode",style:{maxWidth:"100%",maxHeight:"100%"}})),(0,i.yg)("li",{parentName:"ol"},(0,i.yg)("p",{parentName:"li"},"Paste the copied command in your terminal, and execute it to login with ",(0,i.yg)("inlineCode",{parentName:"p"},"oc")," \ud83d\udd11"))),(0,i.yg)("admonition",{title:"Login command",type:"info"},(0,i.yg)("p",{parentName:"admonition"},"The command should look like this:"),(0,i.yg)("pre",{parentName:"admonition"},(0,i.yg)("code",{parentName:"pre",className:"language-bash"},"oc login https://api.dsri2.unimaas.nl:6443 --token=$GENERATED_TOKEN\n"))))}y.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/613f0a4f.04fea9ff.js b/assets/js/613f0a4f.e7d63d6a.js similarity index 99% rename from assets/js/613f0a4f.04fea9ff.js rename to assets/js/613f0a4f.e7d63d6a.js index 1f1296dd3..d68c464a4 100644 --- a/assets/js/613f0a4f.04fea9ff.js +++ b/assets/js/613f0a4f.e7d63d6a.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[6108],{5680:(e,a,t)=>{t.d(a,{xA:()=>c,yg:()=>m});var n=t(6540);function r(e,a,t){return a in e?Object.defineProperty(e,a,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[a]=t,e}function o(e,a){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);a&&(n=n.filter((function(a){return Object.getOwnPropertyDescriptor(e,a).enumerable}))),t.push.apply(t,n)}return t}function s(e){for(var a=1;a=0||(r[t]=e[t]);return r}(e,a);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(r[t]=e[t])}return r}var l=n.createContext({}),u=function(e){var a=n.useContext(l),t=a;return e&&(t="function"==typeof e?e(a):s(s({},a),e)),t},c=function(e){var a=u(e.components);return n.createElement(l.Provider,{value:a},e.children)},d={inlineCode:"code",wrapper:function(e){var a=e.children;return n.createElement(n.Fragment,{},a)}},p=n.forwardRef((function(e,a){var t=e.components,r=e.mdxType,o=e.originalType,l=e.parentName,c=i(e,["components","mdxType","originalType","parentName"]),p=u(t),m=r,g=p["".concat(l,".").concat(m)]||p[m]||d[m]||o;return t?n.createElement(g,s(s({ref:a},c),{},{components:t})):n.createElement(g,s({ref:a},c))}));function m(e,a){var t=arguments,r=a&&a.mdxType;if("string"==typeof e||r){var o=t.length,s=new Array(o);s[0]=p;var i={};for(var l in a)hasOwnProperty.call(a,l)&&(i[l]=a[l]);i.originalType=e,i.mdxType="string"==typeof e?e:r,s[1]=i;for(var u=2;u{t.r(a),t.d(a,{assets:()=>c,contentTitle:()=>l,default:()=>m,frontMatter:()=>i,metadata:()=>u,toc:()=>d});var n=t(9668),r=t(1367),o=(t(6540),t(5680)),s=["components"],i={id:"dask-tutorial",title:"Parallelization using Dask"},l=void 0,u={unversionedId:"dask-tutorial",id:"dask-tutorial",title:"Parallelization using Dask",description:"\ud83e\uddca Installation",source:"@site/docs/dask-tutorial.md",sourceDirName:".",slug:"/dask-tutorial",permalink:"/docs/dask-tutorial",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/dask-tutorial.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"dask-tutorial",title:"Parallelization using Dask"},sidebar:"docs",previous:{title:"Checkpointing Machine Learning Training",permalink:"/docs/checkpointing-ml-training"},next:{title:"Known Issues",permalink:"/docs/guide-known-issues"}},c={},d=[{value:"\ud83e\uddca Installation",id:"-installation",level:2},{value:"\ud83e\ude90 Basic Concepts of Dask",id:"-basic-concepts-of-dask",level:3},{value:"\u2728 Selecting columns and element-wise operations",id:"-selecting-columns-and-element-wise-operations",level:3},{value:"\u26a1\ufe0f Conditional filtering",id:"\ufe0f-conditional-filtering",level:3},{value:"\u2728 Common summary statistics",id:"-common-summary-statistics",level:3},{value:"\u2728 Groupby",id:"-groupby",level:3},{value:"\u26a1\ufe0f Lazy evaluation",id:"\ufe0f-lazy-evaluation",level:3},{value:"\ud83e\ude90 Dask Bags and Dask Delayed for Unstructured Data",id:"-dask-bags-and-dask-delayed-for-unstructured-data",level:4}],p={toc:d};function m(e){var a=e.components,t=(0,r.A)(e,s);return(0,o.yg)("wrapper",(0,n.A)({},p,t,{components:a,mdxType:"MDXLayout"}),(0,o.yg)("h2",{id:"-installation"},"\ud83e\uddca Installation"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},'!pip install "dask[complete]"\n')),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},"import dask\n\ndask.__version__\n")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre"},"'2023.5.0'\n")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},"import dask.array as da\nimport dask.bag as db\nimport dask.dataframe as dd\nimport numpy as np\nimport pandas as pd\n")),(0,o.yg)("h3",{id:"-basic-concepts-of-dask"},"\ud83e\ude90 Basic Concepts of Dask"),(0,o.yg)("p",null,"On a high-level, you can think of Dask as a wrapper that extends the capabilities of traditional tools like pandas, NumPy, and Spark to handle larger-than-memory datasets."),(0,o.yg)("p",null,"When faced with large objects like larger-than-memory arrays (vectors) or matrices (dataframes), Dask breaks them up into chunks, also called partitions."),(0,o.yg)("p",null,"For example, consider the array of 12 random numbers in both NumPy and Dask:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},"narr = np.random.rand(12)\n\nnarr\n")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre"},"array([0.44236558, 0.00504448, 0.87087911, 0.468925 , 0.37513511,\n 0.22607761, 0.83035297, 0.07772372, 0.61587933, 0.82861156,\n 0.66214299, 0.90979423])\n")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},"darr = da.from_array(narr, chunks=3)\ndarr\n")),(0,o.yg)("img",{src:"/img/Screenshot-dask.png",alt:"dask table",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("p",null,"The image above shows that the Dask array contains four chunks as we set chunks to 3. Under the hood, each chunk is a NumPy array in itself."),(0,o.yg)("p",null,"To fully appreciate the benefits of Dask, we need a large dataset, preferably over 1 GB in size. Consider the autogenerated data from the script below:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},"import string\n\n# Set the desired number of rows and columns\nnum_rows = 5_000_000\nnum_cols = 10\nchunk_size = 100_000\n\n# Define an empty DataFrame to store the chunks\ndf_chunks = pd.DataFrame()\n\n# Generate and write the dataset in chunks\nfor i in range(0, num_rows, chunk_size):\n # Generate random numeric data\n numeric_data = np.random.rand(chunk_size, num_cols)\n\n # Generate random categorical data\n letters = list(string.ascii_uppercase)\n categorical_data = np.random.choice(letters, (chunk_size, num_cols))\n\n # Combine numeric and categorical data into a Pandas DataFrame\n df_chunk = pd.DataFrame(np.concatenate([numeric_data, categorical_data], axis=1))\n\n # Set column names for better understanding\n column_names = [f'Numeric_{i}' for i in range(num_cols)] + [f'Categorical_{i}' for i in range(num_cols)]\n df_chunk.columns = column_names\n\n # Append the current chunk to the DataFrame holding all chunks\n df_chunks = pd.concat([df_chunks, df_chunk], ignore_index=True)\n\n # Write the DataFrame chunk to a CSV file incrementally\n if (i + chunk_size) >= num_rows or (i // chunk_size) % 10 == 0:\n df_chunks.to_csv('large_dataset.csv', index=False, mode='a', header=(i == 0))\n df_chunks = pd.DataFrame()\n")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},'dask_df = dd.read_csv("large_dataset.csv")\n\ndask_df.head()\n')),(0,o.yg)("p",null,"Even though the file is large, you will notice that the result is fetched almost instantaneously. For even larger files, you can specify the ",(0,o.yg)("inlineCode",{parentName:"p"},"blocksize")," parameter, which determines the number of bytes to break up the file into."),(0,o.yg)("p",null,"Similar to how Dask Arrays contain chunks of small NumPy arrays, Dask is designed to handle multiple small Pandas DataFrames arranged along the row index."),(0,o.yg)("h3",{id:"-selecting-columns-and-element-wise-operations"},"\u2728 Selecting columns and element-wise operations"),(0,o.yg)("p",null,"In this example, we're doing some pretty straightforward column operations on our Dask DataFrame, called dask_df. We're adding the values from the column Numeric_0 to the result of multiplying the values from Numeric_9 and Numeric_3. We store the outcome in a variable named result."),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},'result = (\n dask_df["Numeric_0"] + dask_df["Numeric_9"] * dask_df["Numeric_3"]\n)\n\nresult.compute().head()\n')),(0,o.yg)("p",null,"As we\u2019ve mentioned, Dask is a bit different from traditional computing tools in that it doesn't immediately execute these operations. Instead, it creates a kind of 'plan' called a task graph to carry out these operations later on. This approach allows Dask to optimize the computations and parallelize them when needed. The compute() function triggers Dask to finally perform these computations, and head() just shows us the first few rows of the result."),(0,o.yg)("h3",{id:"\ufe0f-conditional-filtering"},"\u26a1\ufe0f Conditional filtering"),(0,o.yg)("p",null,'Now, let\'s look at how Dask can filter data. We\'re selecting rows from our DataFrame where the value in the "Categorical_5" column is "A".'),(0,o.yg)("p",null,"This filtering process is similar to how you'd do it in pandas, but with a twist - Dask does this operation lazily. It prepares the task graph for this operation but waits to execute it until we call compute(). When we run head(), we get to see the first few rows of our filtered DataFrame."),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},'dask_df[dask_df["Categorical_5"] == "A"].compute().head()\n')),(0,o.yg)("h3",{id:"-common-summary-statistics"},"\u2728 Common summary statistics"),(0,o.yg)("p",null,"Next, we're going to generate some common summary statistics using Dask's describe() function."),(0,o.yg)("p",null,"It gives us a handful of descriptive statistics for our DataFrame, including the mean, standard deviation, minimum, maximum, and so on. As with our previous examples, Dask prepares the task graph for this operation when we call describe(), but it waits to execute it until we call compute()."),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},"dask_df.describe().compute()\n")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},'dask_df["Categorical_3"].value_counts().compute().head()\n')),(0,o.yg)("p",null,'We also use value_counts() to count the number of occurrences of each unique value in the "Categorical_3" column. We trigger the operation with compute(), and head() shows us the most common values.'),(0,o.yg)("h3",{id:"-groupby"},"\u2728 Groupby"),(0,o.yg)("p",null,'Finally, let\'s use the groupby() function to group our data based on values in the "Categorical_8" column. Then we select the "Numeric_7" column and calculate the mean for each group.'),(0,o.yg)("p",null,'This is similar to how you might use \u2018groupby()\u2019 in pandas, but as you might have guessed, Dask does this lazily. We trigger the operation with compute(), and head() displays the average of the "Numeric_7" column for the first few groups.'),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},'dask_df.groupby("Categorical_8")["Numeric_7"].mean().compute().head()\n')),(0,o.yg)("h3",{id:"\ufe0f-lazy-evaluation"},"\u26a1\ufe0f Lazy evaluation"),(0,o.yg)("p",null,"Now, let\u2019s explore the use of the compute function at the end of each code block."),(0,o.yg)("p",null,"Dask evaluates code blocks in lazy mode compared to Pandas\u2019 eager mode, which returns results immediately."),(0,o.yg)("p",null,"To draw a parallel in cooking, lazy evaluation is like preparing ingredients and chopping vegetables in advance but only combining them to cook when needed. The compute function serves that purpose."),(0,o.yg)("p",null,"In contrast, eager evaluation is like throwing ingredients into the fire to cook as soon as they are ready. This approach ensures everything is ready to serve at once."),(0,o.yg)("p",null,"Lazy evaluation is key to Dask\u2019s excellent performance as it provides:"),(0,o.yg)("ol",null,(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("strong",{parentName:"li"},"Reduced computation.")," Expressions are evaluated only when needed (when compute is called), avoiding unnecessary intermediate results that may not be used in the final result."),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("strong",{parentName:"li"},"Optimal resource allocation.")," Lazy evaluation avoids allocating memory or processing power to intermediate results that may not be required."),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("strong",{parentName:"li"},"Support for large datasets.")," This method processes data elements on-the-fly or in smaller chunks, enabling efficient utilization of memory resources.")),(0,o.yg)("p",null,"When the results of compute are returned, they are given as Pandas Series/DataFrames or NumPy arrays instead of native Dask DataFrames."),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},"type(dask_df)\n")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre"},"dask.dataframe.core.DataFrame\n")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},'type(\n dask_df[["Numeric_5", "Numeric_6", "Numeric_7"]].mean().compute()\n)\n')),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre"},"pandas.core.series.Series\n")),(0,o.yg)("p",null,"The reason for this is that most data manipulation operations return only a subset of the original dataframe, taking up much smaller space. So, there won\u2019t be any need to use parallelism of Dask, and you continue the rest of your workflow either in pandas or NumPy."),(0,o.yg)("h4",{id:"-dask-bags-and-dask-delayed-for-unstructured-data"},"\ud83e\ude90 Dask Bags and Dask Delayed for Unstructured Data"),(0,o.yg)("p",null,"Dask Bags and Dask Delayed are two components of the Dask library that provide powerful tools for working with unstructured or semi-structured data and enabling lazy evaluation."),(0,o.yg)("p",null,"While in the past, tabular data was the most common, today\u2019s datasets often involve unstructured files such as images, text files, videos, and audio. Dask Bags provides the functionality and API to handle such unstructured files in a parallel and scalable manner."),(0,o.yg)("p",null,"For example, let\u2019s consider a simple illustration:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},'# Create a Dask Bag from a list of strings\nb = db.from_sequence(["apple", "banana", "orange", "grape", "kiwi"])\n\n# Filter the strings that start with the letter \'a\'\nfiltered_strings = b.filter(lambda x: x.startswith("a"))\n\n# Map a function to convert each string to uppercase\nuppercase_strings = filtered_strings.map(lambda x: x.upper())\n\n# Compute the result as a list\nresult = uppercase_strings.compute()\n\nprint(result)\n')),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre"},"['APPLE']\n")),(0,o.yg)("p",null,"In this example, we create a Dask Bag b from a list of strings. We then apply operations on the Bag to filter the strings that start with the letter 'a' and convert them to uppercase using the filter() and map() functions, respectively. Finally, we compute the result as a list using the compute() method and print the output."),(0,o.yg)("p",null,"Now imagine that you can perform even more complex operations on billions of similar strings stored in a text file. Without the lazy evaluation and parallelism offered by Dask Bags, you would face significant challenges. "),(0,o.yg)("p",null,"As for Dask Delayed, it provides even more flexibility and introduces lazy evaluation and parallelism to various other scenarios. With Dask Delayed, you can convert any native Python function into a lazy object using the @dask.delayed decorator."),(0,o.yg)("p",null,"Here is a simple example:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},"%%time\n\nimport time\n@dask.delayed\ndef process_data(x):\n # Simulate some computation\n time.sleep(1)\n return x**2\n\n\n# Generate a list of inputs\ninputs = range(1000)\n\n# Apply the delayed function to each input\nresults = [process_data(x) for x in inputs]\n\n# Compute the results in parallel\ncomputed_results = dask.compute(*results)\n")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre"},"CPU times: user 260 ms, sys: 68.1 ms, total: 328 ms\nWall time: 32.2 s\n")),(0,o.yg)("p",null,"In this example, we define a function process_data decorated with @dask.delayed. The function simulates some computational work by sleeping for 1 second and then returning the square of the input value."),(0,o.yg)("p",null,"Without parallelism, performing this computation on 1000 inputs would have taken more than 1000 seconds. However, with Dask Delayed and parallel execution, the computation only took about 42.1 seconds."),(0,o.yg)("p",null,"This example demonstrates the power of parallelism in reducing computation time by efficiently distributing the workload across multiple cores or workers."),(0,o.yg)("p",null,"That\u2019s what parallelism is all about. for more information see ",(0,o.yg)("a",{parentName:"p",href:"https://docs.dask.org/en/stable/"},"https://docs.dask.org/en/stable/")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},"")))}m.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[6108],{5680:(e,a,t)=>{t.d(a,{xA:()=>c,yg:()=>m});var n=t(6540);function r(e,a,t){return a in e?Object.defineProperty(e,a,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[a]=t,e}function o(e,a){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);a&&(n=n.filter((function(a){return Object.getOwnPropertyDescriptor(e,a).enumerable}))),t.push.apply(t,n)}return t}function s(e){for(var a=1;a=0||(r[t]=e[t]);return r}(e,a);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(r[t]=e[t])}return r}var l=n.createContext({}),u=function(e){var a=n.useContext(l),t=a;return e&&(t="function"==typeof e?e(a):s(s({},a),e)),t},c=function(e){var a=u(e.components);return n.createElement(l.Provider,{value:a},e.children)},d={inlineCode:"code",wrapper:function(e){var a=e.children;return n.createElement(n.Fragment,{},a)}},p=n.forwardRef((function(e,a){var t=e.components,r=e.mdxType,o=e.originalType,l=e.parentName,c=i(e,["components","mdxType","originalType","parentName"]),p=u(t),m=r,g=p["".concat(l,".").concat(m)]||p[m]||d[m]||o;return t?n.createElement(g,s(s({ref:a},c),{},{components:t})):n.createElement(g,s({ref:a},c))}));function m(e,a){var t=arguments,r=a&&a.mdxType;if("string"==typeof e||r){var o=t.length,s=new Array(o);s[0]=p;var i={};for(var l in a)hasOwnProperty.call(a,l)&&(i[l]=a[l]);i.originalType=e,i.mdxType="string"==typeof e?e:r,s[1]=i;for(var u=2;u{t.r(a),t.d(a,{assets:()=>c,contentTitle:()=>l,default:()=>m,frontMatter:()=>i,metadata:()=>u,toc:()=>d});var n=t(9668),r=t(1367),o=(t(6540),t(5680)),s=["components"],i={id:"dask-tutorial",title:"Parallelization using Dask"},l=void 0,u={unversionedId:"dask-tutorial",id:"dask-tutorial",title:"Parallelization using Dask",description:"\ud83e\uddca Installation",source:"@site/docs/dask-tutorial.md",sourceDirName:".",slug:"/dask-tutorial",permalink:"/docs/dask-tutorial",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/dask-tutorial.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"dask-tutorial",title:"Parallelization using Dask"},sidebar:"docs",previous:{title:"Checkpointing Machine Learning Training",permalink:"/docs/checkpointing-ml-training"},next:{title:"Known Issues",permalink:"/docs/guide-known-issues"}},c={},d=[{value:"\ud83e\uddca Installation",id:"-installation",level:2},{value:"\ud83e\ude90 Basic Concepts of Dask",id:"-basic-concepts-of-dask",level:3},{value:"\u2728 Selecting columns and element-wise operations",id:"-selecting-columns-and-element-wise-operations",level:3},{value:"\u26a1\ufe0f Conditional filtering",id:"\ufe0f-conditional-filtering",level:3},{value:"\u2728 Common summary statistics",id:"-common-summary-statistics",level:3},{value:"\u2728 Groupby",id:"-groupby",level:3},{value:"\u26a1\ufe0f Lazy evaluation",id:"\ufe0f-lazy-evaluation",level:3},{value:"\ud83e\ude90 Dask Bags and Dask Delayed for Unstructured Data",id:"-dask-bags-and-dask-delayed-for-unstructured-data",level:4}],p={toc:d};function m(e){var a=e.components,t=(0,r.A)(e,s);return(0,o.yg)("wrapper",(0,n.A)({},p,t,{components:a,mdxType:"MDXLayout"}),(0,o.yg)("h2",{id:"-installation"},"\ud83e\uddca Installation"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},'!pip install "dask[complete]"\n')),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},"import dask\n\ndask.__version__\n")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre"},"'2023.5.0'\n")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},"import dask.array as da\nimport dask.bag as db\nimport dask.dataframe as dd\nimport numpy as np\nimport pandas as pd\n")),(0,o.yg)("h3",{id:"-basic-concepts-of-dask"},"\ud83e\ude90 Basic Concepts of Dask"),(0,o.yg)("p",null,"On a high-level, you can think of Dask as a wrapper that extends the capabilities of traditional tools like pandas, NumPy, and Spark to handle larger-than-memory datasets."),(0,o.yg)("p",null,"When faced with large objects like larger-than-memory arrays (vectors) or matrices (dataframes), Dask breaks them up into chunks, also called partitions."),(0,o.yg)("p",null,"For example, consider the array of 12 random numbers in both NumPy and Dask:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},"narr = np.random.rand(12)\n\nnarr\n")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre"},"array([0.44236558, 0.00504448, 0.87087911, 0.468925 , 0.37513511,\n 0.22607761, 0.83035297, 0.07772372, 0.61587933, 0.82861156,\n 0.66214299, 0.90979423])\n")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},"darr = da.from_array(narr, chunks=3)\ndarr\n")),(0,o.yg)("img",{src:"/img/Screenshot-dask.png",alt:"dask table",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("p",null,"The image above shows that the Dask array contains four chunks as we set chunks to 3. Under the hood, each chunk is a NumPy array in itself."),(0,o.yg)("p",null,"To fully appreciate the benefits of Dask, we need a large dataset, preferably over 1 GB in size. Consider the autogenerated data from the script below:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},"import string\n\n# Set the desired number of rows and columns\nnum_rows = 5_000_000\nnum_cols = 10\nchunk_size = 100_000\n\n# Define an empty DataFrame to store the chunks\ndf_chunks = pd.DataFrame()\n\n# Generate and write the dataset in chunks\nfor i in range(0, num_rows, chunk_size):\n # Generate random numeric data\n numeric_data = np.random.rand(chunk_size, num_cols)\n\n # Generate random categorical data\n letters = list(string.ascii_uppercase)\n categorical_data = np.random.choice(letters, (chunk_size, num_cols))\n\n # Combine numeric and categorical data into a Pandas DataFrame\n df_chunk = pd.DataFrame(np.concatenate([numeric_data, categorical_data], axis=1))\n\n # Set column names for better understanding\n column_names = [f'Numeric_{i}' for i in range(num_cols)] + [f'Categorical_{i}' for i in range(num_cols)]\n df_chunk.columns = column_names\n\n # Append the current chunk to the DataFrame holding all chunks\n df_chunks = pd.concat([df_chunks, df_chunk], ignore_index=True)\n\n # Write the DataFrame chunk to a CSV file incrementally\n if (i + chunk_size) >= num_rows or (i // chunk_size) % 10 == 0:\n df_chunks.to_csv('large_dataset.csv', index=False, mode='a', header=(i == 0))\n df_chunks = pd.DataFrame()\n")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},'dask_df = dd.read_csv("large_dataset.csv")\n\ndask_df.head()\n')),(0,o.yg)("p",null,"Even though the file is large, you will notice that the result is fetched almost instantaneously. For even larger files, you can specify the ",(0,o.yg)("inlineCode",{parentName:"p"},"blocksize")," parameter, which determines the number of bytes to break up the file into."),(0,o.yg)("p",null,"Similar to how Dask Arrays contain chunks of small NumPy arrays, Dask is designed to handle multiple small Pandas DataFrames arranged along the row index."),(0,o.yg)("h3",{id:"-selecting-columns-and-element-wise-operations"},"\u2728 Selecting columns and element-wise operations"),(0,o.yg)("p",null,"In this example, we're doing some pretty straightforward column operations on our Dask DataFrame, called dask_df. We're adding the values from the column Numeric_0 to the result of multiplying the values from Numeric_9 and Numeric_3. We store the outcome in a variable named result."),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},'result = (\n dask_df["Numeric_0"] + dask_df["Numeric_9"] * dask_df["Numeric_3"]\n)\n\nresult.compute().head()\n')),(0,o.yg)("p",null,"As we\u2019ve mentioned, Dask is a bit different from traditional computing tools in that it doesn't immediately execute these operations. Instead, it creates a kind of 'plan' called a task graph to carry out these operations later on. This approach allows Dask to optimize the computations and parallelize them when needed. The compute() function triggers Dask to finally perform these computations, and head() just shows us the first few rows of the result."),(0,o.yg)("h3",{id:"\ufe0f-conditional-filtering"},"\u26a1\ufe0f Conditional filtering"),(0,o.yg)("p",null,'Now, let\'s look at how Dask can filter data. We\'re selecting rows from our DataFrame where the value in the "Categorical_5" column is "A".'),(0,o.yg)("p",null,"This filtering process is similar to how you'd do it in pandas, but with a twist - Dask does this operation lazily. It prepares the task graph for this operation but waits to execute it until we call compute(). When we run head(), we get to see the first few rows of our filtered DataFrame."),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},'dask_df[dask_df["Categorical_5"] == "A"].compute().head()\n')),(0,o.yg)("h3",{id:"-common-summary-statistics"},"\u2728 Common summary statistics"),(0,o.yg)("p",null,"Next, we're going to generate some common summary statistics using Dask's describe() function."),(0,o.yg)("p",null,"It gives us a handful of descriptive statistics for our DataFrame, including the mean, standard deviation, minimum, maximum, and so on. As with our previous examples, Dask prepares the task graph for this operation when we call describe(), but it waits to execute it until we call compute()."),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},"dask_df.describe().compute()\n")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},'dask_df["Categorical_3"].value_counts().compute().head()\n')),(0,o.yg)("p",null,'We also use value_counts() to count the number of occurrences of each unique value in the "Categorical_3" column. We trigger the operation with compute(), and head() shows us the most common values.'),(0,o.yg)("h3",{id:"-groupby"},"\u2728 Groupby"),(0,o.yg)("p",null,'Finally, let\'s use the groupby() function to group our data based on values in the "Categorical_8" column. Then we select the "Numeric_7" column and calculate the mean for each group.'),(0,o.yg)("p",null,'This is similar to how you might use \u2018groupby()\u2019 in pandas, but as you might have guessed, Dask does this lazily. We trigger the operation with compute(), and head() displays the average of the "Numeric_7" column for the first few groups.'),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},'dask_df.groupby("Categorical_8")["Numeric_7"].mean().compute().head()\n')),(0,o.yg)("h3",{id:"\ufe0f-lazy-evaluation"},"\u26a1\ufe0f Lazy evaluation"),(0,o.yg)("p",null,"Now, let\u2019s explore the use of the compute function at the end of each code block."),(0,o.yg)("p",null,"Dask evaluates code blocks in lazy mode compared to Pandas\u2019 eager mode, which returns results immediately."),(0,o.yg)("p",null,"To draw a parallel in cooking, lazy evaluation is like preparing ingredients and chopping vegetables in advance but only combining them to cook when needed. The compute function serves that purpose."),(0,o.yg)("p",null,"In contrast, eager evaluation is like throwing ingredients into the fire to cook as soon as they are ready. This approach ensures everything is ready to serve at once."),(0,o.yg)("p",null,"Lazy evaluation is key to Dask\u2019s excellent performance as it provides:"),(0,o.yg)("ol",null,(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("strong",{parentName:"li"},"Reduced computation.")," Expressions are evaluated only when needed (when compute is called), avoiding unnecessary intermediate results that may not be used in the final result."),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("strong",{parentName:"li"},"Optimal resource allocation.")," Lazy evaluation avoids allocating memory or processing power to intermediate results that may not be required."),(0,o.yg)("li",{parentName:"ol"},(0,o.yg)("strong",{parentName:"li"},"Support for large datasets.")," This method processes data elements on-the-fly or in smaller chunks, enabling efficient utilization of memory resources.")),(0,o.yg)("p",null,"When the results of compute are returned, they are given as Pandas Series/DataFrames or NumPy arrays instead of native Dask DataFrames."),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},"type(dask_df)\n")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre"},"dask.dataframe.core.DataFrame\n")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},'type(\n dask_df[["Numeric_5", "Numeric_6", "Numeric_7"]].mean().compute()\n)\n')),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre"},"pandas.core.series.Series\n")),(0,o.yg)("p",null,"The reason for this is that most data manipulation operations return only a subset of the original dataframe, taking up much smaller space. So, there won\u2019t be any need to use parallelism of Dask, and you continue the rest of your workflow either in pandas or NumPy."),(0,o.yg)("h4",{id:"-dask-bags-and-dask-delayed-for-unstructured-data"},"\ud83e\ude90 Dask Bags and Dask Delayed for Unstructured Data"),(0,o.yg)("p",null,"Dask Bags and Dask Delayed are two components of the Dask library that provide powerful tools for working with unstructured or semi-structured data and enabling lazy evaluation."),(0,o.yg)("p",null,"While in the past, tabular data was the most common, today\u2019s datasets often involve unstructured files such as images, text files, videos, and audio. Dask Bags provides the functionality and API to handle such unstructured files in a parallel and scalable manner."),(0,o.yg)("p",null,"For example, let\u2019s consider a simple illustration:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},'# Create a Dask Bag from a list of strings\nb = db.from_sequence(["apple", "banana", "orange", "grape", "kiwi"])\n\n# Filter the strings that start with the letter \'a\'\nfiltered_strings = b.filter(lambda x: x.startswith("a"))\n\n# Map a function to convert each string to uppercase\nuppercase_strings = filtered_strings.map(lambda x: x.upper())\n\n# Compute the result as a list\nresult = uppercase_strings.compute()\n\nprint(result)\n')),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre"},"['APPLE']\n")),(0,o.yg)("p",null,"In this example, we create a Dask Bag b from a list of strings. We then apply operations on the Bag to filter the strings that start with the letter 'a' and convert them to uppercase using the filter() and map() functions, respectively. Finally, we compute the result as a list using the compute() method and print the output."),(0,o.yg)("p",null,"Now imagine that you can perform even more complex operations on billions of similar strings stored in a text file. Without the lazy evaluation and parallelism offered by Dask Bags, you would face significant challenges. "),(0,o.yg)("p",null,"As for Dask Delayed, it provides even more flexibility and introduces lazy evaluation and parallelism to various other scenarios. With Dask Delayed, you can convert any native Python function into a lazy object using the @dask.delayed decorator."),(0,o.yg)("p",null,"Here is a simple example:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},"%%time\n\nimport time\n@dask.delayed\ndef process_data(x):\n # Simulate some computation\n time.sleep(1)\n return x**2\n\n\n# Generate a list of inputs\ninputs = range(1000)\n\n# Apply the delayed function to each input\nresults = [process_data(x) for x in inputs]\n\n# Compute the results in parallel\ncomputed_results = dask.compute(*results)\n")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre"},"CPU times: user 260 ms, sys: 68.1 ms, total: 328 ms\nWall time: 32.2 s\n")),(0,o.yg)("p",null,"In this example, we define a function process_data decorated with @dask.delayed. The function simulates some computational work by sleeping for 1 second and then returning the square of the input value."),(0,o.yg)("p",null,"Without parallelism, performing this computation on 1000 inputs would have taken more than 1000 seconds. However, with Dask Delayed and parallel execution, the computation only took about 42.1 seconds."),(0,o.yg)("p",null,"This example demonstrates the power of parallelism in reducing computation time by efficiently distributing the workload across multiple cores or workers."),(0,o.yg)("p",null,"That\u2019s what parallelism is all about. for more information see ",(0,o.yg)("a",{parentName:"p",href:"https://docs.dask.org/en/stable/"},"https://docs.dask.org/en/stable/")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},"")))}m.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/63528e5f.bfd33598.js b/assets/js/63528e5f.4c209fa2.js similarity index 99% rename from assets/js/63528e5f.bfd33598.js rename to assets/js/63528e5f.4c209fa2.js index 7c771c9c1..f74b211bd 100644 --- a/assets/js/63528e5f.bfd33598.js +++ b/assets/js/63528e5f.4c209fa2.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[5803],{5680:(e,t,n)=>{n.d(t,{xA:()=>s,yg:()=>u});var a=n(6540);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function i(e){for(var t=1;t=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=a.createContext({}),p=function(e){var t=a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},s=function(e){var t=p(e.components);return a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},g=a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,c=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),g=p(n),u=o,y=g["".concat(c,".").concat(u)]||g[u]||d[u]||r;return n?a.createElement(y,i(i({ref:t},s),{},{components:n})):a.createElement(y,i({ref:t},s))}));function u(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,i=new Array(r);i[0]=g;var l={};for(var c in t)hasOwnProperty.call(t,c)&&(l[c]=t[c]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var p=2;p{n.r(t),n.d(t,{assets:()=>s,contentTitle:()=>c,default:()=>u,frontMatter:()=>l,metadata:()=>p,toc:()=>d});var a=n(9668),o=n(1367),r=(n(6540),n(5680)),i=["components"],l={id:"openshift-delete-objects",title:"Delete objects (advanced)"},c=void 0,p={unversionedId:"openshift-delete-objects",id:"openshift-delete-objects",title:"Delete objects (advanced)",description:"This documentation provide guidelines to delete various types of objects in the OpenShift DSRI. Be careful when you are deleting object in your project, as it could be an object required to run an application.",source:"@site/docs/openshift-delete-objects.md",sourceDirName:".",slug:"/openshift-delete-objects",permalink:"/docs/openshift-delete-objects",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/openshift-delete-objects.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"openshift-delete-objects",title:"Delete objects (advanced)"},sidebar:"docs",previous:{title:"Data storage",permalink:"/docs/openshift-storage"},next:{title:"Create a new Project",permalink:"/docs/project-management"}},s={},d=[{value:"Delete an application",id:"delete-an-application",level:2},{value:"Delete pod",id:"delete-pod",level:2},{value:"Delete a project",id:"delete-a-project",level:2},{value:"Delete persistent storage",id:"delete-persistent-storage",level:2},{value:"Fix stuck deletions",id:"fix-stuck-deletions",level:2},{value:"Stuck provisioned service",id:"stuck-provisioned-service",level:3},{value:"Delete stuck project",id:"delete-stuck-project",level:3}],g={toc:d};function u(e){var t=e.components,n=(0,o.A)(e,i);return(0,r.yg)("wrapper",(0,a.A)({},g,n,{components:t,mdxType:"MDXLayout"}),(0,r.yg)("admonition",{title:"Be careful",type:"warning"},(0,r.yg)("p",{parentName:"admonition"},"This documentation provide guidelines to delete various types of objects in the OpenShift DSRI. Be careful when you are deleting object in your project, as it could be an object required to run an application.")),(0,r.yg)("p",null,"It is recommend to use the ",(0,r.yg)("inlineCode",{parentName:"p"},"oc")," tool to delete OpenShift objects, as it will allow to properly delete all objects related to specific deployments."),(0,r.yg)("admonition",{title:"Project",type:"caution"},(0,r.yg)("p",{parentName:"admonition"},"Make sure you are connected to the right project:"),(0,r.yg)("pre",{parentName:"admonition"},(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"oc project my-project\n"))),(0,r.yg)("h2",{id:"delete-an-application"},"Delete an application"),(0,r.yg)("p",null,"The best way to make sure all objects related to your application have been deleted is to use the command line providing your application name."),(0,r.yg)("p",null,"Different selectors can be used to easily delete all objects generated by an application deployment. 2 selectors can easily be found in the template configuration:"),(0,r.yg)("ul",null,(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("inlineCode",{parentName:"li"},"app")," : the name you gave when creating your application"),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("inlineCode",{parentName:"li"},"template")," : the name of the template you used to create the application. Use it only if you want to delete all applications created by a specific template.")),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"oc delete all,secret,configmaps,serviceaccount,rolebinding --selector app=my-application\n")),(0,r.yg)("blockquote",null,(0,r.yg)("p",{parentName:"blockquote"},"Delete storage if necessary from the OpenShift web UI.")),(0,r.yg)("admonition",{title:"Force deletion",type:"caution"},(0,r.yg)("p",{parentName:"admonition"},"You can force the deletion if the objects are not deleting properly:"),(0,r.yg)("pre",{parentName:"admonition"},(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"oc delete all,secret,configmaps,serviceaccount,rolebinding --force --grace-period=0 --selector app=my-application\n"))),(0,r.yg)("h2",{id:"delete-pod"},"Delete pod"),(0,r.yg)("p",null,"Get the ID of the specific pod you want to delete:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"oc get pod\n")),(0,r.yg)("p",null,"Use the pod ID retrieved to delete the pod:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"oc delete pod \n")),(0,r.yg)("admonition",{title:"Force deletion",type:"caution"},(0,r.yg)("p",{parentName:"admonition"},"If the pod is not properly deleted, you can force its deletion:"),(0,r.yg)("pre",{parentName:"admonition"},(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"oc delete pod --force --grace-period=0 \n"))),(0,r.yg)("h2",{id:"delete-a-project"},"Delete a project"),(0,r.yg)("admonition",{title:"Be careful",type:"warning"},(0,r.yg)("p",{parentName:"admonition"},"All objects and persistent storages in this project will be deleted and cannot be retrieved.")),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},"To properly delete a project you need to first delete all objects in this project:")),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc delete all,configmap,pvc,serviceaccount,rolebinding,secret,serviceinstance --all -n \n")),(0,r.yg)("ol",{start:2},(0,r.yg)("li",{parentName:"ol"},"Then delete the project:")),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc delete project \n")),(0,r.yg)("h2",{id:"delete-persistent-storage"},"Delete persistent storage"),(0,r.yg)("admonition",{title:"Be careful",type:"warning"},(0,r.yg)("p",{parentName:"admonition"},"All data stored in this persistent storage will be lost and cannot be retrieved.")),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc delete pvc storage-name\n")),(0,r.yg)("h2",{id:"fix-stuck-deletions"},"Fix stuck deletions"),(0,r.yg)("h3",{id:"stuck-provisioned-service"},"Stuck provisioned service"),(0,r.yg)("p",null,"If a provisioned service is stuck on ",(0,r.yg)("inlineCode",{parentName:"p"},"Marked for deletion")," you might need to set finalizers to null in the YAML."),(0,r.yg)("p",null,"This can be done using the OpenShift web UI:"),(0,r.yg)("ul",null,(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("p",{parentName:"li"},"Go to the ",(0,r.yg)("strong",{parentName:"p"},"Provisionned Service")," in the OpenShift UI overview")),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("p",{parentName:"li"},"Click on ",(0,r.yg)("strong",{parentName:"p"},"Edit YAML"))),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("p",{parentName:"li"},"Remove the ",(0,r.yg)("strong",{parentName:"p"},"finalizers"),":"),(0,r.yg)("pre",{parentName:"li"},(0,r.yg)("code",{parentName:"pre",className:"language-yaml"}," finalizers:\n - kubernetes-incubator/service-catalog\n")))),(0,r.yg)("p",null,"You can also do it using the ",(0,r.yg)("inlineCode",{parentName:"p"},"oc")," CLI:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},'oc get serviceinstance\n \n# Delete problematic line from serviceinstance to delete them\noc get serviceinstance -o yaml | grep Terminating | sed "/kubernetes-incubator/d"| oc apply -f - \n')),(0,r.yg)("admonition",{title:"No global catalog",type:"caution"},(0,r.yg)("p",{parentName:"admonition"},"The OpenShift Catalog does not handle deploying templates globally properly (on all projects). If a template is deployed globally, OpenShift will try to create unnecessary objects such as provisioned service (aka. ServiceInstance), or ClusterClasses. Those services are not used, and some of them cannot be deleted easily. ")),(0,r.yg)("admonition",{title:"Catalog per project ",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"At the moment it is more reliable to create the template in directly in your project if you need to use it multiple time.")),(0,r.yg)("h3",{id:"delete-stuck-project"},"Delete stuck project"),(0,r.yg)("p",null,"Project can get stuck as marked for deletion. Usually due to Objects still present in the project that are not terminated or ",(0,r.yg)("inlineCode",{parentName:"p"},"finalizers")," left in the some objects YAML file."),(0,r.yg)("p",null,"The following commands will allow you to clean up all the projects stuck in terminating state you have access to ."),(0,r.yg)("p",null,"Force deletion of terminating projects:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"for i in $(oc get projects | grep Terminating| awk '{print $1}'); do echo $i; oc delete project --force --grace-period=0 $i ; done\n")),(0,r.yg)("p",null,"Delete all objects in terminating projects:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"for i in $(oc get projects | grep Terminating| awk '{print $1}'); do echo $i; oc delete all,configmap,pvc,serviceaccount,rolebinding,secret,serviceinstance --force --grace-period=0 --all -n $i ; done\n")),(0,r.yg)("p",null,"Remove Kubernetes finalizers from terminating projects:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},'for i in $(oc get projects | grep Terminating| awk \'{print $1}\'); do echo $i; oc get project $i -o yaml | sed "/kubernetes/d" | sed "/finalizers:/d" | oc apply -f - ; done\n')),(0,r.yg)("admonition",{title:"Fix deletion",type:"caution"},(0,r.yg)("p",{parentName:"admonition"},"If ",(0,r.yg)("inlineCode",{parentName:"p"},"ServiceInstances")," refuses to get deleted, try to remove kubernetes finalizers:"),(0,r.yg)("pre",{parentName:"admonition"},(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"for i in $(oc get projects | grep Terminating| awk '{print $1}'); do echo $i; oc get serviceinstance -n $i -o yaml | sed \"/kubernetes-incubator/d\"| oc apply -f - ; done\n"))),(0,r.yg)("admonition",{title:"Check deletion",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"Check if there are still objects in a project:"),(0,r.yg)("pre",{parentName:"admonition"},(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc get all,configmap,pvc,serviceaccount,secret,rolebinding,serviceinstance\n"))))}u.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[5803],{5680:(e,t,n)=>{n.d(t,{xA:()=>s,yg:()=>u});var a=n(6540);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function i(e){for(var t=1;t=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var c=a.createContext({}),p=function(e){var t=a.useContext(c),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},s=function(e){var t=p(e.components);return a.createElement(c.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},g=a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,c=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),g=p(n),u=o,y=g["".concat(c,".").concat(u)]||g[u]||d[u]||r;return n?a.createElement(y,i(i({ref:t},s),{},{components:n})):a.createElement(y,i({ref:t},s))}));function u(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,i=new Array(r);i[0]=g;var l={};for(var c in t)hasOwnProperty.call(t,c)&&(l[c]=t[c]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var p=2;p{n.r(t),n.d(t,{assets:()=>s,contentTitle:()=>c,default:()=>u,frontMatter:()=>l,metadata:()=>p,toc:()=>d});var a=n(9668),o=n(1367),r=(n(6540),n(5680)),i=["components"],l={id:"openshift-delete-objects",title:"Delete objects (advanced)"},c=void 0,p={unversionedId:"openshift-delete-objects",id:"openshift-delete-objects",title:"Delete objects (advanced)",description:"This documentation provide guidelines to delete various types of objects in the OpenShift DSRI. Be careful when you are deleting object in your project, as it could be an object required to run an application.",source:"@site/docs/openshift-delete-objects.md",sourceDirName:".",slug:"/openshift-delete-objects",permalink:"/docs/openshift-delete-objects",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/openshift-delete-objects.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"openshift-delete-objects",title:"Delete objects (advanced)"},sidebar:"docs",previous:{title:"Data storage",permalink:"/docs/openshift-storage"},next:{title:"Create a new Project",permalink:"/docs/project-management"}},s={},d=[{value:"Delete an application",id:"delete-an-application",level:2},{value:"Delete pod",id:"delete-pod",level:2},{value:"Delete a project",id:"delete-a-project",level:2},{value:"Delete persistent storage",id:"delete-persistent-storage",level:2},{value:"Fix stuck deletions",id:"fix-stuck-deletions",level:2},{value:"Stuck provisioned service",id:"stuck-provisioned-service",level:3},{value:"Delete stuck project",id:"delete-stuck-project",level:3}],g={toc:d};function u(e){var t=e.components,n=(0,o.A)(e,i);return(0,r.yg)("wrapper",(0,a.A)({},g,n,{components:t,mdxType:"MDXLayout"}),(0,r.yg)("admonition",{title:"Be careful",type:"warning"},(0,r.yg)("p",{parentName:"admonition"},"This documentation provide guidelines to delete various types of objects in the OpenShift DSRI. Be careful when you are deleting object in your project, as it could be an object required to run an application.")),(0,r.yg)("p",null,"It is recommend to use the ",(0,r.yg)("inlineCode",{parentName:"p"},"oc")," tool to delete OpenShift objects, as it will allow to properly delete all objects related to specific deployments."),(0,r.yg)("admonition",{title:"Project",type:"caution"},(0,r.yg)("p",{parentName:"admonition"},"Make sure you are connected to the right project:"),(0,r.yg)("pre",{parentName:"admonition"},(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"oc project my-project\n"))),(0,r.yg)("h2",{id:"delete-an-application"},"Delete an application"),(0,r.yg)("p",null,"The best way to make sure all objects related to your application have been deleted is to use the command line providing your application name."),(0,r.yg)("p",null,"Different selectors can be used to easily delete all objects generated by an application deployment. 2 selectors can easily be found in the template configuration:"),(0,r.yg)("ul",null,(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("inlineCode",{parentName:"li"},"app")," : the name you gave when creating your application"),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("inlineCode",{parentName:"li"},"template")," : the name of the template you used to create the application. Use it only if you want to delete all applications created by a specific template.")),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"oc delete all,secret,configmaps,serviceaccount,rolebinding --selector app=my-application\n")),(0,r.yg)("blockquote",null,(0,r.yg)("p",{parentName:"blockquote"},"Delete storage if necessary from the OpenShift web UI.")),(0,r.yg)("admonition",{title:"Force deletion",type:"caution"},(0,r.yg)("p",{parentName:"admonition"},"You can force the deletion if the objects are not deleting properly:"),(0,r.yg)("pre",{parentName:"admonition"},(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"oc delete all,secret,configmaps,serviceaccount,rolebinding --force --grace-period=0 --selector app=my-application\n"))),(0,r.yg)("h2",{id:"delete-pod"},"Delete pod"),(0,r.yg)("p",null,"Get the ID of the specific pod you want to delete:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"oc get pod\n")),(0,r.yg)("p",null,"Use the pod ID retrieved to delete the pod:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"oc delete pod \n")),(0,r.yg)("admonition",{title:"Force deletion",type:"caution"},(0,r.yg)("p",{parentName:"admonition"},"If the pod is not properly deleted, you can force its deletion:"),(0,r.yg)("pre",{parentName:"admonition"},(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"oc delete pod --force --grace-period=0 \n"))),(0,r.yg)("h2",{id:"delete-a-project"},"Delete a project"),(0,r.yg)("admonition",{title:"Be careful",type:"warning"},(0,r.yg)("p",{parentName:"admonition"},"All objects and persistent storages in this project will be deleted and cannot be retrieved.")),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},"To properly delete a project you need to first delete all objects in this project:")),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc delete all,configmap,pvc,serviceaccount,rolebinding,secret,serviceinstance --all -n \n")),(0,r.yg)("ol",{start:2},(0,r.yg)("li",{parentName:"ol"},"Then delete the project:")),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc delete project \n")),(0,r.yg)("h2",{id:"delete-persistent-storage"},"Delete persistent storage"),(0,r.yg)("admonition",{title:"Be careful",type:"warning"},(0,r.yg)("p",{parentName:"admonition"},"All data stored in this persistent storage will be lost and cannot be retrieved.")),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc delete pvc storage-name\n")),(0,r.yg)("h2",{id:"fix-stuck-deletions"},"Fix stuck deletions"),(0,r.yg)("h3",{id:"stuck-provisioned-service"},"Stuck provisioned service"),(0,r.yg)("p",null,"If a provisioned service is stuck on ",(0,r.yg)("inlineCode",{parentName:"p"},"Marked for deletion")," you might need to set finalizers to null in the YAML."),(0,r.yg)("p",null,"This can be done using the OpenShift web UI:"),(0,r.yg)("ul",null,(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("p",{parentName:"li"},"Go to the ",(0,r.yg)("strong",{parentName:"p"},"Provisionned Service")," in the OpenShift UI overview")),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("p",{parentName:"li"},"Click on ",(0,r.yg)("strong",{parentName:"p"},"Edit YAML"))),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("p",{parentName:"li"},"Remove the ",(0,r.yg)("strong",{parentName:"p"},"finalizers"),":"),(0,r.yg)("pre",{parentName:"li"},(0,r.yg)("code",{parentName:"pre",className:"language-yaml"}," finalizers:\n - kubernetes-incubator/service-catalog\n")))),(0,r.yg)("p",null,"You can also do it using the ",(0,r.yg)("inlineCode",{parentName:"p"},"oc")," CLI:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-shell"},'oc get serviceinstance\n \n# Delete problematic line from serviceinstance to delete them\noc get serviceinstance -o yaml | grep Terminating | sed "/kubernetes-incubator/d"| oc apply -f - \n')),(0,r.yg)("admonition",{title:"No global catalog",type:"caution"},(0,r.yg)("p",{parentName:"admonition"},"The OpenShift Catalog does not handle deploying templates globally properly (on all projects). If a template is deployed globally, OpenShift will try to create unnecessary objects such as provisioned service (aka. ServiceInstance), or ClusterClasses. Those services are not used, and some of them cannot be deleted easily. ")),(0,r.yg)("admonition",{title:"Catalog per project ",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"At the moment it is more reliable to create the template in directly in your project if you need to use it multiple time.")),(0,r.yg)("h3",{id:"delete-stuck-project"},"Delete stuck project"),(0,r.yg)("p",null,"Project can get stuck as marked for deletion. Usually due to Objects still present in the project that are not terminated or ",(0,r.yg)("inlineCode",{parentName:"p"},"finalizers")," left in the some objects YAML file."),(0,r.yg)("p",null,"The following commands will allow you to clean up all the projects stuck in terminating state you have access to ."),(0,r.yg)("p",null,"Force deletion of terminating projects:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"for i in $(oc get projects | grep Terminating| awk '{print $1}'); do echo $i; oc delete project --force --grace-period=0 $i ; done\n")),(0,r.yg)("p",null,"Delete all objects in terminating projects:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"for i in $(oc get projects | grep Terminating| awk '{print $1}'); do echo $i; oc delete all,configmap,pvc,serviceaccount,rolebinding,secret,serviceinstance --force --grace-period=0 --all -n $i ; done\n")),(0,r.yg)("p",null,"Remove Kubernetes finalizers from terminating projects:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},'for i in $(oc get projects | grep Terminating| awk \'{print $1}\'); do echo $i; oc get project $i -o yaml | sed "/kubernetes/d" | sed "/finalizers:/d" | oc apply -f - ; done\n')),(0,r.yg)("admonition",{title:"Fix deletion",type:"caution"},(0,r.yg)("p",{parentName:"admonition"},"If ",(0,r.yg)("inlineCode",{parentName:"p"},"ServiceInstances")," refuses to get deleted, try to remove kubernetes finalizers:"),(0,r.yg)("pre",{parentName:"admonition"},(0,r.yg)("code",{parentName:"pre",className:"language-shell"},"for i in $(oc get projects | grep Terminating| awk '{print $1}'); do echo $i; oc get serviceinstance -n $i -o yaml | sed \"/kubernetes-incubator/d\"| oc apply -f - ; done\n"))),(0,r.yg)("admonition",{title:"Check deletion",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"Check if there are still objects in a project:"),(0,r.yg)("pre",{parentName:"admonition"},(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc get all,configmap,pvc,serviceaccount,secret,rolebinding,serviceinstance\n"))))}u.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/682467b2.b685bb5c.js b/assets/js/682467b2.a8817d65.js similarity index 99% rename from assets/js/682467b2.b685bb5c.js rename to assets/js/682467b2.a8817d65.js index f5c94777e..145224bd8 100644 --- a/assets/js/682467b2.b685bb5c.js +++ b/assets/js/682467b2.a8817d65.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[6746],{5680:(e,r,t)=>{t.d(r,{xA:()=>p,yg:()=>f});var o=t(6540);function n(e,r,t){return r in e?Object.defineProperty(e,r,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[r]=t,e}function i(e,r){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);r&&(o=o.filter((function(r){return Object.getOwnPropertyDescriptor(e,r).enumerable}))),t.push.apply(t,o)}return t}function a(e){for(var r=1;r=0||(n[t]=e[t]);return n}(e,r);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(o=0;o=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(n[t]=e[t])}return n}var s=o.createContext({}),c=function(e){var r=o.useContext(s),t=r;return e&&(t="function"==typeof e?e(r):a(a({},r),e)),t},p=function(e){var r=c(e.components);return o.createElement(s.Provider,{value:r},e.children)},u={inlineCode:"code",wrapper:function(e){var r=e.children;return o.createElement(o.Fragment,{},r)}},d=o.forwardRef((function(e,r){var t=e.components,n=e.mdxType,i=e.originalType,s=e.parentName,p=l(e,["components","mdxType","originalType","parentName"]),d=c(t),f=n,y=d["".concat(s,".").concat(f)]||d[f]||u[f]||i;return t?o.createElement(y,a(a({ref:r},p),{},{components:t})):o.createElement(y,a({ref:r},p))}));function f(e,r){var t=arguments,n=r&&r.mdxType;if("string"==typeof e||n){var i=t.length,a=new Array(i);a[0]=d;var l={};for(var s in r)hasOwnProperty.call(r,s)&&(l[s]=r[s]);l.originalType=e,l.mdxType="string"==typeof e?e:n,a[1]=l;for(var c=2;c{t.r(r),t.d(r,{assets:()=>p,contentTitle:()=>s,default:()=>f,frontMatter:()=>l,metadata:()=>c,toc:()=>u});var o=t(9668),n=t(1367),i=(t(6540),t(5680)),a=["components"],l={id:"profile-pytorch-code",title:"PyTorch Profiling"},s=void 0,c={unversionedId:"profile-pytorch-code",id:"profile-pytorch-code",title:"PyTorch Profiling",description:"What is profiling?",source:"@site/docs/profile-pytorch-code.md",sourceDirName:".",slug:"/profile-pytorch-code",permalink:"/docs/profile-pytorch-code",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/profile-pytorch-code.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"profile-pytorch-code",title:"PyTorch Profiling"},sidebar:"docs",previous:{title:"Increase your processes speed",permalink:"/docs/increase-process-speed"},next:{title:"Tensorflow Optimization",permalink:"/docs/speeding-tensorflow-dl"}},p={},u=[{value:"What is profiling?",id:"what-is-profiling",level:2},{value:"Why should I care about profiling?",id:"why-should-i-care-about-profiling",level:2},{value:"When should I care about profiling?",id:"when-should-i-care-about-profiling",level:2},{value:"How DSRI team can help you?",id:"how-dsri-team-can-help-you",level:2},{value:"External Resources and references",id:"external-resources-and-references",level:2}],d={toc:u};function f(e){var r=e.components,t=(0,n.A)(e,a);return(0,i.yg)("wrapper",(0,o.A)({},d,t,{components:r,mdxType:"MDXLayout"}),(0,i.yg)("h2",{id:"what-is-profiling"},"What is profiling?"),(0,i.yg)("p",null,"According to wikipedia:"),(0,i.yg)("p",null,'"Profiling is a form of dynamic program analysis that measures, for example, the space (memory) or time complexity of a program, the usage of particular instructions, or the frequency and duration of function calls. Most commonly, profiling information serves to aid program optimization, and more specifically, performance engineering."'),(0,i.yg)("h2",{id:"why-should-i-care-about-profiling"},"Why should I care about profiling?"),(0,i.yg)("p",null,"You may know that training large models like GPT-3 takes several million dollars source and a few hundred MWh source. If the engineers that trained these models did not spend time on optimization, it might have been several million dollars and hunderds of MWh more."),(0,i.yg)("p",null,"Sure, the model you'd like to train is probably not quite as big. But maybe you want to train it 10000 times, because you want to do hyperparameter optimization. And even if you only train it once, it may take quite a bit of compute resources, i.e. money and energy."),(0,i.yg)("h2",{id:"when-should-i-care-about-profiling"},"When should I care about profiling?"),(0,i.yg)("p",null,"Well, you should always care if your code runs efficiently, but there's different levels of caring."),(0,i.yg)("p",null,"From personal experience: if I know I'm going to run a code only once, for a few days, on a single GPU, I'll probably not create a full profile. What I would do is inspect my GPU and CPU utilization during my runs, just to see if it is somewhat efficient, and if I didn't make any obvious mistakes (e.g. accidentally not using the GPU, even if I have one available)."),(0,i.yg)("p",null,"If I know that I'll run my code on multiple GPUs, for multiple days, (potentially) on multiple nodes, and/or I need to run it multiple times, I know that my resource footprint is going to be large, and it's worth spending some time and effort to optimize the code. That's when I'll create a profile. The good part is: the more often you do it, the quicker and more adapt you become at it."),(0,i.yg)("h2",{id:"how-dsri-team-can-help-you"},"How DSRI team can help you?"),(0,i.yg)("p",null,"We can assist you with analyzing the bottleneck/s in your deep learning pipeline and recommend the improvments to speed up your pipeline."),(0,i.yg)("h2",{id:"external-resources-and-references"},"External Resources and references"),(0,i.yg)("ul",null,(0,i.yg)("li",{parentName:"ul"},"This documentation is taken from the Surf's PyTorch profiling wiki (",(0,i.yg)("a",{parentName:"li",href:"https://servicedesk.surf.nl/wiki/display/WIKI/PyTorch+Profiling"},"https://servicedesk.surf.nl/wiki/display/WIKI/PyTorch+Profiling"),")"),(0,i.yg)("li",{parentName:"ul"},"Tutorial on PyTorch profiling can be found here: (",(0,i.yg)("a",{parentName:"li",href:"https://github.com/sara-nl/PraceHPML2022/blob/master/notebooks/PyTorch_profiling/PyTorch_profiling.ipynb"},"https://github.com/sara-nl/PraceHPML2022/blob/master/notebooks/PyTorch_profiling/PyTorch_profiling.ipynb"),")")))}f.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[6746],{5680:(e,r,t)=>{t.d(r,{xA:()=>p,yg:()=>f});var o=t(6540);function n(e,r,t){return r in e?Object.defineProperty(e,r,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[r]=t,e}function i(e,r){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);r&&(o=o.filter((function(r){return Object.getOwnPropertyDescriptor(e,r).enumerable}))),t.push.apply(t,o)}return t}function a(e){for(var r=1;r=0||(n[t]=e[t]);return n}(e,r);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(o=0;o=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(n[t]=e[t])}return n}var s=o.createContext({}),c=function(e){var r=o.useContext(s),t=r;return e&&(t="function"==typeof e?e(r):a(a({},r),e)),t},p=function(e){var r=c(e.components);return o.createElement(s.Provider,{value:r},e.children)},u={inlineCode:"code",wrapper:function(e){var r=e.children;return o.createElement(o.Fragment,{},r)}},d=o.forwardRef((function(e,r){var t=e.components,n=e.mdxType,i=e.originalType,s=e.parentName,p=l(e,["components","mdxType","originalType","parentName"]),d=c(t),f=n,y=d["".concat(s,".").concat(f)]||d[f]||u[f]||i;return t?o.createElement(y,a(a({ref:r},p),{},{components:t})):o.createElement(y,a({ref:r},p))}));function f(e,r){var t=arguments,n=r&&r.mdxType;if("string"==typeof e||n){var i=t.length,a=new Array(i);a[0]=d;var l={};for(var s in r)hasOwnProperty.call(r,s)&&(l[s]=r[s]);l.originalType=e,l.mdxType="string"==typeof e?e:n,a[1]=l;for(var c=2;c{t.r(r),t.d(r,{assets:()=>p,contentTitle:()=>s,default:()=>f,frontMatter:()=>l,metadata:()=>c,toc:()=>u});var o=t(9668),n=t(1367),i=(t(6540),t(5680)),a=["components"],l={id:"profile-pytorch-code",title:"PyTorch Profiling"},s=void 0,c={unversionedId:"profile-pytorch-code",id:"profile-pytorch-code",title:"PyTorch Profiling",description:"What is profiling?",source:"@site/docs/profile-pytorch-code.md",sourceDirName:".",slug:"/profile-pytorch-code",permalink:"/docs/profile-pytorch-code",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/profile-pytorch-code.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"profile-pytorch-code",title:"PyTorch Profiling"},sidebar:"docs",previous:{title:"Increase your processes speed",permalink:"/docs/increase-process-speed"},next:{title:"Tensorflow Optimization",permalink:"/docs/speeding-tensorflow-dl"}},p={},u=[{value:"What is profiling?",id:"what-is-profiling",level:2},{value:"Why should I care about profiling?",id:"why-should-i-care-about-profiling",level:2},{value:"When should I care about profiling?",id:"when-should-i-care-about-profiling",level:2},{value:"How DSRI team can help you?",id:"how-dsri-team-can-help-you",level:2},{value:"External Resources and references",id:"external-resources-and-references",level:2}],d={toc:u};function f(e){var r=e.components,t=(0,n.A)(e,a);return(0,i.yg)("wrapper",(0,o.A)({},d,t,{components:r,mdxType:"MDXLayout"}),(0,i.yg)("h2",{id:"what-is-profiling"},"What is profiling?"),(0,i.yg)("p",null,"According to wikipedia:"),(0,i.yg)("p",null,'"Profiling is a form of dynamic program analysis that measures, for example, the space (memory) or time complexity of a program, the usage of particular instructions, or the frequency and duration of function calls. Most commonly, profiling information serves to aid program optimization, and more specifically, performance engineering."'),(0,i.yg)("h2",{id:"why-should-i-care-about-profiling"},"Why should I care about profiling?"),(0,i.yg)("p",null,"You may know that training large models like GPT-3 takes several million dollars source and a few hundred MWh source. If the engineers that trained these models did not spend time on optimization, it might have been several million dollars and hunderds of MWh more."),(0,i.yg)("p",null,"Sure, the model you'd like to train is probably not quite as big. But maybe you want to train it 10000 times, because you want to do hyperparameter optimization. And even if you only train it once, it may take quite a bit of compute resources, i.e. money and energy."),(0,i.yg)("h2",{id:"when-should-i-care-about-profiling"},"When should I care about profiling?"),(0,i.yg)("p",null,"Well, you should always care if your code runs efficiently, but there's different levels of caring."),(0,i.yg)("p",null,"From personal experience: if I know I'm going to run a code only once, for a few days, on a single GPU, I'll probably not create a full profile. What I would do is inspect my GPU and CPU utilization during my runs, just to see if it is somewhat efficient, and if I didn't make any obvious mistakes (e.g. accidentally not using the GPU, even if I have one available)."),(0,i.yg)("p",null,"If I know that I'll run my code on multiple GPUs, for multiple days, (potentially) on multiple nodes, and/or I need to run it multiple times, I know that my resource footprint is going to be large, and it's worth spending some time and effort to optimize the code. That's when I'll create a profile. The good part is: the more often you do it, the quicker and more adapt you become at it."),(0,i.yg)("h2",{id:"how-dsri-team-can-help-you"},"How DSRI team can help you?"),(0,i.yg)("p",null,"We can assist you with analyzing the bottleneck/s in your deep learning pipeline and recommend the improvments to speed up your pipeline."),(0,i.yg)("h2",{id:"external-resources-and-references"},"External Resources and references"),(0,i.yg)("ul",null,(0,i.yg)("li",{parentName:"ul"},"This documentation is taken from the Surf's PyTorch profiling wiki (",(0,i.yg)("a",{parentName:"li",href:"https://servicedesk.surf.nl/wiki/display/WIKI/PyTorch+Profiling"},"https://servicedesk.surf.nl/wiki/display/WIKI/PyTorch+Profiling"),")"),(0,i.yg)("li",{parentName:"ul"},"Tutorial on PyTorch profiling can be found here: (",(0,i.yg)("a",{parentName:"li",href:"https://github.com/sara-nl/PraceHPML2022/blob/master/notebooks/PyTorch_profiling/PyTorch_profiling.ipynb"},"https://github.com/sara-nl/PraceHPML2022/blob/master/notebooks/PyTorch_profiling/PyTorch_profiling.ipynb"),")")))}f.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/6b741ffd.0142d16a.js b/assets/js/6b741ffd.8a009016.js similarity index 98% rename from assets/js/6b741ffd.0142d16a.js rename to assets/js/6b741ffd.8a009016.js index bc37e9eb7..a0edae87e 100644 --- a/assets/js/6b741ffd.0142d16a.js +++ b/assets/js/6b741ffd.8a009016.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[93],{5680:(e,t,r)=>{r.d(t,{xA:()=>p,yg:()=>m});var o=r(6540);function n(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function a(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,o)}return r}function l(e){for(var t=1;t=0||(n[r]=e[r]);return n}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(o=0;o=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(n[r]=e[r])}return n}var s=o.createContext({}),c=function(e){var t=o.useContext(s),r=t;return e&&(r="function"==typeof e?e(t):l(l({},t),e)),r},p=function(e){var t=c(e.components);return o.createElement(s.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.createElement(o.Fragment,{},t)}},u=o.forwardRef((function(e,t){var r=e.components,n=e.mdxType,a=e.originalType,s=e.parentName,p=i(e,["components","mdxType","originalType","parentName"]),u=c(r),m=n,f=u["".concat(s,".").concat(m)]||u[m]||d[m]||a;return r?o.createElement(f,l(l({ref:t},p),{},{components:r})):o.createElement(f,l({ref:t},p))}));function m(e,t){var r=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var a=r.length,l=new Array(a);l[0]=u;var i={};for(var s in t)hasOwnProperty.call(t,s)&&(i[s]=t[s]);i.originalType=e,i.mdxType="string"==typeof e?e:n,l[1]=i;for(var c=2;c{r.r(t),r.d(t,{assets:()=>p,contentTitle:()=>s,default:()=>m,frontMatter:()=>i,metadata:()=>c,toc:()=>d});var o=r(9668),n=r(1367),a=(r(6540),r(5680)),l=["components"],i={id:"workflows-cwl",title:"Run CWL workflows"},s=void 0,c={unversionedId:"workflows-cwl",id:"workflows-cwl",title:"Run CWL workflows",description:"The Common Workflow Language (CWL) is an open standard for describing analysis workflows and tools in a way that makes them portable and scalable across a variety of software and hardware environments.",source:"@site/docs/workflows-cwl.md",sourceDirName:".",slug:"/workflows-cwl",permalink:"/docs/workflows-cwl",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/workflows-cwl.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"workflows-cwl",title:"Run CWL workflows"},sidebar:"docs",previous:{title:"Run Nextflow workflows",permalink:"/docs/workflows-nextflow"},next:{title:"Contribute",permalink:"/docs/contribute"}},p={},d=[{value:"Clone the repository",id:"clone-the-repository",level:2},{value:"Start pod",id:"start-pod",level:2},{value:"Delete created pod",id:"delete-created-pod",level:2}],u={toc:d};function m(e){var t=e.components,r=(0,n.A)(e,l);return(0,a.yg)("wrapper",(0,o.A)({},u,r,{components:t,mdxType:"MDXLayout"}),(0,a.yg)("p",null,"The ",(0,a.yg)("a",{parentName:"p",href:"https://www.commonwl.org/"},"Common Workflow Language")," (CWL) is an open standard for describing analysis workflows and tools in a way that makes them portable and scalable across a variety of software and hardware environments."),(0,a.yg)("p",null,"We use the ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/Duke-GCB/calrissian"},"CWL Calrissian")," implementation, note that this project is young and still in development, feel free to report issues and contribute to its documentation."),(0,a.yg)("h2",{id:"clone-the-repository"},"Clone the repository"),(0,a.yg)("ol",null,(0,a.yg)("li",{parentName:"ol"},"Git clone in ",(0,a.yg)("inlineCode",{parentName:"li"},"/calrissian")," on a ",(0,a.yg)("a",{parentName:"li",href:"/docs/openshift-storage"},"persistent volume")," on the cluster from a terminal. ")),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-shell"},"cd /data/calrissian\ngit clone --recursive https://github.com/MaastrichtU-IDS/d2s-project-template.git\ncd d2s-project-template\n")),(0,a.yg)("ol",{start:2},(0,a.yg)("li",{parentName:"ol"},"You will need to create the folder for the workflow output data, in our example it is ",(0,a.yg)("inlineCode",{parentName:"li"},"output-data"),":")),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-shell"},"mkdir /data/calrissian/output-data\n")),(0,a.yg)("ol",{start:3},(0,a.yg)("li",{parentName:"ol"},"You might need to give permissions (CWL execution will fail due to permissions issues otherwise).")),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-shell"},"chmod -R 777 /data/calrissian\n")),(0,a.yg)("h2",{id:"start-pod"},"Start pod"),(0,a.yg)("p",null,"Start the CWL execution from your computer using the ",(0,a.yg)("inlineCode",{parentName:"p"},"oc")," client. Define the CWL command arguments to run in ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/d2s-core/blob/master/support/run-workflows-cwl.yaml"},"run-workflows-cwl.yaml")," (be careful to properly define the paths to the CWL files in the pod storage)."),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-shell"},"oc create -f d2s-core/support/run-workflows-cwl.yaml\n")),(0,a.yg)("admonition",{title:"Delete the pod",type:"caution"},(0,a.yg)("p",{parentName:"admonition"},"You will need to delete the pod if you want to re-create it.")),(0,a.yg)("h2",{id:"delete-created-pod"},"Delete created pod"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-shell"},"oc delete -f d2s-core/support/run-workflows-cwl.yaml\n")))}m.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[93],{5680:(e,t,r)=>{r.d(t,{xA:()=>p,yg:()=>m});var o=r(6540);function n(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function a(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,o)}return r}function l(e){for(var t=1;t=0||(n[r]=e[r]);return n}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(o=0;o=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(n[r]=e[r])}return n}var s=o.createContext({}),c=function(e){var t=o.useContext(s),r=t;return e&&(r="function"==typeof e?e(t):l(l({},t),e)),r},p=function(e){var t=c(e.components);return o.createElement(s.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.createElement(o.Fragment,{},t)}},u=o.forwardRef((function(e,t){var r=e.components,n=e.mdxType,a=e.originalType,s=e.parentName,p=i(e,["components","mdxType","originalType","parentName"]),u=c(r),m=n,f=u["".concat(s,".").concat(m)]||u[m]||d[m]||a;return r?o.createElement(f,l(l({ref:t},p),{},{components:r})):o.createElement(f,l({ref:t},p))}));function m(e,t){var r=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var a=r.length,l=new Array(a);l[0]=u;var i={};for(var s in t)hasOwnProperty.call(t,s)&&(i[s]=t[s]);i.originalType=e,i.mdxType="string"==typeof e?e:n,l[1]=i;for(var c=2;c{r.r(t),r.d(t,{assets:()=>p,contentTitle:()=>s,default:()=>m,frontMatter:()=>i,metadata:()=>c,toc:()=>d});var o=r(9668),n=r(1367),a=(r(6540),r(5680)),l=["components"],i={id:"workflows-cwl",title:"Run CWL workflows"},s=void 0,c={unversionedId:"workflows-cwl",id:"workflows-cwl",title:"Run CWL workflows",description:"The Common Workflow Language (CWL) is an open standard for describing analysis workflows and tools in a way that makes them portable and scalable across a variety of software and hardware environments.",source:"@site/docs/workflows-cwl.md",sourceDirName:".",slug:"/workflows-cwl",permalink:"/docs/workflows-cwl",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/workflows-cwl.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"workflows-cwl",title:"Run CWL workflows"},sidebar:"docs",previous:{title:"Run Nextflow workflows",permalink:"/docs/workflows-nextflow"},next:{title:"Contribute",permalink:"/docs/contribute"}},p={},d=[{value:"Clone the repository",id:"clone-the-repository",level:2},{value:"Start pod",id:"start-pod",level:2},{value:"Delete created pod",id:"delete-created-pod",level:2}],u={toc:d};function m(e){var t=e.components,r=(0,n.A)(e,l);return(0,a.yg)("wrapper",(0,o.A)({},u,r,{components:t,mdxType:"MDXLayout"}),(0,a.yg)("p",null,"The ",(0,a.yg)("a",{parentName:"p",href:"https://www.commonwl.org/"},"Common Workflow Language")," (CWL) is an open standard for describing analysis workflows and tools in a way that makes them portable and scalable across a variety of software and hardware environments."),(0,a.yg)("p",null,"We use the ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/Duke-GCB/calrissian"},"CWL Calrissian")," implementation, note that this project is young and still in development, feel free to report issues and contribute to its documentation."),(0,a.yg)("h2",{id:"clone-the-repository"},"Clone the repository"),(0,a.yg)("ol",null,(0,a.yg)("li",{parentName:"ol"},"Git clone in ",(0,a.yg)("inlineCode",{parentName:"li"},"/calrissian")," on a ",(0,a.yg)("a",{parentName:"li",href:"/docs/openshift-storage"},"persistent volume")," on the cluster from a terminal. ")),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-shell"},"cd /data/calrissian\ngit clone --recursive https://github.com/MaastrichtU-IDS/d2s-project-template.git\ncd d2s-project-template\n")),(0,a.yg)("ol",{start:2},(0,a.yg)("li",{parentName:"ol"},"You will need to create the folder for the workflow output data, in our example it is ",(0,a.yg)("inlineCode",{parentName:"li"},"output-data"),":")),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-shell"},"mkdir /data/calrissian/output-data\n")),(0,a.yg)("ol",{start:3},(0,a.yg)("li",{parentName:"ol"},"You might need to give permissions (CWL execution will fail due to permissions issues otherwise).")),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-shell"},"chmod -R 777 /data/calrissian\n")),(0,a.yg)("h2",{id:"start-pod"},"Start pod"),(0,a.yg)("p",null,"Start the CWL execution from your computer using the ",(0,a.yg)("inlineCode",{parentName:"p"},"oc")," client. Define the CWL command arguments to run in ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/d2s-core/blob/master/support/run-workflows-cwl.yaml"},"run-workflows-cwl.yaml")," (be careful to properly define the paths to the CWL files in the pod storage)."),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-shell"},"oc create -f d2s-core/support/run-workflows-cwl.yaml\n")),(0,a.yg)("admonition",{title:"Delete the pod",type:"caution"},(0,a.yg)("p",{parentName:"admonition"},"You will need to delete the pod if you want to re-create it.")),(0,a.yg)("h2",{id:"delete-created-pod"},"Delete created pod"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-shell"},"oc delete -f d2s-core/support/run-workflows-cwl.yaml\n")))}m.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/707d3f57.b2cbcf58.js b/assets/js/707d3f57.9953ff4f.js similarity index 99% rename from assets/js/707d3f57.b2cbcf58.js rename to assets/js/707d3f57.9953ff4f.js index 332b9fbfb..797825081 100644 --- a/assets/js/707d3f57.b2cbcf58.js +++ b/assets/js/707d3f57.9953ff4f.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[248],{5680:(e,t,n)=>{n.d(t,{xA:()=>u,yg:()=>y});var a=n(6540);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function i(e){for(var t=1;t=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var p=a.createContext({}),s=function(e){var t=a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},u=function(e){var t=s(e.components);return a.createElement(p.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},c=a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,p=e.parentName,u=l(e,["components","mdxType","originalType","parentName"]),c=s(n),y=o,g=c["".concat(p,".").concat(y)]||c[y]||d[y]||r;return n?a.createElement(g,i(i({ref:t},u),{},{components:n})):a.createElement(g,i({ref:t},u))}));function y(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,i=new Array(r);i[0]=c;var l={};for(var p in t)hasOwnProperty.call(t,p)&&(l[p]=t[p]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var s=2;s{n.r(t),n.d(t,{assets:()=>u,contentTitle:()=>p,default:()=>y,frontMatter:()=>l,metadata:()=>s,toc:()=>d});var a=n(9668),o=n(1367),r=(n(6540),n(5680)),i=["components"],l={id:"deploy-on-gpu",title:"GPU applications"},p=void 0,s={unversionedId:"deploy-on-gpu",id:"deploy-on-gpu",title:"GPU applications",description:"GPUs on the DSRI can only be used by one workspace at a time, and there is a limited number of GPUs (8).",source:"@site/docs/deploy-on-gpu.md",sourceDirName:".",slug:"/deploy-on-gpu",permalink:"/docs/deploy-on-gpu",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/deploy-on-gpu.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"deploy-on-gpu",title:"GPU applications"},sidebar:"docs",previous:{title:"Monitor your applications",permalink:"/docs/guide-monitoring"},next:{title:"Deploy from a Dockerfile",permalink:"/docs/guide-dockerfile-to-openshift"}},u={},d=[{value:"Prepare your GPU workspace",id:"prepare-your-gpu-workspace",level:2},{value:"About the docker images",id:"about-the-docker-images",level:3},{value:"Deploy the workspace",id:"deploy-the-workspace",level:3},{value:"Prepare the workspace",id:"prepare-the-workspace",level:3},{value:"Enable the GPU",id:"enable-the-gpu",level:2},{value:"Disable the GPU",id:"disable-the-gpu",level:2},{value:"Increase the number of GPUs",id:"increase-the-number-of-gpus",level:2},{value:"Install GPU drivers in any image",id:"install-gpu-drivers-in-any-image",level:2}],c={toc:d};function y(e){var t=e.components,n=(0,o.A)(e,i);return(0,r.yg)("wrapper",(0,a.A)({},c,n,{components:t,mdxType:"MDXLayout"}),(0,r.yg)("p",null,"GPUs on the DSRI can only be used by one workspace at a time, and there is a limited number of GPUs (8)."),(0,r.yg)("p",null,"\u26a0\ufe0f We currently provide a free access to those GPUs, but with the growing demands for GPUs it might get more restricted. As consideration for others, and to help keep this system open, it is important to make a maximum use of your GPUs when you get access to them. "),(0,r.yg)("p",null,"Unfortunately job scheduling is currently not mature enough on Kubernetes, you can look into ",(0,r.yg)("a",{parentName:"p",href:"https://volcano.sh/en/"},"volcano.sh")," if you are interested, but it is still quite experimental."),(0,r.yg)("p",null,"To use the GPU on the DSRI you will go through this process:"),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},"Deploy, prepare and debug your GPU workspace"),(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("a",{parentName:"li",href:"/gpu-booking"},"Book a GPU")),(0,r.yg)("li",{parentName:"ol"},"Once the booking is done you will receive an email about your reservation, and more emails when it starts and before it ends"),(0,r.yg)("li",{parentName:"ol"},"Enable the GPU in workspace when your booking starts, and make the best use of it!")),(0,r.yg)("admonition",{title:"Book a GPU",type:"warning"},(0,r.yg)("p",{parentName:"admonition"},(0,r.yg)("strong",{parentName:"p"},"By default you do not have the permission to run applications on GPU"),", you need to make a reservation."),(0,r.yg)("p",{parentName:"admonition"},"You can check the availability of our GPUs, and reserve GPU slots in the ",(0,r.yg)("a",{parentName:"p",href:"/gpu-booking"},"GPU booking calendar \ud83d\udcc5"))),(0,r.yg)("h2",{id:"prepare-your-gpu-workspace"},"Prepare your GPU workspace"),(0,r.yg)("p",null,"You will first need to start your workspace without the GPU enabled, you can then prepare your experiments: clone the code, download the data, prepare scripts to install all requirements (the workspace will be restarted when you enable the GPU). "),(0,r.yg)("h3",{id:"about-the-docker-images"},"About the docker images"),(0,r.yg)("p",null,"We are mainly using images provided by Nvidia, with all required drivers and optimizations for GPU pre-installed. You can access the workspace with JupyterLab and VisualStudio Code in your browser, and install dependencies with ",(0,r.yg)("inlineCode",{parentName:"p"},"apt-get"),", ",(0,r.yg)("inlineCode",{parentName:"p"},"conda")," or ",(0,r.yg)("inlineCode",{parentName:"p"},"pip")," in the workspace."),(0,r.yg)("p",null,"We currently mainly use Tensorflow, PyTorch and CUDA, but any image available in the ",(0,r.yg)("a",{parentName:"p",href:"https://ngc.nvidia.com/catalog/containers"},"Nvidia catalog")," should be easy to deploy. Checkout ",(0,r.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/jupyterlab#jupyterlab-on-gpu"},"this documentation")," for more details on how we build the optimized docker images for the DSRI GPUs. And feel free to ",(0,r.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/jupyterlab#extend-an-image"},"extend the images")," to install any software you need."),(0,r.yg)("h3",{id:"deploy-the-workspace"},"Deploy the workspace"),(0,r.yg)("p",null,"You can easily deploy your GPU workspace from the DSRI catalog:"),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},"Go to the ",(0,r.yg)("a",{parentName:"li",href:"https://console-openshift-console.apps.dsri2.unimaas.nl/catalog"},"DSRI Catalog web UI"),": Click on ",(0,r.yg)("strong",{parentName:"li"},"Add to Project"),", then ",(0,r.yg)("strong",{parentName:"li"},"Browse Catalog")),(0,r.yg)("li",{parentName:"ol"},'Search the catalog for "GPU", and make sure the Template checkbox is enabled'),(0,r.yg)("li",{parentName:"ol"},"Choose the template: ",(0,r.yg)("strong",{parentName:"li"},"JupyterLab on GPU")),(0,r.yg)("li",{parentName:"ol"},"Follow the instructions to create the template in the DSRI web UI, all information about the images you can use are provided there. The most notable is the base image you want to use for your workspace (",(0,r.yg)("inlineCode",{parentName:"li"},"cuda"),", ",(0,r.yg)("inlineCode",{parentName:"li"},"tensorflow")," or ",(0,r.yg)("inlineCode",{parentName:"li"},"pytorch"),")")),(0,r.yg)("p",null,"Access the workspace from the route created (the small arrow at the top right of your application bubble in the Topology page)."),(0,r.yg)("h3",{id:"prepare-the-workspace"},"Prepare the workspace"),(0,r.yg)("p",null,"You can now add your code and data in the persistent folder to be fully prepared when you will get access to the GPUs."),(0,r.yg)("p",null,"You can install dependencies with ",(0,r.yg)("inlineCode",{parentName:"p"},"apt-get"),", ",(0,r.yg)("inlineCode",{parentName:"p"},"conda")," or ",(0,r.yg)("inlineCode",{parentName:"p"},"pip"),". We recommend your to use scripts stored in the persistent folder to easily install all your requirements, so you can reinstall them when we enable the GPU, as it restarts the workspace."),(0,r.yg)("p",null,"For more information on how to use ",(0,r.yg)("inlineCode",{parentName:"p"},"conda"),"/",(0,r.yg)("inlineCode",{parentName:"p"},"mamba")," to install new dependencies or complete environment (useful if you need to use a different version of python than the one installed by default) checkout ",(0,r.yg)("a",{parentName:"p",href:"/docs/deploy-jupyter#%EF%B8%8F-manage-dependencies-with-conda"},"this page"),". "),(0,r.yg)("p",null,"\u26a0\ufe0f We recommend you to also try and debug your code on small sample using the CPU before getting the GPU, this way you will be able to directly start long running task when you get the GPU, instead of losing time debugging your code (it's probably not going to work on the first try, you know it)."),(0,r.yg)("p",null,"You can find more details on the images we use and how to extend them ",(0,r.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/jupyterlab#jupyterlab-on-gpu"},"in this repository"),"."),(0,r.yg)("admonition",{title:"Storage",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"Use the ",(0,r.yg)("strong",{parentName:"p"},(0,r.yg)("inlineCode",{parentName:"strong"},"/workspace/persistent")," folder"),", which is the JupyterLab workspace, to store your code and data persistently. Note that loading data from the persistent storage will be slowly that what you might expected, this is due to the nature of the distributed storage. So try to optimize this part and avoid reloading multiple time your data, and let us know if it is too much of a problem, we have some solution to improve this")),(0,r.yg)("h2",{id:"enable-the-gpu"},"Enable the GPU"),(0,r.yg)("p",null,"You will receive an email when the GPU has been enabled in your project. You can then update your deployment to use the GPUs using either the ",(0,r.yg)("inlineCode",{parentName:"p"},"oc")," command-line tool, or by editing the deployment configuration from the web UI"),(0,r.yg)("ul",null,(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("strong",{parentName:"li"},"With the Command Line Interface"),", run the following command from the terminal of your laptop after having installed the ",(0,r.yg)("inlineCode",{parentName:"li"},"oc")," command-line tool.")),(0,r.yg)("p",null,"We use ",(0,r.yg)("inlineCode",{parentName:"p"},"jupyterlab-gpu")," as deployment name is in the example, change it to yours if it is different."),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},'oc patch dc/jupyterlab-gpu --type=json -p=\'[{"op": "replace", "path": "/spec/template/spec/containers/0/resources", "value": {"requests": {"nvidia.com/gpu": 1}, "limits": {"nvidia.com/gpu": 1}}}]\'\n')),(0,r.yg)("ul",null,(0,r.yg)("li",{parentName:"ul"},"Or ",(0,r.yg)("strong",{parentName:"li"},"through the web UI"))),(0,r.yg)("p",null,"In the ",(0,r.yg)("strong",{parentName:"p"},"Topology")," view click on the circle representing your GPU application, then click on the ",(0,r.yg)("strong",{parentName:"p"},"Actions")," button in the top right of the screen, and click on ",(0,r.yg)("strong",{parentName:"p"},"Edit Deployment Config")," at the bottom of the list"),(0,r.yg)("p",null,"In the Deployment Config text editor, hit ",(0,r.yg)("inlineCode",{parentName:"p"},"ctrl + f"),' to search for "',(0,r.yg)("strong",{parentName:"p"},"resources"),'". You should see a line ',(0,r.yg)("inlineCode",{parentName:"p"},"- resources: {}")," under ",(0,r.yg)("inlineCode",{parentName:"p"},"containers:"),". You need to change this line to the following to enable GPU in your application (and make sure the indentation match the rest of the file):"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-yaml"}," - resources:\n requests:\n nvidia.com/gpu: 1\n limits:\n nvidia.com/gpu: 1\n")),(0,r.yg)("p",null,"Then wait for the pod to restart, or start it if it was stopped."),(0,r.yg)("p",null,"You can use the following command in the terminal of your container on the DSRI to see the current GPU usage:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"nvidia-smi\n")),(0,r.yg)("admonition",{title:"Windows",type:"info"},(0,r.yg)("p",{parentName:"admonition"},'When using above command with the oc client on windows you might receive an error like:\nerror: unable to parse "\'[{op:": yaml: found unexpected end of stream'),(0,r.yg)("p",{parentName:"admonition"},"This is because the single quotation mark on windows is handled differently. Try replacing the single quotation marks in the command with double quotation marks and the command should work.")),(0,r.yg)("h2",{id:"disable-the-gpu"},"Disable the GPU"),(0,r.yg)("p",null,"The GPU allocated to your workspace will be automatically disabled the after your booking ends at 9:00."),(0,r.yg)("p",null,"You can also manually disable the GPU from your app, the pod will be restarted automatically on a CPU node:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},'oc patch dc/jupyterlab-gpu --type=json -p=\'[{"op": "replace", "path": "/spec/template/spec/containers/0/resources", "value": {}}]\'\n')),(0,r.yg)("h2",{id:"increase-the-number-of-gpus"},"Increase the number of GPUs"),(0,r.yg)("p",null,"If you have been granted a 2nd GPU to speed up your experiment you can easily upgrade the number of GPU used by your workspace:"),(0,r.yg)("p",null,"From the ",(0,r.yg)("strong",{parentName:"p"},"Topology")," view click on your application:"),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},"Stop the application, by decreasing the number of pod to 0 (in the ",(0,r.yg)("strong",{parentName:"li"},"Details")," tab)"),(0,r.yg)("li",{parentName:"ol"},"Click on ",(0,r.yg)("strong",{parentName:"li"},"Options")," > ",(0,r.yg)("strong",{parentName:"li"},"Edit Deployment")," > in the YAML of the deployment search for ",(0,r.yg)("inlineCode",{parentName:"li"},"limits")," and change the number of GPU assigned to your deployment to 2:")),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-yaml"}," resources:\n limits:\n nvidia.com/gpu: '2'\n requests:\n nvidia.com/gpu: '2'\n")),(0,r.yg)("p",null,"You can also do it using the command line, make sure to stop the pod first, and replace ",(0,r.yg)("inlineCode",{parentName:"p"},"jupyterlab-gpu")," by your app name in this command:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},'oc patch dc/jupyterlab-gpu --type=json -p=\'[{"op": "replace", "path": "/spec/template/spec/containers/0/resources", "value": {"requests": {"nvidia.com/gpu": 2}, "limits": {"nvidia.com/gpu": 2}}}]\'\n')),(0,r.yg)("ol",{start:3},(0,r.yg)("li",{parentName:"ol"},"Restart the pod for your application (the same way you stopped it)")),(0,r.yg)("h2",{id:"install-gpu-drivers-in-any-image"},"Install GPU drivers in any image"),(0,r.yg)("p",null,"You can also install the GPU drivers in any image and use this image directly."),(0,r.yg)("p",null,"See the latest official ",(0,r.yg)("a",{parentName:"p",href:"https://nvidia.github.io/nvidia-container-runtime"},"Nvidia docs")," to install the ",(0,r.yg)("inlineCode",{parentName:"p"},"nvidia-container-runtime"),", which should contain all packages and drivers required to access the GPU from your application."),(0,r.yg)("p",null,"Here is an example of commands to add to a debian based ",(0,r.yg)("inlineCode",{parentName:"p"},"Dockerfile")," to install the GPU drivers (note that this is not complete, you will need to check the latest instructions and do some research & development to get it to work):"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-dockerfile"},"RUN curl -s -L https://nvidia.github.io/nvidia-container-runtime/gpgkey | \\\n apt-key add - \\ &&\n distribution=$(. /etc/os-release;echo $ID$VERSION_ID) \\ &&\n curl -s -L https://nvidia.github.io/nvidia-container-runtime/$distribution/nvidia-container-runtime.list | \nRUN apt-get update \\ &&\n apt-get install -y nvidia-container-runtime\n")),(0,r.yg)("p",null,"Then, build your image in your DSRI project using ",(0,r.yg)("inlineCode",{parentName:"p"},"oc")," from the folder where your put the ",(0,r.yg)("inlineCode",{parentName:"p"},"Dockerfile")," (replace ",(0,r.yg)("inlineCode",{parentName:"p"},"custom-app-gpu")," by your app name):"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc new-build --name custom-app-gpu --binary\noc start-build custom-app-gpu --from-dir=. --follow --wait\noc new-app custom-app-gpu\n")),(0,r.yg)("p",null,"You will then need to edit the deployment to the ",(0,r.yg)("inlineCode",{parentName:"p"},"serviceAccountName: anyuid")," and add a persistent storage"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc edit custom-app-gpu\n")),(0,r.yg)("p",null,"Finally for when your reservation start, checkout the section above about how to enable the GPU in workspace "),(0,r.yg)("p",null,"See also: official ",(0,r.yg)("a",{parentName:"p",href:"https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#debian-installation"},"Nvidia docs for CUDA")))}y.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[248],{5680:(e,t,n)=>{n.d(t,{xA:()=>u,yg:()=>y});var a=n(6540);function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function i(e){for(var t=1;t=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}var p=a.createContext({}),s=function(e){var t=a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},u=function(e){var t=s(e.components);return a.createElement(p.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},c=a.forwardRef((function(e,t){var n=e.components,o=e.mdxType,r=e.originalType,p=e.parentName,u=l(e,["components","mdxType","originalType","parentName"]),c=s(n),y=o,g=c["".concat(p,".").concat(y)]||c[y]||d[y]||r;return n?a.createElement(g,i(i({ref:t},u),{},{components:n})):a.createElement(g,i({ref:t},u))}));function y(e,t){var n=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=n.length,i=new Array(r);i[0]=c;var l={};for(var p in t)hasOwnProperty.call(t,p)&&(l[p]=t[p]);l.originalType=e,l.mdxType="string"==typeof e?e:o,i[1]=l;for(var s=2;s{n.r(t),n.d(t,{assets:()=>u,contentTitle:()=>p,default:()=>y,frontMatter:()=>l,metadata:()=>s,toc:()=>d});var a=n(9668),o=n(1367),r=(n(6540),n(5680)),i=["components"],l={id:"deploy-on-gpu",title:"GPU applications"},p=void 0,s={unversionedId:"deploy-on-gpu",id:"deploy-on-gpu",title:"GPU applications",description:"GPUs on the DSRI can only be used by one workspace at a time, and there is a limited number of GPUs (8).",source:"@site/docs/deploy-on-gpu.md",sourceDirName:".",slug:"/deploy-on-gpu",permalink:"/docs/deploy-on-gpu",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/deploy-on-gpu.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"deploy-on-gpu",title:"GPU applications"},sidebar:"docs",previous:{title:"Monitor your applications",permalink:"/docs/guide-monitoring"},next:{title:"Deploy from a Dockerfile",permalink:"/docs/guide-dockerfile-to-openshift"}},u={},d=[{value:"Prepare your GPU workspace",id:"prepare-your-gpu-workspace",level:2},{value:"About the docker images",id:"about-the-docker-images",level:3},{value:"Deploy the workspace",id:"deploy-the-workspace",level:3},{value:"Prepare the workspace",id:"prepare-the-workspace",level:3},{value:"Enable the GPU",id:"enable-the-gpu",level:2},{value:"Disable the GPU",id:"disable-the-gpu",level:2},{value:"Increase the number of GPUs",id:"increase-the-number-of-gpus",level:2},{value:"Install GPU drivers in any image",id:"install-gpu-drivers-in-any-image",level:2}],c={toc:d};function y(e){var t=e.components,n=(0,o.A)(e,i);return(0,r.yg)("wrapper",(0,a.A)({},c,n,{components:t,mdxType:"MDXLayout"}),(0,r.yg)("p",null,"GPUs on the DSRI can only be used by one workspace at a time, and there is a limited number of GPUs (8)."),(0,r.yg)("p",null,"\u26a0\ufe0f We currently provide a free access to those GPUs, but with the growing demands for GPUs it might get more restricted. As consideration for others, and to help keep this system open, it is important to make a maximum use of your GPUs when you get access to them. "),(0,r.yg)("p",null,"Unfortunately job scheduling is currently not mature enough on Kubernetes, you can look into ",(0,r.yg)("a",{parentName:"p",href:"https://volcano.sh/en/"},"volcano.sh")," if you are interested, but it is still quite experimental."),(0,r.yg)("p",null,"To use the GPU on the DSRI you will go through this process:"),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},"Deploy, prepare and debug your GPU workspace"),(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("a",{parentName:"li",href:"/gpu-booking"},"Book a GPU")),(0,r.yg)("li",{parentName:"ol"},"Once the booking is done you will receive an email about your reservation, and more emails when it starts and before it ends"),(0,r.yg)("li",{parentName:"ol"},"Enable the GPU in workspace when your booking starts, and make the best use of it!")),(0,r.yg)("admonition",{title:"Book a GPU",type:"warning"},(0,r.yg)("p",{parentName:"admonition"},(0,r.yg)("strong",{parentName:"p"},"By default you do not have the permission to run applications on GPU"),", you need to make a reservation."),(0,r.yg)("p",{parentName:"admonition"},"You can check the availability of our GPUs, and reserve GPU slots in the ",(0,r.yg)("a",{parentName:"p",href:"/gpu-booking"},"GPU booking calendar \ud83d\udcc5"))),(0,r.yg)("h2",{id:"prepare-your-gpu-workspace"},"Prepare your GPU workspace"),(0,r.yg)("p",null,"You will first need to start your workspace without the GPU enabled, you can then prepare your experiments: clone the code, download the data, prepare scripts to install all requirements (the workspace will be restarted when you enable the GPU). "),(0,r.yg)("h3",{id:"about-the-docker-images"},"About the docker images"),(0,r.yg)("p",null,"We are mainly using images provided by Nvidia, with all required drivers and optimizations for GPU pre-installed. You can access the workspace with JupyterLab and VisualStudio Code in your browser, and install dependencies with ",(0,r.yg)("inlineCode",{parentName:"p"},"apt-get"),", ",(0,r.yg)("inlineCode",{parentName:"p"},"conda")," or ",(0,r.yg)("inlineCode",{parentName:"p"},"pip")," in the workspace."),(0,r.yg)("p",null,"We currently mainly use Tensorflow, PyTorch and CUDA, but any image available in the ",(0,r.yg)("a",{parentName:"p",href:"https://ngc.nvidia.com/catalog/containers"},"Nvidia catalog")," should be easy to deploy. Checkout ",(0,r.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/jupyterlab#jupyterlab-on-gpu"},"this documentation")," for more details on how we build the optimized docker images for the DSRI GPUs. And feel free to ",(0,r.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/jupyterlab#extend-an-image"},"extend the images")," to install any software you need."),(0,r.yg)("h3",{id:"deploy-the-workspace"},"Deploy the workspace"),(0,r.yg)("p",null,"You can easily deploy your GPU workspace from the DSRI catalog:"),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},"Go to the ",(0,r.yg)("a",{parentName:"li",href:"https://console-openshift-console.apps.dsri2.unimaas.nl/catalog"},"DSRI Catalog web UI"),": Click on ",(0,r.yg)("strong",{parentName:"li"},"Add to Project"),", then ",(0,r.yg)("strong",{parentName:"li"},"Browse Catalog")),(0,r.yg)("li",{parentName:"ol"},'Search the catalog for "GPU", and make sure the Template checkbox is enabled'),(0,r.yg)("li",{parentName:"ol"},"Choose the template: ",(0,r.yg)("strong",{parentName:"li"},"JupyterLab on GPU")),(0,r.yg)("li",{parentName:"ol"},"Follow the instructions to create the template in the DSRI web UI, all information about the images you can use are provided there. The most notable is the base image you want to use for your workspace (",(0,r.yg)("inlineCode",{parentName:"li"},"cuda"),", ",(0,r.yg)("inlineCode",{parentName:"li"},"tensorflow")," or ",(0,r.yg)("inlineCode",{parentName:"li"},"pytorch"),")")),(0,r.yg)("p",null,"Access the workspace from the route created (the small arrow at the top right of your application bubble in the Topology page)."),(0,r.yg)("h3",{id:"prepare-the-workspace"},"Prepare the workspace"),(0,r.yg)("p",null,"You can now add your code and data in the persistent folder to be fully prepared when you will get access to the GPUs."),(0,r.yg)("p",null,"You can install dependencies with ",(0,r.yg)("inlineCode",{parentName:"p"},"apt-get"),", ",(0,r.yg)("inlineCode",{parentName:"p"},"conda")," or ",(0,r.yg)("inlineCode",{parentName:"p"},"pip"),". We recommend your to use scripts stored in the persistent folder to easily install all your requirements, so you can reinstall them when we enable the GPU, as it restarts the workspace."),(0,r.yg)("p",null,"For more information on how to use ",(0,r.yg)("inlineCode",{parentName:"p"},"conda"),"/",(0,r.yg)("inlineCode",{parentName:"p"},"mamba")," to install new dependencies or complete environment (useful if you need to use a different version of python than the one installed by default) checkout ",(0,r.yg)("a",{parentName:"p",href:"/docs/deploy-jupyter#%EF%B8%8F-manage-dependencies-with-conda"},"this page"),". "),(0,r.yg)("p",null,"\u26a0\ufe0f We recommend you to also try and debug your code on small sample using the CPU before getting the GPU, this way you will be able to directly start long running task when you get the GPU, instead of losing time debugging your code (it's probably not going to work on the first try, you know it)."),(0,r.yg)("p",null,"You can find more details on the images we use and how to extend them ",(0,r.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/jupyterlab#jupyterlab-on-gpu"},"in this repository"),"."),(0,r.yg)("admonition",{title:"Storage",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"Use the ",(0,r.yg)("strong",{parentName:"p"},(0,r.yg)("inlineCode",{parentName:"strong"},"/workspace/persistent")," folder"),", which is the JupyterLab workspace, to store your code and data persistently. Note that loading data from the persistent storage will be slowly that what you might expected, this is due to the nature of the distributed storage. So try to optimize this part and avoid reloading multiple time your data, and let us know if it is too much of a problem, we have some solution to improve this")),(0,r.yg)("h2",{id:"enable-the-gpu"},"Enable the GPU"),(0,r.yg)("p",null,"You will receive an email when the GPU has been enabled in your project. You can then update your deployment to use the GPUs using either the ",(0,r.yg)("inlineCode",{parentName:"p"},"oc")," command-line tool, or by editing the deployment configuration from the web UI"),(0,r.yg)("ul",null,(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("strong",{parentName:"li"},"With the Command Line Interface"),", run the following command from the terminal of your laptop after having installed the ",(0,r.yg)("inlineCode",{parentName:"li"},"oc")," command-line tool.")),(0,r.yg)("p",null,"We use ",(0,r.yg)("inlineCode",{parentName:"p"},"jupyterlab-gpu")," as deployment name is in the example, change it to yours if it is different."),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},'oc patch dc/jupyterlab-gpu --type=json -p=\'[{"op": "replace", "path": "/spec/template/spec/containers/0/resources", "value": {"requests": {"nvidia.com/gpu": 1}, "limits": {"nvidia.com/gpu": 1}}}]\'\n')),(0,r.yg)("ul",null,(0,r.yg)("li",{parentName:"ul"},"Or ",(0,r.yg)("strong",{parentName:"li"},"through the web UI"))),(0,r.yg)("p",null,"In the ",(0,r.yg)("strong",{parentName:"p"},"Topology")," view click on the circle representing your GPU application, then click on the ",(0,r.yg)("strong",{parentName:"p"},"Actions")," button in the top right of the screen, and click on ",(0,r.yg)("strong",{parentName:"p"},"Edit Deployment Config")," at the bottom of the list"),(0,r.yg)("p",null,"In the Deployment Config text editor, hit ",(0,r.yg)("inlineCode",{parentName:"p"},"ctrl + f"),' to search for "',(0,r.yg)("strong",{parentName:"p"},"resources"),'". You should see a line ',(0,r.yg)("inlineCode",{parentName:"p"},"- resources: {}")," under ",(0,r.yg)("inlineCode",{parentName:"p"},"containers:"),". You need to change this line to the following to enable GPU in your application (and make sure the indentation match the rest of the file):"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-yaml"}," - resources:\n requests:\n nvidia.com/gpu: 1\n limits:\n nvidia.com/gpu: 1\n")),(0,r.yg)("p",null,"Then wait for the pod to restart, or start it if it was stopped."),(0,r.yg)("p",null,"You can use the following command in the terminal of your container on the DSRI to see the current GPU usage:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"nvidia-smi\n")),(0,r.yg)("admonition",{title:"Windows",type:"info"},(0,r.yg)("p",{parentName:"admonition"},'When using above command with the oc client on windows you might receive an error like:\nerror: unable to parse "\'[{op:": yaml: found unexpected end of stream'),(0,r.yg)("p",{parentName:"admonition"},"This is because the single quotation mark on windows is handled differently. Try replacing the single quotation marks in the command with double quotation marks and the command should work.")),(0,r.yg)("h2",{id:"disable-the-gpu"},"Disable the GPU"),(0,r.yg)("p",null,"The GPU allocated to your workspace will be automatically disabled the after your booking ends at 9:00."),(0,r.yg)("p",null,"You can also manually disable the GPU from your app, the pod will be restarted automatically on a CPU node:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},'oc patch dc/jupyterlab-gpu --type=json -p=\'[{"op": "replace", "path": "/spec/template/spec/containers/0/resources", "value": {}}]\'\n')),(0,r.yg)("h2",{id:"increase-the-number-of-gpus"},"Increase the number of GPUs"),(0,r.yg)("p",null,"If you have been granted a 2nd GPU to speed up your experiment you can easily upgrade the number of GPU used by your workspace:"),(0,r.yg)("p",null,"From the ",(0,r.yg)("strong",{parentName:"p"},"Topology")," view click on your application:"),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},"Stop the application, by decreasing the number of pod to 0 (in the ",(0,r.yg)("strong",{parentName:"li"},"Details")," tab)"),(0,r.yg)("li",{parentName:"ol"},"Click on ",(0,r.yg)("strong",{parentName:"li"},"Options")," > ",(0,r.yg)("strong",{parentName:"li"},"Edit Deployment")," > in the YAML of the deployment search for ",(0,r.yg)("inlineCode",{parentName:"li"},"limits")," and change the number of GPU assigned to your deployment to 2:")),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-yaml"}," resources:\n limits:\n nvidia.com/gpu: '2'\n requests:\n nvidia.com/gpu: '2'\n")),(0,r.yg)("p",null,"You can also do it using the command line, make sure to stop the pod first, and replace ",(0,r.yg)("inlineCode",{parentName:"p"},"jupyterlab-gpu")," by your app name in this command:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},'oc patch dc/jupyterlab-gpu --type=json -p=\'[{"op": "replace", "path": "/spec/template/spec/containers/0/resources", "value": {"requests": {"nvidia.com/gpu": 2}, "limits": {"nvidia.com/gpu": 2}}}]\'\n')),(0,r.yg)("ol",{start:3},(0,r.yg)("li",{parentName:"ol"},"Restart the pod for your application (the same way you stopped it)")),(0,r.yg)("h2",{id:"install-gpu-drivers-in-any-image"},"Install GPU drivers in any image"),(0,r.yg)("p",null,"You can also install the GPU drivers in any image and use this image directly."),(0,r.yg)("p",null,"See the latest official ",(0,r.yg)("a",{parentName:"p",href:"https://nvidia.github.io/nvidia-container-runtime"},"Nvidia docs")," to install the ",(0,r.yg)("inlineCode",{parentName:"p"},"nvidia-container-runtime"),", which should contain all packages and drivers required to access the GPU from your application."),(0,r.yg)("p",null,"Here is an example of commands to add to a debian based ",(0,r.yg)("inlineCode",{parentName:"p"},"Dockerfile")," to install the GPU drivers (note that this is not complete, you will need to check the latest instructions and do some research & development to get it to work):"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-dockerfile"},"RUN curl -s -L https://nvidia.github.io/nvidia-container-runtime/gpgkey | \\\n apt-key add - \\ &&\n distribution=$(. /etc/os-release;echo $ID$VERSION_ID) \\ &&\n curl -s -L https://nvidia.github.io/nvidia-container-runtime/$distribution/nvidia-container-runtime.list | \nRUN apt-get update \\ &&\n apt-get install -y nvidia-container-runtime\n")),(0,r.yg)("p",null,"Then, build your image in your DSRI project using ",(0,r.yg)("inlineCode",{parentName:"p"},"oc")," from the folder where your put the ",(0,r.yg)("inlineCode",{parentName:"p"},"Dockerfile")," (replace ",(0,r.yg)("inlineCode",{parentName:"p"},"custom-app-gpu")," by your app name):"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc new-build --name custom-app-gpu --binary\noc start-build custom-app-gpu --from-dir=. --follow --wait\noc new-app custom-app-gpu\n")),(0,r.yg)("p",null,"You will then need to edit the deployment to the ",(0,r.yg)("inlineCode",{parentName:"p"},"serviceAccountName: anyuid")," and add a persistent storage"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc edit custom-app-gpu\n")),(0,r.yg)("p",null,"Finally for when your reservation start, checkout the section above about how to enable the GPU in workspace "),(0,r.yg)("p",null,"See also: official ",(0,r.yg)("a",{parentName:"p",href:"https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#debian-installation"},"Nvidia docs for CUDA")))}y.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/7a61fedb.ef7662df.js b/assets/js/7a61fedb.41549c0c.js similarity index 99% rename from assets/js/7a61fedb.ef7662df.js rename to assets/js/7a61fedb.41549c0c.js index f161cda67..900d3b3c7 100644 --- a/assets/js/7a61fedb.ef7662df.js +++ b/assets/js/7a61fedb.41549c0c.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[5217],{5680:(e,o,t)=>{t.d(o,{xA:()=>p,yg:()=>d});var r=t(6540);function a(e,o,t){return o in e?Object.defineProperty(e,o,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[o]=t,e}function n(e,o){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);o&&(r=r.filter((function(o){return Object.getOwnPropertyDescriptor(e,o).enumerable}))),t.push.apply(t,r)}return t}function l(e){for(var o=1;o=0||(a[t]=e[t]);return a}(e,o);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(a[t]=e[t])}return a}var i=r.createContext({}),g=function(e){var o=r.useContext(i),t=o;return e&&(t="function"==typeof e?e(o):l(l({},o),e)),t},p=function(e){var o=g(e.components);return r.createElement(i.Provider,{value:o},e.children)},u={inlineCode:"code",wrapper:function(e){var o=e.children;return r.createElement(r.Fragment,{},o)}},c=r.forwardRef((function(e,o){var t=e.components,a=e.mdxType,n=e.originalType,i=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),c=g(t),d=a,m=c["".concat(i,".").concat(d)]||c[d]||u[d]||n;return t?r.createElement(m,l(l({ref:o},p),{},{components:t})):r.createElement(m,l({ref:o},p))}));function d(e,o){var t=arguments,a=o&&o.mdxType;if("string"==typeof e||a){var n=t.length,l=new Array(n);l[0]=c;var s={};for(var i in o)hasOwnProperty.call(o,i)&&(s[i]=o[i]);s.originalType=e,s.mdxType="string"==typeof e?e:a,l[1]=s;for(var g=2;g{t.r(o),t.d(o,{assets:()=>p,contentTitle:()=>i,default:()=>d,frontMatter:()=>s,metadata:()=>g,toc:()=>u});var r=t(9668),a=t(1367),n=(t(6540),t(5680)),l=["components"],s={id:"workflows-argo",title:"Run Argo workflows"},i=void 0,g={unversionedId:"workflows-argo",id:"workflows-argo",title:"Run Argo workflows",description:"Argo needs to be installed in your project, contact the DSRI team to request it.",source:"@site/docs/workflows-argo.md",sourceDirName:".",slug:"/workflows-argo",permalink:"/docs/workflows-argo",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/workflows-argo.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"workflows-argo",title:"Run Argo workflows"},sidebar:"docs",previous:{title:"Deploy Airflow",permalink:"/docs/workflows-airflow"},next:{title:"Run Nextflow workflows",permalink:"/docs/workflows-nextflow"}},p={},u=[{value:"Install the argo client",id:"install-the-argo-client",level:2},{value:"On Ubuntu",id:"on-ubuntu",level:3},{value:"On MacOS",id:"on-macos",level:3},{value:"On Windows",id:"on-windows",level:3},{value:"Test Argo",id:"test-argo",level:3},{value:"Install Argo in your project",id:"install-argo-in-your-project",level:2},{value:"Argo workflows with Helm",id:"argo-workflows-with-helm",level:3},{value:"ArgoCD Operator",id:"argocd-operator",level:3},{value:"Uninstall argo",id:"uninstall-argo",level:3},{value:"On Ubuntu",id:"on-ubuntu-1",level:4},{value:"Run workflows to convert structured data to RDF",id:"run-workflows-to-convert-structured-data-to-rdf",level:2},{value:"Clone the repository",id:"clone-the-repository",level:3},{value:"Workflow to convert XML files to RDF",id:"workflow-to-convert-xml-files-to-rdf",level:3},{value:"Workflow to convert CSV files to RDF",id:"workflow-to-convert-csv-files-to-rdf",level:3},{value:"Argo commands",id:"argo-commands",level:2},{value:"List running Argo workflows",id:"list-running-argo-workflows",level:3},{value:"Stop a workflow",id:"stop-a-workflow",level:3},{value:"Delete a workflow",id:"delete-a-workflow",level:3},{value:"Debug a workflow",id:"debug-a-workflow",level:2}],c={toc:u};function d(e){var o=e.components,t=(0,a.A)(e,l);return(0,n.yg)("wrapper",(0,r.A)({},c,t,{components:o,mdxType:"MDXLayout"}),(0,n.yg)("admonition",{title:"Install in your project",type:"warning"},(0,n.yg)("p",{parentName:"admonition"},"Argo needs to be installed in your project, ",(0,n.yg)("a",{parentName:"p",href:"mailto:dsri-support-l@maastrichtuniversity.nl"},"contact the DSRI team")," to request it.")),(0,n.yg)("h2",{id:"install-the-argo-client"},"Install the ",(0,n.yg)("inlineCode",{parentName:"h2"},"argo")," client"),(0,n.yg)("p",null,(0,n.yg)("a",{parentName:"p",href:"https://argoproj.github.io/argo/"},"Argo \ud83e\udd91")," is a container native workflow engine for ",(0,n.yg)("a",{parentName:"p",href:"https://kubernetes.io/"},"Kubernetes")," supporting both ",(0,n.yg)("a",{parentName:"p",href:"https://argoproj.github.io/docs/argo/examples/readme.html#dag"},"DAG")," and ",(0,n.yg)("a",{parentName:"p",href:"https://argoproj.github.io/docs/argo/examples/readme.html#steps"},"step based")," workflows."),(0,n.yg)("p",null,"Download and install the ",(0,n.yg)("a",{parentName:"p",href:"https://github.com/argoproj/argo/blob/master/demo.md#1-download-argo"},"Argo client")," on your computer to ",(0,n.yg)("a",{parentName:"p",href:"https://argoproj.github.io/docs/argo/examples/readme.html"},"start workflows")," on the DSRI."),(0,n.yg)("h3",{id:"on-ubuntu"},"On Ubuntu"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-shell"},"sudo curl -L -o /usr/local/bin/argo https://github.com/argoproj/argo/releases/download/v2.4.2/argo-linux-amd64\nsudo chmod +x /usr/local/bin/argo\n")),(0,n.yg)("h3",{id:"on-macos"},"On MacOS"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-shell"},"brew install argoproj/tap/argo\n")),(0,n.yg)("h3",{id:"on-windows"},"On Windows"),(0,n.yg)("p",null,"Get ",(0,n.yg)("a",{parentName:"p",href:"https://github.com/argoproj/argo/releases/download/v2.4.2/argo-windows-amd64"},"Argo executable version 2.4.2")," from ",(0,n.yg)("a",{parentName:"p",href:"https://github.com/argoproj/argo/releases"},"Argo Releases")," on GitHub."),(0,n.yg)("blockquote",null,(0,n.yg)("p",{parentName:"blockquote"},"See ",(0,n.yg)("a",{parentName:"p",href:"https://argoproj.github.io/docs/argo/demo.html#1-download-argo"},"official Argo documentation"),".")),(0,n.yg)("h3",{id:"test-argo"},"Test Argo"),(0,n.yg)("p",null,"Run Hello world workflow to test if Argo has been properly installed. And take a look at the ",(0,n.yg)("a",{parentName:"p",href:"https://argoproj.github.io/docs/argo/examples/readme.html"},"examples provided in Argo documentation")," to discover how to use the different features available."),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-shell"},"argo submit --watch https://raw.githubusercontent.com/argoproj/argo/master/examples/hello-world.yaml\n")),(0,n.yg)("admonition",{title:"Logged in",type:"caution"},(0,n.yg)("p",{parentName:"admonition"},"You will need to have the ",(0,n.yg)("inlineCode",{parentName:"p"},"oc")," client installed and be logged in with ",(0,n.yg)("inlineCode",{parentName:"p"},"oc login"),", see the ",(0,n.yg)("a",{parentName:"p",href:"/docs/openshift-install"},"install documentation page"),".")),(0,n.yg)("h2",{id:"install-argo-in-your-project"},"Install Argo in your project"),(0,n.yg)("h3",{id:"argo-workflows-with-helm"},"Argo workflows with Helm"),(0,n.yg)("p",null,"Deploy the ",(0,n.yg)("a",{parentName:"p",href:"https://artifacthub.io/packages/helm/argo/argo"},"Argo Helm chart"),"."),(0,n.yg)("ol",null,(0,n.yg)("li",{parentName:"ol"},"Install and use ",(0,n.yg)("a",{parentName:"li",href:"/docs/helm"},(0,n.yg)("inlineCode",{parentName:"a"},"helm"))),(0,n.yg)("li",{parentName:"ol"},"Add the Helm charts repository:")),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-bash"},"helm repo add argo https://argoproj.github.io/argo-helm\n")),(0,n.yg)("ol",{start:3},(0,n.yg)("li",{parentName:"ol"},"Install chart:")),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-bash"},"helm install my-argo argo/argo --version 0.15.2\n")),(0,n.yg)("h3",{id:"argocd-operator"},"ArgoCD Operator"),(0,n.yg)("p",null,"Ask on the DSRI Slack ",(0,n.yg)("strong",{parentName:"p"},"#helpdesk")," channel to have the ",(0,n.yg)("a",{parentName:"p",href:"https://artifacthub.io/packages/olm/community-operators/argocd-operator"},"ArgoCD Operator")," installed in your project."),(0,n.yg)("h3",{id:"uninstall-argo"},"Uninstall ",(0,n.yg)("inlineCode",{parentName:"h3"},"argo")),(0,n.yg)("h4",{id:"on-ubuntu-1"},"On Ubuntu"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-shell"},"sudo rm /usr/local/bin/argo\n")),(0,n.yg)("blockquote",null,(0,n.yg)("p",{parentName:"blockquote"},"You can now reinstall a newer version of Argo.")),(0,n.yg)("hr",null),(0,n.yg)("h2",{id:"run-workflows-to-convert-structured-data-to-rdf"},"Run workflows to convert structured data to RDF"),(0,n.yg)("p",null,"We will use examples from the ",(0,n.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/d2s-core"},"MaastrichtU-IDS/d2s-core")," project."),(0,n.yg)("h3",{id:"clone-the-repository"},"Clone the repository"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-shell"},"git clone --recursive https://github.com/MaastrichtU-IDS/d2s-project-template.git\ncd d2s-project-template\n")),(0,n.yg)("p",null,(0,n.yg)("a",{parentName:"p",href:"/docs/openshift-install"},"Authenticate to the OpenShift cluster")," using ",(0,n.yg)("inlineCode",{parentName:"p"},"oc login")," ."),(0,n.yg)("h3",{id:"workflow-to-convert-xml-files-to-rdf"},"Workflow to convert XML files to RDF"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"Steps-based workflow for XML files, see the example workflow ",(0,n.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/d2s-core/blob/master/argo/d2s-workflow-xml.yaml"},"YAML file on GitHub"),".")),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-shell"},"argo submit d2s-core/argo/workflows/d2s-workflow-transform-xml.yaml \\\n -f support/config/config-transform-xml-drugbank.yml\n")),(0,n.yg)("admonition",{title:"Provide config files",type:"info"},(0,n.yg)("p",{parentName:"admonition"},"Config files can be provided using the ",(0,n.yg)("inlineCode",{parentName:"p"},"-f")," arguments, but are not necessary.")),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"DAG workflow for XML files, see the ",(0,n.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/d2s-core/blob/master/argo/d2s-workflow-xml-dag.yaml"},"YAML file on GitHub"),".")),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-shell"},"argo submit d2s-core/argo/workflows/d2s-workflow-transform-xml-dag.yaml \\\n -f support/config/config-transform-xml-drugbank.yml\n")),(0,n.yg)("h3",{id:"workflow-to-convert-csv-files-to-rdf"},"Workflow to convert CSV files to RDF"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"Steps-based workflow for CSV files")),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-shell"},"argo submit d2s-core/argo/workflows/d2s-workflow-transform-csv.yaml \\\n -f support/config/config-transform-csv-stitch.yml\n")),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"DAG workflow for CSV files")),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-shell"},"argo submit d2s-core/argo/workflows/d2s-workflow-transform-csv-dag.yaml \\\n -f support/config/config-transform-csv-stitch.yml\n")),(0,n.yg)("admonition",{title:"Solve issue",type:"caution"},(0,n.yg)("p",{parentName:"admonition"},"Try this to solve issue related to steps services IP: ",(0,n.yg)("inlineCode",{parentName:"p"},"{{steps.nginx-server.pod-ip}}"))),(0,n.yg)("hr",null),(0,n.yg)("h2",{id:"argo-commands"},"Argo commands"),(0,n.yg)("h3",{id:"list-running-argo-workflows"},"List running Argo workflows"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-shell"},"argo list\n")),(0,n.yg)("h3",{id:"stop-a-workflow"},"Stop a workflow"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-shell"},"argo terminate my-workflow\n")),(0,n.yg)("admonition",{title:"Workflow",type:"caution"},(0,n.yg)("p",{parentName:"admonition"},"This might not stop the workflow, in this case use:"),(0,n.yg)("pre",{parentName:"admonition"},(0,n.yg)("code",{parentName:"pre",className:"language-bash"},"argo delete my-workflow\n"))),(0,n.yg)("h3",{id:"delete-a-workflow"},"Delete a workflow"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-shell"},"argo delete my-workflow\n")),(0,n.yg)("hr",null),(0,n.yg)("h2",{id:"debug-a-workflow"},"Debug a workflow"),(0,n.yg)("p",null,"Get into a container, to understand why it bugs, by creating a YAML with the command ",(0,n.yg)("inlineCode",{parentName:"p"},"tail -f /dev/null")," to keep it hanging."),(0,n.yg)("p",null,"See the ",(0,n.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/d2s-core/blob/master/argo/tests/test-devnull-argo.yaml"},"example in the d2s-argo-workflow repository"),":"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-yaml"},'apiVersion: argoproj.io/v1alpha1\nkind: Workflow\nmetadata:\n generateName: test-devnull-argo-\nspec:\n entrypoint: execute-workflow\n\n # Use existing volume\n volumes:\n - name: workdir\n persistentVolumeClaim:\n claimName: pvc-mapr-projects-test-vincent \n\n templates:\n - name: execute-workflow\n steps:\n - - name: run-rdfunit\n template: rdfunit\n \n - name: rdfunit\n container:\n image: umids/rdfunit:latest\n command: [tail]\n args: ["-f", "/dev/null"]\n volumeMounts:\n - name: workdir\n mountPath: /data\n subPath: dqa-workspace\n')),(0,n.yg)("p",null,"Then start the workflow:"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-shell"},"argo submit --serviceaccount argo tests/test-devnull-argo.yaml\n")),(0,n.yg)("p",null,"And connect with the Shell (change the pod ID to your pod ID):"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-shell"},"oc rsh test-devnull-argo-pod\n")))}d.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[5217],{5680:(e,o,t)=>{t.d(o,{xA:()=>p,yg:()=>d});var r=t(6540);function a(e,o,t){return o in e?Object.defineProperty(e,o,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[o]=t,e}function n(e,o){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);o&&(r=r.filter((function(o){return Object.getOwnPropertyDescriptor(e,o).enumerable}))),t.push.apply(t,r)}return t}function l(e){for(var o=1;o=0||(a[t]=e[t]);return a}(e,o);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(a[t]=e[t])}return a}var i=r.createContext({}),g=function(e){var o=r.useContext(i),t=o;return e&&(t="function"==typeof e?e(o):l(l({},o),e)),t},p=function(e){var o=g(e.components);return r.createElement(i.Provider,{value:o},e.children)},u={inlineCode:"code",wrapper:function(e){var o=e.children;return r.createElement(r.Fragment,{},o)}},c=r.forwardRef((function(e,o){var t=e.components,a=e.mdxType,n=e.originalType,i=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),c=g(t),d=a,m=c["".concat(i,".").concat(d)]||c[d]||u[d]||n;return t?r.createElement(m,l(l({ref:o},p),{},{components:t})):r.createElement(m,l({ref:o},p))}));function d(e,o){var t=arguments,a=o&&o.mdxType;if("string"==typeof e||a){var n=t.length,l=new Array(n);l[0]=c;var s={};for(var i in o)hasOwnProperty.call(o,i)&&(s[i]=o[i]);s.originalType=e,s.mdxType="string"==typeof e?e:a,l[1]=s;for(var g=2;g{t.r(o),t.d(o,{assets:()=>p,contentTitle:()=>i,default:()=>d,frontMatter:()=>s,metadata:()=>g,toc:()=>u});var r=t(9668),a=t(1367),n=(t(6540),t(5680)),l=["components"],s={id:"workflows-argo",title:"Run Argo workflows"},i=void 0,g={unversionedId:"workflows-argo",id:"workflows-argo",title:"Run Argo workflows",description:"Argo needs to be installed in your project, contact the DSRI team to request it.",source:"@site/docs/workflows-argo.md",sourceDirName:".",slug:"/workflows-argo",permalink:"/docs/workflows-argo",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/workflows-argo.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"workflows-argo",title:"Run Argo workflows"},sidebar:"docs",previous:{title:"Deploy Airflow",permalink:"/docs/workflows-airflow"},next:{title:"Run Nextflow workflows",permalink:"/docs/workflows-nextflow"}},p={},u=[{value:"Install the argo client",id:"install-the-argo-client",level:2},{value:"On Ubuntu",id:"on-ubuntu",level:3},{value:"On MacOS",id:"on-macos",level:3},{value:"On Windows",id:"on-windows",level:3},{value:"Test Argo",id:"test-argo",level:3},{value:"Install Argo in your project",id:"install-argo-in-your-project",level:2},{value:"Argo workflows with Helm",id:"argo-workflows-with-helm",level:3},{value:"ArgoCD Operator",id:"argocd-operator",level:3},{value:"Uninstall argo",id:"uninstall-argo",level:3},{value:"On Ubuntu",id:"on-ubuntu-1",level:4},{value:"Run workflows to convert structured data to RDF",id:"run-workflows-to-convert-structured-data-to-rdf",level:2},{value:"Clone the repository",id:"clone-the-repository",level:3},{value:"Workflow to convert XML files to RDF",id:"workflow-to-convert-xml-files-to-rdf",level:3},{value:"Workflow to convert CSV files to RDF",id:"workflow-to-convert-csv-files-to-rdf",level:3},{value:"Argo commands",id:"argo-commands",level:2},{value:"List running Argo workflows",id:"list-running-argo-workflows",level:3},{value:"Stop a workflow",id:"stop-a-workflow",level:3},{value:"Delete a workflow",id:"delete-a-workflow",level:3},{value:"Debug a workflow",id:"debug-a-workflow",level:2}],c={toc:u};function d(e){var o=e.components,t=(0,a.A)(e,l);return(0,n.yg)("wrapper",(0,r.A)({},c,t,{components:o,mdxType:"MDXLayout"}),(0,n.yg)("admonition",{title:"Install in your project",type:"warning"},(0,n.yg)("p",{parentName:"admonition"},"Argo needs to be installed in your project, ",(0,n.yg)("a",{parentName:"p",href:"mailto:dsri-support-l@maastrichtuniversity.nl"},"contact the DSRI team")," to request it.")),(0,n.yg)("h2",{id:"install-the-argo-client"},"Install the ",(0,n.yg)("inlineCode",{parentName:"h2"},"argo")," client"),(0,n.yg)("p",null,(0,n.yg)("a",{parentName:"p",href:"https://argoproj.github.io/argo/"},"Argo \ud83e\udd91")," is a container native workflow engine for ",(0,n.yg)("a",{parentName:"p",href:"https://kubernetes.io/"},"Kubernetes")," supporting both ",(0,n.yg)("a",{parentName:"p",href:"https://argoproj.github.io/docs/argo/examples/readme.html#dag"},"DAG")," and ",(0,n.yg)("a",{parentName:"p",href:"https://argoproj.github.io/docs/argo/examples/readme.html#steps"},"step based")," workflows."),(0,n.yg)("p",null,"Download and install the ",(0,n.yg)("a",{parentName:"p",href:"https://github.com/argoproj/argo/blob/master/demo.md#1-download-argo"},"Argo client")," on your computer to ",(0,n.yg)("a",{parentName:"p",href:"https://argoproj.github.io/docs/argo/examples/readme.html"},"start workflows")," on the DSRI."),(0,n.yg)("h3",{id:"on-ubuntu"},"On Ubuntu"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-shell"},"sudo curl -L -o /usr/local/bin/argo https://github.com/argoproj/argo/releases/download/v2.4.2/argo-linux-amd64\nsudo chmod +x /usr/local/bin/argo\n")),(0,n.yg)("h3",{id:"on-macos"},"On MacOS"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-shell"},"brew install argoproj/tap/argo\n")),(0,n.yg)("h3",{id:"on-windows"},"On Windows"),(0,n.yg)("p",null,"Get ",(0,n.yg)("a",{parentName:"p",href:"https://github.com/argoproj/argo/releases/download/v2.4.2/argo-windows-amd64"},"Argo executable version 2.4.2")," from ",(0,n.yg)("a",{parentName:"p",href:"https://github.com/argoproj/argo/releases"},"Argo Releases")," on GitHub."),(0,n.yg)("blockquote",null,(0,n.yg)("p",{parentName:"blockquote"},"See ",(0,n.yg)("a",{parentName:"p",href:"https://argoproj.github.io/docs/argo/demo.html#1-download-argo"},"official Argo documentation"),".")),(0,n.yg)("h3",{id:"test-argo"},"Test Argo"),(0,n.yg)("p",null,"Run Hello world workflow to test if Argo has been properly installed. And take a look at the ",(0,n.yg)("a",{parentName:"p",href:"https://argoproj.github.io/docs/argo/examples/readme.html"},"examples provided in Argo documentation")," to discover how to use the different features available."),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-shell"},"argo submit --watch https://raw.githubusercontent.com/argoproj/argo/master/examples/hello-world.yaml\n")),(0,n.yg)("admonition",{title:"Logged in",type:"caution"},(0,n.yg)("p",{parentName:"admonition"},"You will need to have the ",(0,n.yg)("inlineCode",{parentName:"p"},"oc")," client installed and be logged in with ",(0,n.yg)("inlineCode",{parentName:"p"},"oc login"),", see the ",(0,n.yg)("a",{parentName:"p",href:"/docs/openshift-install"},"install documentation page"),".")),(0,n.yg)("h2",{id:"install-argo-in-your-project"},"Install Argo in your project"),(0,n.yg)("h3",{id:"argo-workflows-with-helm"},"Argo workflows with Helm"),(0,n.yg)("p",null,"Deploy the ",(0,n.yg)("a",{parentName:"p",href:"https://artifacthub.io/packages/helm/argo/argo"},"Argo Helm chart"),"."),(0,n.yg)("ol",null,(0,n.yg)("li",{parentName:"ol"},"Install and use ",(0,n.yg)("a",{parentName:"li",href:"/docs/helm"},(0,n.yg)("inlineCode",{parentName:"a"},"helm"))),(0,n.yg)("li",{parentName:"ol"},"Add the Helm charts repository:")),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-bash"},"helm repo add argo https://argoproj.github.io/argo-helm\n")),(0,n.yg)("ol",{start:3},(0,n.yg)("li",{parentName:"ol"},"Install chart:")),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-bash"},"helm install my-argo argo/argo --version 0.15.2\n")),(0,n.yg)("h3",{id:"argocd-operator"},"ArgoCD Operator"),(0,n.yg)("p",null,"Ask on the DSRI Slack ",(0,n.yg)("strong",{parentName:"p"},"#helpdesk")," channel to have the ",(0,n.yg)("a",{parentName:"p",href:"https://artifacthub.io/packages/olm/community-operators/argocd-operator"},"ArgoCD Operator")," installed in your project."),(0,n.yg)("h3",{id:"uninstall-argo"},"Uninstall ",(0,n.yg)("inlineCode",{parentName:"h3"},"argo")),(0,n.yg)("h4",{id:"on-ubuntu-1"},"On Ubuntu"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-shell"},"sudo rm /usr/local/bin/argo\n")),(0,n.yg)("blockquote",null,(0,n.yg)("p",{parentName:"blockquote"},"You can now reinstall a newer version of Argo.")),(0,n.yg)("hr",null),(0,n.yg)("h2",{id:"run-workflows-to-convert-structured-data-to-rdf"},"Run workflows to convert structured data to RDF"),(0,n.yg)("p",null,"We will use examples from the ",(0,n.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/d2s-core"},"MaastrichtU-IDS/d2s-core")," project."),(0,n.yg)("h3",{id:"clone-the-repository"},"Clone the repository"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-shell"},"git clone --recursive https://github.com/MaastrichtU-IDS/d2s-project-template.git\ncd d2s-project-template\n")),(0,n.yg)("p",null,(0,n.yg)("a",{parentName:"p",href:"/docs/openshift-install"},"Authenticate to the OpenShift cluster")," using ",(0,n.yg)("inlineCode",{parentName:"p"},"oc login")," ."),(0,n.yg)("h3",{id:"workflow-to-convert-xml-files-to-rdf"},"Workflow to convert XML files to RDF"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"Steps-based workflow for XML files, see the example workflow ",(0,n.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/d2s-core/blob/master/argo/d2s-workflow-xml.yaml"},"YAML file on GitHub"),".")),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-shell"},"argo submit d2s-core/argo/workflows/d2s-workflow-transform-xml.yaml \\\n -f support/config/config-transform-xml-drugbank.yml\n")),(0,n.yg)("admonition",{title:"Provide config files",type:"info"},(0,n.yg)("p",{parentName:"admonition"},"Config files can be provided using the ",(0,n.yg)("inlineCode",{parentName:"p"},"-f")," arguments, but are not necessary.")),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"DAG workflow for XML files, see the ",(0,n.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/d2s-core/blob/master/argo/d2s-workflow-xml-dag.yaml"},"YAML file on GitHub"),".")),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-shell"},"argo submit d2s-core/argo/workflows/d2s-workflow-transform-xml-dag.yaml \\\n -f support/config/config-transform-xml-drugbank.yml\n")),(0,n.yg)("h3",{id:"workflow-to-convert-csv-files-to-rdf"},"Workflow to convert CSV files to RDF"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"Steps-based workflow for CSV files")),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-shell"},"argo submit d2s-core/argo/workflows/d2s-workflow-transform-csv.yaml \\\n -f support/config/config-transform-csv-stitch.yml\n")),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"DAG workflow for CSV files")),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-shell"},"argo submit d2s-core/argo/workflows/d2s-workflow-transform-csv-dag.yaml \\\n -f support/config/config-transform-csv-stitch.yml\n")),(0,n.yg)("admonition",{title:"Solve issue",type:"caution"},(0,n.yg)("p",{parentName:"admonition"},"Try this to solve issue related to steps services IP: ",(0,n.yg)("inlineCode",{parentName:"p"},"{{steps.nginx-server.pod-ip}}"))),(0,n.yg)("hr",null),(0,n.yg)("h2",{id:"argo-commands"},"Argo commands"),(0,n.yg)("h3",{id:"list-running-argo-workflows"},"List running Argo workflows"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-shell"},"argo list\n")),(0,n.yg)("h3",{id:"stop-a-workflow"},"Stop a workflow"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-shell"},"argo terminate my-workflow\n")),(0,n.yg)("admonition",{title:"Workflow",type:"caution"},(0,n.yg)("p",{parentName:"admonition"},"This might not stop the workflow, in this case use:"),(0,n.yg)("pre",{parentName:"admonition"},(0,n.yg)("code",{parentName:"pre",className:"language-bash"},"argo delete my-workflow\n"))),(0,n.yg)("h3",{id:"delete-a-workflow"},"Delete a workflow"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-shell"},"argo delete my-workflow\n")),(0,n.yg)("hr",null),(0,n.yg)("h2",{id:"debug-a-workflow"},"Debug a workflow"),(0,n.yg)("p",null,"Get into a container, to understand why it bugs, by creating a YAML with the command ",(0,n.yg)("inlineCode",{parentName:"p"},"tail -f /dev/null")," to keep it hanging."),(0,n.yg)("p",null,"See the ",(0,n.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/d2s-core/blob/master/argo/tests/test-devnull-argo.yaml"},"example in the d2s-argo-workflow repository"),":"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-yaml"},'apiVersion: argoproj.io/v1alpha1\nkind: Workflow\nmetadata:\n generateName: test-devnull-argo-\nspec:\n entrypoint: execute-workflow\n\n # Use existing volume\n volumes:\n - name: workdir\n persistentVolumeClaim:\n claimName: pvc-mapr-projects-test-vincent \n\n templates:\n - name: execute-workflow\n steps:\n - - name: run-rdfunit\n template: rdfunit\n \n - name: rdfunit\n container:\n image: umids/rdfunit:latest\n command: [tail]\n args: ["-f", "/dev/null"]\n volumeMounts:\n - name: workdir\n mountPath: /data\n subPath: dqa-workspace\n')),(0,n.yg)("p",null,"Then start the workflow:"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-shell"},"argo submit --serviceaccount argo tests/test-devnull-argo.yaml\n")),(0,n.yg)("p",null,"And connect with the Shell (change the pod ID to your pod ID):"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-shell"},"oc rsh test-devnull-argo-pod\n")))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/8cf96c0d.bc65f97a.js b/assets/js/8cf96c0d.d285c328.js similarity index 99% rename from assets/js/8cf96c0d.bc65f97a.js rename to assets/js/8cf96c0d.d285c328.js index 9eefd200c..c1d143e6e 100644 --- a/assets/js/8cf96c0d.bc65f97a.js +++ b/assets/js/8cf96c0d.d285c328.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[4062],{5680:(e,t,a)=>{a.d(t,{xA:()=>c,yg:()=>d});var r=a(6540);function o(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function n(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,r)}return a}function l(e){for(var t=1;t=0||(o[a]=e[a]);return o}(e,t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(o[a]=e[a])}return o}var s=r.createContext({}),p=function(e){var t=r.useContext(s),a=t;return e&&(a="function"==typeof e?e(t):l(l({},t),e)),a},c=function(e){var t=p(e.components);return r.createElement(s.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},m=r.forwardRef((function(e,t){var a=e.components,o=e.mdxType,n=e.originalType,s=e.parentName,c=i(e,["components","mdxType","originalType","parentName"]),m=p(a),d=o,f=m["".concat(s,".").concat(d)]||m[d]||u[d]||n;return a?r.createElement(f,l(l({ref:t},c),{},{components:a})):r.createElement(f,l({ref:t},c))}));function d(e,t){var a=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var n=a.length,l=new Array(n);l[0]=m;var i={};for(var s in t)hasOwnProperty.call(t,s)&&(i[s]=t[s]);i.originalType=e,i.mdxType="string"==typeof e?e:o,l[1]=i;for(var p=2;p{a.r(t),a.d(t,{assets:()=>c,contentTitle:()=>s,default:()=>d,frontMatter:()=>i,metadata:()=>p,toc:()=>u});var r=a(9668),o=a(1367),n=(a(6540),a(5680)),l=["components"],i={id:"workflows-airflow",title:"Deploy Airflow"},s=void 0,p={unversionedId:"workflows-airflow",id:"workflows-airflow",title:"Deploy Airflow",description:"Deploy Apache Airflow to run workflows (aka. DAGs), hosted in a Git repository, on the DSRI.",source:"@site/docs/workflows-airflow.md",sourceDirName:".",slug:"/workflows-airflow",permalink:"/docs/workflows-airflow",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/workflows-airflow.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"workflows-airflow",title:"Deploy Airflow"},sidebar:"docs",previous:{title:"Deploy GitHub Runners",permalink:"/docs/workflows-github-actions"},next:{title:"Run Argo workflows",permalink:"/docs/workflows-argo"}},c={},u=[{value:"Install the chart",id:"install-the-chart",level:2},{value:"Deploy Airflow",id:"deploy-airflow",level:2},{value:"Example workflows",id:"example-workflows",level:2},{value:"Delete the chart",id:"delete-the-chart",level:2},{value:"See also",id:"see-also",level:2}],m={toc:u};function d(e){var t=e.components,a=(0,o.A)(e,l);return(0,n.yg)("wrapper",(0,r.A)({},m,a,{components:t,mdxType:"MDXLayout"}),(0,n.yg)("p",null,"Deploy ",(0,n.yg)("a",{parentName:"p",href:"https://airflow.apache.org"},"Apache Airflow")," to run workflows (aka. DAGs), hosted in a Git repository, on the DSRI. "),(0,n.yg)("h2",{id:"install-the-chart"},"Install the chart"),(0,n.yg)("p",null,"You will need to have Helm installed on your computer to deploy a Helm chart, see the ",(0,n.yg)("a",{parentName:"p",href:"/docs/helm"},"Helm docs")," for more details."),(0,n.yg)("p",null,"Install the Helm chart to be able to deploy Airflow on the DSRI:"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-bash"},"helm repo add apache-airflow https://airflow.apache.org\nhelm repo update\n")),(0,n.yg)("h2",{id:"deploy-airflow"},"Deploy Airflow"),(0,n.yg)("p",null,"You can quickly deploy Airflow on the DSRI, with DAGs automatically synchronized with your Git repository. "),(0,n.yg)("p",null,"We use a ",(0,n.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/blob/master/applications/airflow/values.yml"},(0,n.yg)("inlineCode",{parentName:"a"},"values.yml")," file")," with all default parameters pre-defined for the DSRI, so you just need to edit the password and git repository configuration in this command, and run it:"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-bash"},"helm install airflow apache-airflow/airflow \\\n -f https://raw.githubusercontent.com/MaastrichtU-IDS/dsri-documentation/master/applications/airflow/values.yml \\\n --set webserver.defaultUser.password=yourpassword \\\n --set dags.gitSync.repo=https://github.com/bio2kg/bio2kg-etl.git \\\n --set dags.gitSync.branch=main \\\n --set dags.gitSync.subPath=workflows/dags\n")),(0,n.yg)("admonition",{type:"info"},(0,n.yg)("p",{parentName:"admonition"},"If you need to do more configuration you can download the a ",(0,n.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/blob/master/applications/airflow/values.yml"},(0,n.yg)("inlineCode",{parentName:"a"},"values.yml")," file"),", edit it directly to your settings and use this file locally with ",(0,n.yg)("inlineCode",{parentName:"p"},"-f values.yml"))),(0,n.yg)("p",null,"A few seconds after Airflow started to install, you will need to fix the postgresql deployment in a different terminal window (unfortunately setting the ",(0,n.yg)("inlineCode",{parentName:"p"},"serviceAccount.name")," of the sub chart ",(0,n.yg)("inlineCode",{parentName:"p"},"postgresql")," don't work, even if it should be possible according to the ",(0,n.yg)("a",{parentName:"p",href:"https://helm.sh/docs/chart_template_guide/subcharts_and_globals/"},"official helm docs"),"). Run this command to fix postgresql:"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-bash"},'oc patch statefulset/airflow-postgresql --patch \'{"spec":{"template":{"spec": {"serviceAccountName": "anyuid"}}}}\'\n')),(0,n.yg)("p",null,"Once Airflow finished to deploy, you can access its web interface temporarily by forwarding the webserver on your machine at http://localhost:8080"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-bash"},"oc port-forward svc/airflow-webserver 8080:8080\n")),(0,n.yg)("p",null,"Or permanently expose the interface on a URL accessible when logged to the UM VPN, with HTTPS enabled:"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-bash"},'oc expose svc/airflow-webserver\noc patch route/airflow-webserver --patch \'{"spec":{"tls": {"termination": "edge", "insecureEdgeTerminationPolicy": "Redirect"}}}\'\n')),(0,n.yg)("p",null,"Finally, get the route to the Airflow web interface, or access it via the DSRI web UI:"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-bash"},"oc get routes\n")),(0,n.yg)("h2",{id:"example-workflows"},"Example workflows"),(0,n.yg)("p",null,"You can find example DAGs for bash operator, python operator and Kubernetes pod operator ",(0,n.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/tree/master/applications/airflow/dags"},"here"),"."),(0,n.yg)("p",null,"Here an example of a DAG using the Kubernetes pod operator to run tasks as pods, you will need to change the ",(0,n.yg)("inlineCode",{parentName:"p"},"namespace")," parameter to your DSRI project where Airflow is deployed:"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-python"},"from airflow import DAG\nfrom datetime import datetime, timedelta\nfrom airflow.contrib.operators.kubernetes_pod_operator import KubernetesPodOperator\nfrom airflow.operators.dummy_operator import DummyOperator\n\ndefault_args = {\n 'owner': 'airflow',\n 'depends_on_past': False,\n 'start_date': datetime.utcnow(),\n 'email': ['airflow@example.com'],\n 'email_on_failure': False,\n 'email_on_retry': False,\n 'retries': 1,\n 'retry_delay': timedelta(minutes=5)\n}\ndag = DAG(\n 'kubernetes_pod_operator',\n default_args=default_args, \n schedule_interval=None\n # schedule_interval=timedelta(minutes=10)\n)\n\nstart = DummyOperator(task_id='run_this_first', dag=dag)\n\npassing = KubernetesPodOperator(\n namespace='CHANGEME',\n image=\"python:3.6\",\n cmds=[\"python\",\"-c\"],\n arguments=[\"print('hello world')\"],\n labels={\"app\": \"airflow\"},\n name=\"passing-test\",\n task_id=\"passing-task\",\n get_logs=True,\n dag=dag\n)\n\npassing.set_upstream(start)\n")),(0,n.yg)("h2",{id:"delete-the-chart"},"Delete the chart"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-bash"},"helm uninstall airflow\n")),(0,n.yg)("h2",{id:"see-also"},"See also"),(0,n.yg)("p",null,"Here are a few links for more details on the official Airflow Helm chart:"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("a",{parentName:"li",href:"https://airflow.apache.org/docs/helm-chart/stable/index.html"},"Helm chart docs")),(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("a",{parentName:"li",href:"https://github.com/apache/airflow/tree/main/chart"},"Helm chart source code")),(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("a",{parentName:"li",href:"https://airflow.apache.org/docs/helm-chart/stable/parameters-ref.html"},"Helm chart parameters"))),(0,n.yg)("p",null,"Other ways to deploy Airflow on OpenShift:"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("a",{parentName:"li",href:"https://github.com/airflow-helm/charts/tree/main/charts/airflow"},"Community Helm chart GitHub repo")),(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("a",{parentName:"li",href:"https://github.com/CSCfi/airflow-openshift"},"Airflow template for OpenShift"))))}d.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[4062],{5680:(e,t,a)=>{a.d(t,{xA:()=>c,yg:()=>d});var r=a(6540);function o(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function n(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,r)}return a}function l(e){for(var t=1;t=0||(o[a]=e[a]);return o}(e,t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(o[a]=e[a])}return o}var s=r.createContext({}),p=function(e){var t=r.useContext(s),a=t;return e&&(a="function"==typeof e?e(t):l(l({},t),e)),a},c=function(e){var t=p(e.components);return r.createElement(s.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},m=r.forwardRef((function(e,t){var a=e.components,o=e.mdxType,n=e.originalType,s=e.parentName,c=i(e,["components","mdxType","originalType","parentName"]),m=p(a),d=o,f=m["".concat(s,".").concat(d)]||m[d]||u[d]||n;return a?r.createElement(f,l(l({ref:t},c),{},{components:a})):r.createElement(f,l({ref:t},c))}));function d(e,t){var a=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var n=a.length,l=new Array(n);l[0]=m;var i={};for(var s in t)hasOwnProperty.call(t,s)&&(i[s]=t[s]);i.originalType=e,i.mdxType="string"==typeof e?e:o,l[1]=i;for(var p=2;p{a.r(t),a.d(t,{assets:()=>c,contentTitle:()=>s,default:()=>d,frontMatter:()=>i,metadata:()=>p,toc:()=>u});var r=a(9668),o=a(1367),n=(a(6540),a(5680)),l=["components"],i={id:"workflows-airflow",title:"Deploy Airflow"},s=void 0,p={unversionedId:"workflows-airflow",id:"workflows-airflow",title:"Deploy Airflow",description:"Deploy Apache Airflow to run workflows (aka. DAGs), hosted in a Git repository, on the DSRI.",source:"@site/docs/workflows-airflow.md",sourceDirName:".",slug:"/workflows-airflow",permalink:"/docs/workflows-airflow",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/workflows-airflow.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"workflows-airflow",title:"Deploy Airflow"},sidebar:"docs",previous:{title:"Deploy GitHub Runners",permalink:"/docs/workflows-github-actions"},next:{title:"Run Argo workflows",permalink:"/docs/workflows-argo"}},c={},u=[{value:"Install the chart",id:"install-the-chart",level:2},{value:"Deploy Airflow",id:"deploy-airflow",level:2},{value:"Example workflows",id:"example-workflows",level:2},{value:"Delete the chart",id:"delete-the-chart",level:2},{value:"See also",id:"see-also",level:2}],m={toc:u};function d(e){var t=e.components,a=(0,o.A)(e,l);return(0,n.yg)("wrapper",(0,r.A)({},m,a,{components:t,mdxType:"MDXLayout"}),(0,n.yg)("p",null,"Deploy ",(0,n.yg)("a",{parentName:"p",href:"https://airflow.apache.org"},"Apache Airflow")," to run workflows (aka. DAGs), hosted in a Git repository, on the DSRI. "),(0,n.yg)("h2",{id:"install-the-chart"},"Install the chart"),(0,n.yg)("p",null,"You will need to have Helm installed on your computer to deploy a Helm chart, see the ",(0,n.yg)("a",{parentName:"p",href:"/docs/helm"},"Helm docs")," for more details."),(0,n.yg)("p",null,"Install the Helm chart to be able to deploy Airflow on the DSRI:"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-bash"},"helm repo add apache-airflow https://airflow.apache.org\nhelm repo update\n")),(0,n.yg)("h2",{id:"deploy-airflow"},"Deploy Airflow"),(0,n.yg)("p",null,"You can quickly deploy Airflow on the DSRI, with DAGs automatically synchronized with your Git repository. "),(0,n.yg)("p",null,"We use a ",(0,n.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/blob/master/applications/airflow/values.yml"},(0,n.yg)("inlineCode",{parentName:"a"},"values.yml")," file")," with all default parameters pre-defined for the DSRI, so you just need to edit the password and git repository configuration in this command, and run it:"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-bash"},"helm install airflow apache-airflow/airflow \\\n -f https://raw.githubusercontent.com/MaastrichtU-IDS/dsri-documentation/master/applications/airflow/values.yml \\\n --set webserver.defaultUser.password=yourpassword \\\n --set dags.gitSync.repo=https://github.com/bio2kg/bio2kg-etl.git \\\n --set dags.gitSync.branch=main \\\n --set dags.gitSync.subPath=workflows/dags\n")),(0,n.yg)("admonition",{type:"info"},(0,n.yg)("p",{parentName:"admonition"},"If you need to do more configuration you can download the a ",(0,n.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/blob/master/applications/airflow/values.yml"},(0,n.yg)("inlineCode",{parentName:"a"},"values.yml")," file"),", edit it directly to your settings and use this file locally with ",(0,n.yg)("inlineCode",{parentName:"p"},"-f values.yml"))),(0,n.yg)("p",null,"A few seconds after Airflow started to install, you will need to fix the postgresql deployment in a different terminal window (unfortunately setting the ",(0,n.yg)("inlineCode",{parentName:"p"},"serviceAccount.name")," of the sub chart ",(0,n.yg)("inlineCode",{parentName:"p"},"postgresql")," don't work, even if it should be possible according to the ",(0,n.yg)("a",{parentName:"p",href:"https://helm.sh/docs/chart_template_guide/subcharts_and_globals/"},"official helm docs"),"). Run this command to fix postgresql:"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-bash"},'oc patch statefulset/airflow-postgresql --patch \'{"spec":{"template":{"spec": {"serviceAccountName": "anyuid"}}}}\'\n')),(0,n.yg)("p",null,"Once Airflow finished to deploy, you can access its web interface temporarily by forwarding the webserver on your machine at http://localhost:8080"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-bash"},"oc port-forward svc/airflow-webserver 8080:8080\n")),(0,n.yg)("p",null,"Or permanently expose the interface on a URL accessible when logged to the UM VPN, with HTTPS enabled:"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-bash"},'oc expose svc/airflow-webserver\noc patch route/airflow-webserver --patch \'{"spec":{"tls": {"termination": "edge", "insecureEdgeTerminationPolicy": "Redirect"}}}\'\n')),(0,n.yg)("p",null,"Finally, get the route to the Airflow web interface, or access it via the DSRI web UI:"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-bash"},"oc get routes\n")),(0,n.yg)("h2",{id:"example-workflows"},"Example workflows"),(0,n.yg)("p",null,"You can find example DAGs for bash operator, python operator and Kubernetes pod operator ",(0,n.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/tree/master/applications/airflow/dags"},"here"),"."),(0,n.yg)("p",null,"Here an example of a DAG using the Kubernetes pod operator to run tasks as pods, you will need to change the ",(0,n.yg)("inlineCode",{parentName:"p"},"namespace")," parameter to your DSRI project where Airflow is deployed:"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-python"},"from airflow import DAG\nfrom datetime import datetime, timedelta\nfrom airflow.contrib.operators.kubernetes_pod_operator import KubernetesPodOperator\nfrom airflow.operators.dummy_operator import DummyOperator\n\ndefault_args = {\n 'owner': 'airflow',\n 'depends_on_past': False,\n 'start_date': datetime.utcnow(),\n 'email': ['airflow@example.com'],\n 'email_on_failure': False,\n 'email_on_retry': False,\n 'retries': 1,\n 'retry_delay': timedelta(minutes=5)\n}\ndag = DAG(\n 'kubernetes_pod_operator',\n default_args=default_args, \n schedule_interval=None\n # schedule_interval=timedelta(minutes=10)\n)\n\nstart = DummyOperator(task_id='run_this_first', dag=dag)\n\npassing = KubernetesPodOperator(\n namespace='CHANGEME',\n image=\"python:3.6\",\n cmds=[\"python\",\"-c\"],\n arguments=[\"print('hello world')\"],\n labels={\"app\": \"airflow\"},\n name=\"passing-test\",\n task_id=\"passing-task\",\n get_logs=True,\n dag=dag\n)\n\npassing.set_upstream(start)\n")),(0,n.yg)("h2",{id:"delete-the-chart"},"Delete the chart"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-bash"},"helm uninstall airflow\n")),(0,n.yg)("h2",{id:"see-also"},"See also"),(0,n.yg)("p",null,"Here are a few links for more details on the official Airflow Helm chart:"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("a",{parentName:"li",href:"https://airflow.apache.org/docs/helm-chart/stable/index.html"},"Helm chart docs")),(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("a",{parentName:"li",href:"https://github.com/apache/airflow/tree/main/chart"},"Helm chart source code")),(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("a",{parentName:"li",href:"https://airflow.apache.org/docs/helm-chart/stable/parameters-ref.html"},"Helm chart parameters"))),(0,n.yg)("p",null,"Other ways to deploy Airflow on OpenShift:"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("a",{parentName:"li",href:"https://github.com/airflow-helm/charts/tree/main/charts/airflow"},"Community Helm chart GitHub repo")),(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("a",{parentName:"li",href:"https://github.com/CSCfi/airflow-openshift"},"Airflow template for OpenShift"))))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/94db8302.19c546e0.js b/assets/js/94db8302.e27ad90e.js similarity index 99% rename from assets/js/94db8302.19c546e0.js rename to assets/js/94db8302.e27ad90e.js index 0911ba224..2ad03c7b5 100644 --- a/assets/js/94db8302.19c546e0.js +++ b/assets/js/94db8302.e27ad90e.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[695],{5680:(e,t,r)=>{r.d(t,{xA:()=>u,yg:()=>g});var n=r(6540);function a(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function o(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function i(e){for(var t=1;t=0||(a[r]=e[r]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(a[r]=e[r])}return a}var l=n.createContext({}),p=function(e){var t=n.useContext(l),r=t;return e&&(r="function"==typeof e?e(t):i(i({},t),e)),r},u=function(e){var t=p(e.components);return n.createElement(l.Provider,{value:t},e.children)},c={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},d=n.forwardRef((function(e,t){var r=e.components,a=e.mdxType,o=e.originalType,l=e.parentName,u=s(e,["components","mdxType","originalType","parentName"]),d=p(r),g=a,m=d["".concat(l,".").concat(g)]||d[g]||c[g]||o;return r?n.createElement(m,i(i({ref:t},u),{},{components:r})):n.createElement(m,i({ref:t},u))}));function g(e,t){var r=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=r.length,i=new Array(o);i[0]=d;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s.mdxType="string"==typeof e?e:a,i[1]=s;for(var p=2;p{r.r(t),r.d(t,{assets:()=>u,contentTitle:()=>l,default:()=>g,frontMatter:()=>s,metadata:()=>p,toc:()=>c});var n=r(9668),a=r(1367),o=(r(6540),r(5680)),i=["components"],s={id:"neuroscience",title:"Neuroscience research"},l=void 0,p={unversionedId:"neuroscience",id:"neuroscience",title:"Neuroscience research",description:"We are not expert in Neuroscience ourselves, please contact us if you see any improvements that could be made to this page, or if you need any help to get it working.",source:"@site/docs/deploy-neurodocker.md",sourceDirName:".",slug:"/neuroscience",permalink:"/docs/neuroscience",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/deploy-neurodocker.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"neuroscience",title:"Neuroscience research"},sidebar:"docs",previous:{title:"Run MPI jobs",permalink:"/docs/mpi-jobs"},next:{title:"Genomics",permalink:"/docs/catalog-genomics"}},u={},c=[{value:"JupyterLab with FreeSurfer",id:"jupyterlab-with-freesurfer",level:2},{value:"FreeSurfer and FSL",id:"freesurfer-and-fsl",level:2},{value:"FreeSurfer and AFNI",id:"freesurfer-and-afni",level:2},{value:"Deploy the generated Dockerfile",id:"deploy-the-generated-dockerfile",level:2},{value:"Use the GPUs",id:"use-the-gpus",level:2}],d={toc:c};function g(e){var t=e.components,r=(0,a.A)(e,i);return(0,o.yg)("wrapper",(0,n.A)({},d,r,{components:t,mdxType:"MDXLayout"}),(0,o.yg)("admonition",{title:"Feedbacks welcome",type:"tip"},(0,o.yg)("p",{parentName:"admonition"},"We are not expert in Neuroscience ourselves, please contact us if you see any improvements that could be made to this page, or if you need any help to get it working.")),(0,o.yg)("p",null,"The Neurodocker project helps you to create a Docker image with the Neuroscience softwares you need, such as FSL, FreeSurfer, AFNI or ANTs. "),(0,o.yg)("p",null,"Checkout the Neurodocker documentation for more details: ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/ReproNim/neurodocker"},"https://github.com/ReproNim/neurodocker")),(0,o.yg)("admonition",{title:"Examples",type:"info"},(0,o.yg)("p",{parentName:"admonition"},"In this page we will show you how to generate a Docker image with popular Neuroscience research softwares installed such as FreeSurfer and FSL. Feel free to check the ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/ReproNim/neurodocker"},"Neurodocker documentation"),", and adapt the installation process to your needs.")),(0,o.yg)("h2",{id:"jupyterlab-with-freesurfer"},"JupyterLab with FreeSurfer"),(0,o.yg)("p",null,"Start a JupyterLab container with Freesurfer pre-installed providing admin (",(0,o.yg)("inlineCode",{parentName:"p"},"sudo"),") privileges to install anything you need from the terminal (e.g. pip or apt packages)"),(0,o.yg)("p",null,"When instantiating the template you can provide a few parameters similar to the standard JupyterLab, such as:"),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("strong",{parentName:"li"},"Password")," to access the notebook"),(0,o.yg)("li",{parentName:"ul"},"Optionally you can provide a ",(0,o.yg)("strong",{parentName:"li"},"git repository")," to be automatically cloned in the JupyterLab (if there is a ",(0,o.yg)("inlineCode",{parentName:"li"},"requirements.txt")," packages will be automatically installed with ",(0,o.yg)("inlineCode",{parentName:"li"},"pip"),")"),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("strong",{parentName:"li"},"Docker image")," to use for the notebook (see below for more details on customizing the docker image) "),(0,o.yg)("li",{parentName:"ul"},"Your ",(0,o.yg)("strong",{parentName:"li"},"git username and email")," to automatically configure git")),(0,o.yg)("p",null,"The DSRI will automatically create a persistent volume to store data you will put in the ",(0,o.yg)("inlineCode",{parentName:"p"},"/home/jovyan/work")," folder (the folder used by the notebook interface). You can find the persistent volumes in the DSRI web UI, go to the ",(0,o.yg)("strong",{parentName:"p"},"Administrator")," view > ",(0,o.yg)("strong",{parentName:"p"},"Storage")," > ",(0,o.yg)("strong",{parentName:"p"},"Persistent Volume Claims")),(0,o.yg)("img",{src:"/img/screenshot-freesurfer.png",alt:"Deploy Freesurfer",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("p",null,"You can also link your git repository to the project for automatic deployment see ",(0,o.yg)("a",{parentName:"p",href:"https://dsri.maastrichtuniversity.nl/docs/deploy-jupyter#-use-git-in-jupyterlab"},"using git in JupyterLab")),(0,o.yg)("p",null,"This can also be deployed using Helm from the terminal, the steps are:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"helm repo add dsri https://maastrichtu-ids.github.io/dsri-helm-charts/\nhelm repo update\nhelm install freesurfer dsri/jupyterlab \\\n --set serviceAccount.name=anyuid \\\n --set openshiftRoute.enabled=true \\\n --set image.repository=ghcr.io/maastrichtu-ids/jupyterlab \\\n --set image.tag=freesurfer \\\n --set storage.mountPath=/root \\\n --set password=changeme\noc get route --selector app.kubernetes.io/instance=freesurfer --no-headers -o=custom-columns=HOST:.spec.host\n")),(0,o.yg)("p",null,"Log in to the corresponding jupyter notebook and start the terminal, then enter ",(0,o.yg)("inlineCode",{parentName:"p"},"freesurfer")," as a command"),(0,o.yg)("h2",{id:"freesurfer-and-fsl"},"FreeSurfer and FSL"),(0,o.yg)("p",null,"Generate a ",(0,o.yg)("inlineCode",{parentName:"p"},"Dockerfile")," with:"),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},"FreeSurfer 6.0.1"),(0,o.yg)("li",{parentName:"ul"},"FSL 6.0.3")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"docker run --rm repronim/neurodocker:0.7.0 generate docker \\\n --base debian:stretch --pkg-manager apt \\\n --freesurfer version=6.0.1 --fsl version=6.0.3 > Dockerfile\n")),(0,o.yg)("h2",{id:"freesurfer-and-afni"},"FreeSurfer and AFNI"),(0,o.yg)("p",null,"Generate a ",(0,o.yg)("inlineCode",{parentName:"p"},"Dockerfile")," with:"),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},"FreeSurfer 6.0.1"),(0,o.yg)("li",{parentName:"ul"},"AFNI, R and Python3")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"docker run --rm repronim/neurodocker:0.7.0 generate docker \\\n --base debian:stretch --pkg-manager apt \\\n --afni version=latest install_r=true install_r_pkgs=true install_python3=true \\\n --freesurfer version=6.0.1 > Dockerfile\n")),(0,o.yg)("h2",{id:"deploy-the-generated-dockerfile"},"Deploy the generated Dockerfile"),(0,o.yg)("p",null,"Before deploying the ",(0,o.yg)("inlineCode",{parentName:"p"},"Dockerfile")," to the DSRI you can open it, and add commands to install additional package you are interested in, such as nighres or nipype."),(0,o.yg)("p",null,"Checkout the documentation to ",(0,o.yg)("a",{parentName:"p",href:"https://maastrichtu-ids.github.io/dsri-documentation/docs/guide-dockerfile-to-openshift"},"deploy the ",(0,o.yg)("inlineCode",{parentName:"a"},"Dockerfile")," on DSRI"),". "),(0,o.yg)("admonition",{title:"UI with VNC",type:"note"},(0,o.yg)("p",{parentName:"admonition"},"Running a UI with VNC (e.g. FSLeyes) is still a work in progress. See ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/ReproNim/neurodocker/issues/343"},"this issue for more details"),".")),(0,o.yg)("h2",{id:"use-the-gpus"},"Use the GPUs"),(0,o.yg)("p",null,"More details about using GPU with FSL: ",(0,o.yg)("a",{parentName:"p",href:"https://fsl.fmrib.ox.ac.uk/fsl/fslwiki/GPU"},"https://fsl.fmrib.ox.ac.uk/fsl/fslwiki/GPU")))}g.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[695],{5680:(e,t,r)=>{r.d(t,{xA:()=>u,yg:()=>g});var n=r(6540);function a(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function o(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function i(e){for(var t=1;t=0||(a[r]=e[r]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(a[r]=e[r])}return a}var l=n.createContext({}),p=function(e){var t=n.useContext(l),r=t;return e&&(r="function"==typeof e?e(t):i(i({},t),e)),r},u=function(e){var t=p(e.components);return n.createElement(l.Provider,{value:t},e.children)},c={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},d=n.forwardRef((function(e,t){var r=e.components,a=e.mdxType,o=e.originalType,l=e.parentName,u=s(e,["components","mdxType","originalType","parentName"]),d=p(r),g=a,m=d["".concat(l,".").concat(g)]||d[g]||c[g]||o;return r?n.createElement(m,i(i({ref:t},u),{},{components:r})):n.createElement(m,i({ref:t},u))}));function g(e,t){var r=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=r.length,i=new Array(o);i[0]=d;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s.mdxType="string"==typeof e?e:a,i[1]=s;for(var p=2;p{r.r(t),r.d(t,{assets:()=>u,contentTitle:()=>l,default:()=>g,frontMatter:()=>s,metadata:()=>p,toc:()=>c});var n=r(9668),a=r(1367),o=(r(6540),r(5680)),i=["components"],s={id:"neuroscience",title:"Neuroscience research"},l=void 0,p={unversionedId:"neuroscience",id:"neuroscience",title:"Neuroscience research",description:"We are not expert in Neuroscience ourselves, please contact us if you see any improvements that could be made to this page, or if you need any help to get it working.",source:"@site/docs/deploy-neurodocker.md",sourceDirName:".",slug:"/neuroscience",permalink:"/docs/neuroscience",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/deploy-neurodocker.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"neuroscience",title:"Neuroscience research"},sidebar:"docs",previous:{title:"Run MPI jobs",permalink:"/docs/mpi-jobs"},next:{title:"Genomics",permalink:"/docs/catalog-genomics"}},u={},c=[{value:"JupyterLab with FreeSurfer",id:"jupyterlab-with-freesurfer",level:2},{value:"FreeSurfer and FSL",id:"freesurfer-and-fsl",level:2},{value:"FreeSurfer and AFNI",id:"freesurfer-and-afni",level:2},{value:"Deploy the generated Dockerfile",id:"deploy-the-generated-dockerfile",level:2},{value:"Use the GPUs",id:"use-the-gpus",level:2}],d={toc:c};function g(e){var t=e.components,r=(0,a.A)(e,i);return(0,o.yg)("wrapper",(0,n.A)({},d,r,{components:t,mdxType:"MDXLayout"}),(0,o.yg)("admonition",{title:"Feedbacks welcome",type:"tip"},(0,o.yg)("p",{parentName:"admonition"},"We are not expert in Neuroscience ourselves, please contact us if you see any improvements that could be made to this page, or if you need any help to get it working.")),(0,o.yg)("p",null,"The Neurodocker project helps you to create a Docker image with the Neuroscience softwares you need, such as FSL, FreeSurfer, AFNI or ANTs. "),(0,o.yg)("p",null,"Checkout the Neurodocker documentation for more details: ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/ReproNim/neurodocker"},"https://github.com/ReproNim/neurodocker")),(0,o.yg)("admonition",{title:"Examples",type:"info"},(0,o.yg)("p",{parentName:"admonition"},"In this page we will show you how to generate a Docker image with popular Neuroscience research softwares installed such as FreeSurfer and FSL. Feel free to check the ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/ReproNim/neurodocker"},"Neurodocker documentation"),", and adapt the installation process to your needs.")),(0,o.yg)("h2",{id:"jupyterlab-with-freesurfer"},"JupyterLab with FreeSurfer"),(0,o.yg)("p",null,"Start a JupyterLab container with Freesurfer pre-installed providing admin (",(0,o.yg)("inlineCode",{parentName:"p"},"sudo"),") privileges to install anything you need from the terminal (e.g. pip or apt packages)"),(0,o.yg)("p",null,"When instantiating the template you can provide a few parameters similar to the standard JupyterLab, such as:"),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("strong",{parentName:"li"},"Password")," to access the notebook"),(0,o.yg)("li",{parentName:"ul"},"Optionally you can provide a ",(0,o.yg)("strong",{parentName:"li"},"git repository")," to be automatically cloned in the JupyterLab (if there is a ",(0,o.yg)("inlineCode",{parentName:"li"},"requirements.txt")," packages will be automatically installed with ",(0,o.yg)("inlineCode",{parentName:"li"},"pip"),")"),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("strong",{parentName:"li"},"Docker image")," to use for the notebook (see below for more details on customizing the docker image) "),(0,o.yg)("li",{parentName:"ul"},"Your ",(0,o.yg)("strong",{parentName:"li"},"git username and email")," to automatically configure git")),(0,o.yg)("p",null,"The DSRI will automatically create a persistent volume to store data you will put in the ",(0,o.yg)("inlineCode",{parentName:"p"},"/home/jovyan/work")," folder (the folder used by the notebook interface). You can find the persistent volumes in the DSRI web UI, go to the ",(0,o.yg)("strong",{parentName:"p"},"Administrator")," view > ",(0,o.yg)("strong",{parentName:"p"},"Storage")," > ",(0,o.yg)("strong",{parentName:"p"},"Persistent Volume Claims")),(0,o.yg)("img",{src:"/img/screenshot-freesurfer.png",alt:"Deploy Freesurfer",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("p",null,"You can also link your git repository to the project for automatic deployment see ",(0,o.yg)("a",{parentName:"p",href:"https://dsri.maastrichtuniversity.nl/docs/deploy-jupyter#-use-git-in-jupyterlab"},"using git in JupyterLab")),(0,o.yg)("p",null,"This can also be deployed using Helm from the terminal, the steps are:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"helm repo add dsri https://maastrichtu-ids.github.io/dsri-helm-charts/\nhelm repo update\nhelm install freesurfer dsri/jupyterlab \\\n --set serviceAccount.name=anyuid \\\n --set openshiftRoute.enabled=true \\\n --set image.repository=ghcr.io/maastrichtu-ids/jupyterlab \\\n --set image.tag=freesurfer \\\n --set storage.mountPath=/root \\\n --set password=changeme\noc get route --selector app.kubernetes.io/instance=freesurfer --no-headers -o=custom-columns=HOST:.spec.host\n")),(0,o.yg)("p",null,"Log in to the corresponding jupyter notebook and start the terminal, then enter ",(0,o.yg)("inlineCode",{parentName:"p"},"freesurfer")," as a command"),(0,o.yg)("h2",{id:"freesurfer-and-fsl"},"FreeSurfer and FSL"),(0,o.yg)("p",null,"Generate a ",(0,o.yg)("inlineCode",{parentName:"p"},"Dockerfile")," with:"),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},"FreeSurfer 6.0.1"),(0,o.yg)("li",{parentName:"ul"},"FSL 6.0.3")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"docker run --rm repronim/neurodocker:0.7.0 generate docker \\\n --base debian:stretch --pkg-manager apt \\\n --freesurfer version=6.0.1 --fsl version=6.0.3 > Dockerfile\n")),(0,o.yg)("h2",{id:"freesurfer-and-afni"},"FreeSurfer and AFNI"),(0,o.yg)("p",null,"Generate a ",(0,o.yg)("inlineCode",{parentName:"p"},"Dockerfile")," with:"),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},"FreeSurfer 6.0.1"),(0,o.yg)("li",{parentName:"ul"},"AFNI, R and Python3")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"docker run --rm repronim/neurodocker:0.7.0 generate docker \\\n --base debian:stretch --pkg-manager apt \\\n --afni version=latest install_r=true install_r_pkgs=true install_python3=true \\\n --freesurfer version=6.0.1 > Dockerfile\n")),(0,o.yg)("h2",{id:"deploy-the-generated-dockerfile"},"Deploy the generated Dockerfile"),(0,o.yg)("p",null,"Before deploying the ",(0,o.yg)("inlineCode",{parentName:"p"},"Dockerfile")," to the DSRI you can open it, and add commands to install additional package you are interested in, such as nighres or nipype."),(0,o.yg)("p",null,"Checkout the documentation to ",(0,o.yg)("a",{parentName:"p",href:"https://maastrichtu-ids.github.io/dsri-documentation/docs/guide-dockerfile-to-openshift"},"deploy the ",(0,o.yg)("inlineCode",{parentName:"a"},"Dockerfile")," on DSRI"),". "),(0,o.yg)("admonition",{title:"UI with VNC",type:"note"},(0,o.yg)("p",{parentName:"admonition"},"Running a UI with VNC (e.g. FSLeyes) is still a work in progress. See ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/ReproNim/neurodocker/issues/343"},"this issue for more details"),".")),(0,o.yg)("h2",{id:"use-the-gpus"},"Use the GPUs"),(0,o.yg)("p",null,"More details about using GPU with FSL: ",(0,o.yg)("a",{parentName:"p",href:"https://fsl.fmrib.ox.ac.uk/fsl/fslwiki/GPU"},"https://fsl.fmrib.ox.ac.uk/fsl/fslwiki/GPU")))}g.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/97d64f0a.0a2c2387.js b/assets/js/97d64f0a.3926465f.js similarity index 99% rename from assets/js/97d64f0a.0a2c2387.js rename to assets/js/97d64f0a.3926465f.js index e42d93335..64298baa2 100644 --- a/assets/js/97d64f0a.0a2c2387.js +++ b/assets/js/97d64f0a.3926465f.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[6970],{5680:(e,t,a)=>{a.d(t,{xA:()=>c,yg:()=>d});var n=a(6540);function o(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function r(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function i(e){for(var t=1;t=0||(o[a]=e[a]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(o[a]=e[a])}return o}var l=n.createContext({}),p=function(e){var t=n.useContext(l),a=t;return e&&(a="function"==typeof e?e(t):i(i({},t),e)),a},c=function(e){var t=p(e.components);return n.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},y=n.forwardRef((function(e,t){var a=e.components,o=e.mdxType,r=e.originalType,l=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),y=p(a),d=o,h=y["".concat(l,".").concat(d)]||y[d]||u[d]||r;return a?n.createElement(h,i(i({ref:t},c),{},{components:a})):n.createElement(h,i({ref:t},c))}));function d(e,t){var a=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=a.length,i=new Array(r);i[0]=y;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s.mdxType="string"==typeof e?e:o,i[1]=s;for(var p=2;p{a.r(t),a.d(t,{assets:()=>c,contentTitle:()=>l,default:()=>d,frontMatter:()=>s,metadata:()=>p,toc:()=>u});var n=a(9668),o=a(1367),r=(a(6540),a(5680)),i=["components"],s={id:"start-workspace",title:"Start your workspace"},l=void 0,p={unversionedId:"start-workspace",id:"start-workspace",title:"Start your workspace",description:"This page will help you to start a workspace to run your code and experiments on the DSRI in a container.",source:"@site/docs/start-workspace.md",sourceDirName:".",slug:"/start-workspace",permalink:"/docs/start-workspace",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/start-workspace.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"start-workspace",title:"Start your workspace"},sidebar:"docs",previous:{title:"Access the DSRI",permalink:"/docs/access-dsri"},next:{title:"Install the client",permalink:"/docs/openshift-install"}},c={},u=[{value:"Introduction to containers",id:"introduction-to-containers",level:2},{value:"Choose your interface",id:"choose-your-interface",level:2},{value:"Start your workspace",id:"start-your-workspace",level:2},{value:"Upload your code and data",id:"upload-your-code-and-data",level:2},{value:"Install your dependencies",id:"install-your-dependencies",level:2},{value:"Run your code",id:"run-your-code",level:2},{value:"Stop your application",id:"stop-your-application",level:2},{value:"Start your application",id:"start-your-application",level:2},{value:"Optional: define a docker image",id:"optional-define-a-docker-image",level:2}],y={toc:u};function d(e){var t=e.components,a=(0,o.A)(e,i);return(0,r.yg)("wrapper",(0,n.A)({},y,a,{components:t,mdxType:"MDXLayout"}),(0,r.yg)("p",null,"This page will help you to start a workspace to run your code and experiments on the DSRI in a container. "),(0,r.yg)("h2",{id:"introduction-to-containers"},"Introduction to containers"),(0,r.yg)("p",null,"Anything running in DSRI needs to be running in a docker container. Docker containers are namespaces that share the kernel on a linux system, you can see them as a clean minimalist Linux computers with only what you need to run your programs installed. This allows you to completely control the environment where your code runs, and avoid conflicts."),(0,r.yg)("p",null,"When running experiments we can start from existing images that have been already published for popular data science applications with a web interface. You can use, for example, JupyterLab when running python, RStudio when running R, or VisualStudio Code if you prefer."),(0,r.yg)("p",null,"Once you access a running container, you can install anything you need like if it was a linux/ubuntu computer (most of them runs with admin privileges), and run anything via the notebook/RStudio/VSCode interface, or the terminal."),(0,r.yg)("h2",{id:"choose-your-interface"},"Choose your interface"),(0,r.yg)("p",null,"First step to get your code running on the DSRI is to pick the base interface you want to use to access your workspace on the DSRI."),(0,r.yg)("p",null,"We prepared generic Docker images for data science workspaces with your favorite web UI pre-installed to easily deploy your workspace. So you just need to choose your favorite workspace, start the container, access it, add your code, and install your dependencies."),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},"Login to the DSRI dashboard"),(0,r.yg)("li",{parentName:"ol"},"Select your project, or create one with a meaningful short name representing your project, e.g. ",(0,r.yg)("inlineCode",{parentName:"li"},"workspace-yourname")),(0,r.yg)("li",{parentName:"ol"},"Go to the ",(0,r.yg)("strong",{parentName:"li"},"+Add")," page, and select to add ",(0,r.yg)("strong",{parentName:"li"},"From Developer Catalog => All services"))),(0,r.yg)("img",{src:"/img/screenshot_access_catalog_manu17112024.png",alt:"Access catalog",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("ol",{start:4},(0,r.yg)("li",{parentName:"ol"},"Search for templates corresponding to the application you want to deploy among the one described below (make sure the filter for templates is properly checked). ")),(0,r.yg)("p",null,(0,r.yg)("strong",{parentName:"p"},"JupyterLab"),": Access and run your code using the popular Jupyter notebooks, with kernel for python, java, R, julia. It also provides a good web interface to access the terminal, upload and browse the files."),(0,r.yg)("p",null,(0,r.yg)("strong",{parentName:"p"},"VisualStudio Code"),": Your daily IDE, but in your browser, running on the DSRI. "),(0,r.yg)("p",null,(0,r.yg)("strong",{parentName:"p"},"RStudio"),": R users favorite's."),(0,r.yg)("p",null,(0,r.yg)("strong",{parentName:"p"},"The terminal"),": For people who are used to the terminal and just want to run scripts, it provides smaller and more stable images, which makes installation and deployment easier. You can use the ",(0,r.yg)("strong",{parentName:"p"},"Ubuntu")," template to start a basic ubuntu image and access it from the terminal."),(0,r.yg)("p",null,(0,r.yg)("strong",{parentName:"p"},"Any web interface"),": You can easily run and access most programs with a web interface on the DSRI. You can use the template ",(0,r.yg)("strong",{parentName:"p"},"Custom workspace")," if your application is exposed on port 8888. Otherwise visit the page ",(0,r.yg)("a",{parentName:"p",href:"/docs/anatomy-of-an-application"},"Anatomy of a DSRI application")," for more details. "),(0,r.yg)("p",null,(0,r.yg)("strong",{parentName:"p"},"Desktop interface"),": there is the possibility to start a container as a Linux operating system with a graphical desktop interface. It can be useful to deploy software like Matlab, but the setup can be a bit more complex. You will get an Ubuntu computer with a basic Desktop interface, running on the DSRI, that you can access directly in your web browser. The desktop interface is accessed through a web application by using noVNC, which exposes the VNC connection without needing a VNC client."),(0,r.yg)("admonition",{title:"More applications",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"You can also find more documentation on the different applications that can be deployed from the DSRI under ",(0,r.yg)("strong",{parentName:"p"},"Deploy applications")," in the menu on the left.")),(0,r.yg)("h2",{id:"start-your-workspace"},"Start your workspace"),(0,r.yg)("p",null,"Once you chose your favorite way to run your experiments, you can click on the application you want to use for your workspace. Checkout the description to learn more details about the application that will be deployed. "),(0,r.yg)("p",null,"Then click on ",(0,r.yg)("strong",{parentName:"p"},"Instantiate Template"),", and fill the parameters, such as the password to access the web UI. Note that the application name needs to be unique in the project. Finally click on the ",(0,r.yg)("strong",{parentName:"p"},"Create")," button."),(0,r.yg)("img",{src:"/img/screenshot_template_configuration.png",alt:"Filter templates catalog",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("p",null,"You should see your application in your project dashboard, it can take a few seconds to a few minutes to pull the docker image and start the application. "),(0,r.yg)("p",null,"Once the application has started you will be able to access it by clicking on its circle, then click the ",(0,r.yg)("strong",{parentName:"p"},"Route"),", that has been automatically generated for the web interface, in the Resources tab."),(0,r.yg)("admonition",{title:"Check the workshop",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"For a more detailed tutorial, you can follow the ",(0,r.yg)("a",{parentName:"p",href:"https://maastrichtu-ids.github.io/dsri-workshop-start-app/"},"workshop to start Data Science applications on the DSRI"))),(0,r.yg)("h2",{id:"upload-your-code-and-data"},"Upload your code and data"),(0,r.yg)("p",null,"We recommend you to use ",(0,r.yg)("inlineCode",{parentName:"p"},"git")," to clone your project code in your workspace, as it helps sharing and managing the evolution of your project. "),(0,r.yg)("p",null,"It will be preinstalled in most images, otherwise you can install it easily with ",(0,r.yg)("inlineCode",{parentName:"p"},"apt-get install git")),(0,r.yg)("p",null,"With web interface like JupyterLab, VisualStudio Code and Rstudio you can easily upload small and medium size files directly through the UI with a drag and drop."),(0,r.yg)("p",null,"Otherwise you can use the terminal, install the ",(0,r.yg)("inlineCode",{parentName:"p"},"oc")," client, and use the ",(0,r.yg)("inlineCode",{parentName:"p"},"oc cp")," or ",(0,r.yg)("inlineCode",{parentName:"p"},"oc rsync")," commands to upload large files to your workspace on the DSRI. See the Upload data page for more details."),(0,r.yg)("h2",{id:"install-your-dependencies"},"Install your dependencies"),(0,r.yg)("p",null,"Once the workspace is started, you can install the different dependencies you need to run your experiments."),(0,r.yg)("p",null,"It is recommended to save all the commands you used to install the different requirements in a script (e.g. ",(0,r.yg)("inlineCode",{parentName:"p"},"install.sh"),"). This will insure you can reinstall the environment easily and faithfully if the container is restarted. You can also use them to create a Docker image with everything prepared for your application."),(0,r.yg)("p",null,"Most containers for science are based on debian/ubuntu, so you can install new packages with ",(0,r.yg)("inlineCode",{parentName:"p"},"apt-get"),":"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"apt-get update\napt-get install -y build-essentials wget curl\n")),(0,r.yg)("h2",{id:"run-your-code"},"Run your code"),(0,r.yg)("p",null,"You can use your web interface to run your code as you like to do: notebooks, rstudio, execute via VSCode"),(0,r.yg)("p",null,"Note that for jobs which are running for a long time the web UI is not always the best solution, e.g. Jupyter notebooks can be quite instable when running a 30 min codeblock."),(0,r.yg)("p",null,"A quick solution for that is to run your code in scripts, using the bash terminal. You can use the ",(0,r.yg)("inlineCode",{parentName:"p"},"nohup")," prefix, and ",(0,r.yg)("inlineCode",{parentName:"p"},"&")," suffix to run your script in the background, so that you can even disconnect, and come back later to check the results and logs."),(0,r.yg)("p",null,"For example with a python script, you would do:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"nohup python my_script.py &\n")),(0,r.yg)("p",null,"The script will run in the background, and all terminal output will be stored in the file ",(0,r.yg)("inlineCode",{parentName:"p"},"nohup.out")),(0,r.yg)("p",null,"You can also check if the process is currently running by typing ",(0,r.yg)("inlineCode",{parentName:"p"},"ps aux")," or ",(0,r.yg)("inlineCode",{parentName:"p"},"top")," "),(0,r.yg)("p",null,"You can kill the process by getting the process ID (PID) using the previous commands, and then: ",(0,r.yg)("inlineCode",{parentName:"p"},"kill -9 PID")),(0,r.yg)("h2",{id:"stop-your-application"},"Stop your application"),(0,r.yg)("p",null,"When you are not using your application anymore you can stop the pod. If you are using a Dynamic or Persistent storage you can restart the pod and continue working with all your data in the same state as you left it."),(0,r.yg)("admonition",{title:"Do not waste resources",type:"caution"},(0,r.yg)("p",{parentName:"admonition"},"Please think of stopping applications you are not using to avoid consuming unnecessary resources.")),(0,r.yg)("p",null,"On the ",(0,r.yg)("strong",{parentName:"p"},"Topology")," page click on the down arrow \u2b07\ufe0f next to the number of pods deployed."),(0,r.yg)("img",{src:"/img/screenshot_scaledown_pod.png",alt:"Scale down pod",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("p",null,"You can then restart the pod by clicking the up arrow \u2b06\ufe0f"),(0,r.yg)("p",null,"Note that starting more than 1 pod will not increase the amount of resources you have access to, most of the time it will only waste resources and might ends up in weird behavior on your side. The web UI will randomly assign you to 1 of the pod started when you access it. This only works for clusters with multiple workers, such as Apache Flink and Spark. Or if you connect directly to each pod with the terminal to run different processes."),(0,r.yg)("h2",{id:"start-your-application"},"Start your application"),(0,r.yg)("p",null,"When you try to access your workspace and you encounter the page below, usually this indicates that your pod is not running. For example, this will be the case if you stopped your pod, or if there was maintenance on the cluster."),(0,r.yg)("img",{src:"/img/screenshot_application_unavailable.png",alt:"Screenshot of page that says Application is not available",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("p",null,"To start the pod, go to the ",(0,r.yg)("strong",{parentName:"p"},"Topology")," page, and click on the up arrow \u2b06\ufe0f next to the number of pods deployed. Make sure you scale it to 1. Scaling it to more than 1 will not increase the amount of resources you have access to, most of the time it will only waste resources and causes weird behavior on your side."),(0,r.yg)("img",{src:"/img/screenshot_scaledown_pod.png",alt:"Scale down pod",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("admonition",{title:"Do not waste resources",type:"caution"},(0,r.yg)("p",{parentName:"admonition"},"Please only scale up resources you're using, and scale down when you're not using them anymore. Consuming resources consumes unnecessary power and might prevent other users from using the DSRI.")),(0,r.yg)("h2",{id:"optional-define-a-docker-image"},"Optional: define a docker image"),(0,r.yg)("p",null,"Once you have tested your workspace and you know how to set it up it can be helpful to define a ",(0,r.yg)("inlineCode",{parentName:"p"},"Dockerfile")," to build and publish a Docker image with everything directly installed (instead of installing your requirements after starting a generic workspace)"),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},"Start from an existing generic Docker image, depending on the base technologies you need, such as Debian, Ubuntu, Python, JupyterLab, VisualStudio Code, RStudio..."),(0,r.yg)("li",{parentName:"ol"},"Add your source code in the Docker image using ",(0,r.yg)("inlineCode",{parentName:"li"},"ADD . .")," or ",(0,r.yg)("inlineCode",{parentName:"li"},"COPY . .")),(0,r.yg)("li",{parentName:"ol"},"Install dependencies (e.g. ",(0,r.yg)("inlineCode",{parentName:"li"},"RUN apt-get install gfortran"),")"),(0,r.yg)("li",{parentName:"ol"},"Define which command to run when starting the container (e.g. ",(0,r.yg)("inlineCode",{parentName:"li"},'ENTRYPOINT["jupyter", "lab"]'),")")),(0,r.yg)("p",null,"Here is a simple example ",(0,r.yg)("inlineCode",{parentName:"p"},"Dockerfile")," for a python application:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-dockerfile"},'# The base image to start from, choose the one with everything you need installed\nFROM python:3.8\n\n# Change the user and working directory to make sure we are using root\nUSER root\nWORKDIR /root\n\n# Install additional packages\nRUN apt-get update && \\\n apt-get install build-essentials\n\n# This line will copy all files and folder that are in the same folder as the Dockerfile (usually the code you want to run in the container)\nADD . . \n\n# This line will install all the python packages described in the requirements.txt of your source code\nRUN pip install -r requirements.txt && \\\n pip install notebook jupyterlab\n\n# Command to run when the container is started, here it starts JupyterLab as a service\nENTRYPOINT [ "jupyter", "lab" ]\n')),(0,r.yg)("p",null,"Here are some examples of ",(0,r.yg)("inlineCode",{parentName:"p"},"Dockerfile")," for various type of web applications:"),(0,r.yg)("ul",null,(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/jupyterlab/blob/main/Dockerfile"},"Custom JupyterLab")," based on the official ",(0,r.yg)("a",{parentName:"li",href:"https://github.com/jupyter/docker-stacks"},"jupyter/docker-stacks")),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/rstudio/blob/main/Dockerfile"},"Custom RStudio")),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/code-server/blob/main/Dockerfile"},"VisualStudio Code server")),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/knowledge-collaboratory-api/blob/master/Dockerfile"},"Python web app"))),(0,r.yg)("p",null,"See the guide to ",(0,r.yg)("a",{parentName:"p",href:"/docs/guide-publish-image"},"Publish a Docker image")," for more details on this topic."))}d.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[6970],{5680:(e,t,a)=>{a.d(t,{xA:()=>c,yg:()=>d});var n=a(6540);function o(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function r(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function i(e){for(var t=1;t=0||(o[a]=e[a]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(o[a]=e[a])}return o}var l=n.createContext({}),p=function(e){var t=n.useContext(l),a=t;return e&&(a="function"==typeof e?e(t):i(i({},t),e)),a},c=function(e){var t=p(e.components);return n.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},y=n.forwardRef((function(e,t){var a=e.components,o=e.mdxType,r=e.originalType,l=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),y=p(a),d=o,h=y["".concat(l,".").concat(d)]||y[d]||u[d]||r;return a?n.createElement(h,i(i({ref:t},c),{},{components:a})):n.createElement(h,i({ref:t},c))}));function d(e,t){var a=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=a.length,i=new Array(r);i[0]=y;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s.mdxType="string"==typeof e?e:o,i[1]=s;for(var p=2;p{a.r(t),a.d(t,{assets:()=>c,contentTitle:()=>l,default:()=>d,frontMatter:()=>s,metadata:()=>p,toc:()=>u});var n=a(9668),o=a(1367),r=(a(6540),a(5680)),i=["components"],s={id:"start-workspace",title:"Start your workspace"},l=void 0,p={unversionedId:"start-workspace",id:"start-workspace",title:"Start your workspace",description:"This page will help you to start a workspace to run your code and experiments on the DSRI in a container.",source:"@site/docs/start-workspace.md",sourceDirName:".",slug:"/start-workspace",permalink:"/docs/start-workspace",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/start-workspace.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"start-workspace",title:"Start your workspace"},sidebar:"docs",previous:{title:"Access the DSRI",permalink:"/docs/access-dsri"},next:{title:"Install the client",permalink:"/docs/openshift-install"}},c={},u=[{value:"Introduction to containers",id:"introduction-to-containers",level:2},{value:"Choose your interface",id:"choose-your-interface",level:2},{value:"Start your workspace",id:"start-your-workspace",level:2},{value:"Upload your code and data",id:"upload-your-code-and-data",level:2},{value:"Install your dependencies",id:"install-your-dependencies",level:2},{value:"Run your code",id:"run-your-code",level:2},{value:"Stop your application",id:"stop-your-application",level:2},{value:"Start your application",id:"start-your-application",level:2},{value:"Optional: define a docker image",id:"optional-define-a-docker-image",level:2}],y={toc:u};function d(e){var t=e.components,a=(0,o.A)(e,i);return(0,r.yg)("wrapper",(0,n.A)({},y,a,{components:t,mdxType:"MDXLayout"}),(0,r.yg)("p",null,"This page will help you to start a workspace to run your code and experiments on the DSRI in a container. "),(0,r.yg)("h2",{id:"introduction-to-containers"},"Introduction to containers"),(0,r.yg)("p",null,"Anything running in DSRI needs to be running in a docker container. Docker containers are namespaces that share the kernel on a linux system, you can see them as a clean minimalist Linux computers with only what you need to run your programs installed. This allows you to completely control the environment where your code runs, and avoid conflicts."),(0,r.yg)("p",null,"When running experiments we can start from existing images that have been already published for popular data science applications with a web interface. You can use, for example, JupyterLab when running python, RStudio when running R, or VisualStudio Code if you prefer."),(0,r.yg)("p",null,"Once you access a running container, you can install anything you need like if it was a linux/ubuntu computer (most of them runs with admin privileges), and run anything via the notebook/RStudio/VSCode interface, or the terminal."),(0,r.yg)("h2",{id:"choose-your-interface"},"Choose your interface"),(0,r.yg)("p",null,"First step to get your code running on the DSRI is to pick the base interface you want to use to access your workspace on the DSRI."),(0,r.yg)("p",null,"We prepared generic Docker images for data science workspaces with your favorite web UI pre-installed to easily deploy your workspace. So you just need to choose your favorite workspace, start the container, access it, add your code, and install your dependencies."),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},"Login to the DSRI dashboard"),(0,r.yg)("li",{parentName:"ol"},"Select your project, or create one with a meaningful short name representing your project, e.g. ",(0,r.yg)("inlineCode",{parentName:"li"},"workspace-yourname")),(0,r.yg)("li",{parentName:"ol"},"Go to the ",(0,r.yg)("strong",{parentName:"li"},"+Add")," page, and select to add ",(0,r.yg)("strong",{parentName:"li"},"From Developer Catalog => All services"))),(0,r.yg)("img",{src:"/img/screenshot_access_catalog_manu17112024.png",alt:"Access catalog",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("ol",{start:4},(0,r.yg)("li",{parentName:"ol"},"Search for templates corresponding to the application you want to deploy among the one described below (make sure the filter for templates is properly checked). ")),(0,r.yg)("p",null,(0,r.yg)("strong",{parentName:"p"},"JupyterLab"),": Access and run your code using the popular Jupyter notebooks, with kernel for python, java, R, julia. It also provides a good web interface to access the terminal, upload and browse the files."),(0,r.yg)("p",null,(0,r.yg)("strong",{parentName:"p"},"VisualStudio Code"),": Your daily IDE, but in your browser, running on the DSRI. "),(0,r.yg)("p",null,(0,r.yg)("strong",{parentName:"p"},"RStudio"),": R users favorite's."),(0,r.yg)("p",null,(0,r.yg)("strong",{parentName:"p"},"The terminal"),": For people who are used to the terminal and just want to run scripts, it provides smaller and more stable images, which makes installation and deployment easier. You can use the ",(0,r.yg)("strong",{parentName:"p"},"Ubuntu")," template to start a basic ubuntu image and access it from the terminal."),(0,r.yg)("p",null,(0,r.yg)("strong",{parentName:"p"},"Any web interface"),": You can easily run and access most programs with a web interface on the DSRI. You can use the template ",(0,r.yg)("strong",{parentName:"p"},"Custom workspace")," if your application is exposed on port 8888. Otherwise visit the page ",(0,r.yg)("a",{parentName:"p",href:"/docs/anatomy-of-an-application"},"Anatomy of a DSRI application")," for more details. "),(0,r.yg)("p",null,(0,r.yg)("strong",{parentName:"p"},"Desktop interface"),": there is the possibility to start a container as a Linux operating system with a graphical desktop interface. It can be useful to deploy software like Matlab, but the setup can be a bit more complex. You will get an Ubuntu computer with a basic Desktop interface, running on the DSRI, that you can access directly in your web browser. The desktop interface is accessed through a web application by using noVNC, which exposes the VNC connection without needing a VNC client."),(0,r.yg)("admonition",{title:"More applications",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"You can also find more documentation on the different applications that can be deployed from the DSRI under ",(0,r.yg)("strong",{parentName:"p"},"Deploy applications")," in the menu on the left.")),(0,r.yg)("h2",{id:"start-your-workspace"},"Start your workspace"),(0,r.yg)("p",null,"Once you chose your favorite way to run your experiments, you can click on the application you want to use for your workspace. Checkout the description to learn more details about the application that will be deployed. "),(0,r.yg)("p",null,"Then click on ",(0,r.yg)("strong",{parentName:"p"},"Instantiate Template"),", and fill the parameters, such as the password to access the web UI. Note that the application name needs to be unique in the project. Finally click on the ",(0,r.yg)("strong",{parentName:"p"},"Create")," button."),(0,r.yg)("img",{src:"/img/screenshot_template_configuration.png",alt:"Filter templates catalog",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("p",null,"You should see your application in your project dashboard, it can take a few seconds to a few minutes to pull the docker image and start the application. "),(0,r.yg)("p",null,"Once the application has started you will be able to access it by clicking on its circle, then click the ",(0,r.yg)("strong",{parentName:"p"},"Route"),", that has been automatically generated for the web interface, in the Resources tab."),(0,r.yg)("admonition",{title:"Check the workshop",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"For a more detailed tutorial, you can follow the ",(0,r.yg)("a",{parentName:"p",href:"https://maastrichtu-ids.github.io/dsri-workshop-start-app/"},"workshop to start Data Science applications on the DSRI"))),(0,r.yg)("h2",{id:"upload-your-code-and-data"},"Upload your code and data"),(0,r.yg)("p",null,"We recommend you to use ",(0,r.yg)("inlineCode",{parentName:"p"},"git")," to clone your project code in your workspace, as it helps sharing and managing the evolution of your project. "),(0,r.yg)("p",null,"It will be preinstalled in most images, otherwise you can install it easily with ",(0,r.yg)("inlineCode",{parentName:"p"},"apt-get install git")),(0,r.yg)("p",null,"With web interface like JupyterLab, VisualStudio Code and Rstudio you can easily upload small and medium size files directly through the UI with a drag and drop."),(0,r.yg)("p",null,"Otherwise you can use the terminal, install the ",(0,r.yg)("inlineCode",{parentName:"p"},"oc")," client, and use the ",(0,r.yg)("inlineCode",{parentName:"p"},"oc cp")," or ",(0,r.yg)("inlineCode",{parentName:"p"},"oc rsync")," commands to upload large files to your workspace on the DSRI. See the Upload data page for more details."),(0,r.yg)("h2",{id:"install-your-dependencies"},"Install your dependencies"),(0,r.yg)("p",null,"Once the workspace is started, you can install the different dependencies you need to run your experiments."),(0,r.yg)("p",null,"It is recommended to save all the commands you used to install the different requirements in a script (e.g. ",(0,r.yg)("inlineCode",{parentName:"p"},"install.sh"),"). This will insure you can reinstall the environment easily and faithfully if the container is restarted. You can also use them to create a Docker image with everything prepared for your application."),(0,r.yg)("p",null,"Most containers for science are based on debian/ubuntu, so you can install new packages with ",(0,r.yg)("inlineCode",{parentName:"p"},"apt-get"),":"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"apt-get update\napt-get install -y build-essentials wget curl\n")),(0,r.yg)("h2",{id:"run-your-code"},"Run your code"),(0,r.yg)("p",null,"You can use your web interface to run your code as you like to do: notebooks, rstudio, execute via VSCode"),(0,r.yg)("p",null,"Note that for jobs which are running for a long time the web UI is not always the best solution, e.g. Jupyter notebooks can be quite instable when running a 30 min codeblock."),(0,r.yg)("p",null,"A quick solution for that is to run your code in scripts, using the bash terminal. You can use the ",(0,r.yg)("inlineCode",{parentName:"p"},"nohup")," prefix, and ",(0,r.yg)("inlineCode",{parentName:"p"},"&")," suffix to run your script in the background, so that you can even disconnect, and come back later to check the results and logs."),(0,r.yg)("p",null,"For example with a python script, you would do:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"nohup python my_script.py &\n")),(0,r.yg)("p",null,"The script will run in the background, and all terminal output will be stored in the file ",(0,r.yg)("inlineCode",{parentName:"p"},"nohup.out")),(0,r.yg)("p",null,"You can also check if the process is currently running by typing ",(0,r.yg)("inlineCode",{parentName:"p"},"ps aux")," or ",(0,r.yg)("inlineCode",{parentName:"p"},"top")," "),(0,r.yg)("p",null,"You can kill the process by getting the process ID (PID) using the previous commands, and then: ",(0,r.yg)("inlineCode",{parentName:"p"},"kill -9 PID")),(0,r.yg)("h2",{id:"stop-your-application"},"Stop your application"),(0,r.yg)("p",null,"When you are not using your application anymore you can stop the pod. If you are using a Dynamic or Persistent storage you can restart the pod and continue working with all your data in the same state as you left it."),(0,r.yg)("admonition",{title:"Do not waste resources",type:"caution"},(0,r.yg)("p",{parentName:"admonition"},"Please think of stopping applications you are not using to avoid consuming unnecessary resources.")),(0,r.yg)("p",null,"On the ",(0,r.yg)("strong",{parentName:"p"},"Topology")," page click on the down arrow \u2b07\ufe0f next to the number of pods deployed."),(0,r.yg)("img",{src:"/img/screenshot_scaledown_pod.png",alt:"Scale down pod",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("p",null,"You can then restart the pod by clicking the up arrow \u2b06\ufe0f"),(0,r.yg)("p",null,"Note that starting more than 1 pod will not increase the amount of resources you have access to, most of the time it will only waste resources and might ends up in weird behavior on your side. The web UI will randomly assign you to 1 of the pod started when you access it. This only works for clusters with multiple workers, such as Apache Flink and Spark. Or if you connect directly to each pod with the terminal to run different processes."),(0,r.yg)("h2",{id:"start-your-application"},"Start your application"),(0,r.yg)("p",null,"When you try to access your workspace and you encounter the page below, usually this indicates that your pod is not running. For example, this will be the case if you stopped your pod, or if there was maintenance on the cluster."),(0,r.yg)("img",{src:"/img/screenshot_application_unavailable.png",alt:"Screenshot of page that says Application is not available",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("p",null,"To start the pod, go to the ",(0,r.yg)("strong",{parentName:"p"},"Topology")," page, and click on the up arrow \u2b06\ufe0f next to the number of pods deployed. Make sure you scale it to 1. Scaling it to more than 1 will not increase the amount of resources you have access to, most of the time it will only waste resources and causes weird behavior on your side."),(0,r.yg)("img",{src:"/img/screenshot_scaledown_pod.png",alt:"Scale down pod",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("admonition",{title:"Do not waste resources",type:"caution"},(0,r.yg)("p",{parentName:"admonition"},"Please only scale up resources you're using, and scale down when you're not using them anymore. Consuming resources consumes unnecessary power and might prevent other users from using the DSRI.")),(0,r.yg)("h2",{id:"optional-define-a-docker-image"},"Optional: define a docker image"),(0,r.yg)("p",null,"Once you have tested your workspace and you know how to set it up it can be helpful to define a ",(0,r.yg)("inlineCode",{parentName:"p"},"Dockerfile")," to build and publish a Docker image with everything directly installed (instead of installing your requirements after starting a generic workspace)"),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},"Start from an existing generic Docker image, depending on the base technologies you need, such as Debian, Ubuntu, Python, JupyterLab, VisualStudio Code, RStudio..."),(0,r.yg)("li",{parentName:"ol"},"Add your source code in the Docker image using ",(0,r.yg)("inlineCode",{parentName:"li"},"ADD . .")," or ",(0,r.yg)("inlineCode",{parentName:"li"},"COPY . .")),(0,r.yg)("li",{parentName:"ol"},"Install dependencies (e.g. ",(0,r.yg)("inlineCode",{parentName:"li"},"RUN apt-get install gfortran"),")"),(0,r.yg)("li",{parentName:"ol"},"Define which command to run when starting the container (e.g. ",(0,r.yg)("inlineCode",{parentName:"li"},'ENTRYPOINT["jupyter", "lab"]'),")")),(0,r.yg)("p",null,"Here is a simple example ",(0,r.yg)("inlineCode",{parentName:"p"},"Dockerfile")," for a python application:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-dockerfile"},'# The base image to start from, choose the one with everything you need installed\nFROM python:3.8\n\n# Change the user and working directory to make sure we are using root\nUSER root\nWORKDIR /root\n\n# Install additional packages\nRUN apt-get update && \\\n apt-get install build-essentials\n\n# This line will copy all files and folder that are in the same folder as the Dockerfile (usually the code you want to run in the container)\nADD . . \n\n# This line will install all the python packages described in the requirements.txt of your source code\nRUN pip install -r requirements.txt && \\\n pip install notebook jupyterlab\n\n# Command to run when the container is started, here it starts JupyterLab as a service\nENTRYPOINT [ "jupyter", "lab" ]\n')),(0,r.yg)("p",null,"Here are some examples of ",(0,r.yg)("inlineCode",{parentName:"p"},"Dockerfile")," for various type of web applications:"),(0,r.yg)("ul",null,(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/jupyterlab/blob/main/Dockerfile"},"Custom JupyterLab")," based on the official ",(0,r.yg)("a",{parentName:"li",href:"https://github.com/jupyter/docker-stacks"},"jupyter/docker-stacks")),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/rstudio/blob/main/Dockerfile"},"Custom RStudio")),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/code-server/blob/main/Dockerfile"},"VisualStudio Code server")),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/knowledge-collaboratory-api/blob/master/Dockerfile"},"Python web app"))),(0,r.yg)("p",null,"See the guide to ",(0,r.yg)("a",{parentName:"p",href:"/docs/guide-publish-image"},"Publish a Docker image")," for more details on this topic."))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/99b0f44e.6209a3bf.js b/assets/js/99b0f44e.1708e4dd.js similarity index 99% rename from assets/js/99b0f44e.6209a3bf.js rename to assets/js/99b0f44e.1708e4dd.js index 4ca961354..8044a2ff5 100644 --- a/assets/js/99b0f44e.6209a3bf.js +++ b/assets/js/99b0f44e.1708e4dd.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[7668],{5680:(e,o,t)=>{t.d(o,{xA:()=>u,yg:()=>f});var n=t(6540);function r(e,o,t){return o in e?Object.defineProperty(e,o,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[o]=t,e}function a(e,o){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);o&&(n=n.filter((function(o){return Object.getOwnPropertyDescriptor(e,o).enumerable}))),t.push.apply(t,n)}return t}function i(e){for(var o=1;o=0||(r[t]=e[t]);return r}(e,o);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(r[t]=e[t])}return r}var s=n.createContext({}),p=function(e){var o=n.useContext(s),t=o;return e&&(t="function"==typeof e?e(o):i(i({},o),e)),t},u=function(e){var o=p(e.components);return n.createElement(s.Provider,{value:o},e.children)},c={inlineCode:"code",wrapper:function(e){var o=e.children;return n.createElement(n.Fragment,{},o)}},d=n.forwardRef((function(e,o){var t=e.components,r=e.mdxType,a=e.originalType,s=e.parentName,u=l(e,["components","mdxType","originalType","parentName"]),d=p(t),f=r,g=d["".concat(s,".").concat(f)]||d[f]||c[f]||a;return t?n.createElement(g,i(i({ref:o},u),{},{components:t})):n.createElement(g,i({ref:o},u))}));function f(e,o){var t=arguments,r=o&&o.mdxType;if("string"==typeof e||r){var a=t.length,i=new Array(a);i[0]=d;var l={};for(var s in o)hasOwnProperty.call(o,s)&&(l[s]=o[s]);l.originalType=e,l.mdxType="string"==typeof e?e:r,i[1]=l;for(var p=2;p{t.r(o),t.d(o,{assets:()=>u,contentTitle:()=>s,default:()=>f,frontMatter:()=>l,metadata:()=>p,toc:()=>c});var n=t(9668),r=t(1367),a=(t(6540),t(5680)),i=["components"],l={id:"workflows-introduction",title:"Introduction to workflows"},s=void 0,p={unversionedId:"workflows-introduction",id:"workflows-introduction",title:"Introduction to workflows",description:"Running workflows on the DSRI is a work in progress. It usually requires some knowledge about how to orchestrate containers.",source:"@site/docs/workflows-introduction.md",sourceDirName:".",slug:"/workflows-introduction",permalink:"/docs/workflows-introduction",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/workflows-introduction.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"workflows-introduction",title:"Introduction to workflows"},sidebar:"docs",previous:{title:"Glossary",permalink:"/docs/glossary"},next:{title:"Deploy GitHub Runners",permalink:"/docs/workflows-github-actions"}},u={},c=[{value:"Introduction",id:"introduction",level:2},{value:"Current solutions on the DSRI",id:"current-solutions-on-the-dsri",level:2},{value:"GitHub Actions workflows",id:"github-actions-workflows",level:3},{value:"Apache Airflow",id:"apache-airflow",level:3},{value:"Argo",id:"argo",level:3},{value:"More options",id:"more-options",level:2},{value:"Kubeflow",id:"kubeflow",level:3},{value:"Apache Airflow",id:"apache-airflow-1",level:3},{value:"Volcano",id:"volcano",level:3},{value:"Nextflow",id:"nextflow",level:3},{value:"CWL",id:"cwl",level:3},{value:"KubeGene",id:"kubegene",level:3},{value:"Seldon",id:"seldon",level:3}],d={toc:c};function f(e){var o=e.components,t=(0,r.A)(e,i);return(0,a.yg)("wrapper",(0,n.A)({},d,t,{components:o,mdxType:"MDXLayout"}),(0,a.yg)("admonition",{title:"Work in progress",type:"warning"},(0,a.yg)("p",{parentName:"admonition"},"Running workflows on the DSRI is a work in progress. It usually requires some knowledge about how to orchestrate containers.")),(0,a.yg)("h2",{id:"introduction"},"Introduction"),(0,a.yg)("p",null,"Multiple technologies are available to run workflows on OpenShift/Kubernetes clusters. Each has its strengths and weaknesses in different areas."),(0,a.yg)("admonition",{title:"Use-case dependant",type:"caution"},(0,a.yg)("p",{parentName:"admonition"},"The technology to use needs to be ",(0,a.yg)("strong",{parentName:"p"},"chosen depending on your use-case"),".")),(0,a.yg)("h2",{id:"current-solutions-on-the-dsri"},"Current solutions on the DSRI"),(0,a.yg)("p",null,"Those solutions can easily be deployed on the DSRI. Let"),(0,a.yg)("h3",{id:"github-actions-workflows"},"GitHub Actions workflows"),(0,a.yg)("p",null,"GitHub Actions allows you to define automatically containerized workflows through a simple YAML file hosted in your GitHub repository."),(0,a.yg)("p",null,"See the ",(0,a.yg)("a",{parentName:"p",href:"/docs/workflows-github-actions"},"page about GitHub Actions runners")," for more details, and to deploy runners on the DSRI."),(0,a.yg)("h3",{id:"apache-airflow"},"Apache Airflow"),(0,a.yg)("p",null,"Airflow is a platform to programmatically author, schedule and monitor workflows, aka. DAGs (directed acyclic graphs)."),(0,a.yg)("p",null,"See the ",(0,a.yg)("a",{parentName:"p",href:"/docs/workflows-airflow"},"page about Airflow")," for more details, and to deploy Airflow on the DSRI."),(0,a.yg)("h3",{id:"argo"},"Argo"),(0,a.yg)("p",null,(0,a.yg)("a",{parentName:"p",href:"https://argoproj.github.io/argo/"},"Argo")," is a container native workflow engine for ",(0,a.yg)("a",{parentName:"p",href:"https://kubernetes.io/"},"Kubernetes")," supporting both ",(0,a.yg)("a",{parentName:"p",href:"https://argoproj.github.io/docs/argo/examples/readme.html#dag"},"DAG")," and ",(0,a.yg)("a",{parentName:"p",href:"https://argoproj.github.io/docs/argo/examples/readme.html#steps"},"step based")," workflows."),(0,a.yg)("ul",null,(0,a.yg)("li",{parentName:"ul"},"Workflows easy to define using Kubernetes-like YAML files."),(0,a.yg)("li",{parentName:"ul"},"Easy to define if your workflow is composed of Docker containers to run with arguments.")),(0,a.yg)("admonition",{title:"Contact us",type:"info"},(0,a.yg)("p",{parentName:"admonition"},(0,a.yg)("a",{parentName:"p",href:"/help"},"Contact us")," if you want to run Argo workflow on the DSRI")),(0,a.yg)("h2",{id:"more-options"},"More options"),(0,a.yg)("p",null,"Let us know if you are interested in deploying, and using, any of those workflows on the DSRI."),(0,a.yg)("h3",{id:"kubeflow"},"Kubeflow"),(0,a.yg)("p",null,"Optimized for Tensorflow workflows on Kubernetes."),(0,a.yg)("p",null,"Pipelines written in Python."),(0,a.yg)("h3",{id:"apache-airflow-1"},"Apache Airflow"),(0,a.yg)("p",null,"Define, schedule and run workflows. "),(0,a.yg)("p",null,"Can be deployed with OpenDataHub, see also ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/majordomusio/openshift-airflow"},"this deployment for OpenShift"),"."),(0,a.yg)("p",null,"See also: Airflow on ",(0,a.yg)("a",{parentName:"p",href:"https://kubernetes.io/blog/2018/06/28/airflow-on-kubernetes-part-1-a-different-kind-of-operator/"},"Kubernetes blog"),", and Kubernetes in ",(0,a.yg)("a",{parentName:"p",href:"https://airflow.apache.org/docs/stable/kubernetes.html"},"Airflow documentation"),"."),(0,a.yg)("h3",{id:"volcano"},"Volcano"),(0,a.yg)("p",null,"Run batch pipelines on Kubernetes with ",(0,a.yg)("a",{parentName:"p",href:"https://volcano.sh/"},"Volcano"),". "),(0,a.yg)("ul",null,(0,a.yg)("li",{parentName:"ul"},(0,a.yg)("p",{parentName:"li"},"More a scheduler than a workflow engine. ")),(0,a.yg)("li",{parentName:"ul"},(0,a.yg)("p",{parentName:"li"},"Volcano can be used to run Spark, Kubeflow or KubeGene workflows."))),(0,a.yg)("h3",{id:"nextflow"},"Nextflow"),(0,a.yg)("p",null,(0,a.yg)("a",{parentName:"p",href:"https://www.nextflow.io/"},"Nextflow")," has been developed by the genomic research scientific community and is built to run bioinformatics pipeline."),(0,a.yg)("p",null,"Define your workflow in a Bash script fashion, providing input, output and the command to run. Without the need to create and use Docker container for Conda pipelines."),(0,a.yg)("h3",{id:"cwl"},"CWL"),(0,a.yg)("ul",null,(0,a.yg)("li",{parentName:"ul"},"Developed by the genomic research scientific community."),(0,a.yg)("li",{parentName:"ul"},"Good support for provenance description (export as RDF)."),(0,a.yg)("li",{parentName:"ul"},"Support on OpenShift still in development",(0,a.yg)("ul",{parentName:"li"},(0,a.yg)("li",{parentName:"ul"},(0,a.yg)("a",{parentName:"li",href:"https://airflow.apache.org/docs/stable/kubernetes.html"},"Apache Airflow")),(0,a.yg)("li",{parentName:"ul"},(0,a.yg)("a",{parentName:"li",href:"https://github.com/Duke-GCB/calrissian/"},"workflows-cwl")))),(0,a.yg)("li",{parentName:"ul"},"Propose a GUI to build the workflows: ",(0,a.yg)("a",{parentName:"li",href:"https://rabix.io/"},"Rabix Composer"))),(0,a.yg)("h3",{id:"kubegene"},"KubeGene"),(0,a.yg)("p",null,(0,a.yg)("a",{parentName:"p",href:"https://kubegene.io/"},"KubeGene")," is a turn-key genome sequencing workflow management framework."),(0,a.yg)("p",null,"See the ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/kubegene/kubegene/blob/master/example/simple-sample/simple-sample.yaml"},"Workflow example"),", and how to ",(0,a.yg)("a",{parentName:"p",href:"https://kubegene.io/docs/guides/tool/"},"define a tool"),"."),(0,a.yg)("h3",{id:"seldon"},"Seldon"),(0,a.yg)("p",null,(0,a.yg)("a",{parentName:"p",href:"https://www.seldon.io/tech/"},"Open-source platform")," for rapidly deploying machine learning models on Kubernetes. Manage, serve and scale models built in any framework on Kubernetes."),(0,a.yg)("admonition",{title:"Contact us",type:"info"},(0,a.yg)("p",{parentName:"admonition"},"Feel free to ",(0,a.yg)("a",{parentName:"p",href:"/help"},"contact us")," if you have any questions about running workflows on DSRI or to request the support of a new technology.")))}f.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[7668],{5680:(e,o,t)=>{t.d(o,{xA:()=>u,yg:()=>f});var n=t(6540);function r(e,o,t){return o in e?Object.defineProperty(e,o,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[o]=t,e}function a(e,o){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);o&&(n=n.filter((function(o){return Object.getOwnPropertyDescriptor(e,o).enumerable}))),t.push.apply(t,n)}return t}function i(e){for(var o=1;o=0||(r[t]=e[t]);return r}(e,o);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(r[t]=e[t])}return r}var s=n.createContext({}),p=function(e){var o=n.useContext(s),t=o;return e&&(t="function"==typeof e?e(o):i(i({},o),e)),t},u=function(e){var o=p(e.components);return n.createElement(s.Provider,{value:o},e.children)},c={inlineCode:"code",wrapper:function(e){var o=e.children;return n.createElement(n.Fragment,{},o)}},d=n.forwardRef((function(e,o){var t=e.components,r=e.mdxType,a=e.originalType,s=e.parentName,u=l(e,["components","mdxType","originalType","parentName"]),d=p(t),f=r,g=d["".concat(s,".").concat(f)]||d[f]||c[f]||a;return t?n.createElement(g,i(i({ref:o},u),{},{components:t})):n.createElement(g,i({ref:o},u))}));function f(e,o){var t=arguments,r=o&&o.mdxType;if("string"==typeof e||r){var a=t.length,i=new Array(a);i[0]=d;var l={};for(var s in o)hasOwnProperty.call(o,s)&&(l[s]=o[s]);l.originalType=e,l.mdxType="string"==typeof e?e:r,i[1]=l;for(var p=2;p{t.r(o),t.d(o,{assets:()=>u,contentTitle:()=>s,default:()=>f,frontMatter:()=>l,metadata:()=>p,toc:()=>c});var n=t(9668),r=t(1367),a=(t(6540),t(5680)),i=["components"],l={id:"workflows-introduction",title:"Introduction to workflows"},s=void 0,p={unversionedId:"workflows-introduction",id:"workflows-introduction",title:"Introduction to workflows",description:"Running workflows on the DSRI is a work in progress. It usually requires some knowledge about how to orchestrate containers.",source:"@site/docs/workflows-introduction.md",sourceDirName:".",slug:"/workflows-introduction",permalink:"/docs/workflows-introduction",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/workflows-introduction.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"workflows-introduction",title:"Introduction to workflows"},sidebar:"docs",previous:{title:"Glossary",permalink:"/docs/glossary"},next:{title:"Deploy GitHub Runners",permalink:"/docs/workflows-github-actions"}},u={},c=[{value:"Introduction",id:"introduction",level:2},{value:"Current solutions on the DSRI",id:"current-solutions-on-the-dsri",level:2},{value:"GitHub Actions workflows",id:"github-actions-workflows",level:3},{value:"Apache Airflow",id:"apache-airflow",level:3},{value:"Argo",id:"argo",level:3},{value:"More options",id:"more-options",level:2},{value:"Kubeflow",id:"kubeflow",level:3},{value:"Apache Airflow",id:"apache-airflow-1",level:3},{value:"Volcano",id:"volcano",level:3},{value:"Nextflow",id:"nextflow",level:3},{value:"CWL",id:"cwl",level:3},{value:"KubeGene",id:"kubegene",level:3},{value:"Seldon",id:"seldon",level:3}],d={toc:c};function f(e){var o=e.components,t=(0,r.A)(e,i);return(0,a.yg)("wrapper",(0,n.A)({},d,t,{components:o,mdxType:"MDXLayout"}),(0,a.yg)("admonition",{title:"Work in progress",type:"warning"},(0,a.yg)("p",{parentName:"admonition"},"Running workflows on the DSRI is a work in progress. It usually requires some knowledge about how to orchestrate containers.")),(0,a.yg)("h2",{id:"introduction"},"Introduction"),(0,a.yg)("p",null,"Multiple technologies are available to run workflows on OpenShift/Kubernetes clusters. Each has its strengths and weaknesses in different areas."),(0,a.yg)("admonition",{title:"Use-case dependant",type:"caution"},(0,a.yg)("p",{parentName:"admonition"},"The technology to use needs to be ",(0,a.yg)("strong",{parentName:"p"},"chosen depending on your use-case"),".")),(0,a.yg)("h2",{id:"current-solutions-on-the-dsri"},"Current solutions on the DSRI"),(0,a.yg)("p",null,"Those solutions can easily be deployed on the DSRI. Let"),(0,a.yg)("h3",{id:"github-actions-workflows"},"GitHub Actions workflows"),(0,a.yg)("p",null,"GitHub Actions allows you to define automatically containerized workflows through a simple YAML file hosted in your GitHub repository."),(0,a.yg)("p",null,"See the ",(0,a.yg)("a",{parentName:"p",href:"/docs/workflows-github-actions"},"page about GitHub Actions runners")," for more details, and to deploy runners on the DSRI."),(0,a.yg)("h3",{id:"apache-airflow"},"Apache Airflow"),(0,a.yg)("p",null,"Airflow is a platform to programmatically author, schedule and monitor workflows, aka. DAGs (directed acyclic graphs)."),(0,a.yg)("p",null,"See the ",(0,a.yg)("a",{parentName:"p",href:"/docs/workflows-airflow"},"page about Airflow")," for more details, and to deploy Airflow on the DSRI."),(0,a.yg)("h3",{id:"argo"},"Argo"),(0,a.yg)("p",null,(0,a.yg)("a",{parentName:"p",href:"https://argoproj.github.io/argo/"},"Argo")," is a container native workflow engine for ",(0,a.yg)("a",{parentName:"p",href:"https://kubernetes.io/"},"Kubernetes")," supporting both ",(0,a.yg)("a",{parentName:"p",href:"https://argoproj.github.io/docs/argo/examples/readme.html#dag"},"DAG")," and ",(0,a.yg)("a",{parentName:"p",href:"https://argoproj.github.io/docs/argo/examples/readme.html#steps"},"step based")," workflows."),(0,a.yg)("ul",null,(0,a.yg)("li",{parentName:"ul"},"Workflows easy to define using Kubernetes-like YAML files."),(0,a.yg)("li",{parentName:"ul"},"Easy to define if your workflow is composed of Docker containers to run with arguments.")),(0,a.yg)("admonition",{title:"Contact us",type:"info"},(0,a.yg)("p",{parentName:"admonition"},(0,a.yg)("a",{parentName:"p",href:"/help"},"Contact us")," if you want to run Argo workflow on the DSRI")),(0,a.yg)("h2",{id:"more-options"},"More options"),(0,a.yg)("p",null,"Let us know if you are interested in deploying, and using, any of those workflows on the DSRI."),(0,a.yg)("h3",{id:"kubeflow"},"Kubeflow"),(0,a.yg)("p",null,"Optimized for Tensorflow workflows on Kubernetes."),(0,a.yg)("p",null,"Pipelines written in Python."),(0,a.yg)("h3",{id:"apache-airflow-1"},"Apache Airflow"),(0,a.yg)("p",null,"Define, schedule and run workflows. "),(0,a.yg)("p",null,"Can be deployed with OpenDataHub, see also ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/majordomusio/openshift-airflow"},"this deployment for OpenShift"),"."),(0,a.yg)("p",null,"See also: Airflow on ",(0,a.yg)("a",{parentName:"p",href:"https://kubernetes.io/blog/2018/06/28/airflow-on-kubernetes-part-1-a-different-kind-of-operator/"},"Kubernetes blog"),", and Kubernetes in ",(0,a.yg)("a",{parentName:"p",href:"https://airflow.apache.org/docs/stable/kubernetes.html"},"Airflow documentation"),"."),(0,a.yg)("h3",{id:"volcano"},"Volcano"),(0,a.yg)("p",null,"Run batch pipelines on Kubernetes with ",(0,a.yg)("a",{parentName:"p",href:"https://volcano.sh/"},"Volcano"),". "),(0,a.yg)("ul",null,(0,a.yg)("li",{parentName:"ul"},(0,a.yg)("p",{parentName:"li"},"More a scheduler than a workflow engine. ")),(0,a.yg)("li",{parentName:"ul"},(0,a.yg)("p",{parentName:"li"},"Volcano can be used to run Spark, Kubeflow or KubeGene workflows."))),(0,a.yg)("h3",{id:"nextflow"},"Nextflow"),(0,a.yg)("p",null,(0,a.yg)("a",{parentName:"p",href:"https://www.nextflow.io/"},"Nextflow")," has been developed by the genomic research scientific community and is built to run bioinformatics pipeline."),(0,a.yg)("p",null,"Define your workflow in a Bash script fashion, providing input, output and the command to run. Without the need to create and use Docker container for Conda pipelines."),(0,a.yg)("h3",{id:"cwl"},"CWL"),(0,a.yg)("ul",null,(0,a.yg)("li",{parentName:"ul"},"Developed by the genomic research scientific community."),(0,a.yg)("li",{parentName:"ul"},"Good support for provenance description (export as RDF)."),(0,a.yg)("li",{parentName:"ul"},"Support on OpenShift still in development",(0,a.yg)("ul",{parentName:"li"},(0,a.yg)("li",{parentName:"ul"},(0,a.yg)("a",{parentName:"li",href:"https://airflow.apache.org/docs/stable/kubernetes.html"},"Apache Airflow")),(0,a.yg)("li",{parentName:"ul"},(0,a.yg)("a",{parentName:"li",href:"https://github.com/Duke-GCB/calrissian/"},"workflows-cwl")))),(0,a.yg)("li",{parentName:"ul"},"Propose a GUI to build the workflows: ",(0,a.yg)("a",{parentName:"li",href:"https://rabix.io/"},"Rabix Composer"))),(0,a.yg)("h3",{id:"kubegene"},"KubeGene"),(0,a.yg)("p",null,(0,a.yg)("a",{parentName:"p",href:"https://kubegene.io/"},"KubeGene")," is a turn-key genome sequencing workflow management framework."),(0,a.yg)("p",null,"See the ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/kubegene/kubegene/blob/master/example/simple-sample/simple-sample.yaml"},"Workflow example"),", and how to ",(0,a.yg)("a",{parentName:"p",href:"https://kubegene.io/docs/guides/tool/"},"define a tool"),"."),(0,a.yg)("h3",{id:"seldon"},"Seldon"),(0,a.yg)("p",null,(0,a.yg)("a",{parentName:"p",href:"https://www.seldon.io/tech/"},"Open-source platform")," for rapidly deploying machine learning models on Kubernetes. Manage, serve and scale models built in any framework on Kubernetes."),(0,a.yg)("admonition",{title:"Contact us",type:"info"},(0,a.yg)("p",{parentName:"admonition"},"Feel free to ",(0,a.yg)("a",{parentName:"p",href:"/help"},"contact us")," if you have any questions about running workflows on DSRI or to request the support of a new technology.")))}f.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/9bfda053.1b6abd23.js b/assets/js/9bfda053.264d7928.js similarity index 99% rename from assets/js/9bfda053.1b6abd23.js rename to assets/js/9bfda053.264d7928.js index f904d5c62..271a5b8a1 100644 --- a/assets/js/9bfda053.1b6abd23.js +++ b/assets/js/9bfda053.264d7928.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[8151],{5680:(e,t,a)=>{a.d(t,{xA:()=>m,yg:()=>u});var r=a(6540);function n(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function i(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,r)}return a}function o(e){for(var t=1;t=0||(n[a]=e[a]);return n}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(n[a]=e[a])}return n}var s=r.createContext({}),g=function(e){var t=r.useContext(s),a=t;return e&&(a="function"==typeof e?e(t):o(o({},t),e)),a},m=function(e){var t=g(e.components);return r.createElement(s.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},d=r.forwardRef((function(e,t){var a=e.components,n=e.mdxType,i=e.originalType,s=e.parentName,m=l(e,["components","mdxType","originalType","parentName"]),d=g(a),u=n,c=d["".concat(s,".").concat(u)]||d[u]||p[u]||i;return a?r.createElement(c,o(o({ref:t},m),{},{components:a})):r.createElement(c,o({ref:t},m))}));function u(e,t){var a=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var i=a.length,o=new Array(i);o[0]=d;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:n,o[1]=l;for(var g=2;g{a.r(t),a.d(t,{assets:()=>m,contentTitle:()=>s,default:()=>u,frontMatter:()=>l,metadata:()=>g,toc:()=>p});var r=a(9668),n=a(1367),i=(a(6540),a(5680)),o=["components"],l={id:"dask-cluster",title:"Deploy Dask Cluster"},s=void 0,g={unversionedId:"dask-cluster",id:"dask-cluster",title:"Deploy Dask Cluster",description:"\ud83e\uddca Installation with Helm",source:"@site/docs/dask-cluster.md",sourceDirName:".",slug:"/dask-cluster",permalink:"/docs/dask-cluster",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/dask-cluster.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"dask-cluster",title:"Deploy Dask Cluster"},sidebar:"docs",previous:{title:"JupyterHub",permalink:"/docs/deploy-jupyterhub"},next:{title:"Spark cluster",permalink:"/docs/deploy-spark"}},m={},p=[{value:"\ud83e\uddca Installation with Helm",id:"-installation-with-helm",level:2},{value:"\ud83e\ude90 Configure a Route for the Cluster",id:"-configure-a-route-for-the-cluster",level:3},{value:"\ud83e\ude90 Access the Jupyter Password/Token",id:"-access-the-jupyter-passwordtoken",level:3}],d={toc:p};function u(e){var t=e.components,a=(0,n.A)(e,o);return(0,i.yg)("wrapper",(0,r.A)({},d,a,{components:t,mdxType:"MDXLayout"}),(0,i.yg)("h2",{id:"-installation-with-helm"},"\ud83e\uddca Installation with Helm"),(0,i.yg)("ol",null,(0,i.yg)("li",{parentName:"ol"},"Go to the ",(0,i.yg)("strong",{parentName:"li"},"+Add")," page, and select to add ",(0,i.yg)("strong",{parentName:"li"},"Helm Chart"))),(0,i.yg)("img",{src:"/img/dask-init1.png",alt:"dask init",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("ol",{start:2},(0,i.yg)("li",{parentName:"ol"},"Search and Select the ",(0,i.yg)("strong",{parentName:"li"},"Dask chart")," then click on ",(0,i.yg)("strong",{parentName:"li"},"Create"))),(0,i.yg)("img",{src:"/img/dask-init2.png",alt:"dask init",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("img",{src:"/img/dask-init3.png",alt:"dask init",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("ol",{start:3},(0,i.yg)("li",{parentName:"ol"},"Configure the Yaml file, while under the ",(0,i.yg)("inlineCode",{parentName:"li"},"Jupyter")," section:",(0,i.yg)("ul",{parentName:"li"},(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("inlineCode",{parentName:"li"},'Command: ["jupyter", "lab", "--allow-root", "--ip=0.0.0.0", "--port=8888", "--no-browser"]')),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("inlineCode",{parentName:"li"},"servicePort: 8888"))))),(0,i.yg)("img",{src:"/img/dask-init4.png",alt:"dask init",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("img",{src:"/img/dask-init5.png",alt:"dask init",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("ol",{start:4},(0,i.yg)("li",{parentName:"ol"},(0,i.yg)("strong",{parentName:"li"},"Add Storage")," to the ",(0,i.yg)("strong",{parentName:"li"},"dask-jupyter")," pod as shown below")),(0,i.yg)("img",{src:"/img/dask-init6.png",alt:"dask init",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("ol",{start:5},(0,i.yg)("li",{parentName:"ol"},"Set up a new ",(0,i.yg)("strong",{parentName:"li"},"Persistent Volume Claim")," for the cluster as shown below")),(0,i.yg)("img",{src:"/img/dask-init7.png",alt:"dask init",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("h3",{id:"-configure-a-route-for-the-cluster"},"\ud83e\ude90 Configure a Route for the Cluster"),(0,i.yg)("ol",null,(0,i.yg)("li",{parentName:"ol"},"Switch to the ",(0,i.yg)("strong",{parentName:"li"},"Administrator")," view and navigate to ",(0,i.yg)("strong",{parentName:"li"},"Route"))),(0,i.yg)("img",{src:"/img/dask-route1.png",alt:"dask route",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("img",{src:"/img/dask-route2.png",alt:"dask route",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("ol",{start:2},(0,i.yg)("li",{parentName:"ol"},"Create a new route by clicking the button ",(0,i.yg)("strong",{parentName:"li"},"Create Route")," with the setup as shown below")),(0,i.yg)("img",{src:"/img/dask-route3.png",alt:"dask route",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("img",{src:"/img/dask-route4.png",alt:"dask route",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("img",{src:"/img/dask-route5.png",alt:"dask route",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("ol",{start:3},(0,i.yg)("li",{parentName:"ol"},"Navigate the provided link to access your local cluster")),(0,i.yg)("img",{src:"/img/dask-route6.png",alt:"dask route",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("h3",{id:"-access-the-jupyter-passwordtoken"},"\ud83e\ude90 Access the Jupyter Password/Token"),(0,i.yg)("ol",null,(0,i.yg)("li",{parentName:"ol"},"Start up the ",(0,i.yg)("strong",{parentName:"li"},"terminal"),(0,i.yg)("ul",{parentName:"li"},(0,i.yg)("li",{parentName:"ul"},"Run ",(0,i.yg)("inlineCode",{parentName:"li"},"oc get pods")," to find the full podname of the ",(0,i.yg)("strong",{parentName:"li"},"dask-jupyter")),(0,i.yg)("li",{parentName:"ul"},"Run ",(0,i.yg)("inlineCode",{parentName:"li"},"oc logs ")," and copy the token used to access the jupyter notebook")))),(0,i.yg)("img",{src:"/img/dask-route7.png",alt:"dask route",style:{maxWidth:"100%",maxHeight:"100%"}}))}u.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[8151],{5680:(e,t,a)=>{a.d(t,{xA:()=>m,yg:()=>u});var r=a(6540);function n(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function i(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,r)}return a}function o(e){for(var t=1;t=0||(n[a]=e[a]);return n}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(n[a]=e[a])}return n}var s=r.createContext({}),g=function(e){var t=r.useContext(s),a=t;return e&&(a="function"==typeof e?e(t):o(o({},t),e)),a},m=function(e){var t=g(e.components);return r.createElement(s.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},d=r.forwardRef((function(e,t){var a=e.components,n=e.mdxType,i=e.originalType,s=e.parentName,m=l(e,["components","mdxType","originalType","parentName"]),d=g(a),u=n,c=d["".concat(s,".").concat(u)]||d[u]||p[u]||i;return a?r.createElement(c,o(o({ref:t},m),{},{components:a})):r.createElement(c,o({ref:t},m))}));function u(e,t){var a=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var i=a.length,o=new Array(i);o[0]=d;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:n,o[1]=l;for(var g=2;g{a.r(t),a.d(t,{assets:()=>m,contentTitle:()=>s,default:()=>u,frontMatter:()=>l,metadata:()=>g,toc:()=>p});var r=a(9668),n=a(1367),i=(a(6540),a(5680)),o=["components"],l={id:"dask-cluster",title:"Deploy Dask Cluster"},s=void 0,g={unversionedId:"dask-cluster",id:"dask-cluster",title:"Deploy Dask Cluster",description:"\ud83e\uddca Installation with Helm",source:"@site/docs/dask-cluster.md",sourceDirName:".",slug:"/dask-cluster",permalink:"/docs/dask-cluster",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/dask-cluster.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"dask-cluster",title:"Deploy Dask Cluster"},sidebar:"docs",previous:{title:"JupyterHub",permalink:"/docs/deploy-jupyterhub"},next:{title:"Spark cluster",permalink:"/docs/deploy-spark"}},m={},p=[{value:"\ud83e\uddca Installation with Helm",id:"-installation-with-helm",level:2},{value:"\ud83e\ude90 Configure a Route for the Cluster",id:"-configure-a-route-for-the-cluster",level:3},{value:"\ud83e\ude90 Access the Jupyter Password/Token",id:"-access-the-jupyter-passwordtoken",level:3}],d={toc:p};function u(e){var t=e.components,a=(0,n.A)(e,o);return(0,i.yg)("wrapper",(0,r.A)({},d,a,{components:t,mdxType:"MDXLayout"}),(0,i.yg)("h2",{id:"-installation-with-helm"},"\ud83e\uddca Installation with Helm"),(0,i.yg)("ol",null,(0,i.yg)("li",{parentName:"ol"},"Go to the ",(0,i.yg)("strong",{parentName:"li"},"+Add")," page, and select to add ",(0,i.yg)("strong",{parentName:"li"},"Helm Chart"))),(0,i.yg)("img",{src:"/img/dask-init1.png",alt:"dask init",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("ol",{start:2},(0,i.yg)("li",{parentName:"ol"},"Search and Select the ",(0,i.yg)("strong",{parentName:"li"},"Dask chart")," then click on ",(0,i.yg)("strong",{parentName:"li"},"Create"))),(0,i.yg)("img",{src:"/img/dask-init2.png",alt:"dask init",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("img",{src:"/img/dask-init3.png",alt:"dask init",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("ol",{start:3},(0,i.yg)("li",{parentName:"ol"},"Configure the Yaml file, while under the ",(0,i.yg)("inlineCode",{parentName:"li"},"Jupyter")," section:",(0,i.yg)("ul",{parentName:"li"},(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("inlineCode",{parentName:"li"},'Command: ["jupyter", "lab", "--allow-root", "--ip=0.0.0.0", "--port=8888", "--no-browser"]')),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("inlineCode",{parentName:"li"},"servicePort: 8888"))))),(0,i.yg)("img",{src:"/img/dask-init4.png",alt:"dask init",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("img",{src:"/img/dask-init5.png",alt:"dask init",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("ol",{start:4},(0,i.yg)("li",{parentName:"ol"},(0,i.yg)("strong",{parentName:"li"},"Add Storage")," to the ",(0,i.yg)("strong",{parentName:"li"},"dask-jupyter")," pod as shown below")),(0,i.yg)("img",{src:"/img/dask-init6.png",alt:"dask init",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("ol",{start:5},(0,i.yg)("li",{parentName:"ol"},"Set up a new ",(0,i.yg)("strong",{parentName:"li"},"Persistent Volume Claim")," for the cluster as shown below")),(0,i.yg)("img",{src:"/img/dask-init7.png",alt:"dask init",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("h3",{id:"-configure-a-route-for-the-cluster"},"\ud83e\ude90 Configure a Route for the Cluster"),(0,i.yg)("ol",null,(0,i.yg)("li",{parentName:"ol"},"Switch to the ",(0,i.yg)("strong",{parentName:"li"},"Administrator")," view and navigate to ",(0,i.yg)("strong",{parentName:"li"},"Route"))),(0,i.yg)("img",{src:"/img/dask-route1.png",alt:"dask route",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("img",{src:"/img/dask-route2.png",alt:"dask route",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("ol",{start:2},(0,i.yg)("li",{parentName:"ol"},"Create a new route by clicking the button ",(0,i.yg)("strong",{parentName:"li"},"Create Route")," with the setup as shown below")),(0,i.yg)("img",{src:"/img/dask-route3.png",alt:"dask route",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("img",{src:"/img/dask-route4.png",alt:"dask route",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("img",{src:"/img/dask-route5.png",alt:"dask route",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("ol",{start:3},(0,i.yg)("li",{parentName:"ol"},"Navigate the provided link to access your local cluster")),(0,i.yg)("img",{src:"/img/dask-route6.png",alt:"dask route",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,i.yg)("h3",{id:"-access-the-jupyter-passwordtoken"},"\ud83e\ude90 Access the Jupyter Password/Token"),(0,i.yg)("ol",null,(0,i.yg)("li",{parentName:"ol"},"Start up the ",(0,i.yg)("strong",{parentName:"li"},"terminal"),(0,i.yg)("ul",{parentName:"li"},(0,i.yg)("li",{parentName:"ul"},"Run ",(0,i.yg)("inlineCode",{parentName:"li"},"oc get pods")," to find the full podname of the ",(0,i.yg)("strong",{parentName:"li"},"dask-jupyter")),(0,i.yg)("li",{parentName:"ul"},"Run ",(0,i.yg)("inlineCode",{parentName:"li"},"oc logs ")," and copy the token used to access the jupyter notebook")))),(0,i.yg)("img",{src:"/img/dask-route7.png",alt:"dask route",style:{maxWidth:"100%",maxHeight:"100%"}}))}u.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/9e298cf7.623b3baf.js b/assets/js/9e298cf7.0534294a.js similarity index 98% rename from assets/js/9e298cf7.623b3baf.js rename to assets/js/9e298cf7.0534294a.js index 0d571c541..84199230e 100644 --- a/assets/js/9e298cf7.623b3baf.js +++ b/assets/js/9e298cf7.0534294a.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[7512],{5680:(e,t,r)=>{r.d(t,{xA:()=>c,yg:()=>u});var n=r(6540);function a(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function o(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function i(e){for(var t=1;t=0||(a[r]=e[r]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(a[r]=e[r])}return a}var s=n.createContext({}),p=function(e){var t=n.useContext(s),r=t;return e&&(r="function"==typeof e?e(t):i(i({},t),e)),r},c=function(e){var t=p(e.components);return n.createElement(s.Provider,{value:t},e.children)},g={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},m=n.forwardRef((function(e,t){var r=e.components,a=e.mdxType,o=e.originalType,s=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),m=p(r),u=a,f=m["".concat(s,".").concat(u)]||m[u]||g[u]||o;return r?n.createElement(f,i(i({ref:t},c),{},{components:r})):n.createElement(f,i({ref:t},c))}));function u(e,t){var r=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=r.length,i=new Array(o);i[0]=m;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:a,i[1]=l;for(var p=2;p{r.r(t),r.d(t,{assets:()=>c,contentTitle:()=>s,default:()=>u,frontMatter:()=>l,metadata:()=>p,toc:()=>g});var n=r(9668),a=r(1367),o=(r(6540),r(5680)),i=["components"],l={id:"catalog-imaging",title:"Imaging softwares"},s=void 0,p={unversionedId:"catalog-imaging",id:"catalog-imaging",title:"Imaging softwares",description:"Feel free to propose new services using pull requests, or to request them by creating new issues.",source:"@site/docs/catalog-imaging.md",sourceDirName:".",slug:"/catalog-imaging",permalink:"/docs/catalog-imaging",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/catalog-imaging.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"catalog-imaging",title:"Imaging softwares"},sidebar:"docs",previous:{title:"Genomics",permalink:"/docs/catalog-genomics"},next:{title:"Utilities",permalink:"/docs/catalog-utilities"}},c={},g=[{value:"CellProfiler",id:"cellprofiler",level:2}],m={toc:g};function u(e){var t=e.components,r=(0,a.A)(e,i);return(0,o.yg)("wrapper",(0,n.A)({},m,r,{components:t,mdxType:"MDXLayout"}),(0,o.yg)("p",null,"Feel free to propose new services using ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/pulls"},"pull requests"),", or to request them by creating ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/issues"},"new issues"),"."),(0,o.yg)("h2",{id:"cellprofiler"},"CellProfiler"),(0,o.yg)("p",null,"Cell image analysis software. See ",(0,o.yg)("a",{parentName:"p",href:"https://cellprofiler.org/home"},"their website"),"."),(0,o.yg)("p",null,"You can start a container using the ",(0,o.yg)("strong",{parentName:"p"},"CellProfiler")," template in the ",(0,o.yg)("a",{parentName:"p",href:"https://console-openshift-console.apps.dsri2.unimaas.nl/catalog"},"Catalog web UI")," (make sure the ",(0,o.yg)("strong",{parentName:"p"},"Templates")," checkbox is checked)"),(0,o.yg)("p",null,"This template uses the ",(0,o.yg)("a",{parentName:"p",href:"https://hub.docker.com/r/cellprofiler/cellprofiler"},"official CellProfiler image")," hosted on DockerHub"),(0,o.yg)("admonition",{title:"Persistent data folder",type:"info"},(0,o.yg)("p",{parentName:"admonition"},"\ud83d\udcc2 Use the ",(0,o.yg)("inlineCode",{parentName:"p"},"/usr/local/src/work")," folder (home of the root user) to store your data in the existing persistent storage. You can find the persistent volumes in the DSRI web UI, go to the ",(0,o.yg)("strong",{parentName:"p"},"Administrator")," view > ",(0,o.yg)("strong",{parentName:"p"},"Storage")," > ",(0,o.yg)("strong",{parentName:"p"},"Persistent Volume Claims"),".")),(0,o.yg)("p",null,"Once the CellProfiler has been started you can access it through the pod terminal (in the DSRI web UI, or using ",(0,o.yg)("inlineCode",{parentName:"p"},"oc rsh POD_ID"),")"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"cellprofiler --help\ncellprofiler --run\ncellprofiler --run-headless\n")),(0,o.yg)("admonition",{title:"Getting Started",type:"info"},(0,o.yg)("p",{parentName:"admonition"},"\ud83d\udc2c For more information using cell profiler from the command line see ",(0,o.yg)("a",{parentName:"p",href:"https://carpenter-singh-lab.broadinstitute.org/blog/getting-started-using-cellprofiler-command-line"},"this post"))))}u.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[7512],{5680:(e,t,r)=>{r.d(t,{xA:()=>c,yg:()=>u});var n=r(6540);function a(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function o(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function i(e){for(var t=1;t=0||(a[r]=e[r]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(a[r]=e[r])}return a}var s=n.createContext({}),p=function(e){var t=n.useContext(s),r=t;return e&&(r="function"==typeof e?e(t):i(i({},t),e)),r},c=function(e){var t=p(e.components);return n.createElement(s.Provider,{value:t},e.children)},g={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},m=n.forwardRef((function(e,t){var r=e.components,a=e.mdxType,o=e.originalType,s=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),m=p(r),u=a,f=m["".concat(s,".").concat(u)]||m[u]||g[u]||o;return r?n.createElement(f,i(i({ref:t},c),{},{components:r})):n.createElement(f,i({ref:t},c))}));function u(e,t){var r=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=r.length,i=new Array(o);i[0]=m;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:a,i[1]=l;for(var p=2;p{r.r(t),r.d(t,{assets:()=>c,contentTitle:()=>s,default:()=>u,frontMatter:()=>l,metadata:()=>p,toc:()=>g});var n=r(9668),a=r(1367),o=(r(6540),r(5680)),i=["components"],l={id:"catalog-imaging",title:"Imaging softwares"},s=void 0,p={unversionedId:"catalog-imaging",id:"catalog-imaging",title:"Imaging softwares",description:"Feel free to propose new services using pull requests, or to request them by creating new issues.",source:"@site/docs/catalog-imaging.md",sourceDirName:".",slug:"/catalog-imaging",permalink:"/docs/catalog-imaging",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/catalog-imaging.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"catalog-imaging",title:"Imaging softwares"},sidebar:"docs",previous:{title:"Genomics",permalink:"/docs/catalog-genomics"},next:{title:"Utilities",permalink:"/docs/catalog-utilities"}},c={},g=[{value:"CellProfiler",id:"cellprofiler",level:2}],m={toc:g};function u(e){var t=e.components,r=(0,a.A)(e,i);return(0,o.yg)("wrapper",(0,n.A)({},m,r,{components:t,mdxType:"MDXLayout"}),(0,o.yg)("p",null,"Feel free to propose new services using ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/pulls"},"pull requests"),", or to request them by creating ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/issues"},"new issues"),"."),(0,o.yg)("h2",{id:"cellprofiler"},"CellProfiler"),(0,o.yg)("p",null,"Cell image analysis software. See ",(0,o.yg)("a",{parentName:"p",href:"https://cellprofiler.org/home"},"their website"),"."),(0,o.yg)("p",null,"You can start a container using the ",(0,o.yg)("strong",{parentName:"p"},"CellProfiler")," template in the ",(0,o.yg)("a",{parentName:"p",href:"https://console-openshift-console.apps.dsri2.unimaas.nl/catalog"},"Catalog web UI")," (make sure the ",(0,o.yg)("strong",{parentName:"p"},"Templates")," checkbox is checked)"),(0,o.yg)("p",null,"This template uses the ",(0,o.yg)("a",{parentName:"p",href:"https://hub.docker.com/r/cellprofiler/cellprofiler"},"official CellProfiler image")," hosted on DockerHub"),(0,o.yg)("admonition",{title:"Persistent data folder",type:"info"},(0,o.yg)("p",{parentName:"admonition"},"\ud83d\udcc2 Use the ",(0,o.yg)("inlineCode",{parentName:"p"},"/usr/local/src/work")," folder (home of the root user) to store your data in the existing persistent storage. You can find the persistent volumes in the DSRI web UI, go to the ",(0,o.yg)("strong",{parentName:"p"},"Administrator")," view > ",(0,o.yg)("strong",{parentName:"p"},"Storage")," > ",(0,o.yg)("strong",{parentName:"p"},"Persistent Volume Claims"),".")),(0,o.yg)("p",null,"Once the CellProfiler has been started you can access it through the pod terminal (in the DSRI web UI, or using ",(0,o.yg)("inlineCode",{parentName:"p"},"oc rsh POD_ID"),")"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"cellprofiler --help\ncellprofiler --run\ncellprofiler --run-headless\n")),(0,o.yg)("admonition",{title:"Getting Started",type:"info"},(0,o.yg)("p",{parentName:"admonition"},"\ud83d\udc2c For more information using cell profiler from the command line see ",(0,o.yg)("a",{parentName:"p",href:"https://carpenter-singh-lab.broadinstitute.org/blog/getting-started-using-cellprofiler-command-line"},"this post"))))}u.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/9f389101.0b162e92.js b/assets/js/9f389101.01cc133a.js similarity index 98% rename from assets/js/9f389101.0b162e92.js rename to assets/js/9f389101.01cc133a.js index 60e657fb8..8f7b629fe 100644 --- a/assets/js/9f389101.0b162e92.js +++ b/assets/js/9f389101.01cc133a.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[5020],{5680:(e,t,n)=>{n.d(t,{xA:()=>p,yg:()=>u});var o=n(6540);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function a(e){for(var t=1;t=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);for(o=0;o=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var s=o.createContext({}),c=function(e){var t=o.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):a(a({},t),e)),n},p=function(e){var t=c(e.components);return o.createElement(s.Provider,{value:t},e.children)},f={inlineCode:"code",wrapper:function(e){var t=e.children;return o.createElement(o.Fragment,{},t)}},w=o.forwardRef((function(e,t){var n=e.components,r=e.mdxType,l=e.originalType,s=e.parentName,p=i(e,["components","mdxType","originalType","parentName"]),w=c(n),u=r,d=w["".concat(s,".").concat(u)]||w[u]||f[u]||l;return n?o.createElement(d,a(a({ref:t},p),{},{components:n})):o.createElement(d,a({ref:t},p))}));function u(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var l=n.length,a=new Array(l);a[0]=w;var i={};for(var s in t)hasOwnProperty.call(t,s)&&(i[s]=t[s]);i.originalType=e,i.mdxType="string"==typeof e?e:r,a[1]=i;for(var c=2;c{n.r(t),n.d(t,{assets:()=>p,contentTitle:()=>s,default:()=>u,frontMatter:()=>i,metadata:()=>c,toc:()=>f});var o=n(9668),r=n(1367),l=(n(6540),n(5680)),a=["components"],i={id:"workflows-nextflow",title:"Run Nextflow workflows"},s=void 0,c={unversionedId:"workflows-nextflow",id:"workflows-nextflow",title:"Run Nextflow workflows",description:"Nextflow enables scalable and reproducible scientific workflows using software containers. It allows the adaptation of pipelines written in the most common scripting languages.",source:"@site/docs/workflows-nextflow.md",sourceDirName:".",slug:"/workflows-nextflow",permalink:"/docs/workflows-nextflow",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/workflows-nextflow.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"workflows-nextflow",title:"Run Nextflow workflows"},sidebar:"docs",previous:{title:"Run Argo workflows",permalink:"/docs/workflows-argo"},next:{title:"Run CWL workflows",permalink:"/docs/workflows-cwl"}},p={},f=[{value:"Install Nextflow",id:"install-nextflow",level:2},{value:"Run workflow",id:"run-workflow",level:2}],w={toc:f};function u(e){var t=e.components,n=(0,r.A)(e,a);return(0,l.yg)("wrapper",(0,o.A)({},w,n,{components:t,mdxType:"MDXLayout"}),(0,l.yg)("p",null,(0,l.yg)("a",{parentName:"p",href:"https://www.nextflow.io/"},"Nextflow")," enables scalable and reproducible scientific workflows using software containers. It allows the adaptation of pipelines written in the most common scripting languages."),(0,l.yg)("p",null,"Nextflow has been developed by the genomic research scientific community and is built to run bioinformatics pipeline."),(0,l.yg)("p",null,"Define your workflow in a Bash script fashion, providing input, output and the command to run. Without the need to create and use Docker container for Conda pipelines"),(0,l.yg)("h2",{id:"install-nextflow"},"Install Nextflow"),(0,l.yg)("p",null,"Install the ",(0,l.yg)("inlineCode",{parentName:"p"},"nextflow")," client on your computer:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"wget -qO- https://get.nextflow.io | bash\n")),(0,l.yg)("admonition",{title:"Official documentation",type:"info"},(0,l.yg)("p",{parentName:"admonition"},"See the ",(0,l.yg)("a",{parentName:"p",href:"https://www.nextflow.io/docs/latest/getstarted.html#installation"},"Nextflow documentation"),".")),(0,l.yg)("h2",{id:"run-workflow"},"Run workflow"),(0,l.yg)("p",null,"Try the hello world workflow from Nextflow using an existing storage:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"nextflow kuberun https://github.com/nextflow-io/hello -v pvc-mapr-projects-showcase:/data\n")),(0,l.yg)("admonition",{title:"Use Conda environments",type:"tip"},(0,l.yg)("p",{parentName:"admonition"},"You can easily ",(0,l.yg)("a",{parentName:"p",href:"https://www.nextflow.io/docs/latest/conda.html"},"define Conda environments and workflows")," with Nextflow.")))}u.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[5020],{5680:(e,t,n)=>{n.d(t,{xA:()=>p,yg:()=>u});var o=n(6540);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function l(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function a(e){for(var t=1;t=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);for(o=0;o=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var s=o.createContext({}),c=function(e){var t=o.useContext(s),n=t;return e&&(n="function"==typeof e?e(t):a(a({},t),e)),n},p=function(e){var t=c(e.components);return o.createElement(s.Provider,{value:t},e.children)},f={inlineCode:"code",wrapper:function(e){var t=e.children;return o.createElement(o.Fragment,{},t)}},w=o.forwardRef((function(e,t){var n=e.components,r=e.mdxType,l=e.originalType,s=e.parentName,p=i(e,["components","mdxType","originalType","parentName"]),w=c(n),u=r,d=w["".concat(s,".").concat(u)]||w[u]||f[u]||l;return n?o.createElement(d,a(a({ref:t},p),{},{components:n})):o.createElement(d,a({ref:t},p))}));function u(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var l=n.length,a=new Array(l);a[0]=w;var i={};for(var s in t)hasOwnProperty.call(t,s)&&(i[s]=t[s]);i.originalType=e,i.mdxType="string"==typeof e?e:r,a[1]=i;for(var c=2;c{n.r(t),n.d(t,{assets:()=>p,contentTitle:()=>s,default:()=>u,frontMatter:()=>i,metadata:()=>c,toc:()=>f});var o=n(9668),r=n(1367),l=(n(6540),n(5680)),a=["components"],i={id:"workflows-nextflow",title:"Run Nextflow workflows"},s=void 0,c={unversionedId:"workflows-nextflow",id:"workflows-nextflow",title:"Run Nextflow workflows",description:"Nextflow enables scalable and reproducible scientific workflows using software containers. It allows the adaptation of pipelines written in the most common scripting languages.",source:"@site/docs/workflows-nextflow.md",sourceDirName:".",slug:"/workflows-nextflow",permalink:"/docs/workflows-nextflow",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/workflows-nextflow.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"workflows-nextflow",title:"Run Nextflow workflows"},sidebar:"docs",previous:{title:"Run Argo workflows",permalink:"/docs/workflows-argo"},next:{title:"Run CWL workflows",permalink:"/docs/workflows-cwl"}},p={},f=[{value:"Install Nextflow",id:"install-nextflow",level:2},{value:"Run workflow",id:"run-workflow",level:2}],w={toc:f};function u(e){var t=e.components,n=(0,r.A)(e,a);return(0,l.yg)("wrapper",(0,o.A)({},w,n,{components:t,mdxType:"MDXLayout"}),(0,l.yg)("p",null,(0,l.yg)("a",{parentName:"p",href:"https://www.nextflow.io/"},"Nextflow")," enables scalable and reproducible scientific workflows using software containers. It allows the adaptation of pipelines written in the most common scripting languages."),(0,l.yg)("p",null,"Nextflow has been developed by the genomic research scientific community and is built to run bioinformatics pipeline."),(0,l.yg)("p",null,"Define your workflow in a Bash script fashion, providing input, output and the command to run. Without the need to create and use Docker container for Conda pipelines"),(0,l.yg)("h2",{id:"install-nextflow"},"Install Nextflow"),(0,l.yg)("p",null,"Install the ",(0,l.yg)("inlineCode",{parentName:"p"},"nextflow")," client on your computer:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"wget -qO- https://get.nextflow.io | bash\n")),(0,l.yg)("admonition",{title:"Official documentation",type:"info"},(0,l.yg)("p",{parentName:"admonition"},"See the ",(0,l.yg)("a",{parentName:"p",href:"https://www.nextflow.io/docs/latest/getstarted.html#installation"},"Nextflow documentation"),".")),(0,l.yg)("h2",{id:"run-workflow"},"Run workflow"),(0,l.yg)("p",null,"Try the hello world workflow from Nextflow using an existing storage:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"nextflow kuberun https://github.com/nextflow-io/hello -v pvc-mapr-projects-showcase:/data\n")),(0,l.yg)("admonition",{title:"Use Conda environments",type:"tip"},(0,l.yg)("p",{parentName:"admonition"},"You can easily ",(0,l.yg)("a",{parentName:"p",href:"https://www.nextflow.io/docs/latest/conda.html"},"define Conda environments and workflows")," with Nextflow.")))}u.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/a09c2993.8b69eef8.js b/assets/js/a09c2993.3df8ebed.js similarity index 99% rename from assets/js/a09c2993.8b69eef8.js rename to assets/js/a09c2993.3df8ebed.js index bec51d59c..83dfa6f3a 100644 --- a/assets/js/a09c2993.8b69eef8.js +++ b/assets/js/a09c2993.3df8ebed.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[5899],{5680:(e,t,a)=>{a.d(t,{xA:()=>c,yg:()=>g});var r=a(6540);function n(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function o(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,r)}return a}function i(e){for(var t=1;t=0||(n[a]=e[a]);return n}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(n[a]=e[a])}return n}var s=r.createContext({}),u=function(e){var t=r.useContext(s),a=t;return e&&(a="function"==typeof e?e(t):i(i({},t),e)),a},c=function(e){var t=u(e.components);return r.createElement(s.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},d=r.forwardRef((function(e,t){var a=e.components,n=e.mdxType,o=e.originalType,s=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),d=u(a),g=n,y=d["".concat(s,".").concat(g)]||d[g]||p[g]||o;return a?r.createElement(y,i(i({ref:t},c),{},{components:a})):r.createElement(y,i({ref:t},c))}));function g(e,t){var a=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var o=a.length,i=new Array(o);i[0]=d;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:n,i[1]=l;for(var u=2;u{a.r(t),a.d(t,{assets:()=>c,contentTitle:()=>s,default:()=>g,frontMatter:()=>l,metadata:()=>u,toc:()=>p});var r=a(9668),n=a(1367),o=(a(6540),a(5680)),i=["components"],l={id:"introduction",title:"Introduction",slug:"/"},s=void 0,u={unversionedId:"introduction",id:"introduction",title:"Introduction",description:"The Data Science Research Infrastructure is a cluster of servers to deploy workspaces and applications for Data Science.",source:"@site/docs/introduction.md",sourceDirName:".",slug:"/",permalink:"/docs/",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/introduction.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"introduction",title:"Introduction",slug:"/"},sidebar:"docs",next:{title:"Access the DSRI",permalink:"/docs/access-dsri"}},c={},p=[{value:"Getting started",id:"getting-started",level:2},{value:"\u2705 What can be done on the DSRI",id:"-what-can-be-done-on-the-dsri",level:3},{value:"\u274c What cannot be done",id:"-what-cannot-be-done",level:3},{value:"The DSRI architecture",id:"the-dsri-architecture",level:2},{value:"The DSRI specifications",id:"the-dsri-specifications",level:2},{value:"Software",id:"software",level:3},{value:"Hardware",id:"hardware",level:3},{value:"Learn more about DSRI",id:"learn-more-about-dsri",level:2}],d={toc:p};function g(e){var t=e.components,a=(0,n.A)(e,i);return(0,o.yg)("wrapper",(0,r.A)({},d,a,{components:t,mdxType:"MDXLayout"}),(0,o.yg)("p",null,"The Data Science Research Infrastructure is a cluster of servers to deploy workspaces and applications for Data Science. "),(0,o.yg)("p",null,"It works by starting workspaces and applications in Docker containers that are automatically deployed to a powerful server on the cluster using Kubernetes, a container orchestration system. You can then access your workspace or application through an URL automatically generated."),(0,o.yg)("h2",{id:"getting-started"},"Getting started"),(0,o.yg)("h3",{id:"-what-can-be-done-on-the-dsri"},"\u2705 What can be done on the DSRI"),(0,o.yg)("p",null,"The DSRI is particularly useful if you need to:"),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},"Gain access to more computing resources (memory and CPUs), which enables you to load larger amount of data, or use more threads for parallelized tasks"),(0,o.yg)("li",{parentName:"ul"},"Run jobs that takes a long time to complete"),(0,o.yg)("li",{parentName:"ul"},"Deploy any database or service you need, and connect to it from your workspace easily"),(0,o.yg)("li",{parentName:"ul"},"Book and start a workspace that uses one of our GPUs")),(0,o.yg)("p",null,"The DSRI proposes a number of popular workspaces to work with data:"),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},"Multiple flavors of ",(0,o.yg)("strong",{parentName:"li"},"JupyterLab")," (scipy, tensorflow, all-spark, and more)"),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("strong",{parentName:"li"},"VisualStudio Code")," server (also available within the JupyterLab workspaces)"),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("strong",{parentName:"li"},"RStudio"),", with a complementary Shiny server"),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("strong",{parentName:"li"},"Matlab")),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("strong",{parentName:"li"},"Ubuntu")," Desktop ")),(0,o.yg)("p",null,"You can then install anything you want in your workspace using ",(0,o.yg)("inlineCode",{parentName:"p"},"conda"),", ",(0,o.yg)("inlineCode",{parentName:"p"},"pip"),", or ",(0,o.yg)("inlineCode",{parentName:"p"},"apt"),"."),(0,o.yg)("admonition",{title:"Data storage",type:"caution"},(0,o.yg)("p",{parentName:"admonition"},(0,o.yg)("strong",{parentName:"p"},"DSRI is a computing infrastructure"),", built and used to run data science workloads. DSRI stores data in a persistent manner, but all data stored on the DSRI is susceptible to be altered by the workloads you are running, and we cannot guarantee its immutability."),(0,o.yg)("p",{parentName:"admonition"},(0,o.yg)("strong",{parentName:"p"},"Always keep a safe copy of your data outside the DSRI"),". And don't rely on the DSRI for long term storage.")),(0,o.yg)("h3",{id:"-what-cannot-be-done"},"\u274c What cannot be done"),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},"Since DSRI can only be accessed when using the ",(0,o.yg)("a",{parentName:"li",href:"https://vpn.maastrichtuniversity.nl/"},"UM VPN"),", deployed services will not be available on the public Internet \ud83d\udd12"),(0,o.yg)("li",{parentName:"ul"},"All activities must be legal in basis. You must closely examine and abide by the terms and conditions of any data, software, or web service that you use as part of your work \ud83d\udcdc"),(0,o.yg)("li",{parentName:"ul"},"You cannot reach data or servers hosted at Maastricht University from the DSRI by default. You will need to request access in advance ",(0,o.yg)("a",{parentName:"li",href:"/docs/prepare-project-for-dsri#request-access-to-internal-um-servers"},"here \ud83d\udcec\ufe0f")),(0,o.yg)("li",{parentName:"ul"},"Right now it is not possible to reach the central UM fileservices (MFS)")),(0,o.yg)("admonition",{title:"Request an account",type:"info"},(0,o.yg)("p",{parentName:"admonition"},"If you are working at Maastricht University, ",(0,o.yg)("strong",{parentName:"p"},"see ",(0,o.yg)("a",{parentName:"strong",href:"https://maastrichtu-ids.github.io/dsri-documentation/docs/access-dsri"},"this page")," to request an account"),", and run your services on the DSRI.")),(0,o.yg)("h2",{id:"the-dsri-architecture"},"The DSRI architecture"),(0,o.yg)("p",null,"Here is a diagram providing a simplified explanation of how the DSRI works, using popular data science applications as examples (JupyterLab, RStudio, VSCode server)"),(0,o.yg)("img",{src:"/img/dsri_simplified_overview.png",alt:"DSRI in a nutshell ",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("h2",{id:"the-dsri-specifications"},"The DSRI specifications"),(0,o.yg)("h3",{id:"software"},"Software"),(0,o.yg)("p",null,"We use ",(0,o.yg)("a",{parentName:"p",href:"https://www.okd.io/"},(0,o.yg)("strong",{parentName:"a"},"OKD 4.11")),", the Origin Community Distribution of Kubernetes that powers ",(0,o.yg)("a",{parentName:"p",href:"https://www.openshift.com/"},"RedHat OpenShift"),", a distribution of the Kubernetes container orchestration tool. Kubernetes takes care of deploying the Docker containers on the cluster of servers, the OKD distribution extends it to improve security, and provide a user-friendly web UI to manage your applications."),(0,o.yg)("p",null,"We use ",(0,o.yg)("a",{parentName:"p",href:"https://www.redhat.com/fr/technologies/storage/ceph"},(0,o.yg)("strong",{parentName:"a"},"RedHat Ceph storage"))," for the distributed storage."),(0,o.yg)("h3",{id:"hardware"},"Hardware"),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},"16 CPU nodes")),(0,o.yg)("table",null,(0,o.yg)("thead",{parentName:"table"},(0,o.yg)("tr",{parentName:"thead"},(0,o.yg)("th",{parentName:"tr",align:null}),(0,o.yg)("th",{parentName:"tr",align:null},"RAM (GB)"),(0,o.yg)("th",{parentName:"tr",align:null},"CPU (cores)"),(0,o.yg)("th",{parentName:"tr",align:null},"Storage (TB)"))),(0,o.yg)("tbody",{parentName:"table"},(0,o.yg)("tr",{parentName:"tbody"},(0,o.yg)("td",{parentName:"tr",align:null},"Node capacity"),(0,o.yg)("td",{parentName:"tr",align:null},"512 GB"),(0,o.yg)("td",{parentName:"tr",align:null},"64 cores (128 threads)"),(0,o.yg)("td",{parentName:"tr",align:null},"120 TB")),(0,o.yg)("tr",{parentName:"tbody"},(0,o.yg)("td",{parentName:"tr",align:null},"Total capacity"),(0,o.yg)("td",{parentName:"tr",align:null},"8 192 GB"),(0,o.yg)("td",{parentName:"tr",align:null},"1 024 cores"),(0,o.yg)("td",{parentName:"tr",align:null},"1 920 TB")))),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},"1 GPU node: ",(0,o.yg)("strong",{parentName:"li"},"Nvidia DGX1")," 8x Tesla V100 - 32GB GPU")),(0,o.yg)("table",null,(0,o.yg)("thead",{parentName:"table"},(0,o.yg)("tr",{parentName:"thead"},(0,o.yg)("th",{parentName:"tr",align:null}),(0,o.yg)("th",{parentName:"tr",align:null},"GPUs"),(0,o.yg)("th",{parentName:"tr",align:null},"RAM (GB)"),(0,o.yg)("th",{parentName:"tr",align:null},"CPU (cores)"))),(0,o.yg)("tbody",{parentName:"table"},(0,o.yg)("tr",{parentName:"tbody"},(0,o.yg)("td",{parentName:"tr",align:null},"GPU node capacity"),(0,o.yg)("td",{parentName:"tr",align:null},"8"),(0,o.yg)("td",{parentName:"tr",align:null},"512 GB"),(0,o.yg)("td",{parentName:"tr",align:null},"40 cores")))),(0,o.yg)("img",{src:"/img/DSRI_infrastructure_architecture_overview.png",alt:"DSRI infrastructure ",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("h2",{id:"learn-more-about-dsri"},"Learn more about DSRI"),(0,o.yg)("p",null,"See the following presentation about the Data Science Research Infrastructure "),(0,o.yg)("a",{href:"/resource/2021-04-DSRI-Community-Event.pdf",target:"_blank",rel:"noopener noreferrer"},(0,o.yg)("img",{src:"/resource/DSRI-community-event.png",style:{maxWidth:"100%",maxHeight:"100%"},alt:"DSRI April 2021 Community Event Presentation"})))}g.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[5899],{5680:(e,t,a)=>{a.d(t,{xA:()=>c,yg:()=>g});var r=a(6540);function n(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function o(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,r)}return a}function i(e){for(var t=1;t=0||(n[a]=e[a]);return n}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(n[a]=e[a])}return n}var s=r.createContext({}),u=function(e){var t=r.useContext(s),a=t;return e&&(a="function"==typeof e?e(t):i(i({},t),e)),a},c=function(e){var t=u(e.components);return r.createElement(s.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},d=r.forwardRef((function(e,t){var a=e.components,n=e.mdxType,o=e.originalType,s=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),d=u(a),g=n,y=d["".concat(s,".").concat(g)]||d[g]||p[g]||o;return a?r.createElement(y,i(i({ref:t},c),{},{components:a})):r.createElement(y,i({ref:t},c))}));function g(e,t){var a=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var o=a.length,i=new Array(o);i[0]=d;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:n,i[1]=l;for(var u=2;u{a.r(t),a.d(t,{assets:()=>c,contentTitle:()=>s,default:()=>g,frontMatter:()=>l,metadata:()=>u,toc:()=>p});var r=a(9668),n=a(1367),o=(a(6540),a(5680)),i=["components"],l={id:"introduction",title:"Introduction",slug:"/"},s=void 0,u={unversionedId:"introduction",id:"introduction",title:"Introduction",description:"The Data Science Research Infrastructure is a cluster of servers to deploy workspaces and applications for Data Science.",source:"@site/docs/introduction.md",sourceDirName:".",slug:"/",permalink:"/docs/",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/introduction.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"introduction",title:"Introduction",slug:"/"},sidebar:"docs",next:{title:"Access the DSRI",permalink:"/docs/access-dsri"}},c={},p=[{value:"Getting started",id:"getting-started",level:2},{value:"\u2705 What can be done on the DSRI",id:"-what-can-be-done-on-the-dsri",level:3},{value:"\u274c What cannot be done",id:"-what-cannot-be-done",level:3},{value:"The DSRI architecture",id:"the-dsri-architecture",level:2},{value:"The DSRI specifications",id:"the-dsri-specifications",level:2},{value:"Software",id:"software",level:3},{value:"Hardware",id:"hardware",level:3},{value:"Learn more about DSRI",id:"learn-more-about-dsri",level:2}],d={toc:p};function g(e){var t=e.components,a=(0,n.A)(e,i);return(0,o.yg)("wrapper",(0,r.A)({},d,a,{components:t,mdxType:"MDXLayout"}),(0,o.yg)("p",null,"The Data Science Research Infrastructure is a cluster of servers to deploy workspaces and applications for Data Science. "),(0,o.yg)("p",null,"It works by starting workspaces and applications in Docker containers that are automatically deployed to a powerful server on the cluster using Kubernetes, a container orchestration system. You can then access your workspace or application through an URL automatically generated."),(0,o.yg)("h2",{id:"getting-started"},"Getting started"),(0,o.yg)("h3",{id:"-what-can-be-done-on-the-dsri"},"\u2705 What can be done on the DSRI"),(0,o.yg)("p",null,"The DSRI is particularly useful if you need to:"),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},"Gain access to more computing resources (memory and CPUs), which enables you to load larger amount of data, or use more threads for parallelized tasks"),(0,o.yg)("li",{parentName:"ul"},"Run jobs that takes a long time to complete"),(0,o.yg)("li",{parentName:"ul"},"Deploy any database or service you need, and connect to it from your workspace easily"),(0,o.yg)("li",{parentName:"ul"},"Book and start a workspace that uses one of our GPUs")),(0,o.yg)("p",null,"The DSRI proposes a number of popular workspaces to work with data:"),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},"Multiple flavors of ",(0,o.yg)("strong",{parentName:"li"},"JupyterLab")," (scipy, tensorflow, all-spark, and more)"),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("strong",{parentName:"li"},"VisualStudio Code")," server (also available within the JupyterLab workspaces)"),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("strong",{parentName:"li"},"RStudio"),", with a complementary Shiny server"),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("strong",{parentName:"li"},"Matlab")),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("strong",{parentName:"li"},"Ubuntu")," Desktop ")),(0,o.yg)("p",null,"You can then install anything you want in your workspace using ",(0,o.yg)("inlineCode",{parentName:"p"},"conda"),", ",(0,o.yg)("inlineCode",{parentName:"p"},"pip"),", or ",(0,o.yg)("inlineCode",{parentName:"p"},"apt"),"."),(0,o.yg)("admonition",{title:"Data storage",type:"caution"},(0,o.yg)("p",{parentName:"admonition"},(0,o.yg)("strong",{parentName:"p"},"DSRI is a computing infrastructure"),", built and used to run data science workloads. DSRI stores data in a persistent manner, but all data stored on the DSRI is susceptible to be altered by the workloads you are running, and we cannot guarantee its immutability."),(0,o.yg)("p",{parentName:"admonition"},(0,o.yg)("strong",{parentName:"p"},"Always keep a safe copy of your data outside the DSRI"),". And don't rely on the DSRI for long term storage.")),(0,o.yg)("h3",{id:"-what-cannot-be-done"},"\u274c What cannot be done"),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},"Since DSRI can only be accessed when using the ",(0,o.yg)("a",{parentName:"li",href:"https://vpn.maastrichtuniversity.nl/"},"UM VPN"),", deployed services will not be available on the public Internet \ud83d\udd12"),(0,o.yg)("li",{parentName:"ul"},"All activities must be legal in basis. You must closely examine and abide by the terms and conditions of any data, software, or web service that you use as part of your work \ud83d\udcdc"),(0,o.yg)("li",{parentName:"ul"},"You cannot reach data or servers hosted at Maastricht University from the DSRI by default. You will need to request access in advance ",(0,o.yg)("a",{parentName:"li",href:"/docs/prepare-project-for-dsri#request-access-to-internal-um-servers"},"here \ud83d\udcec\ufe0f")),(0,o.yg)("li",{parentName:"ul"},"Right now it is not possible to reach the central UM fileservices (MFS)")),(0,o.yg)("admonition",{title:"Request an account",type:"info"},(0,o.yg)("p",{parentName:"admonition"},"If you are working at Maastricht University, ",(0,o.yg)("strong",{parentName:"p"},"see ",(0,o.yg)("a",{parentName:"strong",href:"https://maastrichtu-ids.github.io/dsri-documentation/docs/access-dsri"},"this page")," to request an account"),", and run your services on the DSRI.")),(0,o.yg)("h2",{id:"the-dsri-architecture"},"The DSRI architecture"),(0,o.yg)("p",null,"Here is a diagram providing a simplified explanation of how the DSRI works, using popular data science applications as examples (JupyterLab, RStudio, VSCode server)"),(0,o.yg)("img",{src:"/img/dsri_simplified_overview.png",alt:"DSRI in a nutshell ",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("h2",{id:"the-dsri-specifications"},"The DSRI specifications"),(0,o.yg)("h3",{id:"software"},"Software"),(0,o.yg)("p",null,"We use ",(0,o.yg)("a",{parentName:"p",href:"https://www.okd.io/"},(0,o.yg)("strong",{parentName:"a"},"OKD 4.11")),", the Origin Community Distribution of Kubernetes that powers ",(0,o.yg)("a",{parentName:"p",href:"https://www.openshift.com/"},"RedHat OpenShift"),", a distribution of the Kubernetes container orchestration tool. Kubernetes takes care of deploying the Docker containers on the cluster of servers, the OKD distribution extends it to improve security, and provide a user-friendly web UI to manage your applications."),(0,o.yg)("p",null,"We use ",(0,o.yg)("a",{parentName:"p",href:"https://www.redhat.com/fr/technologies/storage/ceph"},(0,o.yg)("strong",{parentName:"a"},"RedHat Ceph storage"))," for the distributed storage."),(0,o.yg)("h3",{id:"hardware"},"Hardware"),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},"16 CPU nodes")),(0,o.yg)("table",null,(0,o.yg)("thead",{parentName:"table"},(0,o.yg)("tr",{parentName:"thead"},(0,o.yg)("th",{parentName:"tr",align:null}),(0,o.yg)("th",{parentName:"tr",align:null},"RAM (GB)"),(0,o.yg)("th",{parentName:"tr",align:null},"CPU (cores)"),(0,o.yg)("th",{parentName:"tr",align:null},"Storage (TB)"))),(0,o.yg)("tbody",{parentName:"table"},(0,o.yg)("tr",{parentName:"tbody"},(0,o.yg)("td",{parentName:"tr",align:null},"Node capacity"),(0,o.yg)("td",{parentName:"tr",align:null},"512 GB"),(0,o.yg)("td",{parentName:"tr",align:null},"64 cores (128 threads)"),(0,o.yg)("td",{parentName:"tr",align:null},"120 TB")),(0,o.yg)("tr",{parentName:"tbody"},(0,o.yg)("td",{parentName:"tr",align:null},"Total capacity"),(0,o.yg)("td",{parentName:"tr",align:null},"8 192 GB"),(0,o.yg)("td",{parentName:"tr",align:null},"1 024 cores"),(0,o.yg)("td",{parentName:"tr",align:null},"1 920 TB")))),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},"1 GPU node: ",(0,o.yg)("strong",{parentName:"li"},"Nvidia DGX1")," 8x Tesla V100 - 32GB GPU")),(0,o.yg)("table",null,(0,o.yg)("thead",{parentName:"table"},(0,o.yg)("tr",{parentName:"thead"},(0,o.yg)("th",{parentName:"tr",align:null}),(0,o.yg)("th",{parentName:"tr",align:null},"GPUs"),(0,o.yg)("th",{parentName:"tr",align:null},"RAM (GB)"),(0,o.yg)("th",{parentName:"tr",align:null},"CPU (cores)"))),(0,o.yg)("tbody",{parentName:"table"},(0,o.yg)("tr",{parentName:"tbody"},(0,o.yg)("td",{parentName:"tr",align:null},"GPU node capacity"),(0,o.yg)("td",{parentName:"tr",align:null},"8"),(0,o.yg)("td",{parentName:"tr",align:null},"512 GB"),(0,o.yg)("td",{parentName:"tr",align:null},"40 cores")))),(0,o.yg)("img",{src:"/img/DSRI_infrastructure_architecture_overview.png",alt:"DSRI infrastructure ",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("h2",{id:"learn-more-about-dsri"},"Learn more about DSRI"),(0,o.yg)("p",null,"See the following presentation about the Data Science Research Infrastructure "),(0,o.yg)("a",{href:"/resource/2021-04-DSRI-Community-Event.pdf",target:"_blank",rel:"noopener noreferrer"},(0,o.yg)("img",{src:"/resource/DSRI-community-event.png",style:{maxWidth:"100%",maxHeight:"100%"},alt:"DSRI April 2021 Community Event Presentation"})))}g.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/a1c3d222.b96a0769.js b/assets/js/a1c3d222.79c79900.js similarity index 99% rename from assets/js/a1c3d222.b96a0769.js rename to assets/js/a1c3d222.79c79900.js index 8d6c54072..c2d307132 100644 --- a/assets/js/a1c3d222.b96a0769.js +++ b/assets/js/a1c3d222.79c79900.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[8433],{5680:(e,t,n)=>{n.d(t,{xA:()=>c,yg:()=>y});var o=n(6540);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function i(e){for(var t=1;t=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(o=0;o=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var l=o.createContext({}),p=function(e){var t=o.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},c=function(e){var t=p(e.components);return o.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.createElement(o.Fragment,{},t)}},g=o.forwardRef((function(e,t){var n=e.components,a=e.mdxType,r=e.originalType,l=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),g=p(n),y=a,m=g["".concat(l,".").concat(y)]||g[y]||u[y]||r;return n?o.createElement(m,i(i({ref:t},c),{},{components:n})):o.createElement(m,i({ref:t},c))}));function y(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var r=n.length,i=new Array(r);i[0]=g;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s.mdxType="string"==typeof e?e:a,i[1]=s;for(var p=2;p{n.r(t),n.d(t,{assets:()=>c,contentTitle:()=>l,default:()=>y,frontMatter:()=>s,metadata:()=>p,toc:()=>u});var o=n(9668),a=n(1367),r=(n(6540),n(5680)),i=["components"],s={id:"access-dsri",title:"Access the DSRI"},l=void 0,p={unversionedId:"access-dsri",id:"access-dsri",title:"Access the DSRI",description:"Request an account",source:"@site/docs/access-dsri.md",sourceDirName:".",slug:"/access-dsri",permalink:"/docs/access-dsri",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/access-dsri.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"access-dsri",title:"Access the DSRI"},sidebar:"docs",previous:{title:"Introduction",permalink:"/docs/"},next:{title:"Start your workspace",permalink:"/docs/start-workspace"}},c={},u=[{value:"Request an account",id:"request-an-account",level:2},{value:"Connect to the UM network",id:"connect-to-the-um-network",level:2},{value:"Access the web UI",id:"access-the-web-ui",level:2},{value:"Access your project",id:"access-your-project",level:2},{value:"About the web UI",id:"about-the-web-ui",level:2},{value:"Accessing the Developer perspective",id:"accessing-the-developer-perspective",level:3}],g={toc:u};function y(e){var t=e.components,n=(0,a.A)(e,i);return(0,r.yg)("wrapper",(0,o.A)({},g,n,{components:t,mdxType:"MDXLayout"}),(0,r.yg)("h2",{id:"request-an-account"},"Request an account"),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"You will need to have an account at Maastricht University with an email ending with ",(0,r.yg)("inlineCode",{parentName:"p"},"@maastrichtuniversity.nl")," or ",(0,r.yg)("inlineCode",{parentName:"p"},"@student.maastrichtuniversity.nl"),".")),(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"Request access to the DSRI for your account Please fill this ",(0,r.yg)("a",{parentName:"p",href:"/register"},"form \ud83d\udcec")," to provide us some information on what you plan to do with the DSRI. Once you fill the form, you will receive an email with detailed instructions on how to log in."))),(0,r.yg)("h2",{id:"connect-to-the-um-network"},"Connect to the UM network"),(0,r.yg)("p",null,"You need to be connected to the UM network to access the DSRI."),(0,r.yg)("p",null,(0,r.yg)("strong",{parentName:"p"},"\ud83d\udc27 On Linux"),": use ",(0,r.yg)("inlineCode",{parentName:"p"},"openconnect")," to connect to the UM VPN. You can easily install it on Ubuntu and Debian distributions with ",(0,r.yg)("inlineCode",{parentName:"p"},"apt"),":"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"sudo apt install openconnect\nsudo openconnect -u YOUR.USER --authgroup 01-Employees vpn.maastrichtuniversity.nl\n")),(0,r.yg)("p",null,(0,r.yg)("strong",{parentName:"p"},"\ud83c\udf4e On MacOS and Windows"),": download and install the ",(0,r.yg)("strong",{parentName:"p"},"Maastricht University VPN")," client available at ",(0,r.yg)("strong",{parentName:"p"},(0,r.yg)("a",{parentName:"strong",href:"https://vpn.maastrichtuniversity.nl/"},"vpn.maastrichtuniversity.nl"))),(0,r.yg)("details",null,(0,r.yg)("summary",null,"\u26a0\ufe0f If your are a ",(0,r.yg)("b",null,"student")," you will need to request access to the UM VPN first"),(0,r.yg)("ul",null,(0,r.yg)("li",null,"You can try to use the Athena Student Desktop at ",(0,r.yg)("a",{href:"https://athenadesktop.maastrichtuniversity.nl"},"athenadesktop.maastrichtuniversity.nl"),", to access the VPN through a virtual desktop"),(0,r.yg)("li",null,"Or ask one of your teachers to request VPN access for you. You will need to send an email to the IT helpdesk of your department with the following information: "),(0,r.yg)("ul",null,(0,r.yg)("li",null,"Email of the student who will get VPN"),(0,r.yg)("li",null," for which course (provide the course ID) or project does the student need the VPN"),(0,r.yg)("li",null,"until which date the student will need the VPN.")))),(0,r.yg)("h2",{id:"access-the-web-ui"},"Access the web UI"),(0,r.yg)("p",null,"Access the DSRI web UI at ",(0,r.yg)("strong",{parentName:"p"},(0,r.yg)("a",{parentName:"strong",href:"https://console-openshift-console.apps.dsri2.unimaas.nl"},"https://console-openshift-console.apps.dsri2.unimaas.nl"))),(0,r.yg)("admonition",{title:"Password",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"Use your general UM password.")),(0,r.yg)("p",null,"If you do not have access to the DSRI ",(0,r.yg)("a",{parentName:"p",href:"mailto:dsri-support-l@maastrichtuniversity.nl"},"contact us"),"."),(0,r.yg)("p",null,"You will be able to login at ",(0,r.yg)("strong",{parentName:"p"},(0,r.yg)("a",{parentName:"strong",href:"https://console-openshift-console.apps.dsri2.unimaas.nl"},"https://console-openshift-console.apps.dsri2.unimaas.nl"))," using the standard maastricht portal upon clicking the login button:"),(0,r.yg)("img",{src:"/img/screenshot_login_screen.png",alt:"Login screen",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("img",{src:"/img/screenshot_um_login_screen.png",alt:"Login screen",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("admonition",{title:"Command line interface",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"We recommend you to install the ",(0,r.yg)("inlineCode",{parentName:"p"},"oc")," command line interface to perform additional operations on your applications, such as loading large amount of data using ",(0,r.yg)("inlineCode",{parentName:"p"},"oc cp"),", or deploying an application from a local ",(0,r.yg)("inlineCode",{parentName:"p"},"Dockerfile"),". Instructions on installing the client can be found ",(0,r.yg)("a",{parentName:"p",href:"/docs/openshift-install"},"\u27a1 here"))),(0,r.yg)("h2",{id:"access-your-project"},"Access your project"),(0,r.yg)("p",null,"In the DSRI OpenShift web UI, applications are deployed in projects."),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"Create a new project with a meaningful name describing what you are doing, such as ",(0,r.yg)("inlineCode",{parentName:"p"},"workspace-yourname"),".")),(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"Go to your project (applications are deployed in a project)."))),(0,r.yg)("admonition",{title:"Reuse your project",type:"caution"},(0,r.yg)("p",{parentName:"admonition"},"Only create new projects when it is necessary (for a new project). You can easily ",(0,r.yg)("a",{parentName:"p",href:"https://maastrichtu-ids.github.io/dsri-documentation/docs/project-management#delete-a-project-using-the-web-ui"},"clean up your current project")," instead of creating a new one every time you want to try something.")),(0,r.yg)("img",{src:"/img/screenshot_go_to_project.png",alt:"Login screen",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("admonition",{title:"Access permissions for developers to your project",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"You can use the ",(0,r.yg)("strong",{parentName:"p"},"Project")," view in the ",(0,r.yg)("strong",{parentName:"p"},"Developer")," perspective to grant or revoke access permissions to your project collaborators. For More Info: ",(0,r.yg)("a",{parentName:"p",href:"https://maastrichtu-ids.github.io/dsri-documentation/docs/project-management/#access-permissions-for-developers-to-your-project"},"Access permissions for developers to your project"))),(0,r.yg)("h2",{id:"about-the-web-ui"},"About the web UI"),(0,r.yg)("p",null,"Developers can use the web console to ",(0,r.yg)("strong",{parentName:"p"},"visualize"),", ",(0,r.yg)("strong",{parentName:"p"},"browse"),", and ",(0,r.yg)("strong",{parentName:"p"},"manage")," the contents of projects in new version of OKD4. "),(0,r.yg)("p",null,"The ",(0,r.yg)("a",{parentName:"p",href:"https://docs.openshift.com/container-platform/4.6/web_console/odc-about-developer-perspective.html"},"OpenShift Container Platform web console")," provides two perspectives; "),(0,r.yg)("ul",null,(0,r.yg)("li",{parentName:"ul"},"the ",(0,r.yg)("strong",{parentName:"li"},"Administrator")," perspective "),(0,r.yg)("li",{parentName:"ul"},"the ",(0,r.yg)("strong",{parentName:"li"},"Developer")," perspective.")),(0,r.yg)("p",null,"The Developer perspective provides workflows specific to developer use cases, such as the ability to:"),(0,r.yg)("ul",null,(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("p",{parentName:"li"},"Create and deploy applications on OpenShift Container Platform by importing existing codebases, images, and dockerfiles.")),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("p",{parentName:"li"},"Visually interact with applications, components, and services associated with them within a project and monitor their deployment and build status.")),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("p",{parentName:"li"},"Group components within an application and connect the components within and across applications."))),(0,r.yg)("h3",{id:"accessing-the-developer-perspective"},"Accessing the Developer perspective"),(0,r.yg)("p",null,"You can access the ",(0,r.yg)("strong",{parentName:"p"},"Developer")," perspective from the web console as follows:"),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"Log in to the OpenShift Container Platform web console using your login credentials. "),(0,r.yg)("ul",{parentName:"li"},(0,r.yg)("li",{parentName:"ul"},"The default view for the OpenShift Container Platform web console is the ",(0,r.yg)("strong",{parentName:"li"},"Administrator")," perspective."))),(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"Use the perspective switcher to switch to the ",(0,r.yg)("strong",{parentName:"p"},"Developer")," perspective. The ",(0,r.yg)("strong",{parentName:"p"},"Topology")," view with a list of all the projects in your cluster is displayed."),(0,r.yg)("img",{src:"/img/screenshot_developer_perspective.png",alt:"Developer Perspective",style:{maxWidth:"100%",maxHeight:"100%"}})),(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"Select an existing project from the list or use the ",(0,r.yg)("strong",{parentName:"p"},"Project")," drop-down list to create a new project."))),(0,r.yg)("admonition",{type:"info"},(0,r.yg)("p",{parentName:"admonition"},"If you have no workloads or applications in the project, the ",(0,r.yg)("strong",{parentName:"p"},"Topology")," view displays the available options to create applications. If you have existing workloads, the ",(0,r.yg)("strong",{parentName:"p"},"Topology")," view graphically displays your workload nodes.")),(0,r.yg)("img",{src:"/img/screenshot_topology_view.png",alt:"Topology View",style:{maxWidth:"100%",maxHeight:"100%"}}))}y.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[8433],{5680:(e,t,n)=>{n.d(t,{xA:()=>c,yg:()=>y});var o=n(6540);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function i(e){for(var t=1;t=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(o=0;o=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var l=o.createContext({}),p=function(e){var t=o.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},c=function(e){var t=p(e.components);return o.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.createElement(o.Fragment,{},t)}},g=o.forwardRef((function(e,t){var n=e.components,a=e.mdxType,r=e.originalType,l=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),g=p(n),y=a,m=g["".concat(l,".").concat(y)]||g[y]||u[y]||r;return n?o.createElement(m,i(i({ref:t},c),{},{components:n})):o.createElement(m,i({ref:t},c))}));function y(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var r=n.length,i=new Array(r);i[0]=g;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s.mdxType="string"==typeof e?e:a,i[1]=s;for(var p=2;p{n.r(t),n.d(t,{assets:()=>c,contentTitle:()=>l,default:()=>y,frontMatter:()=>s,metadata:()=>p,toc:()=>u});var o=n(9668),a=n(1367),r=(n(6540),n(5680)),i=["components"],s={id:"access-dsri",title:"Access the DSRI"},l=void 0,p={unversionedId:"access-dsri",id:"access-dsri",title:"Access the DSRI",description:"Request an account",source:"@site/docs/access-dsri.md",sourceDirName:".",slug:"/access-dsri",permalink:"/docs/access-dsri",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/access-dsri.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"access-dsri",title:"Access the DSRI"},sidebar:"docs",previous:{title:"Introduction",permalink:"/docs/"},next:{title:"Start your workspace",permalink:"/docs/start-workspace"}},c={},u=[{value:"Request an account",id:"request-an-account",level:2},{value:"Connect to the UM network",id:"connect-to-the-um-network",level:2},{value:"Access the web UI",id:"access-the-web-ui",level:2},{value:"Access your project",id:"access-your-project",level:2},{value:"About the web UI",id:"about-the-web-ui",level:2},{value:"Accessing the Developer perspective",id:"accessing-the-developer-perspective",level:3}],g={toc:u};function y(e){var t=e.components,n=(0,a.A)(e,i);return(0,r.yg)("wrapper",(0,o.A)({},g,n,{components:t,mdxType:"MDXLayout"}),(0,r.yg)("h2",{id:"request-an-account"},"Request an account"),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"You will need to have an account at Maastricht University with an email ending with ",(0,r.yg)("inlineCode",{parentName:"p"},"@maastrichtuniversity.nl")," or ",(0,r.yg)("inlineCode",{parentName:"p"},"@student.maastrichtuniversity.nl"),".")),(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"Request access to the DSRI for your account Please fill this ",(0,r.yg)("a",{parentName:"p",href:"/register"},"form \ud83d\udcec")," to provide us some information on what you plan to do with the DSRI. Once you fill the form, you will receive an email with detailed instructions on how to log in."))),(0,r.yg)("h2",{id:"connect-to-the-um-network"},"Connect to the UM network"),(0,r.yg)("p",null,"You need to be connected to the UM network to access the DSRI."),(0,r.yg)("p",null,(0,r.yg)("strong",{parentName:"p"},"\ud83d\udc27 On Linux"),": use ",(0,r.yg)("inlineCode",{parentName:"p"},"openconnect")," to connect to the UM VPN. You can easily install it on Ubuntu and Debian distributions with ",(0,r.yg)("inlineCode",{parentName:"p"},"apt"),":"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"sudo apt install openconnect\nsudo openconnect -u YOUR.USER --authgroup 01-Employees vpn.maastrichtuniversity.nl\n")),(0,r.yg)("p",null,(0,r.yg)("strong",{parentName:"p"},"\ud83c\udf4e On MacOS and Windows"),": download and install the ",(0,r.yg)("strong",{parentName:"p"},"Maastricht University VPN")," client available at ",(0,r.yg)("strong",{parentName:"p"},(0,r.yg)("a",{parentName:"strong",href:"https://vpn.maastrichtuniversity.nl/"},"vpn.maastrichtuniversity.nl"))),(0,r.yg)("details",null,(0,r.yg)("summary",null,"\u26a0\ufe0f If your are a ",(0,r.yg)("b",null,"student")," you will need to request access to the UM VPN first"),(0,r.yg)("ul",null,(0,r.yg)("li",null,"You can try to use the Athena Student Desktop at ",(0,r.yg)("a",{href:"https://athenadesktop.maastrichtuniversity.nl"},"athenadesktop.maastrichtuniversity.nl"),", to access the VPN through a virtual desktop"),(0,r.yg)("li",null,"Or ask one of your teachers to request VPN access for you. You will need to send an email to the IT helpdesk of your department with the following information: "),(0,r.yg)("ul",null,(0,r.yg)("li",null,"Email of the student who will get VPN"),(0,r.yg)("li",null," for which course (provide the course ID) or project does the student need the VPN"),(0,r.yg)("li",null,"until which date the student will need the VPN.")))),(0,r.yg)("h2",{id:"access-the-web-ui"},"Access the web UI"),(0,r.yg)("p",null,"Access the DSRI web UI at ",(0,r.yg)("strong",{parentName:"p"},(0,r.yg)("a",{parentName:"strong",href:"https://console-openshift-console.apps.dsri2.unimaas.nl"},"https://console-openshift-console.apps.dsri2.unimaas.nl"))),(0,r.yg)("admonition",{title:"Password",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"Use your general UM password.")),(0,r.yg)("p",null,"If you do not have access to the DSRI ",(0,r.yg)("a",{parentName:"p",href:"mailto:dsri-support-l@maastrichtuniversity.nl"},"contact us"),"."),(0,r.yg)("p",null,"You will be able to login at ",(0,r.yg)("strong",{parentName:"p"},(0,r.yg)("a",{parentName:"strong",href:"https://console-openshift-console.apps.dsri2.unimaas.nl"},"https://console-openshift-console.apps.dsri2.unimaas.nl"))," using the standard maastricht portal upon clicking the login button:"),(0,r.yg)("img",{src:"/img/screenshot_login_screen.png",alt:"Login screen",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("img",{src:"/img/screenshot_um_login_screen.png",alt:"Login screen",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("admonition",{title:"Command line interface",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"We recommend you to install the ",(0,r.yg)("inlineCode",{parentName:"p"},"oc")," command line interface to perform additional operations on your applications, such as loading large amount of data using ",(0,r.yg)("inlineCode",{parentName:"p"},"oc cp"),", or deploying an application from a local ",(0,r.yg)("inlineCode",{parentName:"p"},"Dockerfile"),". Instructions on installing the client can be found ",(0,r.yg)("a",{parentName:"p",href:"/docs/openshift-install"},"\u27a1 here"))),(0,r.yg)("h2",{id:"access-your-project"},"Access your project"),(0,r.yg)("p",null,"In the DSRI OpenShift web UI, applications are deployed in projects."),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"Create a new project with a meaningful name describing what you are doing, such as ",(0,r.yg)("inlineCode",{parentName:"p"},"workspace-yourname"),".")),(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"Go to your project (applications are deployed in a project)."))),(0,r.yg)("admonition",{title:"Reuse your project",type:"caution"},(0,r.yg)("p",{parentName:"admonition"},"Only create new projects when it is necessary (for a new project). You can easily ",(0,r.yg)("a",{parentName:"p",href:"https://maastrichtu-ids.github.io/dsri-documentation/docs/project-management#delete-a-project-using-the-web-ui"},"clean up your current project")," instead of creating a new one every time you want to try something.")),(0,r.yg)("img",{src:"/img/screenshot_go_to_project.png",alt:"Login screen",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("admonition",{title:"Access permissions for developers to your project",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"You can use the ",(0,r.yg)("strong",{parentName:"p"},"Project")," view in the ",(0,r.yg)("strong",{parentName:"p"},"Developer")," perspective to grant or revoke access permissions to your project collaborators. For More Info: ",(0,r.yg)("a",{parentName:"p",href:"https://maastrichtu-ids.github.io/dsri-documentation/docs/project-management/#access-permissions-for-developers-to-your-project"},"Access permissions for developers to your project"))),(0,r.yg)("h2",{id:"about-the-web-ui"},"About the web UI"),(0,r.yg)("p",null,"Developers can use the web console to ",(0,r.yg)("strong",{parentName:"p"},"visualize"),", ",(0,r.yg)("strong",{parentName:"p"},"browse"),", and ",(0,r.yg)("strong",{parentName:"p"},"manage")," the contents of projects in new version of OKD4. "),(0,r.yg)("p",null,"The ",(0,r.yg)("a",{parentName:"p",href:"https://docs.openshift.com/container-platform/4.6/web_console/odc-about-developer-perspective.html"},"OpenShift Container Platform web console")," provides two perspectives; "),(0,r.yg)("ul",null,(0,r.yg)("li",{parentName:"ul"},"the ",(0,r.yg)("strong",{parentName:"li"},"Administrator")," perspective "),(0,r.yg)("li",{parentName:"ul"},"the ",(0,r.yg)("strong",{parentName:"li"},"Developer")," perspective.")),(0,r.yg)("p",null,"The Developer perspective provides workflows specific to developer use cases, such as the ability to:"),(0,r.yg)("ul",null,(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("p",{parentName:"li"},"Create and deploy applications on OpenShift Container Platform by importing existing codebases, images, and dockerfiles.")),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("p",{parentName:"li"},"Visually interact with applications, components, and services associated with them within a project and monitor their deployment and build status.")),(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("p",{parentName:"li"},"Group components within an application and connect the components within and across applications."))),(0,r.yg)("h3",{id:"accessing-the-developer-perspective"},"Accessing the Developer perspective"),(0,r.yg)("p",null,"You can access the ",(0,r.yg)("strong",{parentName:"p"},"Developer")," perspective from the web console as follows:"),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"Log in to the OpenShift Container Platform web console using your login credentials. "),(0,r.yg)("ul",{parentName:"li"},(0,r.yg)("li",{parentName:"ul"},"The default view for the OpenShift Container Platform web console is the ",(0,r.yg)("strong",{parentName:"li"},"Administrator")," perspective."))),(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"Use the perspective switcher to switch to the ",(0,r.yg)("strong",{parentName:"p"},"Developer")," perspective. The ",(0,r.yg)("strong",{parentName:"p"},"Topology")," view with a list of all the projects in your cluster is displayed."),(0,r.yg)("img",{src:"/img/screenshot_developer_perspective.png",alt:"Developer Perspective",style:{maxWidth:"100%",maxHeight:"100%"}})),(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"Select an existing project from the list or use the ",(0,r.yg)("strong",{parentName:"p"},"Project")," drop-down list to create a new project."))),(0,r.yg)("admonition",{type:"info"},(0,r.yg)("p",{parentName:"admonition"},"If you have no workloads or applications in the project, the ",(0,r.yg)("strong",{parentName:"p"},"Topology")," view displays the available options to create applications. If you have existing workloads, the ",(0,r.yg)("strong",{parentName:"p"},"Topology")," view graphically displays your workload nodes.")),(0,r.yg)("img",{src:"/img/screenshot_topology_view.png",alt:"Topology View",style:{maxWidth:"100%",maxHeight:"100%"}}))}y.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/b258fc9c.eaf50fef.js b/assets/js/b258fc9c.021f53f5.js similarity index 99% rename from assets/js/b258fc9c.eaf50fef.js rename to assets/js/b258fc9c.021f53f5.js index 2d78dcf50..7fa8d9ff5 100644 --- a/assets/js/b258fc9c.eaf50fef.js +++ b/assets/js/b258fc9c.021f53f5.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[9725],{5680:(e,t,n)=>{n.d(t,{xA:()=>c,yg:()=>g});var l=n(6540);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);t&&(l=l.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,l)}return n}function r(e){for(var t=1;t=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(l=0;l=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var i=l.createContext({}),u=function(e){var t=l.useContext(i),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},c=function(e){var t=u(e.components);return l.createElement(i.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return l.createElement(l.Fragment,{},t)}},d=l.forwardRef((function(e,t){var n=e.components,a=e.mdxType,o=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),d=u(n),g=a,m=d["".concat(i,".").concat(g)]||d[g]||p[g]||o;return n?l.createElement(m,r(r({ref:t},c),{},{components:n})):l.createElement(m,r({ref:t},c))}));function g(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=n.length,r=new Array(o);r[0]=d;var s={};for(var i in t)hasOwnProperty.call(t,i)&&(s[i]=t[i]);s.originalType=e,s.mdxType="string"==typeof e?e:a,r[1]=s;for(var u=2;u{n.r(t),n.d(t,{assets:()=>c,contentTitle:()=>i,default:()=>g,frontMatter:()=>s,metadata:()=>u,toc:()=>p});var l=n(9668),a=n(1367),o=(n(6540),n(5680)),r=["components"],s={id:"guide-local-install",title:"Install local OpenShift"},i=void 0,u={unversionedId:"guide-local-install",id:"guide-local-install",title:"Install local OpenShift",description:"OpenShift and Kubernetes can be installed locally on a single machine for test purpose. The installation requires knowledge of your OS administration, and can be quite complex. We recommend to install it locally only if really required. Otherwise we recommend you to simply use Docker to test images, then deploy them on the DSRI.",source:"@site/docs/guide-local-install.md",sourceDirName:".",slug:"/guide-local-install",permalink:"/docs/guide-local-install",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/guide-local-install.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"guide-local-install",title:"Install local OpenShift"}},c={},p=[{value:"Install MiniShift",id:"install-minishift",level:2},{value:"Start MiniShift",id:"start-minishift",level:3},{value:"Login",id:"login",level:3},{value:"Stop",id:"stop",level:3},{value:"Reset",id:"reset",level:3},{value:"Install kubectl",id:"install-kubectl",level:2},{value:"kubectl on Ubuntu",id:"kubectl-on-ubuntu",level:3},{value:"kubectl on MacOS & Windows",id:"kubectl-on-macos--windows",level:3},{value:"Install the Dashboard UI",id:"install-the-dashboard-ui",level:3},{value:"Run kubectl",id:"run-kubectl",level:3},{value:"Enable internet",id:"enable-internet",level:3},{value:"Create persistent volume",id:"create-persistent-volume",level:3},{value:"Uninstall",id:"uninstall",level:3},{value:"Install Argo workflows",id:"install-argo-workflows",level:2},{value:"Install on your local Kubernetes",id:"install-on-your-local-kubernetes",level:3},{value:"Install the client",id:"install-the-client",level:3},{value:"Expose the UI",id:"expose-the-ui",level:3}],d={toc:p};function g(e){var t=e.components,s=(0,a.A)(e,r);return(0,o.yg)("wrapper",(0,l.A)({},d,s,{components:t,mdxType:"MDXLayout"}),(0,o.yg)("p",null,"OpenShift and Kubernetes can be installed locally on a single machine for test purpose. The installation requires knowledge of your OS administration, and can be quite complex. We recommend to install it locally only if really required. Otherwise we recommend you to simply use Docker to test images, then deploy them on the DSRI."),(0,o.yg)("h2",{id:"install-minishift"},"Install MiniShift"),(0,o.yg)("p",null,(0,o.yg)("a",{parentName:"p",href:"https://docs.okd.io/latest/minishift/getting-started/installing.html"})),(0,o.yg)("p",null,"You will need to ",(0,o.yg)("a",{parentName:"p",href:"https://docs.okd.io/latest/minishift/getting-started/setting-up-virtualization-environment.html"},"set up the virtualization environment")," before ",(0,o.yg)("a",{parentName:"p",href:"https://docs.okd.io/latest/minishift/getting-started/installing.html"},"installing MiniShift"),"."),(0,o.yg)("p",null,"Download ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/minishift/minishift/releases"},"MiniShift")," and unzip it."),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"# For Ubuntu 18.04 and older\nsudo apt install -y libvirt-bin qemu-kvm\n# For Ubuntu 18.10 and newer (replace libvirtd by libvirt in next commands)\nsudo apt install -y qemu-kvm libvirt-daemon libvirt-daemon-system\n\n# Create group if does not exist\nsudo addgroup libvirtd\nsudo adduser $(whoami) libvirtd\n\nsudo usermod -a -G libvirtd $(whoami)\nnewgrp libvirtd\ncurl -L https://github.com/dhiltgen/docker-machine-kvm/releases/download/v0.10.0/docker-machine-driver-kvm-ubuntu16.04 -o /usr/local/bin/docker-machine-driver-kvm\nsudo chmod +x /usr/local/bin/docker-machine-driver-kvm\n\n# Check if libvirtd running\nsystemctl is-active libvirtd\n# Start if inactive\nsudo systemctl start libvirtd\n\n# Copy MiniShift in your path\ncp minishift-1.34.1-linux-amd64/minishift /usr/local/bin\n")),(0,o.yg)("h3",{id:"start-minishift"},"Start MiniShift"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"minishift start\n")),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},"Get your local OpenShift cluster URL after the command complete.")),(0,o.yg)("h3",{id:"login"},"Login"),(0,o.yg)("p",null,"Go to your local cluster URL."),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},"E.g. ",(0,o.yg)("a",{parentName:"p",href:"https://192.168.42.58:8443/console/catalog"},"https://192.168.42.58:8443/console/catalog"),".")),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},"Username: ",(0,o.yg)("inlineCode",{parentName:"p"},"admin")," or ",(0,o.yg)("inlineCode",{parentName:"p"},"developer")),(0,o.yg)("p",{parentName:"blockquote"},"Password: anything will work")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"# As admin\noc login -u system:admin\n")),(0,o.yg)("h3",{id:"stop"},"Stop"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"minishift stop\n")),(0,o.yg)("h3",{id:"reset"},"Reset"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"minishift delete -f\n")),(0,o.yg)("hr",null),(0,o.yg)("h2",{id:"install-kubectl"},"Install kubectl"),(0,o.yg)("p",null,(0,o.yg)("a",{parentName:"p",href:"https://kubernetes.io/"},(0,o.yg)("img",{alt:"Kubernetes",src:n(647).A,width:"209",height:"203"}))),(0,o.yg)("h3",{id:"kubectl-on-ubuntu"},"kubectl on Ubuntu"),(0,o.yg)("p",null,"For more details: read the official ",(0,o.yg)("a",{parentName:"p",href:"https://tutorials.ubuntu.com/tutorial/install-a-local-kubernetes-with-microk8s#0"},"install Kubernetes on Ubuntu tutorial")," or see the official ",(0,o.yg)("a",{parentName:"p",href:"https://ubuntu.com/kubernetes/install"},"Ubuntu Kubernetes install documentation"),"."),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"sudo snap install microk8s --classic\nsudo usermod -a -G microk8s $USER\n# Restart your machine\nmkdir -p ~/.kube\nmicrok8s.kubectl config view --raw > $HOME/.kube/config\n\n# Make sure this works for dashboard on Ubuntu\nmicrok8s.enable dashboard dns\n")),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},"To do only if kubectl is not already installed on your machine:"),(0,o.yg)("pre",{parentName:"blockquote"},(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"sudo snap alias microk8s.kubectl kubectl\n"))),(0,o.yg)("h3",{id:"kubectl-on-macos--windows"},"kubectl on MacOS & Windows"),(0,o.yg)("p",null,"Included in Docker installation. Use the installer provided by DockerHub."),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},"Activate it in Docker Preferences > Kubernetes.")),(0,o.yg)("p",null,"For Windows you will need to download the ",(0,o.yg)("inlineCode",{parentName:"p"},"kubectl.exe")," and place it in your PATH."),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("a",{parentName:"li",href:"https://storage.googleapis.com/kubernetes-release/release/v1.17.0/bin/windows/amd64/kubectl.exe"},"https://storage.googleapis.com/kubernetes-release/release/v1.17.0/bin/windows/amd64/kubectl.exe"))),(0,o.yg)("p",null,"We recommend to create a ",(0,o.yg)("inlineCode",{parentName:"p"},"kubectl")," directory in ",(0,o.yg)("inlineCode",{parentName:"p"},"C:/")," and add this ",(0,o.yg)("inlineCode",{parentName:"p"},"C:/kubectl")," to the Path environment variable in System properties > Advanced > Environment Variables > Path"),(0,o.yg)("h3",{id:"install-the-dashboard-ui"},"Install the Dashboard UI"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"# Install Kubernetes UI\nkubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0-beta4/aio/deploy/recommended.yaml\nkubectl apply -f https://raw.githubusercontent.com/MaastrichtU-IDS/d2s-core/master/argo/roles/kube-dashboard-adminuser-sa.yml\nkubectl apply -f https://raw.githubusercontent.com/MaastrichtU-IDS/d2s-core/master/argo/roles/kube-adminuser-rolebinding.yml\n\n# Get the Token to access the dashboard\nkubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep admin-user | awk '{print $1}')\n\n\n# Windows user will need to execute the 2 commands manually:\nkubectl -n kube-system get secret \n# And get the token containing 'admin-user'\nkubectl -n kube-system describe secret\n# For Windows: give the anonymous user global access\nkubectl create clusterrolebinding cluster-system-anonymous --clusterrole=admin --user=system:anonymous\n# Note: this could be improved. I think only the Dashboard UI didn't have the required permissions.\n\n# Finally, start the web UI, and chose the Token connection\nkubectl proxy\n")),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},"Then visit: http://localhost:8001/api/v1/namespaces/kubernetes-dashboard/services/https:kubernetes-dashboard:/proxy/"),(0,o.yg)("p",{parentName:"blockquote"},"And provide the previously obtained token.")),(0,o.yg)("p",null,(0,o.yg)("strong",{parentName:"p"},"Warning:")," you will need to save the token to login again next time (use the password save from your browser if possible)."),(0,o.yg)("h3",{id:"run-kubectl"},"Run kubectl"),(0,o.yg)("p",null,(0,o.yg)("inlineCode",{parentName:"p"},"kubectl")," should be running at start."),(0,o.yg)("p",null,"Just restart the web UI"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"kubectl proxy\n")),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},"Then visit: http://localhost:8001/api/v1/namespaces/kubernetes-dashboard/services/https:kubernetes-dashboard:/proxy/")),(0,o.yg)("h3",{id:"enable-internet"},"Enable internet"),(0,o.yg)("p",null,(0,o.yg)("a",{parentName:"p",href:"https://kubernetes.io/docs/tasks/administer-cluster/dns-debugging-resolution/"},"Debug DNS on Ubuntu")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"microk8s.enable dns\n")),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},"Restart your machine.")),(0,o.yg)("p",null,"You might need to change your firewall configuration"),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},"On Ubuntu")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"sudo ufw allow in on cni0\nsudo ufw allow out on cni0\nsudo ufw default allow routed\n")),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},"Try to connect to the internet from Kubernetes with the ",(0,o.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/d2s-core/blob/master/argo/tests/test-busybox.yaml"},"test-busybox pod"),".")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"kubectl exec -ti busybox -- /bin/sh\nping google.com\n")),(0,o.yg)("h3",{id:"create-persistent-volume"},"Create persistent volume"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"# Create volume\nkubectl apply -n argo -f d2s-core/argo/storage/storage-mac.yaml\n")),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},"Not working at the moment.")),(0,o.yg)("h3",{id:"uninstall"},"Uninstall"),(0,o.yg)("p",null,"Clean uninstall before 2.2."),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"kubectl get cm workflow-controller-configmap -o yaml -n kube-system --export | kubectl apply -n argo -f -\nkubectl delete -n kube-system cm workflow-controller-configmap\nkubectl delete -n kube-system deploy workflow-controller argo-ui\nkubectl delete -n kube-system sa argo argo-ui\nkubectl delete -n kube-system svc argo-ui\n")),(0,o.yg)("hr",null),(0,o.yg)("h2",{id:"install-argo-workflows"},"Install Argo workflows"),(0,o.yg)("p",null,(0,o.yg)("a",{parentName:"p",href:"https://argoproj.github.io/argo/"},(0,o.yg)("img",{alt:"Argo project",src:n(4122).A,width:"300",height:"137"}))),(0,o.yg)("h3",{id:"install-on-your-local-kubernetes"},"Install on your local Kubernetes"),(0,o.yg)("p",null,"Argo workflows will be installed on the ",(0,o.yg)("inlineCode",{parentName:"p"},"argo")," namespace. See the ",(0,o.yg)("a",{parentName:"p",href:"https://argoproj.github.io/docs/argo/demo.html#2-install-the-controller-and-ui"},"official Argo documentation")," for more details."),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"kubectl create ns argo\nkubectl apply -n argo -f https://raw.githubusercontent.com/argoproj/argo/v2.4.2/manifests/install.yaml\n\n# Configure service account to run workflow\nkubectl create rolebinding default-admin --clusterrole=admin --serviceaccount=default:default\n\n# Test run\nargo submit --watch https://raw.githubusercontent.com/argoproj/argo/master/examples/hello-world.yaml\n")),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},"See ",(0,o.yg)("a",{parentName:"p",href:"https://raw.githubusercontent.com/vemonet/argo/master/manifests/namespace-install.yaml"},"custom configuration")," for namespace install."),(0,o.yg)("pre",{parentName:"blockquote"},(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"kubectl apply -n argo -f https://raw.githubusercontent.com/vemonet/argo/master/manifests/namespace-install.yaml\n"))),(0,o.yg)("h3",{id:"install-the-client"},"Install the client"),(0,o.yg)("p",null,"See the ",(0,o.yg)("a",{parentName:"p",href:"/docs/workflows-argo"},"Argo workflows documentation"),"."),(0,o.yg)("h3",{id:"expose-the-ui"},"Expose the UI"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"kubectl -n argo port-forward deployment/argo-ui 8002:8001\n")),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},"Access on http://localhost:8002.")))}g.isMDXComponent=!0},647:(e,t,n)=>{n.d(t,{A:()=>l});const l=n.p+"assets/images/Kubernetes-c5f36b415dd16a8fcdae01fbc8d9c940.png"},4122:(e,t,n)=>{n.d(t,{A:()=>l});const l=n.p+"assets/images/argo-logo-c091bfee39aec37120d0e879edac74f6.png"}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[9725],{5680:(e,t,n)=>{n.d(t,{xA:()=>c,yg:()=>g});var l=n(6540);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);t&&(l=l.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,l)}return n}function r(e){for(var t=1;t=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(l=0;l=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var i=l.createContext({}),u=function(e){var t=l.useContext(i),n=t;return e&&(n="function"==typeof e?e(t):r(r({},t),e)),n},c=function(e){var t=u(e.components);return l.createElement(i.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return l.createElement(l.Fragment,{},t)}},d=l.forwardRef((function(e,t){var n=e.components,a=e.mdxType,o=e.originalType,i=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),d=u(n),g=a,m=d["".concat(i,".").concat(g)]||d[g]||p[g]||o;return n?l.createElement(m,r(r({ref:t},c),{},{components:n})):l.createElement(m,r({ref:t},c))}));function g(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=n.length,r=new Array(o);r[0]=d;var s={};for(var i in t)hasOwnProperty.call(t,i)&&(s[i]=t[i]);s.originalType=e,s.mdxType="string"==typeof e?e:a,r[1]=s;for(var u=2;u{n.r(t),n.d(t,{assets:()=>c,contentTitle:()=>i,default:()=>g,frontMatter:()=>s,metadata:()=>u,toc:()=>p});var l=n(9668),a=n(1367),o=(n(6540),n(5680)),r=["components"],s={id:"guide-local-install",title:"Install local OpenShift"},i=void 0,u={unversionedId:"guide-local-install",id:"guide-local-install",title:"Install local OpenShift",description:"OpenShift and Kubernetes can be installed locally on a single machine for test purpose. The installation requires knowledge of your OS administration, and can be quite complex. We recommend to install it locally only if really required. Otherwise we recommend you to simply use Docker to test images, then deploy them on the DSRI.",source:"@site/docs/guide-local-install.md",sourceDirName:".",slug:"/guide-local-install",permalink:"/docs/guide-local-install",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/guide-local-install.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"guide-local-install",title:"Install local OpenShift"}},c={},p=[{value:"Install MiniShift",id:"install-minishift",level:2},{value:"Start MiniShift",id:"start-minishift",level:3},{value:"Login",id:"login",level:3},{value:"Stop",id:"stop",level:3},{value:"Reset",id:"reset",level:3},{value:"Install kubectl",id:"install-kubectl",level:2},{value:"kubectl on Ubuntu",id:"kubectl-on-ubuntu",level:3},{value:"kubectl on MacOS & Windows",id:"kubectl-on-macos--windows",level:3},{value:"Install the Dashboard UI",id:"install-the-dashboard-ui",level:3},{value:"Run kubectl",id:"run-kubectl",level:3},{value:"Enable internet",id:"enable-internet",level:3},{value:"Create persistent volume",id:"create-persistent-volume",level:3},{value:"Uninstall",id:"uninstall",level:3},{value:"Install Argo workflows",id:"install-argo-workflows",level:2},{value:"Install on your local Kubernetes",id:"install-on-your-local-kubernetes",level:3},{value:"Install the client",id:"install-the-client",level:3},{value:"Expose the UI",id:"expose-the-ui",level:3}],d={toc:p};function g(e){var t=e.components,s=(0,a.A)(e,r);return(0,o.yg)("wrapper",(0,l.A)({},d,s,{components:t,mdxType:"MDXLayout"}),(0,o.yg)("p",null,"OpenShift and Kubernetes can be installed locally on a single machine for test purpose. The installation requires knowledge of your OS administration, and can be quite complex. We recommend to install it locally only if really required. Otherwise we recommend you to simply use Docker to test images, then deploy them on the DSRI."),(0,o.yg)("h2",{id:"install-minishift"},"Install MiniShift"),(0,o.yg)("p",null,(0,o.yg)("a",{parentName:"p",href:"https://docs.okd.io/latest/minishift/getting-started/installing.html"})),(0,o.yg)("p",null,"You will need to ",(0,o.yg)("a",{parentName:"p",href:"https://docs.okd.io/latest/minishift/getting-started/setting-up-virtualization-environment.html"},"set up the virtualization environment")," before ",(0,o.yg)("a",{parentName:"p",href:"https://docs.okd.io/latest/minishift/getting-started/installing.html"},"installing MiniShift"),"."),(0,o.yg)("p",null,"Download ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/minishift/minishift/releases"},"MiniShift")," and unzip it."),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"# For Ubuntu 18.04 and older\nsudo apt install -y libvirt-bin qemu-kvm\n# For Ubuntu 18.10 and newer (replace libvirtd by libvirt in next commands)\nsudo apt install -y qemu-kvm libvirt-daemon libvirt-daemon-system\n\n# Create group if does not exist\nsudo addgroup libvirtd\nsudo adduser $(whoami) libvirtd\n\nsudo usermod -a -G libvirtd $(whoami)\nnewgrp libvirtd\ncurl -L https://github.com/dhiltgen/docker-machine-kvm/releases/download/v0.10.0/docker-machine-driver-kvm-ubuntu16.04 -o /usr/local/bin/docker-machine-driver-kvm\nsudo chmod +x /usr/local/bin/docker-machine-driver-kvm\n\n# Check if libvirtd running\nsystemctl is-active libvirtd\n# Start if inactive\nsudo systemctl start libvirtd\n\n# Copy MiniShift in your path\ncp minishift-1.34.1-linux-amd64/minishift /usr/local/bin\n")),(0,o.yg)("h3",{id:"start-minishift"},"Start MiniShift"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"minishift start\n")),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},"Get your local OpenShift cluster URL after the command complete.")),(0,o.yg)("h3",{id:"login"},"Login"),(0,o.yg)("p",null,"Go to your local cluster URL."),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},"E.g. ",(0,o.yg)("a",{parentName:"p",href:"https://192.168.42.58:8443/console/catalog"},"https://192.168.42.58:8443/console/catalog"),".")),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},"Username: ",(0,o.yg)("inlineCode",{parentName:"p"},"admin")," or ",(0,o.yg)("inlineCode",{parentName:"p"},"developer")),(0,o.yg)("p",{parentName:"blockquote"},"Password: anything will work")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"# As admin\noc login -u system:admin\n")),(0,o.yg)("h3",{id:"stop"},"Stop"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"minishift stop\n")),(0,o.yg)("h3",{id:"reset"},"Reset"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"minishift delete -f\n")),(0,o.yg)("hr",null),(0,o.yg)("h2",{id:"install-kubectl"},"Install kubectl"),(0,o.yg)("p",null,(0,o.yg)("a",{parentName:"p",href:"https://kubernetes.io/"},(0,o.yg)("img",{alt:"Kubernetes",src:n(647).A,width:"209",height:"203"}))),(0,o.yg)("h3",{id:"kubectl-on-ubuntu"},"kubectl on Ubuntu"),(0,o.yg)("p",null,"For more details: read the official ",(0,o.yg)("a",{parentName:"p",href:"https://tutorials.ubuntu.com/tutorial/install-a-local-kubernetes-with-microk8s#0"},"install Kubernetes on Ubuntu tutorial")," or see the official ",(0,o.yg)("a",{parentName:"p",href:"https://ubuntu.com/kubernetes/install"},"Ubuntu Kubernetes install documentation"),"."),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"sudo snap install microk8s --classic\nsudo usermod -a -G microk8s $USER\n# Restart your machine\nmkdir -p ~/.kube\nmicrok8s.kubectl config view --raw > $HOME/.kube/config\n\n# Make sure this works for dashboard on Ubuntu\nmicrok8s.enable dashboard dns\n")),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},"To do only if kubectl is not already installed on your machine:"),(0,o.yg)("pre",{parentName:"blockquote"},(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"sudo snap alias microk8s.kubectl kubectl\n"))),(0,o.yg)("h3",{id:"kubectl-on-macos--windows"},"kubectl on MacOS & Windows"),(0,o.yg)("p",null,"Included in Docker installation. Use the installer provided by DockerHub."),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},"Activate it in Docker Preferences > Kubernetes.")),(0,o.yg)("p",null,"For Windows you will need to download the ",(0,o.yg)("inlineCode",{parentName:"p"},"kubectl.exe")," and place it in your PATH."),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("a",{parentName:"li",href:"https://storage.googleapis.com/kubernetes-release/release/v1.17.0/bin/windows/amd64/kubectl.exe"},"https://storage.googleapis.com/kubernetes-release/release/v1.17.0/bin/windows/amd64/kubectl.exe"))),(0,o.yg)("p",null,"We recommend to create a ",(0,o.yg)("inlineCode",{parentName:"p"},"kubectl")," directory in ",(0,o.yg)("inlineCode",{parentName:"p"},"C:/")," and add this ",(0,o.yg)("inlineCode",{parentName:"p"},"C:/kubectl")," to the Path environment variable in System properties > Advanced > Environment Variables > Path"),(0,o.yg)("h3",{id:"install-the-dashboard-ui"},"Install the Dashboard UI"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"# Install Kubernetes UI\nkubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0-beta4/aio/deploy/recommended.yaml\nkubectl apply -f https://raw.githubusercontent.com/MaastrichtU-IDS/d2s-core/master/argo/roles/kube-dashboard-adminuser-sa.yml\nkubectl apply -f https://raw.githubusercontent.com/MaastrichtU-IDS/d2s-core/master/argo/roles/kube-adminuser-rolebinding.yml\n\n# Get the Token to access the dashboard\nkubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep admin-user | awk '{print $1}')\n\n\n# Windows user will need to execute the 2 commands manually:\nkubectl -n kube-system get secret \n# And get the token containing 'admin-user'\nkubectl -n kube-system describe secret\n# For Windows: give the anonymous user global access\nkubectl create clusterrolebinding cluster-system-anonymous --clusterrole=admin --user=system:anonymous\n# Note: this could be improved. I think only the Dashboard UI didn't have the required permissions.\n\n# Finally, start the web UI, and chose the Token connection\nkubectl proxy\n")),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},"Then visit: http://localhost:8001/api/v1/namespaces/kubernetes-dashboard/services/https:kubernetes-dashboard:/proxy/"),(0,o.yg)("p",{parentName:"blockquote"},"And provide the previously obtained token.")),(0,o.yg)("p",null,(0,o.yg)("strong",{parentName:"p"},"Warning:")," you will need to save the token to login again next time (use the password save from your browser if possible)."),(0,o.yg)("h3",{id:"run-kubectl"},"Run kubectl"),(0,o.yg)("p",null,(0,o.yg)("inlineCode",{parentName:"p"},"kubectl")," should be running at start."),(0,o.yg)("p",null,"Just restart the web UI"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"kubectl proxy\n")),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},"Then visit: http://localhost:8001/api/v1/namespaces/kubernetes-dashboard/services/https:kubernetes-dashboard:/proxy/")),(0,o.yg)("h3",{id:"enable-internet"},"Enable internet"),(0,o.yg)("p",null,(0,o.yg)("a",{parentName:"p",href:"https://kubernetes.io/docs/tasks/administer-cluster/dns-debugging-resolution/"},"Debug DNS on Ubuntu")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"microk8s.enable dns\n")),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},"Restart your machine.")),(0,o.yg)("p",null,"You might need to change your firewall configuration"),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},"On Ubuntu")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"sudo ufw allow in on cni0\nsudo ufw allow out on cni0\nsudo ufw default allow routed\n")),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},"Try to connect to the internet from Kubernetes with the ",(0,o.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/d2s-core/blob/master/argo/tests/test-busybox.yaml"},"test-busybox pod"),".")),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"kubectl exec -ti busybox -- /bin/sh\nping google.com\n")),(0,o.yg)("h3",{id:"create-persistent-volume"},"Create persistent volume"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"# Create volume\nkubectl apply -n argo -f d2s-core/argo/storage/storage-mac.yaml\n")),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},"Not working at the moment.")),(0,o.yg)("h3",{id:"uninstall"},"Uninstall"),(0,o.yg)("p",null,"Clean uninstall before 2.2."),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"kubectl get cm workflow-controller-configmap -o yaml -n kube-system --export | kubectl apply -n argo -f -\nkubectl delete -n kube-system cm workflow-controller-configmap\nkubectl delete -n kube-system deploy workflow-controller argo-ui\nkubectl delete -n kube-system sa argo argo-ui\nkubectl delete -n kube-system svc argo-ui\n")),(0,o.yg)("hr",null),(0,o.yg)("h2",{id:"install-argo-workflows"},"Install Argo workflows"),(0,o.yg)("p",null,(0,o.yg)("a",{parentName:"p",href:"https://argoproj.github.io/argo/"},(0,o.yg)("img",{alt:"Argo project",src:n(4122).A,width:"300",height:"137"}))),(0,o.yg)("h3",{id:"install-on-your-local-kubernetes"},"Install on your local Kubernetes"),(0,o.yg)("p",null,"Argo workflows will be installed on the ",(0,o.yg)("inlineCode",{parentName:"p"},"argo")," namespace. See the ",(0,o.yg)("a",{parentName:"p",href:"https://argoproj.github.io/docs/argo/demo.html#2-install-the-controller-and-ui"},"official Argo documentation")," for more details."),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"kubectl create ns argo\nkubectl apply -n argo -f https://raw.githubusercontent.com/argoproj/argo/v2.4.2/manifests/install.yaml\n\n# Configure service account to run workflow\nkubectl create rolebinding default-admin --clusterrole=admin --serviceaccount=default:default\n\n# Test run\nargo submit --watch https://raw.githubusercontent.com/argoproj/argo/master/examples/hello-world.yaml\n")),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},"See ",(0,o.yg)("a",{parentName:"p",href:"https://raw.githubusercontent.com/vemonet/argo/master/manifests/namespace-install.yaml"},"custom configuration")," for namespace install."),(0,o.yg)("pre",{parentName:"blockquote"},(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"kubectl apply -n argo -f https://raw.githubusercontent.com/vemonet/argo/master/manifests/namespace-install.yaml\n"))),(0,o.yg)("h3",{id:"install-the-client"},"Install the client"),(0,o.yg)("p",null,"See the ",(0,o.yg)("a",{parentName:"p",href:"/docs/workflows-argo"},"Argo workflows documentation"),"."),(0,o.yg)("h3",{id:"expose-the-ui"},"Expose the UI"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-shell"},"kubectl -n argo port-forward deployment/argo-ui 8002:8001\n")),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},"Access on http://localhost:8002.")))}g.isMDXComponent=!0},647:(e,t,n)=>{n.d(t,{A:()=>l});const l=n.p+"assets/images/Kubernetes-c5f36b415dd16a8fcdae01fbc8d9c940.png"},4122:(e,t,n)=>{n.d(t,{A:()=>l});const l=n.p+"assets/images/argo-logo-c091bfee39aec37120d0e879edac74f6.png"}}]); \ No newline at end of file diff --git a/assets/js/b6e2013e.c193906a.js b/assets/js/b6e2013e.b7dc1bcf.js similarity index 98% rename from assets/js/b6e2013e.c193906a.js rename to assets/js/b6e2013e.b7dc1bcf.js index 9bcac689b..0791f7eb1 100644 --- a/assets/js/b6e2013e.c193906a.js +++ b/assets/js/b6e2013e.b7dc1bcf.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[9759],{5680:(e,t,r)=>{r.d(t,{xA:()=>u,yg:()=>d});var n=r(6540);function s(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function a(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function c(e){for(var t=1;t=0||(s[r]=e[r]);return s}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(s[r]=e[r])}return s}var i=n.createContext({}),l=function(e){var t=n.useContext(i),r=t;return e&&(r="function"==typeof e?e(t):c(c({},t),e)),r},u=function(e){var t=l(e.components);return n.createElement(i.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},m=n.forwardRef((function(e,t){var r=e.components,s=e.mdxType,a=e.originalType,i=e.parentName,u=o(e,["components","mdxType","originalType","parentName"]),m=l(r),d=s,v=m["".concat(i,".").concat(d)]||m[d]||p[d]||a;return r?n.createElement(v,c(c({ref:t},u),{},{components:r})):n.createElement(v,c({ref:t},u))}));function d(e,t){var r=arguments,s=t&&t.mdxType;if("string"==typeof e||s){var a=r.length,c=new Array(a);c[0]=m;var o={};for(var i in t)hasOwnProperty.call(t,i)&&(o[i]=t[i]);o.originalType=e,o.mdxType="string"==typeof e?e:s,c[1]=o;for(var l=2;l{r.r(t),r.d(t,{assets:()=>u,contentTitle:()=>i,default:()=>d,frontMatter:()=>o,metadata:()=>l,toc:()=>p});var n=r(9668),s=r(1367),a=(r(6540),r(5680)),c=["components"],o={id:"access-um-servers",title:"Access UM servers"},i=void 0,l={unversionedId:"access-um-servers",id:"access-um-servers",title:"Access UM servers",description:"Request access to internal UM servers",source:"@site/docs/access-um-servers.md",sourceDirName:".",slug:"/access-um-servers",permalink:"/docs/access-um-servers",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/access-um-servers.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"access-um-servers",title:"Access UM servers"},sidebar:"docs",previous:{title:"Utilities",permalink:"/docs/catalog-utilities"},next:{title:"Checkpointing Machine Learning Training",permalink:"/docs/checkpointing-ml-training"}},u={},p=[{value:"Request access to internal UM servers",id:"request-access-to-internal-um-servers",level:2}],m={toc:p};function d(e){var t=e.components,r=(0,s.A)(e,c);return(0,a.yg)("wrapper",(0,n.A)({},m,r,{components:t,mdxType:"MDXLayout"}),(0,a.yg)("h2",{id:"request-access-to-internal-um-servers"},"Request access to internal UM servers"),(0,a.yg)("p",null,"In certain cases, UM servers are not accessible by default from the DSRI. This is even the case for servers that are normally publicly accessible. To be able to access these UM servers from the DSRI, we need to put in the request to open the connection. "),(0,a.yg)("p",null,"Please let us know either the servername and port you like to access, or the URL (e.g. um-vm0057.unimaas.nl on port 443 or ",(0,a.yg)("a",{parentName:"p",href:"https://gitlab.maastrichtuniversity.nl"},"https://gitlab.maastrichtuniversity.nl"),"). You can reach out to us either by mail or by Slack."),(0,a.yg)("p",null,"UM services that are not accessible from DSRI right now:"),(0,a.yg)("ul",null,(0,a.yg)("li",{parentName:"ul"},"central UM fileservices (MFS)")),(0,a.yg)("p",null,"The procedure is described in the diagram below:"),(0,a.yg)("img",{src:"/img/request-access-um-servers.svg",alt:"Access procedure UM servers",style:{maxWidth:"100%",maxHeight:"100%"}}))}d.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[9759],{5680:(e,t,r)=>{r.d(t,{xA:()=>u,yg:()=>d});var n=r(6540);function s(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function a(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function c(e){for(var t=1;t=0||(s[r]=e[r]);return s}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(s[r]=e[r])}return s}var i=n.createContext({}),l=function(e){var t=n.useContext(i),r=t;return e&&(r="function"==typeof e?e(t):c(c({},t),e)),r},u=function(e){var t=l(e.components);return n.createElement(i.Provider,{value:t},e.children)},p={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},m=n.forwardRef((function(e,t){var r=e.components,s=e.mdxType,a=e.originalType,i=e.parentName,u=o(e,["components","mdxType","originalType","parentName"]),m=l(r),d=s,v=m["".concat(i,".").concat(d)]||m[d]||p[d]||a;return r?n.createElement(v,c(c({ref:t},u),{},{components:r})):n.createElement(v,c({ref:t},u))}));function d(e,t){var r=arguments,s=t&&t.mdxType;if("string"==typeof e||s){var a=r.length,c=new Array(a);c[0]=m;var o={};for(var i in t)hasOwnProperty.call(t,i)&&(o[i]=t[i]);o.originalType=e,o.mdxType="string"==typeof e?e:s,c[1]=o;for(var l=2;l{r.r(t),r.d(t,{assets:()=>u,contentTitle:()=>i,default:()=>d,frontMatter:()=>o,metadata:()=>l,toc:()=>p});var n=r(9668),s=r(1367),a=(r(6540),r(5680)),c=["components"],o={id:"access-um-servers",title:"Access UM servers"},i=void 0,l={unversionedId:"access-um-servers",id:"access-um-servers",title:"Access UM servers",description:"Request access to internal UM servers",source:"@site/docs/access-um-servers.md",sourceDirName:".",slug:"/access-um-servers",permalink:"/docs/access-um-servers",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/access-um-servers.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"access-um-servers",title:"Access UM servers"},sidebar:"docs",previous:{title:"Utilities",permalink:"/docs/catalog-utilities"},next:{title:"Checkpointing Machine Learning Training",permalink:"/docs/checkpointing-ml-training"}},u={},p=[{value:"Request access to internal UM servers",id:"request-access-to-internal-um-servers",level:2}],m={toc:p};function d(e){var t=e.components,r=(0,s.A)(e,c);return(0,a.yg)("wrapper",(0,n.A)({},m,r,{components:t,mdxType:"MDXLayout"}),(0,a.yg)("h2",{id:"request-access-to-internal-um-servers"},"Request access to internal UM servers"),(0,a.yg)("p",null,"In certain cases, UM servers are not accessible by default from the DSRI. This is even the case for servers that are normally publicly accessible. To be able to access these UM servers from the DSRI, we need to put in the request to open the connection. "),(0,a.yg)("p",null,"Please let us know either the servername and port you like to access, or the URL (e.g. um-vm0057.unimaas.nl on port 443 or ",(0,a.yg)("a",{parentName:"p",href:"https://gitlab.maastrichtuniversity.nl"},"https://gitlab.maastrichtuniversity.nl"),"). You can reach out to us either by mail or by Slack."),(0,a.yg)("p",null,"UM services that are not accessible from DSRI right now:"),(0,a.yg)("ul",null,(0,a.yg)("li",{parentName:"ul"},"central UM fileservices (MFS)")),(0,a.yg)("p",null,"The procedure is described in the diagram below:"),(0,a.yg)("img",{src:"/img/request-access-um-servers.svg",alt:"Access procedure UM servers",style:{maxWidth:"100%",maxHeight:"100%"}}))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/bbb26d62.fd8b549d.js b/assets/js/bbb26d62.9288eeb9.js similarity index 99% rename from assets/js/bbb26d62.fd8b549d.js rename to assets/js/bbb26d62.9288eeb9.js index 8222c8422..33326cf0d 100644 --- a/assets/js/bbb26d62.fd8b549d.js +++ b/assets/js/bbb26d62.9288eeb9.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[4969],{5680:(e,n,t)=>{t.d(n,{xA:()=>d,yg:()=>g});var r=t(6540);function a(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function i(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);n&&(r=r.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,r)}return t}function o(e){for(var n=1;n=0||(a[t]=e[t]);return a}(e,n);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(a[t]=e[t])}return a}var s=r.createContext({}),p=function(e){var n=r.useContext(s),t=n;return e&&(t="function"==typeof e?e(n):o(o({},n),e)),t},d=function(e){var n=p(e.components);return r.createElement(s.Provider,{value:n},e.children)},u={inlineCode:"code",wrapper:function(e){var n=e.children;return r.createElement(r.Fragment,{},n)}},c=r.forwardRef((function(e,n){var t=e.components,a=e.mdxType,i=e.originalType,s=e.parentName,d=l(e,["components","mdxType","originalType","parentName"]),c=p(t),g=a,f=c["".concat(s,".").concat(g)]||c[g]||u[g]||i;return t?r.createElement(f,o(o({ref:n},d),{},{components:t})):r.createElement(f,o({ref:n},d))}));function g(e,n){var t=arguments,a=n&&n.mdxType;if("string"==typeof e||a){var i=t.length,o=new Array(i);o[0]=c;var l={};for(var s in n)hasOwnProperty.call(n,s)&&(l[s]=n[s]);l.originalType=e,l.mdxType="string"==typeof e?e:a,o[1]=l;for(var p=2;p{t.r(n),t.d(n,{assets:()=>d,contentTitle:()=>s,default:()=>g,frontMatter:()=>l,metadata:()=>p,toc:()=>u});var r=t(9668),a=t(1367),i=(t(6540),t(5680)),o=["components"],l={id:"speeding-tensorflow-dl",title:"Tensorflow Optimization"},s=void 0,p={unversionedId:"speeding-tensorflow-dl",id:"speeding-tensorflow-dl",title:"Tensorflow Optimization",description:"\ud83d\udd36 Speeding up Tensorflow based deep learning pipelines",source:"@site/docs/speeding-tensorflow-dl.md",sourceDirName:".",slug:"/speeding-tensorflow-dl",permalink:"/docs/speeding-tensorflow-dl",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/speeding-tensorflow-dl.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"speeding-tensorflow-dl",title:"Tensorflow Optimization"},sidebar:"docs",previous:{title:"PyTorch Profiling",permalink:"/docs/profile-pytorch-code"},next:{title:"SURF Offerings",permalink:"/docs/surf-offerings"}},d={},u=[{value:"\ud83d\udd36 Speeding up Tensorflow based deep learning pipelines",id:"-speeding-up-tensorflow-based-deep-learning-pipelines",level:2},{value:"A possible checklist for speeding up your deep learning pipeline in Tensorflow?",id:"a-possible-checklist-for-speeding-up-your-deep-learning-pipeline-in-tensorflow",level:2},{value:"Data Preparation",id:"data-preparation",level:2},{value:"Data Reading",id:"data-reading",level:2},{value:"Data Augmentation",id:"data-augmentation",level:2},{value:"Training",id:"training",level:2},{value:"Inference",id:"inference",level:2},{value:"How DSRI team can help you?",id:"how-dsri-team-can-help-you",level:2},{value:"External Resources and references",id:"external-resources-and-references",level:2}],c={toc:u};function g(e){var n=e.components,t=(0,a.A)(e,o);return(0,i.yg)("wrapper",(0,r.A)({},c,t,{components:n,mdxType:"MDXLayout"}),(0,i.yg)("h2",{id:"-speeding-up-tensorflow-based-deep-learning-pipelines"},"\ud83d\udd36 Speeding up Tensorflow based deep learning pipelines"),(0,i.yg)("p",null,"The amount of resources that you have is not nearly as important as using them to their maximum potential. It\u2019s all about doing more with less.In this write up, we discuss optimizations related to data preparation, data reading, data augmentation,training, and inference. "),(0,i.yg)("h2",{id:"a-possible-checklist-for-speeding-up-your-deep-learning-pipeline-in-tensorflow"},"A possible checklist for speeding up your deep learning pipeline in Tensorflow?"),(0,i.yg)("p",null,"Let\u2019s look at each area of the deep learning pipeline step by step, including data preparation, data reading, data augmentation, training, and, finally, inference."),(0,i.yg)("h2",{id:"data-preparation"},"Data Preparation"),(0,i.yg)("p",null,"1) Store as TFRecords\n2) Reduce Size of Input Data\n3) Use TensorFlow Datasets"),(0,i.yg)("h2",{id:"data-reading"},"Data Reading"),(0,i.yg)("p",null,"1) Use tf.data\n2) Prefetch Data\n3) Parallelize CPU Processing\n4) Parallelize I/O and Processing\n5) Enable Nondeterministic Ordering\n6) Cache Data\n7) Turn on Experimental Optimizations\n8) Autotune Parameter Values"),(0,i.yg)("h2",{id:"data-augmentation"},"Data Augmentation"),(0,i.yg)("p",null,"1) Use GPU for Augmentation"),(0,i.yg)("h2",{id:"training"},"Training"),(0,i.yg)("p",null,"1) Use Automatic Mixed Precision\n2) Use Larger Batch Size\n3) Use Multiples of Eight\n4) Find the Optimal Learning Rate\n5) Use tf.function\n6) Overtrain, and Then Generalize"),(0,i.yg)("p",null," 6a) Use progressive sampling"),(0,i.yg)("p",null," 6b) Use progressive augmentation"),(0,i.yg)("p",null," 6c) Use progressive resizing\u201d"),(0,i.yg)("p",null," 7) Install an Optimized Stack for the Hardware\n8) Optimize the Number of Parallel CPU Threads\n9) Use Better Hardware\n10) Distribute Training\n11) Examine Industry Benchmarks"),(0,i.yg)("h2",{id:"inference"},"Inference"),(0,i.yg)("p",null,"1) Use an Efficient Model\n2) Quantize the Model\n3) Prune the Model\n4) Use Fused Operations\n5) Enable GPU Persistence"),(0,i.yg)("h2",{id:"how-dsri-team-can-help-you"},"How DSRI team can help you?"),(0,i.yg)("p",null,"We can assist you with analyzing the bottleneck/s in your deep learning pipeline and recommend the improvments to speed up your pipeline."),(0,i.yg)("h2",{id:"external-resources-and-references"},"External Resources and references"),(0,i.yg)("ul",null,(0,i.yg)("li",{parentName:"ul"},'This documentation is adopted from the "Practical Deep Learning for Cloud, Mobile, and Edge by Koul etl (publish by O\u2019Reilly)')))}g.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[4969],{5680:(e,n,t)=>{t.d(n,{xA:()=>d,yg:()=>g});var r=t(6540);function a(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function i(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);n&&(r=r.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,r)}return t}function o(e){for(var n=1;n=0||(a[t]=e[t]);return a}(e,n);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(a[t]=e[t])}return a}var s=r.createContext({}),p=function(e){var n=r.useContext(s),t=n;return e&&(t="function"==typeof e?e(n):o(o({},n),e)),t},d=function(e){var n=p(e.components);return r.createElement(s.Provider,{value:n},e.children)},u={inlineCode:"code",wrapper:function(e){var n=e.children;return r.createElement(r.Fragment,{},n)}},c=r.forwardRef((function(e,n){var t=e.components,a=e.mdxType,i=e.originalType,s=e.parentName,d=l(e,["components","mdxType","originalType","parentName"]),c=p(t),g=a,f=c["".concat(s,".").concat(g)]||c[g]||u[g]||i;return t?r.createElement(f,o(o({ref:n},d),{},{components:t})):r.createElement(f,o({ref:n},d))}));function g(e,n){var t=arguments,a=n&&n.mdxType;if("string"==typeof e||a){var i=t.length,o=new Array(i);o[0]=c;var l={};for(var s in n)hasOwnProperty.call(n,s)&&(l[s]=n[s]);l.originalType=e,l.mdxType="string"==typeof e?e:a,o[1]=l;for(var p=2;p{t.r(n),t.d(n,{assets:()=>d,contentTitle:()=>s,default:()=>g,frontMatter:()=>l,metadata:()=>p,toc:()=>u});var r=t(9668),a=t(1367),i=(t(6540),t(5680)),o=["components"],l={id:"speeding-tensorflow-dl",title:"Tensorflow Optimization"},s=void 0,p={unversionedId:"speeding-tensorflow-dl",id:"speeding-tensorflow-dl",title:"Tensorflow Optimization",description:"\ud83d\udd36 Speeding up Tensorflow based deep learning pipelines",source:"@site/docs/speeding-tensorflow-dl.md",sourceDirName:".",slug:"/speeding-tensorflow-dl",permalink:"/docs/speeding-tensorflow-dl",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/speeding-tensorflow-dl.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"speeding-tensorflow-dl",title:"Tensorflow Optimization"},sidebar:"docs",previous:{title:"PyTorch Profiling",permalink:"/docs/profile-pytorch-code"},next:{title:"SURF Offerings",permalink:"/docs/surf-offerings"}},d={},u=[{value:"\ud83d\udd36 Speeding up Tensorflow based deep learning pipelines",id:"-speeding-up-tensorflow-based-deep-learning-pipelines",level:2},{value:"A possible checklist for speeding up your deep learning pipeline in Tensorflow?",id:"a-possible-checklist-for-speeding-up-your-deep-learning-pipeline-in-tensorflow",level:2},{value:"Data Preparation",id:"data-preparation",level:2},{value:"Data Reading",id:"data-reading",level:2},{value:"Data Augmentation",id:"data-augmentation",level:2},{value:"Training",id:"training",level:2},{value:"Inference",id:"inference",level:2},{value:"How DSRI team can help you?",id:"how-dsri-team-can-help-you",level:2},{value:"External Resources and references",id:"external-resources-and-references",level:2}],c={toc:u};function g(e){var n=e.components,t=(0,a.A)(e,o);return(0,i.yg)("wrapper",(0,r.A)({},c,t,{components:n,mdxType:"MDXLayout"}),(0,i.yg)("h2",{id:"-speeding-up-tensorflow-based-deep-learning-pipelines"},"\ud83d\udd36 Speeding up Tensorflow based deep learning pipelines"),(0,i.yg)("p",null,"The amount of resources that you have is not nearly as important as using them to their maximum potential. It\u2019s all about doing more with less.In this write up, we discuss optimizations related to data preparation, data reading, data augmentation,training, and inference. "),(0,i.yg)("h2",{id:"a-possible-checklist-for-speeding-up-your-deep-learning-pipeline-in-tensorflow"},"A possible checklist for speeding up your deep learning pipeline in Tensorflow?"),(0,i.yg)("p",null,"Let\u2019s look at each area of the deep learning pipeline step by step, including data preparation, data reading, data augmentation, training, and, finally, inference."),(0,i.yg)("h2",{id:"data-preparation"},"Data Preparation"),(0,i.yg)("p",null,"1) Store as TFRecords\n2) Reduce Size of Input Data\n3) Use TensorFlow Datasets"),(0,i.yg)("h2",{id:"data-reading"},"Data Reading"),(0,i.yg)("p",null,"1) Use tf.data\n2) Prefetch Data\n3) Parallelize CPU Processing\n4) Parallelize I/O and Processing\n5) Enable Nondeterministic Ordering\n6) Cache Data\n7) Turn on Experimental Optimizations\n8) Autotune Parameter Values"),(0,i.yg)("h2",{id:"data-augmentation"},"Data Augmentation"),(0,i.yg)("p",null,"1) Use GPU for Augmentation"),(0,i.yg)("h2",{id:"training"},"Training"),(0,i.yg)("p",null,"1) Use Automatic Mixed Precision\n2) Use Larger Batch Size\n3) Use Multiples of Eight\n4) Find the Optimal Learning Rate\n5) Use tf.function\n6) Overtrain, and Then Generalize"),(0,i.yg)("p",null," 6a) Use progressive sampling"),(0,i.yg)("p",null," 6b) Use progressive augmentation"),(0,i.yg)("p",null," 6c) Use progressive resizing\u201d"),(0,i.yg)("p",null," 7) Install an Optimized Stack for the Hardware\n8) Optimize the Number of Parallel CPU Threads\n9) Use Better Hardware\n10) Distribute Training\n11) Examine Industry Benchmarks"),(0,i.yg)("h2",{id:"inference"},"Inference"),(0,i.yg)("p",null,"1) Use an Efficient Model\n2) Quantize the Model\n3) Prune the Model\n4) Use Fused Operations\n5) Enable GPU Persistence"),(0,i.yg)("h2",{id:"how-dsri-team-can-help-you"},"How DSRI team can help you?"),(0,i.yg)("p",null,"We can assist you with analyzing the bottleneck/s in your deep learning pipeline and recommend the improvments to speed up your pipeline."),(0,i.yg)("h2",{id:"external-resources-and-references"},"External Resources and references"),(0,i.yg)("ul",null,(0,i.yg)("li",{parentName:"ul"},'This documentation is adopted from the "Practical Deep Learning for Cloud, Mobile, and Edge by Koul etl (publish by O\u2019Reilly)')))}g.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/bd7c3f6d.ace780eb.js b/assets/js/bd7c3f6d.1733c6a7.js similarity index 98% rename from assets/js/bd7c3f6d.ace780eb.js rename to assets/js/bd7c3f6d.1733c6a7.js index 52a2b8861..2c8c0fcaa 100644 --- a/assets/js/bd7c3f6d.ace780eb.js +++ b/assets/js/bd7c3f6d.1733c6a7.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[3762],{5680:(e,t,n)=>{n.d(t,{xA:()=>s,yg:()=>u});var o=n(6540);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function i(e){for(var t=1;t=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(o=0;o=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var p=o.createContext({}),c=function(e){var t=o.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},s=function(e){var t=c(e.components);return o.createElement(p.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.createElement(o.Fragment,{},t)}},m=o.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,p=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),m=c(n),u=r,y=m["".concat(p,".").concat(u)]||m[u]||d[u]||a;return n?o.createElement(y,i(i({ref:t},s),{},{components:n})):o.createElement(y,i({ref:t},s))}));function u(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=m;var l={};for(var p in t)hasOwnProperty.call(t,p)&&(l[p]=t[p]);l.originalType=e,l.mdxType="string"==typeof e?e:r,i[1]=l;for(var c=2;c{n.r(t),n.d(t,{assets:()=>s,contentTitle:()=>p,default:()=>u,frontMatter:()=>l,metadata:()=>c,toc:()=>d});var o=n(9668),r=n(1367),a=(n(6540),n(5680)),i=["components"],l={id:"openshift-delete-services",title:"Delete an application"},p=void 0,c={unversionedId:"openshift-delete-services",id:"openshift-delete-services",title:"Delete an application",description:"It is recommend to use the oc tool to delete an application, as it will allow to properly delete all objects related to the application deployment.",source:"@site/docs/openshift-delete-services.md",sourceDirName:".",slug:"/openshift-delete-services",permalink:"/docs/openshift-delete-services",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/openshift-delete-services.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"openshift-delete-services",title:"Delete an application"},sidebar:"docs",previous:{title:"Upload data",permalink:"/docs/openshift-load-data"},next:{title:"Monitor your applications",permalink:"/docs/guide-monitoring"}},s={},d=[{value:"From the terminal",id:"from-the-terminal",level:2},{value:"From the web UI",id:"from-the-web-ui",level:2}],m={toc:d};function u(e){var t=e.components,n=(0,r.A)(e,i);return(0,a.yg)("wrapper",(0,o.A)({},m,n,{components:t,mdxType:"MDXLayout"}),(0,a.yg)("p",null,"It is recommend to use the ",(0,a.yg)("inlineCode",{parentName:"p"},"oc")," tool to delete an application, as it will allow to properly delete all objects related to the application deployment."),(0,a.yg)("admonition",{title:"Project",type:"caution"},(0,a.yg)("p",{parentName:"admonition"},"Make sure you are connected to the right project:"),(0,a.yg)("pre",{parentName:"admonition"},(0,a.yg)("code",{parentName:"pre",className:"language-shell"},"oc project my-project\n"))),(0,a.yg)("h2",{id:"from-the-terminal"},"From the terminal"),(0,a.yg)("p",null,"The best way to make sure all objects related to your application have been deleted is to use the command line providing your application name:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-shell"},"oc delete all,secret,configmaps,serviceaccount,rolebinding --selector app=my-application\n")),(0,a.yg)("admonition",{title:"Force deletion",type:"info"},(0,a.yg)("p",{parentName:"admonition"},"You can force the deletion if the objects are not deleting properly:"),(0,a.yg)("pre",{parentName:"admonition"},(0,a.yg)("code",{parentName:"pre",className:"language-shell"},"oc delete all,secret,configmaps,serviceaccount,rolebinding --force --grace-period=0 --selector app=my-application\n"))),(0,a.yg)("h2",{id:"from-the-web-ui"},"From the web UI"),(0,a.yg)("p",null,"We recommend to use the ",(0,a.yg)("inlineCode",{parentName:"p"},"oc")," CLI to easily delete an application. But in the case you cannot install ",(0,a.yg)("inlineCode",{parentName:"p"},"oc")," on your computer you can delete the different objects created by the application (easy to find in the ",(0,a.yg)("strong",{parentName:"p"},"Topology")," page):"),(0,a.yg)("ol",null,(0,a.yg)("li",{parentName:"ol"},"Delete the ",(0,a.yg)("strong",{parentName:"li"},"Route")),(0,a.yg)("li",{parentName:"ol"},"Delete the ",(0,a.yg)("strong",{parentName:"li"},"Service")),(0,a.yg)("li",{parentName:"ol"},"Delete the ",(0,a.yg)("strong",{parentName:"li"},"Deployment Config")," ")),(0,a.yg)("img",{src:"/img/screenshot_delete_application.png",alt:"Delete application from the web UI",style:{maxWidth:"100%",maxHeight:"100%"}}))}u.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[3762],{5680:(e,t,n)=>{n.d(t,{xA:()=>s,yg:()=>u});var o=n(6540);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function i(e){for(var t=1;t=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(o=0;o=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var p=o.createContext({}),c=function(e){var t=o.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},s=function(e){var t=c(e.components);return o.createElement(p.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return o.createElement(o.Fragment,{},t)}},m=o.forwardRef((function(e,t){var n=e.components,r=e.mdxType,a=e.originalType,p=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),m=c(n),u=r,y=m["".concat(p,".").concat(u)]||m[u]||d[u]||a;return n?o.createElement(y,i(i({ref:t},s),{},{components:n})):o.createElement(y,i({ref:t},s))}));function u(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=n.length,i=new Array(a);i[0]=m;var l={};for(var p in t)hasOwnProperty.call(t,p)&&(l[p]=t[p]);l.originalType=e,l.mdxType="string"==typeof e?e:r,i[1]=l;for(var c=2;c{n.r(t),n.d(t,{assets:()=>s,contentTitle:()=>p,default:()=>u,frontMatter:()=>l,metadata:()=>c,toc:()=>d});var o=n(9668),r=n(1367),a=(n(6540),n(5680)),i=["components"],l={id:"openshift-delete-services",title:"Delete an application"},p=void 0,c={unversionedId:"openshift-delete-services",id:"openshift-delete-services",title:"Delete an application",description:"It is recommend to use the oc tool to delete an application, as it will allow to properly delete all objects related to the application deployment.",source:"@site/docs/openshift-delete-services.md",sourceDirName:".",slug:"/openshift-delete-services",permalink:"/docs/openshift-delete-services",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/openshift-delete-services.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"openshift-delete-services",title:"Delete an application"},sidebar:"docs",previous:{title:"Upload data",permalink:"/docs/openshift-load-data"},next:{title:"Monitor your applications",permalink:"/docs/guide-monitoring"}},s={},d=[{value:"From the terminal",id:"from-the-terminal",level:2},{value:"From the web UI",id:"from-the-web-ui",level:2}],m={toc:d};function u(e){var t=e.components,n=(0,r.A)(e,i);return(0,a.yg)("wrapper",(0,o.A)({},m,n,{components:t,mdxType:"MDXLayout"}),(0,a.yg)("p",null,"It is recommend to use the ",(0,a.yg)("inlineCode",{parentName:"p"},"oc")," tool to delete an application, as it will allow to properly delete all objects related to the application deployment."),(0,a.yg)("admonition",{title:"Project",type:"caution"},(0,a.yg)("p",{parentName:"admonition"},"Make sure you are connected to the right project:"),(0,a.yg)("pre",{parentName:"admonition"},(0,a.yg)("code",{parentName:"pre",className:"language-shell"},"oc project my-project\n"))),(0,a.yg)("h2",{id:"from-the-terminal"},"From the terminal"),(0,a.yg)("p",null,"The best way to make sure all objects related to your application have been deleted is to use the command line providing your application name:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-shell"},"oc delete all,secret,configmaps,serviceaccount,rolebinding --selector app=my-application\n")),(0,a.yg)("admonition",{title:"Force deletion",type:"info"},(0,a.yg)("p",{parentName:"admonition"},"You can force the deletion if the objects are not deleting properly:"),(0,a.yg)("pre",{parentName:"admonition"},(0,a.yg)("code",{parentName:"pre",className:"language-shell"},"oc delete all,secret,configmaps,serviceaccount,rolebinding --force --grace-period=0 --selector app=my-application\n"))),(0,a.yg)("h2",{id:"from-the-web-ui"},"From the web UI"),(0,a.yg)("p",null,"We recommend to use the ",(0,a.yg)("inlineCode",{parentName:"p"},"oc")," CLI to easily delete an application. But in the case you cannot install ",(0,a.yg)("inlineCode",{parentName:"p"},"oc")," on your computer you can delete the different objects created by the application (easy to find in the ",(0,a.yg)("strong",{parentName:"p"},"Topology")," page):"),(0,a.yg)("ol",null,(0,a.yg)("li",{parentName:"ol"},"Delete the ",(0,a.yg)("strong",{parentName:"li"},"Route")),(0,a.yg)("li",{parentName:"ol"},"Delete the ",(0,a.yg)("strong",{parentName:"li"},"Service")),(0,a.yg)("li",{parentName:"ol"},"Delete the ",(0,a.yg)("strong",{parentName:"li"},"Deployment Config")," ")),(0,a.yg)("img",{src:"/img/screenshot_delete_application.png",alt:"Delete application from the web UI",style:{maxWidth:"100%",maxHeight:"100%"}}))}u.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/cdd0b013.6d9a7602.js b/assets/js/cdd0b013.5e9bf89a.js similarity index 99% rename from assets/js/cdd0b013.6d9a7602.js rename to assets/js/cdd0b013.5e9bf89a.js index 7c5f3bdc1..5df5635ee 100644 --- a/assets/js/cdd0b013.6d9a7602.js +++ b/assets/js/cdd0b013.5e9bf89a.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[6236],{5680:(e,t,n)=>{n.d(t,{xA:()=>u,yg:()=>d});var a=n(6540);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function o(e){for(var t=1;t=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var p=a.createContext({}),s=function(e){var t=a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):o(o({},t),e)),n},u=function(e){var t=s(e.components);return a.createElement(p.Provider,{value:t},e.children)},y={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},c=a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,p=e.parentName,u=l(e,["components","mdxType","originalType","parentName"]),c=s(n),d=r,m=c["".concat(p,".").concat(d)]||c[d]||y[d]||i;return n?a.createElement(m,o(o({ref:t},u),{},{components:n})):a.createElement(m,o({ref:t},u))}));function d(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,o=new Array(i);o[0]=c;var l={};for(var p in t)hasOwnProperty.call(t,p)&&(l[p]=t[p]);l.originalType=e,l.mdxType="string"==typeof e?e:r,o[1]=l;for(var s=2;s{n.r(t),n.d(t,{assets:()=>u,contentTitle:()=>p,default:()=>d,frontMatter:()=>l,metadata:()=>s,toc:()=>y});var a=n(9668),r=n(1367),i=(n(6540),n(5680)),o=["components"],l={id:"jupyterhub-workspace",title:"JupyterHub workspace"},p=void 0,s={unversionedId:"jupyterhub-workspace",id:"jupyterhub-workspace",title:"JupyterHub workspace",description:"\ud83e\ude90 Start your workspace",source:"@site/docs/jupyterhub-workspace.md",sourceDirName:".",slug:"/jupyterhub-workspace",permalink:"/docs/jupyterhub-workspace",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/jupyterhub-workspace.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"jupyterhub-workspace",title:"JupyterHub workspace"}},u={},y=[{value:"\ud83e\ude90 Start your workspace",id:"-start-your-workspace",level:2},{value:"\ud83d\udce6\ufe0f Manage dependencies with Conda",id:"\ufe0f-manage-dependencies-with-conda",level:2},{value:"\ud83d\udc19 Use git in JupyterLab",id:"-use-git-in-jupyterlab",level:2}],c={toc:y};function d(e){var t=e.components,n=(0,r.A)(e,o);return(0,i.yg)("wrapper",(0,a.A)({},c,n,{components:t,mdxType:"MDXLayout"}),(0,i.yg)("h2",{id:"-start-your-workspace"},"\ud83e\ude90 Start your workspace"),(0,i.yg)("p",null,"You can easily start a data science workspace with JupyterLab, VisualStudio Code and Conda pre-installed on the DSRI with JupyterHub:"),(0,i.yg)("ol",null,(0,i.yg)("li",{parentName:"ol"},"Connect to the UM VPN"),(0,i.yg)("li",{parentName:"ol"},"Go to ",(0,i.yg)("strong",{parentName:"li"},(0,i.yg)("a",{parentName:"strong",href:"https://jupyterhub-github.apps.dsri2.unimaas.nl"},"https://jupyterhub-github.apps.dsri2.unimaas.nl"))),(0,i.yg)("li",{parentName:"ol"},"Login with your ",(0,i.yg)("a",{parentName:"li",href:"https://github.com"},"GitHub")," account"),(0,i.yg)("li",{parentName:"ol"},"Choose the type of workspace, and the resources limitations "),(0,i.yg)("li",{parentName:"ol"},"Optionally you can provide additional parameters as environment variables:",(0,i.yg)("ol",{parentName:"li"},(0,i.yg)("li",{parentName:"ol"},(0,i.yg)("inlineCode",{parentName:"li"},"GIT_NAME")," and ",(0,i.yg)("inlineCode",{parentName:"li"},"GIT_EMAIL"),": your name and email that will be used when committing with git"),(0,i.yg)("li",{parentName:"ol"},(0,i.yg)("inlineCode",{parentName:"li"},"GIT_URL"),": the URL of a git repository to be automatically cloned in the workspace, if there is a ",(0,i.yg)("inlineCode",{parentName:"li"},"requirements.txt")," it will be automatically installed with ",(0,i.yg)("inlineCode",{parentName:"li"},"pip"))))),(0,i.yg)("p",null,"Once your workspace has started you can:"),(0,i.yg)("ul",null,(0,i.yg)("li",{parentName:"ul"},"Use the ",(0,i.yg)("inlineCode",{parentName:"li"},"persistent")," folder to put data that will be kept even when the server is stopped, or if you use a different type of workspace"),(0,i.yg)("li",{parentName:"ul"},"Clone your code repository with ",(0,i.yg)("inlineCode",{parentName:"li"},"git")),(0,i.yg)("li",{parentName:"ul"},"Install packages with ",(0,i.yg)("inlineCode",{parentName:"li"},"mamba"),"/",(0,i.yg)("inlineCode",{parentName:"li"},"conda")," or ",(0,i.yg)("inlineCode",{parentName:"li"},"pip")),(0,i.yg)("li",{parentName:"ul"},"Go to the workspace overview: ",(0,i.yg)("a",{parentName:"li",href:"https://jupyterhub-github.apps.dsri2.unimaas.nl/hub/home"},"https://jupyterhub-github.apps.dsri2.unimaas.nl/hub/home")," to see your workspace, and stop it.")),(0,i.yg)("admonition",{type:"tip"},(0,i.yg)("p",{parentName:"admonition"},"Put all the commands you use to install the packages required to run your code in a file in the ",(0,i.yg)("inlineCode",{parentName:"p"},"persistent")," folder (ideally in the git repository with your code), so you can easily reinstall your environment if your workspace is stopped.")),(0,i.yg)("h2",{id:"\ufe0f-manage-dependencies-with-conda"},"\ud83d\udce6\ufe0f Manage dependencies with Conda"),(0,i.yg)("p",null,"In your workspace you can install new ",(0,i.yg)("inlineCode",{parentName:"p"},"conda")," environments, if they include the packages ",(0,i.yg)("inlineCode",{parentName:"p"},"nb_conda_kernels")," and ",(0,i.yg)("inlineCode",{parentName:"p"},"ipykernel"),", then you will be able to easily start notebooks in those environments from the JupyterLab Launcher page."),(0,i.yg)("p",null,"Install a conda environment from a file with ",(0,i.yg)("inlineCode",{parentName:"p"},"mamba")," (it is like ",(0,i.yg)("inlineCode",{parentName:"p"},"conda")," but faster):"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-bash"},"mamba env create -f environment.yml\n")),(0,i.yg)("p",null,"You'll need to wait for 1 minute before the new conda environment becomes available on the JupyterLab Launcher page."),(0,i.yg)("p",null,"You can easily install an environment with a different version of Python if you need it. Here is an example of an ",(0,i.yg)("inlineCode",{parentName:"p"},"environment.yml")," file to create an environment with Python 3.9, install the minimal dependencies required to easily starts notebooks in this environment with ",(0,i.yg)("inlineCode",{parentName:"p"},"conda"),", and install a ",(0,i.yg)("inlineCode",{parentName:"p"},"pip")," package:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},"name: py39\nchannels:\n - defaults\n - conda-forge\n - anaconda\ndependencies:\n - python=3.9\n - ipykernel\n - nb_conda_kernels\n - pip\n - pip:\n - matplotlib\n")),(0,i.yg)("h2",{id:"-use-git-in-jupyterlab"},"\ud83d\udc19 Use git in JupyterLab"),(0,i.yg)("p",null,"You can use ",(0,i.yg)("inlineCode",{parentName:"p"},"git")," from the terminal."),(0,i.yg)("p",null,"You can also use the ",(0,i.yg)("a",{parentName:"p",href:"https://github.com/jupyterlab/jupyterlab-git"},"JupyterLab Git extension")," or the VisualStudio Code git integration to clone and manage your ",(0,i.yg)("inlineCode",{parentName:"p"},"git")," repositories."),(0,i.yg)("p",null,"They will ask you for a username and personal access token if the repository is private, or the first time you want to push changes."))}d.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[6236],{5680:(e,t,n)=>{n.d(t,{xA:()=>u,yg:()=>d});var a=n(6540);function r(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function i(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,a)}return n}function o(e){for(var t=1;t=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var p=a.createContext({}),s=function(e){var t=a.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):o(o({},t),e)),n},u=function(e){var t=s(e.components);return a.createElement(p.Provider,{value:t},e.children)},y={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},c=a.forwardRef((function(e,t){var n=e.components,r=e.mdxType,i=e.originalType,p=e.parentName,u=l(e,["components","mdxType","originalType","parentName"]),c=s(n),d=r,m=c["".concat(p,".").concat(d)]||c[d]||y[d]||i;return n?a.createElement(m,o(o({ref:t},u),{},{components:n})):a.createElement(m,o({ref:t},u))}));function d(e,t){var n=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var i=n.length,o=new Array(i);o[0]=c;var l={};for(var p in t)hasOwnProperty.call(t,p)&&(l[p]=t[p]);l.originalType=e,l.mdxType="string"==typeof e?e:r,o[1]=l;for(var s=2;s{n.r(t),n.d(t,{assets:()=>u,contentTitle:()=>p,default:()=>d,frontMatter:()=>l,metadata:()=>s,toc:()=>y});var a=n(9668),r=n(1367),i=(n(6540),n(5680)),o=["components"],l={id:"jupyterhub-workspace",title:"JupyterHub workspace"},p=void 0,s={unversionedId:"jupyterhub-workspace",id:"jupyterhub-workspace",title:"JupyterHub workspace",description:"\ud83e\ude90 Start your workspace",source:"@site/docs/jupyterhub-workspace.md",sourceDirName:".",slug:"/jupyterhub-workspace",permalink:"/docs/jupyterhub-workspace",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/jupyterhub-workspace.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"jupyterhub-workspace",title:"JupyterHub workspace"}},u={},y=[{value:"\ud83e\ude90 Start your workspace",id:"-start-your-workspace",level:2},{value:"\ud83d\udce6\ufe0f Manage dependencies with Conda",id:"\ufe0f-manage-dependencies-with-conda",level:2},{value:"\ud83d\udc19 Use git in JupyterLab",id:"-use-git-in-jupyterlab",level:2}],c={toc:y};function d(e){var t=e.components,n=(0,r.A)(e,o);return(0,i.yg)("wrapper",(0,a.A)({},c,n,{components:t,mdxType:"MDXLayout"}),(0,i.yg)("h2",{id:"-start-your-workspace"},"\ud83e\ude90 Start your workspace"),(0,i.yg)("p",null,"You can easily start a data science workspace with JupyterLab, VisualStudio Code and Conda pre-installed on the DSRI with JupyterHub:"),(0,i.yg)("ol",null,(0,i.yg)("li",{parentName:"ol"},"Connect to the UM VPN"),(0,i.yg)("li",{parentName:"ol"},"Go to ",(0,i.yg)("strong",{parentName:"li"},(0,i.yg)("a",{parentName:"strong",href:"https://jupyterhub-github.apps.dsri2.unimaas.nl"},"https://jupyterhub-github.apps.dsri2.unimaas.nl"))),(0,i.yg)("li",{parentName:"ol"},"Login with your ",(0,i.yg)("a",{parentName:"li",href:"https://github.com"},"GitHub")," account"),(0,i.yg)("li",{parentName:"ol"},"Choose the type of workspace, and the resources limitations "),(0,i.yg)("li",{parentName:"ol"},"Optionally you can provide additional parameters as environment variables:",(0,i.yg)("ol",{parentName:"li"},(0,i.yg)("li",{parentName:"ol"},(0,i.yg)("inlineCode",{parentName:"li"},"GIT_NAME")," and ",(0,i.yg)("inlineCode",{parentName:"li"},"GIT_EMAIL"),": your name and email that will be used when committing with git"),(0,i.yg)("li",{parentName:"ol"},(0,i.yg)("inlineCode",{parentName:"li"},"GIT_URL"),": the URL of a git repository to be automatically cloned in the workspace, if there is a ",(0,i.yg)("inlineCode",{parentName:"li"},"requirements.txt")," it will be automatically installed with ",(0,i.yg)("inlineCode",{parentName:"li"},"pip"))))),(0,i.yg)("p",null,"Once your workspace has started you can:"),(0,i.yg)("ul",null,(0,i.yg)("li",{parentName:"ul"},"Use the ",(0,i.yg)("inlineCode",{parentName:"li"},"persistent")," folder to put data that will be kept even when the server is stopped, or if you use a different type of workspace"),(0,i.yg)("li",{parentName:"ul"},"Clone your code repository with ",(0,i.yg)("inlineCode",{parentName:"li"},"git")),(0,i.yg)("li",{parentName:"ul"},"Install packages with ",(0,i.yg)("inlineCode",{parentName:"li"},"mamba"),"/",(0,i.yg)("inlineCode",{parentName:"li"},"conda")," or ",(0,i.yg)("inlineCode",{parentName:"li"},"pip")),(0,i.yg)("li",{parentName:"ul"},"Go to the workspace overview: ",(0,i.yg)("a",{parentName:"li",href:"https://jupyterhub-github.apps.dsri2.unimaas.nl/hub/home"},"https://jupyterhub-github.apps.dsri2.unimaas.nl/hub/home")," to see your workspace, and stop it.")),(0,i.yg)("admonition",{type:"tip"},(0,i.yg)("p",{parentName:"admonition"},"Put all the commands you use to install the packages required to run your code in a file in the ",(0,i.yg)("inlineCode",{parentName:"p"},"persistent")," folder (ideally in the git repository with your code), so you can easily reinstall your environment if your workspace is stopped.")),(0,i.yg)("h2",{id:"\ufe0f-manage-dependencies-with-conda"},"\ud83d\udce6\ufe0f Manage dependencies with Conda"),(0,i.yg)("p",null,"In your workspace you can install new ",(0,i.yg)("inlineCode",{parentName:"p"},"conda")," environments, if they include the packages ",(0,i.yg)("inlineCode",{parentName:"p"},"nb_conda_kernels")," and ",(0,i.yg)("inlineCode",{parentName:"p"},"ipykernel"),", then you will be able to easily start notebooks in those environments from the JupyterLab Launcher page."),(0,i.yg)("p",null,"Install a conda environment from a file with ",(0,i.yg)("inlineCode",{parentName:"p"},"mamba")," (it is like ",(0,i.yg)("inlineCode",{parentName:"p"},"conda")," but faster):"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-bash"},"mamba env create -f environment.yml\n")),(0,i.yg)("p",null,"You'll need to wait for 1 minute before the new conda environment becomes available on the JupyterLab Launcher page."),(0,i.yg)("p",null,"You can easily install an environment with a different version of Python if you need it. Here is an example of an ",(0,i.yg)("inlineCode",{parentName:"p"},"environment.yml")," file to create an environment with Python 3.9, install the minimal dependencies required to easily starts notebooks in this environment with ",(0,i.yg)("inlineCode",{parentName:"p"},"conda"),", and install a ",(0,i.yg)("inlineCode",{parentName:"p"},"pip")," package:"),(0,i.yg)("pre",null,(0,i.yg)("code",{parentName:"pre",className:"language-yaml"},"name: py39\nchannels:\n - defaults\n - conda-forge\n - anaconda\ndependencies:\n - python=3.9\n - ipykernel\n - nb_conda_kernels\n - pip\n - pip:\n - matplotlib\n")),(0,i.yg)("h2",{id:"-use-git-in-jupyterlab"},"\ud83d\udc19 Use git in JupyterLab"),(0,i.yg)("p",null,"You can use ",(0,i.yg)("inlineCode",{parentName:"p"},"git")," from the terminal."),(0,i.yg)("p",null,"You can also use the ",(0,i.yg)("a",{parentName:"p",href:"https://github.com/jupyterlab/jupyterlab-git"},"JupyterLab Git extension")," or the VisualStudio Code git integration to clone and manage your ",(0,i.yg)("inlineCode",{parentName:"p"},"git")," repositories."),(0,i.yg)("p",null,"They will ask you for a username and personal access token if the repository is private, or the first time you want to push changes."))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/cfec30f8.9ccf0700.js b/assets/js/cfec30f8.c5153a87.js similarity index 99% rename from assets/js/cfec30f8.9ccf0700.js rename to assets/js/cfec30f8.c5153a87.js index 1636e7347..5d4ba48d6 100644 --- a/assets/js/cfec30f8.9ccf0700.js +++ b/assets/js/cfec30f8.c5153a87.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[9215],{5680:(e,t,a)=>{a.d(t,{xA:()=>d,yg:()=>g});var n=a(6540);function o(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function r(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function s(e){for(var t=1;t=0||(o[a]=e[a]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(o[a]=e[a])}return o}var p=n.createContext({}),i=function(e){var t=n.useContext(p),a=t;return e&&(a="function"==typeof e?e(t):s(s({},t),e)),a},d=function(e){var t=i(e.components);return n.createElement(p.Provider,{value:t},e.children)},c={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},m=n.forwardRef((function(e,t){var a=e.components,o=e.mdxType,r=e.originalType,p=e.parentName,d=l(e,["components","mdxType","originalType","parentName"]),m=i(a),g=o,u=m["".concat(p,".").concat(g)]||m[g]||c[g]||r;return a?n.createElement(u,s(s({ref:t},d),{},{components:a})):n.createElement(u,s({ref:t},d))}));function g(e,t){var a=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=a.length,s=new Array(r);s[0]=m;var l={};for(var p in t)hasOwnProperty.call(t,p)&&(l[p]=t[p]);l.originalType=e,l.mdxType="string"==typeof e?e:o,s[1]=l;for(var i=2;i{a.r(t),a.d(t,{assets:()=>d,contentTitle:()=>p,default:()=>g,frontMatter:()=>l,metadata:()=>i,toc:()=>c});var n=a(9668),o=a(1367),r=(a(6540),a(5680)),s=["components"],l={id:"deploy-database",title:"Databases"},p=void 0,i={unversionedId:"deploy-database",id:"deploy-database",title:"Databases",description:"SQL databases",source:"@site/docs/deploy-database.md",sourceDirName:".",slug:"/deploy-database",permalink:"/docs/deploy-database",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/deploy-database.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"deploy-database",title:"Databases"},sidebar:"docs",previous:{title:"VisualStudio Code",permalink:"/docs/deploy-vscode"},next:{title:"Matlab",permalink:"/docs/deploy-matlab"}},d={},c=[{value:"SQL databases",id:"sql-databases",level:2},{value:"Start PostgreSQL \ud83d\udc18",id:"start-postgresql-",level:3},{value:"Start MySQL \ud83d\udc2c",id:"start-mysql-",level:3},{value:"NoSQL databases",id:"nosql-databases",level:2},{value:"MongoDB \ud83c\udf3f",id:"mongodb-",level:3},{value:"Redis \ud83c\udfb2",id:"redis-",level:3},{value:"Graph databases",id:"graph-databases",level:2},{value:"OpenLink Virtuoso triplestore",id:"openlink-virtuoso-triplestore",level:3},{value:"Ontotext GraphDB triplestore",id:"ontotext-graphdb-triplestore",level:3}],m={toc:c};function g(e){var t=e.components,a=(0,o.A)(e,s);return(0,r.yg)("wrapper",(0,n.A)({},m,a,{components:t,mdxType:"MDXLayout"}),(0,r.yg)("h2",{id:"sql-databases"},"SQL databases"),(0,r.yg)("p",null,"You can easily create a database from the templates available in the DSRI OpenShift web UI catalog:"),(0,r.yg)("img",{src:"/img/screenshot-databases.png",alt:"Databases in catalog web UI",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("p",null,"You can connect to a database from another application in the same project by using the database service name as hostname:"),(0,r.yg)("img",{src:"/img/screenshot_database_service.png",alt:"Databases in catalog web UI",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("p",null,"You can also use the ",(0,r.yg)("inlineCode",{parentName:"p"},"oc")," CLI to get the services in your project:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc get services\n")),(0,r.yg)("h3",{id:"start-postgresql-"},"Start PostgreSQL \ud83d\udc18"),(0,r.yg)("p",null,"Use the ",(0,r.yg)("strong",{parentName:"p"},"Postgresql")," template in the DSRI OpenShift web UI catalog to start a SQL database. "),(0,r.yg)("admonition",{title:"Connect to the database",type:"tip"},(0,r.yg)("p",{parentName:"admonition"},"When the database has been deployed, you can connect from another pod using your favorite language and connector.")),(0,r.yg)("p",null,"Example with the ",(0,r.yg)("inlineCode",{parentName:"p"},"psql")," Command Line Interface:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"apt-get update && apt-get install postgresql-client -y\n")),(0,r.yg)("p",null,"Connect to the Postgresql database using the service name (change depending on the username and database name you chose):"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"psql -h postgresql-db -U postgres db\n")),(0,r.yg)("p",null,"Checkout the ",(0,r.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-demo"},"dsri-demo repository")," for a quick demo for accessing and using a PostgreSQL database from a Jupyter notebook on the DSRI."),(0,r.yg)("h3",{id:"start-mysql-"},"Start MySQL \ud83d\udc2c"),(0,r.yg)("p",null,"Use the ",(0,r.yg)("strong",{parentName:"p"},"MySQL")," template in the DSRI OpenShift web UI catalog."),(0,r.yg)("admonition",{title:"Connect to the database",type:"tip"},(0,r.yg)("p",{parentName:"admonition"},"When the database has been deployed, you can connect from another pod using your favorite language and connector.")),(0,r.yg)("p",null,"Example with the ",(0,r.yg)("inlineCode",{parentName:"p"},"mysql")," Command Line Interface:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"apt-get update && apt-get install mariadb-client -y\n")),(0,r.yg)("p",null,"Connect to the MySQL database using the service name:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"mysql -h example-mysql -p\n")),(0,r.yg)("p",null,"Checkout the ",(0,r.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-demo"},"dsri-demo repository")," for a quick demo for accessing and using a MySQL database from a Jupyter notebook on the DSRI."),(0,r.yg)("blockquote",null,(0,r.yg)("p",{parentName:"blockquote"},"Alternatively, MySQL databases can be started using Helm, see the ",(0,r.yg)("a",{parentName:"p",href:"/docs/helm#install-a-helm-chart"},"Helm documentation page")," for more details.")),(0,r.yg)("h2",{id:"nosql-databases"},"NoSQL databases"),(0,r.yg)("h3",{id:"mongodb-"},"MongoDB \ud83c\udf3f"),(0,r.yg)("p",null,(0,r.yg)("a",{parentName:"p",href:"https://www.mongodb.com/"},"MongoDB")," is a general purpose, document-based, distributed database built for modern application developers and for the cloud era. "),(0,r.yg)("p",null,(0,r.yg)("strong",{parentName:"p"},"MongoDB")," can be deployed via Helm Chart as follows:"),(0,r.yg)("p",null,"1) Install the Helm Client as explained in (",(0,r.yg)("a",{parentName:"p",href:"https://dsri.maastrichtuniversity.nl/docs/helm"},"https://dsri.maastrichtuniversity.nl/docs/helm"),")"),(0,r.yg)("p",null,"2) Add the Bitnami Helm Repository. Bitnami offers a wide range of Helm charts, and you can add their repository with the following command:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"}," helm repo add bitnami https://charts.bitnami.com/bitnami\n\n helm repo update\n")),(0,r.yg)("p",null,"3) you can ",(0,r.yg)("strong",{parentName:"p"},"deploy MongoDB")," as follows:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"}," helm install my-mongodb bitnami/mongodb\n")),(0,r.yg)("p",null,"4) You can connect with ",(0,r.yg)("strong",{parentName:"p"},"my-mongodb")," as follows (within the cluster, for ex; via a terminal session opened at ",(0,r.yg)("a",{parentName:"p",href:"https://console-openshift-console.apps.dsri2.unimaas.nl/"},"https://console-openshift-console.apps.dsri2.unimaas.nl/"),"):"),(0,r.yg)("p",null,"In the follwoing scripts, we assume that we are working with a project/namespace named as 'manu-test'. Replace this with your own project/namespace name.\nMongoDB can be accessed on the following DNS name(s) and ports from within your cluster:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre"},"my-mongodb.manu-test.svc.cluster.local\n")),(0,r.yg)("p",null,"To get the root password run:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre"},'export MONGODB_ROOT_PASSWORD=$(kubectl get secret --namespace manu-test my-mongodb -o jsonpath="{.data.mongodb-root-password}" | base64 -d)\n')),(0,r.yg)("p",null,"To connect to your database, create a MongoDB; client container:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre"},"kubectl run --namespace manu-test my-mongodb-client --rm --tty -i --restart='Never' --env=\"MONGODB_ROOT_PASSWORD=$MONGODB_ROOT_PASSWORD\" --image docker.io/bitnami/mongodb:8.0.4-debian-12-r0 --command -- bash\n")),(0,r.yg)("p",null,"Then, run the following command:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre"},'export MONGODB_ROOT_USER="root"\nmongosh admin --host "my-mongodb" --authenticationDatabase admin --username $MONGODB_ROOT_USER --password $MONGODB_ROOT_PASSWORD\n')),(0,r.yg)("p",null,"To connect to your database from outside the cluster execute the following commands:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre"},"kubectl port-forward --namespace manu-test svc/my-mongodb 27017:27017 &\nmongosh --host 127.0.0.1 --authenticationDatabase admin --username $MONGODB_ROOT_USER --password $MONGODB_ROOT_PASSWORD\n")),(0,r.yg)("h3",{id:"redis-"},"Redis \ud83c\udfb2"),(0,r.yg)("p",null,(0,r.yg)("a",{parentName:"p",href:"http://redis.io/"},"Redis")," is an advanced key-value cache and store. It is often referred to as a data structure server since keys can contain strings, hashes, lists, sets, sorted sets, bitmaps and hyperlog."),(0,r.yg)("p",null,"Use the ",(0,r.yg)("strong",{parentName:"p"},"Redis")," template in the DSRI OpenShift web UI catalog."),(0,r.yg)("admonition",{title:"Connect to the database",type:"tip"},(0,r.yg)("p",{parentName:"admonition"},"Use the service name as hostname to connect from another pod in the same project.")),(0,r.yg)("h2",{id:"graph-databases"},"Graph databases"),(0,r.yg)("h3",{id:"openlink-virtuoso-triplestore"},"OpenLink Virtuoso triplestore"),(0,r.yg)("p",null,"Search for the ",(0,r.yg)("strong",{parentName:"p"},"Virtuoso triplestore")," template in the DSRI web UI catalog. Instantiate the template to create a Virtuoso triplestore in your project."),(0,r.yg)("p",null,"The deployment is based on the latest open source version of Virtuoso: ",(0,r.yg)("a",{parentName:"p",href:"https://hub.docker.com/r/openlink/virtuoso-opensource-7"},"https://hub.docker.com/r/openlink/virtuoso-opensource-7")),(0,r.yg)("admonition",{title:"Connect to the database",type:"tip"},(0,r.yg)("p",{parentName:"admonition"},"Use the service name as hostname to connect from another pod in the same project.")),(0,r.yg)("h3",{id:"ontotext-graphdb-triplestore"},"Ontotext GraphDB triplestore"),(0,r.yg)("p",null,"Use the official DockerHub image if you have an enterprise license. Or ",(0,r.yg)("a",{parentName:"p",href:"https://maastrichtu-ids.github.io/dsri-documentation/docs/guide-dockerfile-to-openshift"},"build")," GraphDB free edition image from ",(0,r.yg)("a",{parentName:"p",href:"https://github.com/Ontotext-AD/graphdb-docker"},"graphdb-docker on GitHub"),"."),(0,r.yg)("p",null,"After downloading the ",(0,r.yg)("inlineCode",{parentName:"p"},".zip")," file from the github to your laptop, unzip it and go the ",(0,r.yg)("inlineCode",{parentName:"p"},"graphdb-docker-master")," folder in your terminal:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"cd graphdb-docker-master\n")),(0,r.yg)("p",null,"Before creating your GraphDB ImageStream, make sure you are in the right project:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc project my-project\n")),(0,r.yg)("p",null,"Create the ImageStream for GraphDB:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc new-build --name graphdb --binary\n")),(0,r.yg)("p",null,"Build the image on the DSRI and save it in the ImageStream:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc start-build graphdb --from-dir=. --follow --wait\n")),(0,r.yg)("p",null,"You can now use the ",(0,r.yg)("strong",{parentName:"p"},"Ontotext GraphDB")," template to deploy a GraphDB instance on DSRI. "),(0,r.yg)("p",null,"Use the name of the ImageStream when instantiating the template, you can check if the image was properly built in ",(0,r.yg)("strong",{parentName:"p"},"Search")," > Filter ",(0,r.yg)("strong",{parentName:"p"},"Resources")," for ImageStreams"),(0,r.yg)("admonition",{title:"Connect to the database",type:"tip"},(0,r.yg)("p",{parentName:"admonition"},"Use the service name as hostname to connect from another pod in the same project.")))}g.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[9215],{5680:(e,t,a)=>{a.d(t,{xA:()=>d,yg:()=>g});var n=a(6540);function o(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function r(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function s(e){for(var t=1;t=0||(o[a]=e[a]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(o[a]=e[a])}return o}var p=n.createContext({}),i=function(e){var t=n.useContext(p),a=t;return e&&(a="function"==typeof e?e(t):s(s({},t),e)),a},d=function(e){var t=i(e.components);return n.createElement(p.Provider,{value:t},e.children)},c={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},m=n.forwardRef((function(e,t){var a=e.components,o=e.mdxType,r=e.originalType,p=e.parentName,d=l(e,["components","mdxType","originalType","parentName"]),m=i(a),g=o,u=m["".concat(p,".").concat(g)]||m[g]||c[g]||r;return a?n.createElement(u,s(s({ref:t},d),{},{components:a})):n.createElement(u,s({ref:t},d))}));function g(e,t){var a=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=a.length,s=new Array(r);s[0]=m;var l={};for(var p in t)hasOwnProperty.call(t,p)&&(l[p]=t[p]);l.originalType=e,l.mdxType="string"==typeof e?e:o,s[1]=l;for(var i=2;i{a.r(t),a.d(t,{assets:()=>d,contentTitle:()=>p,default:()=>g,frontMatter:()=>l,metadata:()=>i,toc:()=>c});var n=a(9668),o=a(1367),r=(a(6540),a(5680)),s=["components"],l={id:"deploy-database",title:"Databases"},p=void 0,i={unversionedId:"deploy-database",id:"deploy-database",title:"Databases",description:"SQL databases",source:"@site/docs/deploy-database.md",sourceDirName:".",slug:"/deploy-database",permalink:"/docs/deploy-database",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/deploy-database.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"deploy-database",title:"Databases"},sidebar:"docs",previous:{title:"VisualStudio Code",permalink:"/docs/deploy-vscode"},next:{title:"Matlab",permalink:"/docs/deploy-matlab"}},d={},c=[{value:"SQL databases",id:"sql-databases",level:2},{value:"Start PostgreSQL \ud83d\udc18",id:"start-postgresql-",level:3},{value:"Start MySQL \ud83d\udc2c",id:"start-mysql-",level:3},{value:"NoSQL databases",id:"nosql-databases",level:2},{value:"MongoDB \ud83c\udf3f",id:"mongodb-",level:3},{value:"Redis \ud83c\udfb2",id:"redis-",level:3},{value:"Graph databases",id:"graph-databases",level:2},{value:"OpenLink Virtuoso triplestore",id:"openlink-virtuoso-triplestore",level:3},{value:"Ontotext GraphDB triplestore",id:"ontotext-graphdb-triplestore",level:3}],m={toc:c};function g(e){var t=e.components,a=(0,o.A)(e,s);return(0,r.yg)("wrapper",(0,n.A)({},m,a,{components:t,mdxType:"MDXLayout"}),(0,r.yg)("h2",{id:"sql-databases"},"SQL databases"),(0,r.yg)("p",null,"You can easily create a database from the templates available in the DSRI OpenShift web UI catalog:"),(0,r.yg)("img",{src:"/img/screenshot-databases.png",alt:"Databases in catalog web UI",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("p",null,"You can connect to a database from another application in the same project by using the database service name as hostname:"),(0,r.yg)("img",{src:"/img/screenshot_database_service.png",alt:"Databases in catalog web UI",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("p",null,"You can also use the ",(0,r.yg)("inlineCode",{parentName:"p"},"oc")," CLI to get the services in your project:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc get services\n")),(0,r.yg)("h3",{id:"start-postgresql-"},"Start PostgreSQL \ud83d\udc18"),(0,r.yg)("p",null,"Use the ",(0,r.yg)("strong",{parentName:"p"},"Postgresql")," template in the DSRI OpenShift web UI catalog to start a SQL database. "),(0,r.yg)("admonition",{title:"Connect to the database",type:"tip"},(0,r.yg)("p",{parentName:"admonition"},"When the database has been deployed, you can connect from another pod using your favorite language and connector.")),(0,r.yg)("p",null,"Example with the ",(0,r.yg)("inlineCode",{parentName:"p"},"psql")," Command Line Interface:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"apt-get update && apt-get install postgresql-client -y\n")),(0,r.yg)("p",null,"Connect to the Postgresql database using the service name (change depending on the username and database name you chose):"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"psql -h postgresql-db -U postgres db\n")),(0,r.yg)("p",null,"Checkout the ",(0,r.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-demo"},"dsri-demo repository")," for a quick demo for accessing and using a PostgreSQL database from a Jupyter notebook on the DSRI."),(0,r.yg)("h3",{id:"start-mysql-"},"Start MySQL \ud83d\udc2c"),(0,r.yg)("p",null,"Use the ",(0,r.yg)("strong",{parentName:"p"},"MySQL")," template in the DSRI OpenShift web UI catalog."),(0,r.yg)("admonition",{title:"Connect to the database",type:"tip"},(0,r.yg)("p",{parentName:"admonition"},"When the database has been deployed, you can connect from another pod using your favorite language and connector.")),(0,r.yg)("p",null,"Example with the ",(0,r.yg)("inlineCode",{parentName:"p"},"mysql")," Command Line Interface:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"apt-get update && apt-get install mariadb-client -y\n")),(0,r.yg)("p",null,"Connect to the MySQL database using the service name:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"mysql -h example-mysql -p\n")),(0,r.yg)("p",null,"Checkout the ",(0,r.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-demo"},"dsri-demo repository")," for a quick demo for accessing and using a MySQL database from a Jupyter notebook on the DSRI."),(0,r.yg)("blockquote",null,(0,r.yg)("p",{parentName:"blockquote"},"Alternatively, MySQL databases can be started using Helm, see the ",(0,r.yg)("a",{parentName:"p",href:"/docs/helm#install-a-helm-chart"},"Helm documentation page")," for more details.")),(0,r.yg)("h2",{id:"nosql-databases"},"NoSQL databases"),(0,r.yg)("h3",{id:"mongodb-"},"MongoDB \ud83c\udf3f"),(0,r.yg)("p",null,(0,r.yg)("a",{parentName:"p",href:"https://www.mongodb.com/"},"MongoDB")," is a general purpose, document-based, distributed database built for modern application developers and for the cloud era. "),(0,r.yg)("p",null,(0,r.yg)("strong",{parentName:"p"},"MongoDB")," can be deployed via Helm Chart as follows:"),(0,r.yg)("p",null,"1) Install the Helm Client as explained in (",(0,r.yg)("a",{parentName:"p",href:"https://dsri.maastrichtuniversity.nl/docs/helm"},"https://dsri.maastrichtuniversity.nl/docs/helm"),")"),(0,r.yg)("p",null,"2) Add the Bitnami Helm Repository. Bitnami offers a wide range of Helm charts, and you can add their repository with the following command:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"}," helm repo add bitnami https://charts.bitnami.com/bitnami\n\n helm repo update\n")),(0,r.yg)("p",null,"3) you can ",(0,r.yg)("strong",{parentName:"p"},"deploy MongoDB")," as follows:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"}," helm install my-mongodb bitnami/mongodb\n")),(0,r.yg)("p",null,"4) You can connect with ",(0,r.yg)("strong",{parentName:"p"},"my-mongodb")," as follows (within the cluster, for ex; via a terminal session opened at ",(0,r.yg)("a",{parentName:"p",href:"https://console-openshift-console.apps.dsri2.unimaas.nl/"},"https://console-openshift-console.apps.dsri2.unimaas.nl/"),"):"),(0,r.yg)("p",null,"In the follwoing scripts, we assume that we are working with a project/namespace named as 'manu-test'. Replace this with your own project/namespace name.\nMongoDB can be accessed on the following DNS name(s) and ports from within your cluster:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre"},"my-mongodb.manu-test.svc.cluster.local\n")),(0,r.yg)("p",null,"To get the root password run:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre"},'export MONGODB_ROOT_PASSWORD=$(kubectl get secret --namespace manu-test my-mongodb -o jsonpath="{.data.mongodb-root-password}" | base64 -d)\n')),(0,r.yg)("p",null,"To connect to your database, create a MongoDB; client container:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre"},"kubectl run --namespace manu-test my-mongodb-client --rm --tty -i --restart='Never' --env=\"MONGODB_ROOT_PASSWORD=$MONGODB_ROOT_PASSWORD\" --image docker.io/bitnami/mongodb:8.0.4-debian-12-r0 --command -- bash\n")),(0,r.yg)("p",null,"Then, run the following command:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre"},'export MONGODB_ROOT_USER="root"\nmongosh admin --host "my-mongodb" --authenticationDatabase admin --username $MONGODB_ROOT_USER --password $MONGODB_ROOT_PASSWORD\n')),(0,r.yg)("p",null,"To connect to your database from outside the cluster execute the following commands:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre"},"kubectl port-forward --namespace manu-test svc/my-mongodb 27017:27017 &\nmongosh --host 127.0.0.1 --authenticationDatabase admin --username $MONGODB_ROOT_USER --password $MONGODB_ROOT_PASSWORD\n")),(0,r.yg)("h3",{id:"redis-"},"Redis \ud83c\udfb2"),(0,r.yg)("p",null,(0,r.yg)("a",{parentName:"p",href:"http://redis.io/"},"Redis")," is an advanced key-value cache and store. It is often referred to as a data structure server since keys can contain strings, hashes, lists, sets, sorted sets, bitmaps and hyperlog."),(0,r.yg)("p",null,"Use the ",(0,r.yg)("strong",{parentName:"p"},"Redis")," template in the DSRI OpenShift web UI catalog."),(0,r.yg)("admonition",{title:"Connect to the database",type:"tip"},(0,r.yg)("p",{parentName:"admonition"},"Use the service name as hostname to connect from another pod in the same project.")),(0,r.yg)("h2",{id:"graph-databases"},"Graph databases"),(0,r.yg)("h3",{id:"openlink-virtuoso-triplestore"},"OpenLink Virtuoso triplestore"),(0,r.yg)("p",null,"Search for the ",(0,r.yg)("strong",{parentName:"p"},"Virtuoso triplestore")," template in the DSRI web UI catalog. Instantiate the template to create a Virtuoso triplestore in your project."),(0,r.yg)("p",null,"The deployment is based on the latest open source version of Virtuoso: ",(0,r.yg)("a",{parentName:"p",href:"https://hub.docker.com/r/openlink/virtuoso-opensource-7"},"https://hub.docker.com/r/openlink/virtuoso-opensource-7")),(0,r.yg)("admonition",{title:"Connect to the database",type:"tip"},(0,r.yg)("p",{parentName:"admonition"},"Use the service name as hostname to connect from another pod in the same project.")),(0,r.yg)("h3",{id:"ontotext-graphdb-triplestore"},"Ontotext GraphDB triplestore"),(0,r.yg)("p",null,"Use the official DockerHub image if you have an enterprise license. Or ",(0,r.yg)("a",{parentName:"p",href:"https://maastrichtu-ids.github.io/dsri-documentation/docs/guide-dockerfile-to-openshift"},"build")," GraphDB free edition image from ",(0,r.yg)("a",{parentName:"p",href:"https://github.com/Ontotext-AD/graphdb-docker"},"graphdb-docker on GitHub"),"."),(0,r.yg)("p",null,"After downloading the ",(0,r.yg)("inlineCode",{parentName:"p"},".zip")," file from the github to your laptop, unzip it and go the ",(0,r.yg)("inlineCode",{parentName:"p"},"graphdb-docker-master")," folder in your terminal:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"cd graphdb-docker-master\n")),(0,r.yg)("p",null,"Before creating your GraphDB ImageStream, make sure you are in the right project:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc project my-project\n")),(0,r.yg)("p",null,"Create the ImageStream for GraphDB:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc new-build --name graphdb --binary\n")),(0,r.yg)("p",null,"Build the image on the DSRI and save it in the ImageStream:"),(0,r.yg)("pre",null,(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc start-build graphdb --from-dir=. --follow --wait\n")),(0,r.yg)("p",null,"You can now use the ",(0,r.yg)("strong",{parentName:"p"},"Ontotext GraphDB")," template to deploy a GraphDB instance on DSRI. "),(0,r.yg)("p",null,"Use the name of the ImageStream when instantiating the template, you can check if the image was properly built in ",(0,r.yg)("strong",{parentName:"p"},"Search")," > Filter ",(0,r.yg)("strong",{parentName:"p"},"Resources")," for ImageStreams"),(0,r.yg)("admonition",{title:"Connect to the database",type:"tip"},(0,r.yg)("p",{parentName:"admonition"},"Use the service name as hostname to connect from another pod in the same project.")))}g.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/d013d563.345d2301.js b/assets/js/d013d563.fc3369b8.js similarity index 99% rename from assets/js/d013d563.345d2301.js rename to assets/js/d013d563.fc3369b8.js index f76bc08e6..e78b33a78 100644 --- a/assets/js/d013d563.345d2301.js +++ b/assets/js/d013d563.fc3369b8.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[5299],{5680:(e,t,r)=>{r.d(t,{xA:()=>c,yg:()=>y});var a=r(6540);function n(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function o(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,a)}return r}function i(e){for(var t=1;t=0||(n[r]=e[r]);return n}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(n[r]=e[r])}return n}var p=a.createContext({}),s=function(e){var t=a.useContext(p),r=t;return e&&(r="function"==typeof e?e(t):i(i({},t),e)),r},c=function(e){var t=s(e.components);return a.createElement(p.Provider,{value:t},e.children)},g={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},d=a.forwardRef((function(e,t){var r=e.components,n=e.mdxType,o=e.originalType,p=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),d=s(r),y=n,h=d["".concat(p,".").concat(y)]||d[y]||g[y]||o;return r?a.createElement(h,i(i({ref:t},c),{},{components:r})):a.createElement(h,i({ref:t},c))}));function y(e,t){var r=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var o=r.length,i=new Array(o);i[0]=d;var l={};for(var p in t)hasOwnProperty.call(t,p)&&(l[p]=t[p]);l.originalType=e,l.mdxType="string"==typeof e?e:n,i[1]=l;for(var s=2;s{r.r(t),r.d(t,{assets:()=>c,contentTitle:()=>p,default:()=>y,frontMatter:()=>l,metadata:()=>s,toc:()=>g});var a=r(9668),n=r(1367),o=(r(6540),r(5680)),i=["components"],l={id:"tools-machine-learning",title:"Libraries for Machine Learning"},p=void 0,s={unversionedId:"tools-machine-learning",id:"tools-machine-learning",title:"Libraries for Machine Learning",description:"This page is in development, feel free to edit it to add more information.",source:"@site/docs/tools-machine-learning.md",sourceDirName:".",slug:"/tools-machine-learning",permalink:"/docs/tools-machine-learning",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/tools-machine-learning.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"tools-machine-learning",title:"Libraries for Machine Learning"},sidebar:"docs",previous:{title:"Create a new Project",permalink:"/docs/project-management"},next:{title:"Glossary",permalink:"/docs/glossary"}},c={},g=[{value:"Machine Learning libraries",id:"machine-learning-libraries",level:2},{value:"SciKit Learn",id:"scikit-learn",level:3},{value:"Deep Learning libraries",id:"deep-learning-libraries",level:2},{value:"Tensorflow",id:"tensorflow",level:3},{value:"PyTorch",id:"pytorch",level:3},{value:"Deep Java Library",id:"deep-java-library",level:3},{value:"Sonnet",id:"sonnet",level:3},{value:"Keras",id:"keras",level:3},{value:"Metaflow",id:"metaflow",level:3}],d={toc:g};function y(e){var t=e.components,r=(0,n.A)(e,i);return(0,o.yg)("wrapper",(0,a.A)({},d,r,{components:t,mdxType:"MDXLayout"}),(0,o.yg)("admonition",{title:"Work in progress",type:"caution"},(0,o.yg)("p",{parentName:"admonition"},"This page is in development, feel free to ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/docs/tools-machine-learning.md"},"edit it")," to add more information.")),(0,o.yg)("h2",{id:"machine-learning-libraries"},"Machine Learning libraries"),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},"See ",(0,o.yg)("a",{parentName:"li",href:"https://www.saagie.com/blog/machine-learning-for-grandmas/"},"this vulgarisation article")," explaining the different principles of Machine Learning."),(0,o.yg)("li",{parentName:"ul"},"The ",(0,o.yg)("a",{parentName:"li",href:"https://docs.microsoft.com/en-us/azure/machine-learning/algorithm-cheat-sheet"},"Azure Machine Learning Algorithm Cheat Sheet")," helps you choose the right algorithm for a predictive analytics model."),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("a",{parentName:"li",href:"https://github.com/TarrySingh/Artificial-Intelligence-Deep-Learning-Machine-Learning-Tutorials"},"This repository")," provides tutorials and examples to a vast number of Machine / Deep Learning library.")),(0,o.yg)("h3",{id:"scikit-learn"},"SciKit Learn"),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},(0,o.yg)("a",{parentName:"p",href:"https://scikit-learn.org/stable/"},"https://scikit-learn.org/stable/"))),(0,o.yg)("h2",{id:"deep-learning-libraries"},"Deep Learning libraries"),(0,o.yg)("p",null,"See ",(0,o.yg)("a",{parentName:"p",href:"https://towardsdatascience.com/top-10-best-deep-learning-frameworks-in-2019-5ccb90ea6de"},"this article")," for more details about modern Deep Learning libraries."),(0,o.yg)("h3",{id:"tensorflow"},"Tensorflow"),(0,o.yg)("p",null,"Python library developed by Google."),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},(0,o.yg)("a",{parentName:"p",href:"https://www.tensorflow.org/"},"https://www.tensorflow.org/"))),(0,o.yg)("h3",{id:"pytorch"},"PyTorch"),(0,o.yg)("p",null,"Python library developed by Facebook."),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},(0,o.yg)("a",{parentName:"p",href:"https://pytorch.org/"},"https://pytorch.org/"))),(0,o.yg)("h3",{id:"deep-java-library"},"Deep Java Library"),(0,o.yg)("p",null,"Java library developed by Amazon. See the ",(0,o.yg)("a",{parentName:"p",href:"https://towardsdatascience.com/introducing-deep-java-library-djl-9de98de8c6ca"},"introduction article"),"."),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},(0,o.yg)("a",{parentName:"p",href:"https://djl.ai/"},"https://djl.ai/"))),(0,o.yg)("h3",{id:"sonnet"},"Sonnet"),(0,o.yg)("p",null,"Layer on top of Tensorflow."),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},(0,o.yg)("a",{parentName:"p",href:"https://sonnet.readthedocs.io/en/latest/"},"https://sonnet.readthedocs.io/en/latest/"))),(0,o.yg)("h3",{id:"keras"},"Keras"),(0,o.yg)("p",null,"Python library. Layer on top of Tensorflow, CNTK, Theano."),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},(0,o.yg)("a",{parentName:"p",href:"https://keras.io/"},"https://keras.io/"))),(0,o.yg)("h3",{id:"metaflow"},"Metaflow"),(0,o.yg)("p",null,"Layer on top of Tensorflow, PyTorch, SciKit Learn developed by Netflix. "),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},(0,o.yg)("a",{parentName:"p",href:"https://metaflow.org/"},"https://metaflow.org/"))))}y.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[5299],{5680:(e,t,r)=>{r.d(t,{xA:()=>c,yg:()=>y});var a=r(6540);function n(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function o(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,a)}return r}function i(e){for(var t=1;t=0||(n[r]=e[r]);return n}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(n[r]=e[r])}return n}var p=a.createContext({}),s=function(e){var t=a.useContext(p),r=t;return e&&(r="function"==typeof e?e(t):i(i({},t),e)),r},c=function(e){var t=s(e.components);return a.createElement(p.Provider,{value:t},e.children)},g={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},d=a.forwardRef((function(e,t){var r=e.components,n=e.mdxType,o=e.originalType,p=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),d=s(r),y=n,h=d["".concat(p,".").concat(y)]||d[y]||g[y]||o;return r?a.createElement(h,i(i({ref:t},c),{},{components:r})):a.createElement(h,i({ref:t},c))}));function y(e,t){var r=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var o=r.length,i=new Array(o);i[0]=d;var l={};for(var p in t)hasOwnProperty.call(t,p)&&(l[p]=t[p]);l.originalType=e,l.mdxType="string"==typeof e?e:n,i[1]=l;for(var s=2;s{r.r(t),r.d(t,{assets:()=>c,contentTitle:()=>p,default:()=>y,frontMatter:()=>l,metadata:()=>s,toc:()=>g});var a=r(9668),n=r(1367),o=(r(6540),r(5680)),i=["components"],l={id:"tools-machine-learning",title:"Libraries for Machine Learning"},p=void 0,s={unversionedId:"tools-machine-learning",id:"tools-machine-learning",title:"Libraries for Machine Learning",description:"This page is in development, feel free to edit it to add more information.",source:"@site/docs/tools-machine-learning.md",sourceDirName:".",slug:"/tools-machine-learning",permalink:"/docs/tools-machine-learning",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/tools-machine-learning.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"tools-machine-learning",title:"Libraries for Machine Learning"},sidebar:"docs",previous:{title:"Create a new Project",permalink:"/docs/project-management"},next:{title:"Glossary",permalink:"/docs/glossary"}},c={},g=[{value:"Machine Learning libraries",id:"machine-learning-libraries",level:2},{value:"SciKit Learn",id:"scikit-learn",level:3},{value:"Deep Learning libraries",id:"deep-learning-libraries",level:2},{value:"Tensorflow",id:"tensorflow",level:3},{value:"PyTorch",id:"pytorch",level:3},{value:"Deep Java Library",id:"deep-java-library",level:3},{value:"Sonnet",id:"sonnet",level:3},{value:"Keras",id:"keras",level:3},{value:"Metaflow",id:"metaflow",level:3}],d={toc:g};function y(e){var t=e.components,r=(0,n.A)(e,i);return(0,o.yg)("wrapper",(0,a.A)({},d,r,{components:t,mdxType:"MDXLayout"}),(0,o.yg)("admonition",{title:"Work in progress",type:"caution"},(0,o.yg)("p",{parentName:"admonition"},"This page is in development, feel free to ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/docs/tools-machine-learning.md"},"edit it")," to add more information.")),(0,o.yg)("h2",{id:"machine-learning-libraries"},"Machine Learning libraries"),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},"See ",(0,o.yg)("a",{parentName:"li",href:"https://www.saagie.com/blog/machine-learning-for-grandmas/"},"this vulgarisation article")," explaining the different principles of Machine Learning."),(0,o.yg)("li",{parentName:"ul"},"The ",(0,o.yg)("a",{parentName:"li",href:"https://docs.microsoft.com/en-us/azure/machine-learning/algorithm-cheat-sheet"},"Azure Machine Learning Algorithm Cheat Sheet")," helps you choose the right algorithm for a predictive analytics model."),(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("a",{parentName:"li",href:"https://github.com/TarrySingh/Artificial-Intelligence-Deep-Learning-Machine-Learning-Tutorials"},"This repository")," provides tutorials and examples to a vast number of Machine / Deep Learning library.")),(0,o.yg)("h3",{id:"scikit-learn"},"SciKit Learn"),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},(0,o.yg)("a",{parentName:"p",href:"https://scikit-learn.org/stable/"},"https://scikit-learn.org/stable/"))),(0,o.yg)("h2",{id:"deep-learning-libraries"},"Deep Learning libraries"),(0,o.yg)("p",null,"See ",(0,o.yg)("a",{parentName:"p",href:"https://towardsdatascience.com/top-10-best-deep-learning-frameworks-in-2019-5ccb90ea6de"},"this article")," for more details about modern Deep Learning libraries."),(0,o.yg)("h3",{id:"tensorflow"},"Tensorflow"),(0,o.yg)("p",null,"Python library developed by Google."),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},(0,o.yg)("a",{parentName:"p",href:"https://www.tensorflow.org/"},"https://www.tensorflow.org/"))),(0,o.yg)("h3",{id:"pytorch"},"PyTorch"),(0,o.yg)("p",null,"Python library developed by Facebook."),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},(0,o.yg)("a",{parentName:"p",href:"https://pytorch.org/"},"https://pytorch.org/"))),(0,o.yg)("h3",{id:"deep-java-library"},"Deep Java Library"),(0,o.yg)("p",null,"Java library developed by Amazon. See the ",(0,o.yg)("a",{parentName:"p",href:"https://towardsdatascience.com/introducing-deep-java-library-djl-9de98de8c6ca"},"introduction article"),"."),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},(0,o.yg)("a",{parentName:"p",href:"https://djl.ai/"},"https://djl.ai/"))),(0,o.yg)("h3",{id:"sonnet"},"Sonnet"),(0,o.yg)("p",null,"Layer on top of Tensorflow."),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},(0,o.yg)("a",{parentName:"p",href:"https://sonnet.readthedocs.io/en/latest/"},"https://sonnet.readthedocs.io/en/latest/"))),(0,o.yg)("h3",{id:"keras"},"Keras"),(0,o.yg)("p",null,"Python library. Layer on top of Tensorflow, CNTK, Theano."),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},(0,o.yg)("a",{parentName:"p",href:"https://keras.io/"},"https://keras.io/"))),(0,o.yg)("h3",{id:"metaflow"},"Metaflow"),(0,o.yg)("p",null,"Layer on top of Tensorflow, PyTorch, SciKit Learn developed by Netflix. "),(0,o.yg)("blockquote",null,(0,o.yg)("p",{parentName:"blockquote"},(0,o.yg)("a",{parentName:"p",href:"https://metaflow.org/"},"https://metaflow.org/"))))}y.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/d306cda8.711f7b65.js b/assets/js/d306cda8.f464980f.js similarity index 99% rename from assets/js/d306cda8.711f7b65.js rename to assets/js/d306cda8.f464980f.js index e3d0ab0c8..e2856cf16 100644 --- a/assets/js/d306cda8.711f7b65.js +++ b/assets/js/d306cda8.f464980f.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[4100],{5680:(e,t,n)=>{n.d(t,{xA:()=>c,yg:()=>m});var o=n(6540);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function i(e){for(var t=1;t=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(o=0;o=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var p=o.createContext({}),s=function(e){var t=o.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},c=function(e){var t=s(e.components);return o.createElement(p.Provider,{value:t},e.children)},g={inlineCode:"code",wrapper:function(e){var t=e.children;return o.createElement(o.Fragment,{},t)}},u=o.forwardRef((function(e,t){var n=e.components,a=e.mdxType,r=e.originalType,p=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),u=s(n),m=a,y=u["".concat(p,".").concat(m)]||u[m]||g[m]||r;return n?o.createElement(y,i(i({ref:t},c),{},{components:n})):o.createElement(y,i({ref:t},c))}));function m(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var r=n.length,i=new Array(r);i[0]=u;var l={};for(var p in t)hasOwnProperty.call(t,p)&&(l[p]=t[p]);l.originalType=e,l.mdxType="string"==typeof e?e:a,i[1]=l;for(var s=2;s{n.r(t),n.d(t,{assets:()=>c,contentTitle:()=>p,default:()=>m,frontMatter:()=>l,metadata:()=>s,toc:()=>g});var o=n(9668),a=n(1367),r=(n(6540),n(5680)),i=["components"],l={id:"guide-monitoring",title:"Monitor your applications"},p=void 0,s={unversionedId:"guide-monitoring",id:"guide-monitoring",title:"Monitor your applications",description:"Monitor your application resources use",source:"@site/docs/guide-monitoring.md",sourceDirName:".",slug:"/guide-monitoring",permalink:"/docs/guide-monitoring",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/guide-monitoring.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"guide-monitoring",title:"Monitor your applications"},sidebar:"docs",previous:{title:"Delete an application",permalink:"/docs/openshift-delete-services"},next:{title:"GPU applications",permalink:"/docs/deploy-on-gpu"}},c={},g=[{value:"Monitor your application resources use",id:"monitor-your-application-resources-use",level:2},{value:"Debug an application deployment",id:"debug-an-application-deployment",level:2}],u={toc:g};function m(e){var t=e.components,n=(0,a.A)(e,i);return(0,r.yg)("wrapper",(0,o.A)({},u,n,{components:t,mdxType:"MDXLayout"}),(0,r.yg)("h2",{id:"monitor-your-application-resources-use"},"Monitor your application resources use"),(0,r.yg)("p",null,"You can have an overview of the different resources consumed by the applications running in your project by going to the ",(0,r.yg)("strong",{parentName:"p"},"Monitoring")," tab (in the developer view)"),(0,r.yg)("img",{src:"/img/screenshot_monitoring.png",alt:"Filter templates catalog",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("p",null,"You can also check the CPU and memory usage directly from the terminal inside a specific container"),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"Go to your application terminal, and run:"),(0,r.yg)("pre",{parentName:"li"},(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"top\n"))),(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"Check the number of Cpu(s) used at the top:"),(0,r.yg)("blockquote",{parentName:"li"},(0,r.yg)("p",{parentName:"blockquote"},"%Cpu(s): ",(0,r.yg)("strong",{parentName:"p"},"3,3")," us,"))),(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"Check the memory usage with the ",(0,r.yg)("inlineCode",{parentName:"p"},"used")," column:"),(0,r.yg)("blockquote",{parentName:"li"},(0,r.yg)("p",{parentName:"blockquote"},"MiB Mem : ",(0,r.yg)("strong",{parentName:"p"},"515543.2")," total, ",(0,r.yg)("strong",{parentName:"p"},"403486.8")," free, ",(0,r.yg)("strong",{parentName:"p"},"98612.0")," used, ",(0,r.yg)("strong",{parentName:"p"},"13444.5")," buff/cache")))),(0,r.yg)("h2",{id:"debug-an-application-deployment"},"Debug an application deployment"),(0,r.yg)("p",null,"If your application is facing issues when deployed:"),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},"If the pod is not building, or not deploying properly, take a look at the ",(0,r.yg)("strong",{parentName:"li"},"Events")," tab of the deployment. It shows a log of all events faced by the deployment (assign to node, pull image, build, etc). Additionally, all ",(0,r.yg)("strong",{parentName:"li"},"Events")," in your project can be accessed in ",(0,r.yg)("strong",{parentName:"li"},"Monitoring"),".")),(0,r.yg)("img",{src:"/img/screenshot_debug_event.png",alt:"Filter templates catalog",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("admonition",{title:"Various ways to check the events",type:"tip"},(0,r.yg)("p",{parentName:"admonition"},"You can also check the ",(0,r.yg)("inlineCode",{parentName:"p"},"Monitoring")," page in the left side menu to see all events in a project."),(0,r.yg)("p",{parentName:"admonition"},"Or use the terminal:"),(0,r.yg)("pre",{parentName:"admonition"},(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc get events\n"))),(0,r.yg)("ol",{start:2},(0,r.yg)("li",{parentName:"ol"},"When a pod is running you can check its logs in the ",(0,r.yg)("strong",{parentName:"li"},"Logs")," tab (after going to the pod page). It will show the logs output of the container, equivalent to doing ",(0,r.yg)("inlineCode",{parentName:"li"},"docker logs"),".")),(0,r.yg)("admonition",{title:"Get help",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"If you cannot figure out the issue by yourself:"),(0,r.yg)("ol",{parentName:"admonition"},(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("strong",{parentName:"li"},"Gather relevant information to help the DSRI team")," to solve your issue: URL to the faulty application, which error was shown in the ",(0,r.yg)("strong",{parentName:"li"},"Events")," tab? Or in the ",(0,r.yg)("strong",{parentName:"li"},"Logs")," tab?"),(0,r.yg)("li",{parentName:"ol"},"Seek help on the ",(0,r.yg)("inlineCode",{parentName:"li"},"#helpdesk")," DSRI Slack channel"),(0,r.yg)("li",{parentName:"ol"},"Checkout if an issue have already been created for this problem, or create a new one: ",(0,r.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/issues"},"https://github.com/MaastrichtU-IDS/dsri-documentation/issues")))))}m.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[4100],{5680:(e,t,n)=>{n.d(t,{xA:()=>c,yg:()=>m});var o=n(6540);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function r(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,o)}return n}function i(e){for(var t=1;t=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(o=0;o=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var p=o.createContext({}),s=function(e){var t=o.useContext(p),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},c=function(e){var t=s(e.components);return o.createElement(p.Provider,{value:t},e.children)},g={inlineCode:"code",wrapper:function(e){var t=e.children;return o.createElement(o.Fragment,{},t)}},u=o.forwardRef((function(e,t){var n=e.components,a=e.mdxType,r=e.originalType,p=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),u=s(n),m=a,y=u["".concat(p,".").concat(m)]||u[m]||g[m]||r;return n?o.createElement(y,i(i({ref:t},c),{},{components:n})):o.createElement(y,i({ref:t},c))}));function m(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var r=n.length,i=new Array(r);i[0]=u;var l={};for(var p in t)hasOwnProperty.call(t,p)&&(l[p]=t[p]);l.originalType=e,l.mdxType="string"==typeof e?e:a,i[1]=l;for(var s=2;s{n.r(t),n.d(t,{assets:()=>c,contentTitle:()=>p,default:()=>m,frontMatter:()=>l,metadata:()=>s,toc:()=>g});var o=n(9668),a=n(1367),r=(n(6540),n(5680)),i=["components"],l={id:"guide-monitoring",title:"Monitor your applications"},p=void 0,s={unversionedId:"guide-monitoring",id:"guide-monitoring",title:"Monitor your applications",description:"Monitor your application resources use",source:"@site/docs/guide-monitoring.md",sourceDirName:".",slug:"/guide-monitoring",permalink:"/docs/guide-monitoring",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/guide-monitoring.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"guide-monitoring",title:"Monitor your applications"},sidebar:"docs",previous:{title:"Delete an application",permalink:"/docs/openshift-delete-services"},next:{title:"GPU applications",permalink:"/docs/deploy-on-gpu"}},c={},g=[{value:"Monitor your application resources use",id:"monitor-your-application-resources-use",level:2},{value:"Debug an application deployment",id:"debug-an-application-deployment",level:2}],u={toc:g};function m(e){var t=e.components,n=(0,a.A)(e,i);return(0,r.yg)("wrapper",(0,o.A)({},u,n,{components:t,mdxType:"MDXLayout"}),(0,r.yg)("h2",{id:"monitor-your-application-resources-use"},"Monitor your application resources use"),(0,r.yg)("p",null,"You can have an overview of the different resources consumed by the applications running in your project by going to the ",(0,r.yg)("strong",{parentName:"p"},"Monitoring")," tab (in the developer view)"),(0,r.yg)("img",{src:"/img/screenshot_monitoring.png",alt:"Filter templates catalog",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("p",null,"You can also check the CPU and memory usage directly from the terminal inside a specific container"),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"Go to your application terminal, and run:"),(0,r.yg)("pre",{parentName:"li"},(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"top\n"))),(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"Check the number of Cpu(s) used at the top:"),(0,r.yg)("blockquote",{parentName:"li"},(0,r.yg)("p",{parentName:"blockquote"},"%Cpu(s): ",(0,r.yg)("strong",{parentName:"p"},"3,3")," us,"))),(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"Check the memory usage with the ",(0,r.yg)("inlineCode",{parentName:"p"},"used")," column:"),(0,r.yg)("blockquote",{parentName:"li"},(0,r.yg)("p",{parentName:"blockquote"},"MiB Mem : ",(0,r.yg)("strong",{parentName:"p"},"515543.2")," total, ",(0,r.yg)("strong",{parentName:"p"},"403486.8")," free, ",(0,r.yg)("strong",{parentName:"p"},"98612.0")," used, ",(0,r.yg)("strong",{parentName:"p"},"13444.5")," buff/cache")))),(0,r.yg)("h2",{id:"debug-an-application-deployment"},"Debug an application deployment"),(0,r.yg)("p",null,"If your application is facing issues when deployed:"),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},"If the pod is not building, or not deploying properly, take a look at the ",(0,r.yg)("strong",{parentName:"li"},"Events")," tab of the deployment. It shows a log of all events faced by the deployment (assign to node, pull image, build, etc). Additionally, all ",(0,r.yg)("strong",{parentName:"li"},"Events")," in your project can be accessed in ",(0,r.yg)("strong",{parentName:"li"},"Monitoring"),".")),(0,r.yg)("img",{src:"/img/screenshot_debug_event.png",alt:"Filter templates catalog",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("admonition",{title:"Various ways to check the events",type:"tip"},(0,r.yg)("p",{parentName:"admonition"},"You can also check the ",(0,r.yg)("inlineCode",{parentName:"p"},"Monitoring")," page in the left side menu to see all events in a project."),(0,r.yg)("p",{parentName:"admonition"},"Or use the terminal:"),(0,r.yg)("pre",{parentName:"admonition"},(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc get events\n"))),(0,r.yg)("ol",{start:2},(0,r.yg)("li",{parentName:"ol"},"When a pod is running you can check its logs in the ",(0,r.yg)("strong",{parentName:"li"},"Logs")," tab (after going to the pod page). It will show the logs output of the container, equivalent to doing ",(0,r.yg)("inlineCode",{parentName:"li"},"docker logs"),".")),(0,r.yg)("admonition",{title:"Get help",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"If you cannot figure out the issue by yourself:"),(0,r.yg)("ol",{parentName:"admonition"},(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("strong",{parentName:"li"},"Gather relevant information to help the DSRI team")," to solve your issue: URL to the faulty application, which error was shown in the ",(0,r.yg)("strong",{parentName:"li"},"Events")," tab? Or in the ",(0,r.yg)("strong",{parentName:"li"},"Logs")," tab?"),(0,r.yg)("li",{parentName:"ol"},"Seek help on the ",(0,r.yg)("inlineCode",{parentName:"li"},"#helpdesk")," DSRI Slack channel"),(0,r.yg)("li",{parentName:"ol"},"Checkout if an issue have already been created for this problem, or create a new one: ",(0,r.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/issues"},"https://github.com/MaastrichtU-IDS/dsri-documentation/issues")))))}m.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/d8f096f7.1b502123.js b/assets/js/d8f096f7.883525b5.js similarity index 99% rename from assets/js/d8f096f7.1b502123.js rename to assets/js/d8f096f7.883525b5.js index 034b0dc03..02738c831 100644 --- a/assets/js/d8f096f7.1b502123.js +++ b/assets/js/d8f096f7.883525b5.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[145],{5680:(e,n,t)=>{t.d(n,{xA:()=>c,yg:()=>d});var o=t(6540);function r(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function a(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);n&&(o=o.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,o)}return t}function i(e){for(var n=1;n=0||(r[t]=e[t]);return r}(e,n);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(o=0;o=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(r[t]=e[t])}return r}var l=o.createContext({}),s=function(e){var n=o.useContext(l),t=n;return e&&(t="function"==typeof e?e(n):i(i({},n),e)),t},c=function(e){var n=s(e.components);return o.createElement(l.Provider,{value:n},e.children)},m={inlineCode:"code",wrapper:function(e){var n=e.children;return o.createElement(o.Fragment,{},n)}},u=o.forwardRef((function(e,n){var t=e.components,r=e.mdxType,a=e.originalType,l=e.parentName,c=p(e,["components","mdxType","originalType","parentName"]),u=s(t),d=r,y=u["".concat(l,".").concat(d)]||u[d]||m[d]||a;return t?o.createElement(y,i(i({ref:n},c),{},{components:t})):o.createElement(y,i({ref:n},c))}));function d(e,n){var t=arguments,r=n&&n.mdxType;if("string"==typeof e||r){var a=t.length,i=new Array(a);i[0]=u;var p={};for(var l in n)hasOwnProperty.call(n,l)&&(p[l]=n[l]);p.originalType=e,p.mdxType="string"==typeof e?e:r,i[1]=p;for(var s=2;s{t.r(n),t.d(n,{assets:()=>c,contentTitle:()=>l,default:()=>d,frontMatter:()=>p,metadata:()=>s,toc:()=>m});var o=t(9668),r=t(1367),a=(t(6540),t(5680)),i=["components"],p={id:"mpi-jobs",title:"Run MPI jobs"},l=void 0,s={unversionedId:"mpi-jobs",id:"mpi-jobs",title:"Run MPI jobs",description:"We deployed the MPI Operator from Kubeflow to run MPI jobs on the DSRI.",source:"@site/docs/mpi-jobs.md",sourceDirName:".",slug:"/mpi-jobs",permalink:"/docs/mpi-jobs",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/mpi-jobs.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"mpi-jobs",title:"Run MPI jobs"},sidebar:"docs",previous:{title:"Spark cluster",permalink:"/docs/deploy-spark"},next:{title:"Neuroscience research",permalink:"/docs/neuroscience"}},c={},m=[{value:"Run MPI-jobs on CPU",id:"run-mpi-jobs-on-cpu",level:2}],u={toc:m};function d(e){var n=e.components,t=(0,r.A)(e,i);return(0,a.yg)("wrapper",(0,o.A)({},u,t,{components:n,mdxType:"MDXLayout"}),(0,a.yg)("p",null,"We deployed the ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/kubeflow/mpi-operator"},"MPI Operator")," from Kubeflow to run MPI jobs on the DSRI."),(0,a.yg)("admonition",{title:"Request access to the MPI Operator",type:"warning"},(0,a.yg)("p",{parentName:"admonition"},"To be able to deploy MPI-Jobs you will need to ",(0,a.yg)("a",{parentName:"p",href:"https://dsri.maastrichtuniversity.nl/help"},"ask the DSRI admins")," to enable the MPI Operator in your project. Once enabled you will be able to start an MPI-job in a few clicks.")),(0,a.yg)("blockquote",null,(0,a.yg)("p",{parentName:"blockquote"},"The MPI Operator makes it easy to run allreduce-style distributed training on Kubernetes. Please check out ",(0,a.yg)("a",{parentName:"p",href:"https://medium.com/kubeflow/introduction-to-kubeflow-mpi-operator-and-industry-adoption-296d5f2e6edc"},"this blog post")," for an introduction to MPI Operator and its industry adoption.")),(0,a.yg)("h2",{id:"run-mpi-jobs-on-cpu"},"Run MPI-jobs on CPU"),(0,a.yg)("p",null,"Checkout the ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/kubeflow/mpi-operator/tree/master/examples/v2beta1/pi"},"repository of the example")," for a complete example of an MPI-job: python script, ",(0,a.yg)("inlineCode",{parentName:"p"},"Dockerfile"),", and the job deployment YAML."),(0,a.yg)("ol",null,(0,a.yg)("li",{parentName:"ol"},"Clone the repository, and go to the example folder:")),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},"git clone https://github.com/kubeflow/mpi-operator.git\ncd mpi-operator/examples/v2beta1/pi\n")),(0,a.yg)("ol",{start:2},(0,a.yg)("li",{parentName:"ol"},"Open the ",(0,a.yg)("inlineCode",{parentName:"li"},"pi.yaml")," file, and make sure the ",(0,a.yg)("inlineCode",{parentName:"li"},"apiVersion")," on the first line is the following:")),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-yaml"},"apiVersion: kubeflow.org/v2beta1\n")),(0,a.yg)("p",null,"You will also need to specify those containers can run with the ",(0,a.yg)("inlineCode",{parentName:"p"},"root")," user by adding the ",(0,a.yg)("inlineCode",{parentName:"p"},"serviceAccountName")," between ",(0,a.yg)("inlineCode",{parentName:"p"},"spec:")," and ",(0,a.yg)("inlineCode",{parentName:"p"},"container:")," for the launcher and the worker templates:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-yaml"},"#...\n Launcher:\n replicas: 1\n template:\n spec:\n serviceAccountName: anyuid\n containers:\n - image: mpioperator/mpi-pi:openmpi\n\n#...\n Worker:\n replicas: 2\n template:\n spec:\n serviceAccountName: anyuid\n containers:\n - image: mpioperator/mpi-pi:openmpi\n")),(0,a.yg)("p",null,"Your ",(0,a.yg)("inlineCode",{parentName:"p"},"tensorflow-mnist.yaml")," file should look like this: "),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-yaml"},'apiVersion: kubeflow.org/v2beta1\nkind: MPIJob\nmetadata:\n name: pi\nspec:\n slotsPerWorker: 1\n runPolicy:\n cleanPodPolicy: Running\n ttlSecondsAfterFinished: 60\n sshAuthMountPath: /home/mpiuser/.ssh\n mpiReplicaSpecs:\n Launcher:\n replicas: 1\n template:\n spec:\n serviceAccountName: anyuid\n containers:\n - image: mpioperator/mpi-pi:openmpi\n name: mpi-launcher\n securityContext:\n runAsUser: 1000\n command:\n - mpirun\n args:\n - -n\n - "2"\n - /home/mpiuser/pi\n resources:\n limits:\n cpu: 1\n memory: 1Gi\n Worker:\n replicas: 2\n template:\n spec:\n serviceAccountName: anyuid\n containers:\n - image: mpioperator/mpi-pi:openmpi\n name: mpi-worker\n securityContext:\n runAsUser: 1000\n command:\n - /usr/sbin/sshd\n args:\n - -De\n - -f\n - /home/mpiuser/.sshd_config\n resources:\n limits:\n cpu: 1\n memory: 1Gi\n\n')),(0,a.yg)("ol",{start:3},(0,a.yg)("li",{parentName:"ol"},"Once this has been set, create the job in your current project on the DSRI (change with ",(0,a.yg)("inlineCode",{parentName:"li"},"oc project my-project"),"):")),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},"oc create -f pi.yaml\n")),(0,a.yg)("p",null,"You should see the 2 workers and the main job running in your project ",(0,a.yg)("strong",{parentName:"p"},"Topology")," page in the DSRI web UI. You can then easily check the logs of the launcher and workers."),(0,a.yg)("p",null,"To run your own MPI job on the DSRI, you can take a look at, and edit, the different files provided by the ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/kubeflow/mpi-operator/tree/master/examples/horovod"},"MPI Operator example"),":"),(0,a.yg)("p",null,"\ud83d\udc0d ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/kubeflow/mpi-operator/blob/master/examples/horovod/tensorflow_mnist.py"},(0,a.yg)("inlineCode",{parentName:"a"},"tensorflow_mnist.py")),": the python script with the actual job to run"),(0,a.yg)("p",null,"\ud83d\udc33 ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/kubeflow/mpi-operator/blob/master/examples/horovod/Dockerfile.cpu"},(0,a.yg)("inlineCode",{parentName:"a"},"Dockerfile.cpu")),": the Dockerfile to define the image of the containers in which your job will run (install dependencies)"),(0,a.yg)("p",null,"\u26f5\ufe0f ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/kubeflow/mpi-operator/blob/master/examples/horovod/tensorflow-mnist.yaml"},(0,a.yg)("inlineCode",{parentName:"a"},"tensorflow-mnist.yaml")),": the YAML file to define the MPI deployment on Kubernetes (number and limits of workers, ",(0,a.yg)("inlineCode",{parentName:"p"},"mpirun")," command, etc)"),(0,a.yg)("p",null,"Visit the ",(0,a.yg)("a",{parentName:"p",href:"https://www.kubeflow.org/docs/components/training/mpi/#creating-an-mpi-job"},"Kubeflow documentation to create a MPI job")," for more details."),(0,a.yg)("admonition",{title:"Contact us",type:"info"},(0,a.yg)("p",{parentName:"admonition"},"Feel free to contact us on the DSRI Slack ",(0,a.yg)("strong",{parentName:"p"},"#helpdesk")," channel to discuss the use of MPI on the DSRI.")))}d.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[145],{5680:(e,n,t)=>{t.d(n,{xA:()=>c,yg:()=>d});var o=t(6540);function r(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function a(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);n&&(o=o.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,o)}return t}function i(e){for(var n=1;n=0||(r[t]=e[t]);return r}(e,n);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(o=0;o=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(r[t]=e[t])}return r}var l=o.createContext({}),s=function(e){var n=o.useContext(l),t=n;return e&&(t="function"==typeof e?e(n):i(i({},n),e)),t},c=function(e){var n=s(e.components);return o.createElement(l.Provider,{value:n},e.children)},m={inlineCode:"code",wrapper:function(e){var n=e.children;return o.createElement(o.Fragment,{},n)}},u=o.forwardRef((function(e,n){var t=e.components,r=e.mdxType,a=e.originalType,l=e.parentName,c=p(e,["components","mdxType","originalType","parentName"]),u=s(t),d=r,y=u["".concat(l,".").concat(d)]||u[d]||m[d]||a;return t?o.createElement(y,i(i({ref:n},c),{},{components:t})):o.createElement(y,i({ref:n},c))}));function d(e,n){var t=arguments,r=n&&n.mdxType;if("string"==typeof e||r){var a=t.length,i=new Array(a);i[0]=u;var p={};for(var l in n)hasOwnProperty.call(n,l)&&(p[l]=n[l]);p.originalType=e,p.mdxType="string"==typeof e?e:r,i[1]=p;for(var s=2;s{t.r(n),t.d(n,{assets:()=>c,contentTitle:()=>l,default:()=>d,frontMatter:()=>p,metadata:()=>s,toc:()=>m});var o=t(9668),r=t(1367),a=(t(6540),t(5680)),i=["components"],p={id:"mpi-jobs",title:"Run MPI jobs"},l=void 0,s={unversionedId:"mpi-jobs",id:"mpi-jobs",title:"Run MPI jobs",description:"We deployed the MPI Operator from Kubeflow to run MPI jobs on the DSRI.",source:"@site/docs/mpi-jobs.md",sourceDirName:".",slug:"/mpi-jobs",permalink:"/docs/mpi-jobs",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/mpi-jobs.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"mpi-jobs",title:"Run MPI jobs"},sidebar:"docs",previous:{title:"Spark cluster",permalink:"/docs/deploy-spark"},next:{title:"Neuroscience research",permalink:"/docs/neuroscience"}},c={},m=[{value:"Run MPI-jobs on CPU",id:"run-mpi-jobs-on-cpu",level:2}],u={toc:m};function d(e){var n=e.components,t=(0,r.A)(e,i);return(0,a.yg)("wrapper",(0,o.A)({},u,t,{components:n,mdxType:"MDXLayout"}),(0,a.yg)("p",null,"We deployed the ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/kubeflow/mpi-operator"},"MPI Operator")," from Kubeflow to run MPI jobs on the DSRI."),(0,a.yg)("admonition",{title:"Request access to the MPI Operator",type:"warning"},(0,a.yg)("p",{parentName:"admonition"},"To be able to deploy MPI-Jobs you will need to ",(0,a.yg)("a",{parentName:"p",href:"https://dsri.maastrichtuniversity.nl/help"},"ask the DSRI admins")," to enable the MPI Operator in your project. Once enabled you will be able to start an MPI-job in a few clicks.")),(0,a.yg)("blockquote",null,(0,a.yg)("p",{parentName:"blockquote"},"The MPI Operator makes it easy to run allreduce-style distributed training on Kubernetes. Please check out ",(0,a.yg)("a",{parentName:"p",href:"https://medium.com/kubeflow/introduction-to-kubeflow-mpi-operator-and-industry-adoption-296d5f2e6edc"},"this blog post")," for an introduction to MPI Operator and its industry adoption.")),(0,a.yg)("h2",{id:"run-mpi-jobs-on-cpu"},"Run MPI-jobs on CPU"),(0,a.yg)("p",null,"Checkout the ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/kubeflow/mpi-operator/tree/master/examples/v2beta1/pi"},"repository of the example")," for a complete example of an MPI-job: python script, ",(0,a.yg)("inlineCode",{parentName:"p"},"Dockerfile"),", and the job deployment YAML."),(0,a.yg)("ol",null,(0,a.yg)("li",{parentName:"ol"},"Clone the repository, and go to the example folder:")),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},"git clone https://github.com/kubeflow/mpi-operator.git\ncd mpi-operator/examples/v2beta1/pi\n")),(0,a.yg)("ol",{start:2},(0,a.yg)("li",{parentName:"ol"},"Open the ",(0,a.yg)("inlineCode",{parentName:"li"},"pi.yaml")," file, and make sure the ",(0,a.yg)("inlineCode",{parentName:"li"},"apiVersion")," on the first line is the following:")),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-yaml"},"apiVersion: kubeflow.org/v2beta1\n")),(0,a.yg)("p",null,"You will also need to specify those containers can run with the ",(0,a.yg)("inlineCode",{parentName:"p"},"root")," user by adding the ",(0,a.yg)("inlineCode",{parentName:"p"},"serviceAccountName")," between ",(0,a.yg)("inlineCode",{parentName:"p"},"spec:")," and ",(0,a.yg)("inlineCode",{parentName:"p"},"container:")," for the launcher and the worker templates:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-yaml"},"#...\n Launcher:\n replicas: 1\n template:\n spec:\n serviceAccountName: anyuid\n containers:\n - image: mpioperator/mpi-pi:openmpi\n\n#...\n Worker:\n replicas: 2\n template:\n spec:\n serviceAccountName: anyuid\n containers:\n - image: mpioperator/mpi-pi:openmpi\n")),(0,a.yg)("p",null,"Your ",(0,a.yg)("inlineCode",{parentName:"p"},"tensorflow-mnist.yaml")," file should look like this: "),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-yaml"},'apiVersion: kubeflow.org/v2beta1\nkind: MPIJob\nmetadata:\n name: pi\nspec:\n slotsPerWorker: 1\n runPolicy:\n cleanPodPolicy: Running\n ttlSecondsAfterFinished: 60\n sshAuthMountPath: /home/mpiuser/.ssh\n mpiReplicaSpecs:\n Launcher:\n replicas: 1\n template:\n spec:\n serviceAccountName: anyuid\n containers:\n - image: mpioperator/mpi-pi:openmpi\n name: mpi-launcher\n securityContext:\n runAsUser: 1000\n command:\n - mpirun\n args:\n - -n\n - "2"\n - /home/mpiuser/pi\n resources:\n limits:\n cpu: 1\n memory: 1Gi\n Worker:\n replicas: 2\n template:\n spec:\n serviceAccountName: anyuid\n containers:\n - image: mpioperator/mpi-pi:openmpi\n name: mpi-worker\n securityContext:\n runAsUser: 1000\n command:\n - /usr/sbin/sshd\n args:\n - -De\n - -f\n - /home/mpiuser/.sshd_config\n resources:\n limits:\n cpu: 1\n memory: 1Gi\n\n')),(0,a.yg)("ol",{start:3},(0,a.yg)("li",{parentName:"ol"},"Once this has been set, create the job in your current project on the DSRI (change with ",(0,a.yg)("inlineCode",{parentName:"li"},"oc project my-project"),"):")),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},"oc create -f pi.yaml\n")),(0,a.yg)("p",null,"You should see the 2 workers and the main job running in your project ",(0,a.yg)("strong",{parentName:"p"},"Topology")," page in the DSRI web UI. You can then easily check the logs of the launcher and workers."),(0,a.yg)("p",null,"To run your own MPI job on the DSRI, you can take a look at, and edit, the different files provided by the ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/kubeflow/mpi-operator/tree/master/examples/horovod"},"MPI Operator example"),":"),(0,a.yg)("p",null,"\ud83d\udc0d ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/kubeflow/mpi-operator/blob/master/examples/horovod/tensorflow_mnist.py"},(0,a.yg)("inlineCode",{parentName:"a"},"tensorflow_mnist.py")),": the python script with the actual job to run"),(0,a.yg)("p",null,"\ud83d\udc33 ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/kubeflow/mpi-operator/blob/master/examples/horovod/Dockerfile.cpu"},(0,a.yg)("inlineCode",{parentName:"a"},"Dockerfile.cpu")),": the Dockerfile to define the image of the containers in which your job will run (install dependencies)"),(0,a.yg)("p",null,"\u26f5\ufe0f ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/kubeflow/mpi-operator/blob/master/examples/horovod/tensorflow-mnist.yaml"},(0,a.yg)("inlineCode",{parentName:"a"},"tensorflow-mnist.yaml")),": the YAML file to define the MPI deployment on Kubernetes (number and limits of workers, ",(0,a.yg)("inlineCode",{parentName:"p"},"mpirun")," command, etc)"),(0,a.yg)("p",null,"Visit the ",(0,a.yg)("a",{parentName:"p",href:"https://www.kubeflow.org/docs/components/training/mpi/#creating-an-mpi-job"},"Kubeflow documentation to create a MPI job")," for more details."),(0,a.yg)("admonition",{title:"Contact us",type:"info"},(0,a.yg)("p",{parentName:"admonition"},"Feel free to contact us on the DSRI Slack ",(0,a.yg)("strong",{parentName:"p"},"#helpdesk")," channel to discuss the use of MPI on the DSRI.")))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/dbeba2b5.8b85ab16.js b/assets/js/dbeba2b5.d25107ca.js similarity index 99% rename from assets/js/dbeba2b5.8b85ab16.js rename to assets/js/dbeba2b5.d25107ca.js index 11cb026e5..dac4124ac 100644 --- a/assets/js/dbeba2b5.8b85ab16.js +++ b/assets/js/dbeba2b5.d25107ca.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[4070],{5680:(e,t,r)=>{r.d(t,{xA:()=>c,yg:()=>d});var o=r(6540);function a(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function n(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,o)}return r}function i(e){for(var t=1;t=0||(a[r]=e[r]);return a}(e,t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);for(o=0;o=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(a[r]=e[r])}return a}var p=o.createContext({}),s=function(e){var t=o.useContext(p),r=t;return e&&(r="function"==typeof e?e(t):i(i({},t),e)),r},c=function(e){var t=s(e.components);return o.createElement(p.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.createElement(o.Fragment,{},t)}},m=o.forwardRef((function(e,t){var r=e.components,a=e.mdxType,n=e.originalType,p=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),m=s(r),d=a,g=m["".concat(p,".").concat(d)]||m[d]||u[d]||n;return r?o.createElement(g,i(i({ref:t},c),{},{components:r})):o.createElement(g,i({ref:t},c))}));function d(e,t){var r=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var n=r.length,i=new Array(n);i[0]=m;var l={};for(var p in t)hasOwnProperty.call(t,p)&&(l[p]=t[p]);l.originalType=e,l.mdxType="string"==typeof e?e:a,i[1]=l;for(var s=2;s{r.r(t),r.d(t,{assets:()=>c,contentTitle:()=>p,default:()=>d,frontMatter:()=>l,metadata:()=>s,toc:()=>u});var o=r(9668),a=r(1367),n=(r(6540),r(5680)),i=["components"],l={id:"operators",title:"Install from Operators"},p=void 0,s={unversionedId:"operators",id:"operators",title:"Install from Operators",description:"The Operator Framework is an open source toolkit to manage Kubernetes native applications, called Operators, in an effective, automated, and scalable way.",source:"@site/docs/operators.md",sourceDirName:".",slug:"/operators",permalink:"/docs/operators",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/operators.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"operators",title:"Install from Operators"},sidebar:"docs",previous:{title:"Install from Helm charts",permalink:"/docs/helm"},next:{title:"Jupyter Notebooks",permalink:"/docs/deploy-jupyter"}},c={},u=[{value:"Install existing Operators",id:"install-existing-operators",level:2},{value:"Build Operators",id:"build-operators",level:2},{value:"External resources",id:"external-resources",level:3}],m={toc:u};function d(e){var t=e.components,r=(0,a.A)(e,i);return(0,n.yg)("wrapper",(0,o.A)({},m,r,{components:t,mdxType:"MDXLayout"}),(0,n.yg)("p",null,"The ",(0,n.yg)("a",{parentName:"p",href:"https://operatorframework.io/"},"Operator Framework")," is an open source toolkit to manage Kubernetes native applications, called Operators, in an effective, automated, and scalable way."),(0,n.yg)("admonition",{title:"Use existing Operators",type:"tip"},(0,n.yg)("p",{parentName:"admonition"},"You can explore published Operators at ",(0,n.yg)("a",{parentName:"p",href:"https://operatorhub.io"},"https://operatorhub.io"))),(0,n.yg)("h2",{id:"install-existing-operators"},"Install existing Operators"),(0,n.yg)("admonition",{title:"Contact us",type:"info"},(0,n.yg)("p",{parentName:"admonition"},"Contact us on the DSRI Slack ",(0,n.yg)("strong",{parentName:"p"},"#helpdesk")," channel, if you want to install a new Operator on the DSRI.")),(0,n.yg)("h2",{id:"build-operators"},"Build Operators"),(0,n.yg)("p",null,"Install the ",(0,n.yg)("inlineCode",{parentName:"p"},"operator-sdk")," tool. See the ",(0,n.yg)("a",{parentName:"p",href:"https://sdk.operatorframework.io/docs/installation/"},"official documentation"),"."),(0,n.yg)("p",null,"Operators can be built using 3 different approaches:"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("strong",{parentName:"li"},"Helm"),": a framework to define the deployment logic based on regular kubernetes YAML, but less capabilities for complete auto-update and insights. "),(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("strong",{parentName:"li"},"Ansible"),": define the deployment logic with Ansible, provide maximum capabilities."),(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("strong",{parentName:"li"},"Golang"),": define the deployment logic in Golang, provide maximum capabilities, but require more code.")),(0,n.yg)("h3",{id:"external-resources"},"External resources"),(0,n.yg)("p",null,"Documentation:"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("a",{parentName:"li",href:"https://sdk.operatorframework.io"},"Official docs")," to build Operators",(0,n.yg)("ul",{parentName:"li"},(0,n.yg)("li",{parentName:"ul"},"Official docs to build Operator from Helm charts: ",(0,n.yg)("a",{parentName:"li",href:"https://sdk.operatorframework.io/docs/building-operators/helm/tutorial"},"https://sdk.operatorframework.io/docs/building-operators/helm/tutorial")),(0,n.yg)("li",{parentName:"ul"},"Official docs to build Operator with Ansible: ",(0,n.yg)("a",{parentName:"li",href:"https://sdk.operatorframework.io/docs/building-operators/ansible/quickstart"},"https://sdk.operatorframework.io/docs/building-operators/ansible/quickstart")))),(0,n.yg)("li",{parentName:"ul"},"RedHat Certified Operator guide",(0,n.yg)("ul",{parentName:"li"},(0,n.yg)("li",{parentName:"ul"},"Make an operator use ",(0,n.yg)("inlineCode",{parentName:"li"},"anyuid"),": ",(0,n.yg)("a",{parentName:"li",href:"https://redhat-connect.gitbook.io/certified-operator-guide/what-if-ive-already-published-a-community-operator/applying-security-context-constraints"},"https://redhat-connect.gitbook.io/certified-operator-guide/what-if-ive-already-published-a-community-operator/applying-security-context-constraints")),(0,n.yg)("li",{parentName:"ul"},"Submit community Operators: ",(0,n.yg)("a",{parentName:"li",href:"https://redhat-connect.gitbook.io/certified-operator-guide/troubleshooting-and-resources/submitting-a-community-operator-to-operatorhub.io"},"https://redhat-connect.gitbook.io/certified-operator-guide/troubleshooting-and-resources/submitting-a-community-operator-to-operatorhub.io"))))),(0,n.yg)("p",null,"Examples:"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("p",{parentName:"li"},"Deployment example: ",(0,n.yg)("a",{parentName:"p",href:"https://github.com/microcks/microcks-ansible-operator/blob/master/roles/microcks/tasks/main.yml"},"https://github.com/microcks/microcks-ansible-operator/blob/master/roles/microcks/tasks/main.yml"))),(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("p",{parentName:"li"},"Older OpenShift guide: ",(0,n.yg)("a",{parentName:"p",href:"https://docs.openshift.com/container-platform/4.1/applications/operator_sdk/osdk-ansible.html"},"https://docs.openshift.com/container-platform/4.1/applications/operator_sdk/osdk-ansible.html"))),(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("p",{parentName:"li"},"Simple older example with route: ",(0,n.yg)("a",{parentName:"p",href:"https://github.com/djzager/ansible-role-hello-world-k8s"},"https://github.com/djzager/ansible-role-hello-world-k8s")))))}d.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[4070],{5680:(e,t,r)=>{r.d(t,{xA:()=>c,yg:()=>d});var o=r(6540);function a(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function n(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,o)}return r}function i(e){for(var t=1;t=0||(a[r]=e[r]);return a}(e,t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);for(o=0;o=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(a[r]=e[r])}return a}var p=o.createContext({}),s=function(e){var t=o.useContext(p),r=t;return e&&(r="function"==typeof e?e(t):i(i({},t),e)),r},c=function(e){var t=s(e.components);return o.createElement(p.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return o.createElement(o.Fragment,{},t)}},m=o.forwardRef((function(e,t){var r=e.components,a=e.mdxType,n=e.originalType,p=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),m=s(r),d=a,g=m["".concat(p,".").concat(d)]||m[d]||u[d]||n;return r?o.createElement(g,i(i({ref:t},c),{},{components:r})):o.createElement(g,i({ref:t},c))}));function d(e,t){var r=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var n=r.length,i=new Array(n);i[0]=m;var l={};for(var p in t)hasOwnProperty.call(t,p)&&(l[p]=t[p]);l.originalType=e,l.mdxType="string"==typeof e?e:a,i[1]=l;for(var s=2;s{r.r(t),r.d(t,{assets:()=>c,contentTitle:()=>p,default:()=>d,frontMatter:()=>l,metadata:()=>s,toc:()=>u});var o=r(9668),a=r(1367),n=(r(6540),r(5680)),i=["components"],l={id:"operators",title:"Install from Operators"},p=void 0,s={unversionedId:"operators",id:"operators",title:"Install from Operators",description:"The Operator Framework is an open source toolkit to manage Kubernetes native applications, called Operators, in an effective, automated, and scalable way.",source:"@site/docs/operators.md",sourceDirName:".",slug:"/operators",permalink:"/docs/operators",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/operators.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"operators",title:"Install from Operators"},sidebar:"docs",previous:{title:"Install from Helm charts",permalink:"/docs/helm"},next:{title:"Jupyter Notebooks",permalink:"/docs/deploy-jupyter"}},c={},u=[{value:"Install existing Operators",id:"install-existing-operators",level:2},{value:"Build Operators",id:"build-operators",level:2},{value:"External resources",id:"external-resources",level:3}],m={toc:u};function d(e){var t=e.components,r=(0,a.A)(e,i);return(0,n.yg)("wrapper",(0,o.A)({},m,r,{components:t,mdxType:"MDXLayout"}),(0,n.yg)("p",null,"The ",(0,n.yg)("a",{parentName:"p",href:"https://operatorframework.io/"},"Operator Framework")," is an open source toolkit to manage Kubernetes native applications, called Operators, in an effective, automated, and scalable way."),(0,n.yg)("admonition",{title:"Use existing Operators",type:"tip"},(0,n.yg)("p",{parentName:"admonition"},"You can explore published Operators at ",(0,n.yg)("a",{parentName:"p",href:"https://operatorhub.io"},"https://operatorhub.io"))),(0,n.yg)("h2",{id:"install-existing-operators"},"Install existing Operators"),(0,n.yg)("admonition",{title:"Contact us",type:"info"},(0,n.yg)("p",{parentName:"admonition"},"Contact us on the DSRI Slack ",(0,n.yg)("strong",{parentName:"p"},"#helpdesk")," channel, if you want to install a new Operator on the DSRI.")),(0,n.yg)("h2",{id:"build-operators"},"Build Operators"),(0,n.yg)("p",null,"Install the ",(0,n.yg)("inlineCode",{parentName:"p"},"operator-sdk")," tool. See the ",(0,n.yg)("a",{parentName:"p",href:"https://sdk.operatorframework.io/docs/installation/"},"official documentation"),"."),(0,n.yg)("p",null,"Operators can be built using 3 different approaches:"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("strong",{parentName:"li"},"Helm"),": a framework to define the deployment logic based on regular kubernetes YAML, but less capabilities for complete auto-update and insights. "),(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("strong",{parentName:"li"},"Ansible"),": define the deployment logic with Ansible, provide maximum capabilities."),(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("strong",{parentName:"li"},"Golang"),": define the deployment logic in Golang, provide maximum capabilities, but require more code.")),(0,n.yg)("h3",{id:"external-resources"},"External resources"),(0,n.yg)("p",null,"Documentation:"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("a",{parentName:"li",href:"https://sdk.operatorframework.io"},"Official docs")," to build Operators",(0,n.yg)("ul",{parentName:"li"},(0,n.yg)("li",{parentName:"ul"},"Official docs to build Operator from Helm charts: ",(0,n.yg)("a",{parentName:"li",href:"https://sdk.operatorframework.io/docs/building-operators/helm/tutorial"},"https://sdk.operatorframework.io/docs/building-operators/helm/tutorial")),(0,n.yg)("li",{parentName:"ul"},"Official docs to build Operator with Ansible: ",(0,n.yg)("a",{parentName:"li",href:"https://sdk.operatorframework.io/docs/building-operators/ansible/quickstart"},"https://sdk.operatorframework.io/docs/building-operators/ansible/quickstart")))),(0,n.yg)("li",{parentName:"ul"},"RedHat Certified Operator guide",(0,n.yg)("ul",{parentName:"li"},(0,n.yg)("li",{parentName:"ul"},"Make an operator use ",(0,n.yg)("inlineCode",{parentName:"li"},"anyuid"),": ",(0,n.yg)("a",{parentName:"li",href:"https://redhat-connect.gitbook.io/certified-operator-guide/what-if-ive-already-published-a-community-operator/applying-security-context-constraints"},"https://redhat-connect.gitbook.io/certified-operator-guide/what-if-ive-already-published-a-community-operator/applying-security-context-constraints")),(0,n.yg)("li",{parentName:"ul"},"Submit community Operators: ",(0,n.yg)("a",{parentName:"li",href:"https://redhat-connect.gitbook.io/certified-operator-guide/troubleshooting-and-resources/submitting-a-community-operator-to-operatorhub.io"},"https://redhat-connect.gitbook.io/certified-operator-guide/troubleshooting-and-resources/submitting-a-community-operator-to-operatorhub.io"))))),(0,n.yg)("p",null,"Examples:"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("p",{parentName:"li"},"Deployment example: ",(0,n.yg)("a",{parentName:"p",href:"https://github.com/microcks/microcks-ansible-operator/blob/master/roles/microcks/tasks/main.yml"},"https://github.com/microcks/microcks-ansible-operator/blob/master/roles/microcks/tasks/main.yml"))),(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("p",{parentName:"li"},"Older OpenShift guide: ",(0,n.yg)("a",{parentName:"p",href:"https://docs.openshift.com/container-platform/4.1/applications/operator_sdk/osdk-ansible.html"},"https://docs.openshift.com/container-platform/4.1/applications/operator_sdk/osdk-ansible.html"))),(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("p",{parentName:"li"},"Simple older example with route: ",(0,n.yg)("a",{parentName:"p",href:"https://github.com/djzager/ansible-role-hello-world-k8s"},"https://github.com/djzager/ansible-role-hello-world-k8s")))))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/dca73612.3d45d971.js b/assets/js/dca73612.478ee8d2.js similarity index 99% rename from assets/js/dca73612.3d45d971.js rename to assets/js/dca73612.478ee8d2.js index 825d84470..acefb5630 100644 --- a/assets/js/dca73612.3d45d971.js +++ b/assets/js/dca73612.478ee8d2.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[6690],{5680:(e,t,o)=>{o.d(t,{xA:()=>s,yg:()=>u});var a=o(6540);function n(e,t,o){return t in e?Object.defineProperty(e,t,{value:o,enumerable:!0,configurable:!0,writable:!0}):e[t]=o,e}function l(e,t){var o=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),o.push.apply(o,a)}return o}function r(e){for(var t=1;t=0||(n[o]=e[o]);return n}(e,t);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,o)&&(n[o]=e[o])}return n}var p=a.createContext({}),d=function(e){var t=a.useContext(p),o=t;return e&&(o="function"==typeof e?e(t):r(r({},t),e)),o},s=function(e){var t=d(e.components);return a.createElement(p.Provider,{value:t},e.children)},c={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},y=a.forwardRef((function(e,t){var o=e.components,n=e.mdxType,l=e.originalType,p=e.parentName,s=i(e,["components","mdxType","originalType","parentName"]),y=d(o),u=n,g=y["".concat(p,".").concat(u)]||y[u]||c[u]||l;return o?a.createElement(g,r(r({ref:t},s),{},{components:o})):a.createElement(g,r({ref:t},s))}));function u(e,t){var o=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var l=o.length,r=new Array(l);r[0]=y;var i={};for(var p in t)hasOwnProperty.call(t,p)&&(i[p]=t[p]);i.originalType=e,i.mdxType="string"==typeof e?e:n,r[1]=i;for(var d=2;d{o.r(t),o.d(t,{assets:()=>s,contentTitle:()=>p,default:()=>u,frontMatter:()=>i,metadata:()=>d,toc:()=>c});var a=o(9668),n=o(1367),l=(o(6540),o(5680)),r=["components"],i={id:"openshift-load-data",title:"Upload data"},p=void 0,d={unversionedId:"openshift-load-data",id:"openshift-load-data",title:"Upload data",description:"In RStudio, JupyterLab and VSCode",source:"@site/docs/openshift-load-data.md",sourceDirName:".",slug:"/openshift-load-data",permalink:"/docs/openshift-load-data",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/openshift-load-data.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"openshift-load-data",title:"Upload data"},sidebar:"docs",previous:{title:"Prepare your project",permalink:"/docs/prepare-project-for-dsri"},next:{title:"Delete an application",permalink:"/docs/openshift-delete-services"}},s={},c=[{value:"In RStudio, JupyterLab and VSCode",id:"in-rstudio-jupyterlab-and-vscode",level:2},{value:"Copy large files with the terminal",id:"copy-large-files-with-the-terminal",level:2},{value:"Copy from local to pod",id:"copy-from-local-to-pod",level:3},{value:"Copy from pod to local",id:"copy-from-pod-to-local",level:3},{value:"Download data from SURFdrive",id:"download-data-from-surfdrive",level:3},{value:"Synchronizes files with oc rsync",id:"synchronizes-files-with-oc-rsync",level:2},{value:"Sync local to pod",id:"sync-local-to-pod",level:3},{value:"Sync pod to local",id:"sync-pod-to-local",level:3},{value:"More options",id:"more-options",level:3},{value:"One-liner",id:"one-liner",level:2}],y={toc:c};function u(e){var t=e.components,o=(0,n.A)(e,r);return(0,l.yg)("wrapper",(0,a.A)({},y,o,{components:t,mdxType:"MDXLayout"}),(0,l.yg)("h2",{id:"in-rstudio-jupyterlab-and-vscode"},"In RStudio, JupyterLab and VSCode"),(0,l.yg)("ul",null,(0,l.yg)("li",{parentName:"ul"},"If you are using ",(0,l.yg)("strong",{parentName:"li"},"JupyterLab")," or ",(0,l.yg)("strong",{parentName:"li"},"VSCode")," you should be able to load data to the container by simply ",(0,l.yg)("strong",{parentName:"li"},"drag and drop the files to upload")," in the JupyterLab/VSCode web UI."),(0,l.yg)("li",{parentName:"ul"},"For ",(0,l.yg)("strong",{parentName:"li"},"RStudio"),", use the Upload file button in the RStudio web UI to upload files from your computer to the RStudio workspace.")),(0,l.yg)("admonition",{title:"File too big",type:"caution"},(0,l.yg)("p",{parentName:"admonition"},"If those solutions don't work due to the files size, try one of the solutions below.")),(0,l.yg)("h2",{id:"copy-large-files-with-the-terminal"},"Copy large files with the terminal"),(0,l.yg)("p",null,"The quickest way to upload large files or folders from a laptop or server to the DSRI is to use the ",(0,l.yg)("inlineCode",{parentName:"p"},"oc")," command line interface."),(0,l.yg)("admonition",{title:"Install the client",type:"tip"},(0,l.yg)("p",{parentName:"admonition"},"To install the ",(0,l.yg)("inlineCode",{parentName:"p"},"oc")," client on your laptop/server, visit the ",(0,l.yg)("a",{parentName:"p",href:"/docs/openshift-install"},"Install the client")," page")),(0,l.yg)("p",null,(0,l.yg)("inlineCode",{parentName:"p"},"oc cp")," directly copy, and overwrite existing files, from a laptop or server to an Application pod on the DSRI."),(0,l.yg)("p",null,"First get the ",(0,l.yg)("inlineCode",{parentName:"p"},"")," using your application name:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"oc get pod --selector app=\n")),(0,l.yg)("h3",{id:"copy-from-local-to-pod"},"Copy from local to pod"),(0,l.yg)("p",null,"Folders are uploaded recursively by default:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"oc cp :\n")),(0,l.yg)("admonition",{title:"Use absolute path in the pod",type:"caution"},(0,l.yg)("p",{parentName:"admonition"},"You need to provide the absolute (full) path where you want to copy it in the pod. Use your application workspace path, e.g. ",(0,l.yg)("inlineCode",{parentName:"p"},"/home/jovyan")," for JupyterLab or ",(0,l.yg)("inlineCode",{parentName:"p"},"/home/rstudio")," for RStudio)")),(0,l.yg)("p",null,"For example:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"oc cp my-folder jupyterlab-000:/home/jovyan\n")),(0,l.yg)("p",null,"You can also use this one-liner to automatically get the pod ID based on your app label:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-bash"},"oc get pod --selector app= | xargs -I{} oc cp {}:\n")),(0,l.yg)("h3",{id:"copy-from-pod-to-local"},"Copy from pod to local"),(0,l.yg)("p",null,"Just do the inverse:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"oc cp : \n")),(0,l.yg)("h3",{id:"download-data-from-surfdrive"},"Download data from SURFdrive"),(0,l.yg)("p",null,"You can download data from your SURFdrive to your pod by creating a public link to the file:"),(0,l.yg)("ol",null,(0,l.yg)("li",{parentName:"ol"},"Go to the file in SURFdrive you'd like to share"),(0,l.yg)("li",{parentName:"ol"},"Click share and the create public link"),(0,l.yg)("li",{parentName:"ol"},"Fill in a name for the public link (like DSRI). The name does not matter much, but it can help you keep track of the goal of the public link."),(0,l.yg)("li",{parentName:"ol"},"Click copy to clipboard"),(0,l.yg)("li",{parentName:"ol"},"Visit link in browser and copy the direct URL displayed on that page."),(0,l.yg)("li",{parentName:"ol"},'Use the direct URL you just copied to download the file using either wget or curl (e.g. "wget ',(0,l.yg)("a",{parentName:"li",href:"https://surfdrive.surf.nl/files/index.php/s/5mFwyAKj4UexlJb/download%22"},'https://surfdrive.surf.nl/files/index.php/s/5mFwyAKj4UexlJb/download"'),")"),(0,l.yg)("li",{parentName:"ol"},"Revoke link in the SURFdrive portal")),(0,l.yg)("h2",{id:"synchronizes-files-with-oc-rsync"},"Synchronizes files with ",(0,l.yg)("inlineCode",{parentName:"h2"},"oc rsync")),(0,l.yg)("p",null,"If you have a lot of large files and/or they are updated regularly, you can use ",(0,l.yg)("inlineCode",{parentName:"p"},"rsync")," as it synchronizes the files if they already exist, preventing duplication and making synchronization faster. You can also see the progress with ",(0,l.yg)("inlineCode",{parentName:"p"},"rsync")," which you cannot with ",(0,l.yg)("inlineCode",{parentName:"p"},"cp"),". And if the upload is stopped for any reason ",(0,l.yg)("inlineCode",{parentName:"p"},"rsync")," should pick it up from where it stopped (instead of restarting from scratch like ",(0,l.yg)("inlineCode",{parentName:"p"},"oc cp")," does)"),(0,l.yg)("admonition",{type:"caution"},(0,l.yg)("p",{parentName:"admonition"},"Rsync does not work with symlinks (created with ",(0,l.yg)("inlineCode",{parentName:"p"},"ln -s"),")")),(0,l.yg)("h3",{id:"sync-local-to-pod"},"Sync local to pod"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"oc rsync --progress :\n")),(0,l.yg)("p",null,"You can also use this one-liner to automatically get the pod ID based on your app label:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-bash"},"oc get pod --selector app= | xargs -I{} oc rsync --progress {}:\n")),(0,l.yg)("h3",{id:"sync-pod-to-local"},"Sync pod to local"),(0,l.yg)("p",null,"Again, do the inverse:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"oc rsync --progress : \n")),(0,l.yg)("h3",{id:"more-options"},"More options"),(0,l.yg)("p",null,"You can use more options to improve the upload of large files:"),(0,l.yg)("table",null,(0,l.yg)("thead",{parentName:"table"},(0,l.yg)("tr",{parentName:"thead"},(0,l.yg)("th",{parentName:"tr",align:null},(0,l.yg)("inlineCode",{parentName:"th"},"--compress")),(0,l.yg)("th",{parentName:"tr",align:null},"compress file data during the transfer"))),(0,l.yg)("tbody",{parentName:"table"},(0,l.yg)("tr",{parentName:"tbody"},(0,l.yg)("td",{parentName:"tr",align:null},(0,l.yg)("inlineCode",{parentName:"td"},"--delete")),(0,l.yg)("td",{parentName:"tr",align:null},"delete files not present in source")),(0,l.yg)("tr",{parentName:"tbody"},(0,l.yg)("td",{parentName:"tr",align:null},(0,l.yg)("inlineCode",{parentName:"td"},"--watch")),(0,l.yg)("td",{parentName:"tr",align:null},"Watch directory for changes and resync automatically")))),(0,l.yg)("h2",{id:"one-liner"},"One-liner"))}u.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[6690],{5680:(e,t,o)=>{o.d(t,{xA:()=>s,yg:()=>u});var a=o(6540);function n(e,t,o){return t in e?Object.defineProperty(e,t,{value:o,enumerable:!0,configurable:!0,writable:!0}):e[t]=o,e}function l(e,t){var o=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),o.push.apply(o,a)}return o}function r(e){for(var t=1;t=0||(n[o]=e[o]);return n}(e,t);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,o)&&(n[o]=e[o])}return n}var p=a.createContext({}),d=function(e){var t=a.useContext(p),o=t;return e&&(o="function"==typeof e?e(t):r(r({},t),e)),o},s=function(e){var t=d(e.components);return a.createElement(p.Provider,{value:t},e.children)},c={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},y=a.forwardRef((function(e,t){var o=e.components,n=e.mdxType,l=e.originalType,p=e.parentName,s=i(e,["components","mdxType","originalType","parentName"]),y=d(o),u=n,g=y["".concat(p,".").concat(u)]||y[u]||c[u]||l;return o?a.createElement(g,r(r({ref:t},s),{},{components:o})):a.createElement(g,r({ref:t},s))}));function u(e,t){var o=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var l=o.length,r=new Array(l);r[0]=y;var i={};for(var p in t)hasOwnProperty.call(t,p)&&(i[p]=t[p]);i.originalType=e,i.mdxType="string"==typeof e?e:n,r[1]=i;for(var d=2;d{o.r(t),o.d(t,{assets:()=>s,contentTitle:()=>p,default:()=>u,frontMatter:()=>i,metadata:()=>d,toc:()=>c});var a=o(9668),n=o(1367),l=(o(6540),o(5680)),r=["components"],i={id:"openshift-load-data",title:"Upload data"},p=void 0,d={unversionedId:"openshift-load-data",id:"openshift-load-data",title:"Upload data",description:"In RStudio, JupyterLab and VSCode",source:"@site/docs/openshift-load-data.md",sourceDirName:".",slug:"/openshift-load-data",permalink:"/docs/openshift-load-data",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/openshift-load-data.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"openshift-load-data",title:"Upload data"},sidebar:"docs",previous:{title:"Prepare your project",permalink:"/docs/prepare-project-for-dsri"},next:{title:"Delete an application",permalink:"/docs/openshift-delete-services"}},s={},c=[{value:"In RStudio, JupyterLab and VSCode",id:"in-rstudio-jupyterlab-and-vscode",level:2},{value:"Copy large files with the terminal",id:"copy-large-files-with-the-terminal",level:2},{value:"Copy from local to pod",id:"copy-from-local-to-pod",level:3},{value:"Copy from pod to local",id:"copy-from-pod-to-local",level:3},{value:"Download data from SURFdrive",id:"download-data-from-surfdrive",level:3},{value:"Synchronizes files with oc rsync",id:"synchronizes-files-with-oc-rsync",level:2},{value:"Sync local to pod",id:"sync-local-to-pod",level:3},{value:"Sync pod to local",id:"sync-pod-to-local",level:3},{value:"More options",id:"more-options",level:3},{value:"One-liner",id:"one-liner",level:2}],y={toc:c};function u(e){var t=e.components,o=(0,n.A)(e,r);return(0,l.yg)("wrapper",(0,a.A)({},y,o,{components:t,mdxType:"MDXLayout"}),(0,l.yg)("h2",{id:"in-rstudio-jupyterlab-and-vscode"},"In RStudio, JupyterLab and VSCode"),(0,l.yg)("ul",null,(0,l.yg)("li",{parentName:"ul"},"If you are using ",(0,l.yg)("strong",{parentName:"li"},"JupyterLab")," or ",(0,l.yg)("strong",{parentName:"li"},"VSCode")," you should be able to load data to the container by simply ",(0,l.yg)("strong",{parentName:"li"},"drag and drop the files to upload")," in the JupyterLab/VSCode web UI."),(0,l.yg)("li",{parentName:"ul"},"For ",(0,l.yg)("strong",{parentName:"li"},"RStudio"),", use the Upload file button in the RStudio web UI to upload files from your computer to the RStudio workspace.")),(0,l.yg)("admonition",{title:"File too big",type:"caution"},(0,l.yg)("p",{parentName:"admonition"},"If those solutions don't work due to the files size, try one of the solutions below.")),(0,l.yg)("h2",{id:"copy-large-files-with-the-terminal"},"Copy large files with the terminal"),(0,l.yg)("p",null,"The quickest way to upload large files or folders from a laptop or server to the DSRI is to use the ",(0,l.yg)("inlineCode",{parentName:"p"},"oc")," command line interface."),(0,l.yg)("admonition",{title:"Install the client",type:"tip"},(0,l.yg)("p",{parentName:"admonition"},"To install the ",(0,l.yg)("inlineCode",{parentName:"p"},"oc")," client on your laptop/server, visit the ",(0,l.yg)("a",{parentName:"p",href:"/docs/openshift-install"},"Install the client")," page")),(0,l.yg)("p",null,(0,l.yg)("inlineCode",{parentName:"p"},"oc cp")," directly copy, and overwrite existing files, from a laptop or server to an Application pod on the DSRI."),(0,l.yg)("p",null,"First get the ",(0,l.yg)("inlineCode",{parentName:"p"},"")," using your application name:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"oc get pod --selector app=\n")),(0,l.yg)("h3",{id:"copy-from-local-to-pod"},"Copy from local to pod"),(0,l.yg)("p",null,"Folders are uploaded recursively by default:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"oc cp :\n")),(0,l.yg)("admonition",{title:"Use absolute path in the pod",type:"caution"},(0,l.yg)("p",{parentName:"admonition"},"You need to provide the absolute (full) path where you want to copy it in the pod. Use your application workspace path, e.g. ",(0,l.yg)("inlineCode",{parentName:"p"},"/home/jovyan")," for JupyterLab or ",(0,l.yg)("inlineCode",{parentName:"p"},"/home/rstudio")," for RStudio)")),(0,l.yg)("p",null,"For example:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"oc cp my-folder jupyterlab-000:/home/jovyan\n")),(0,l.yg)("p",null,"You can also use this one-liner to automatically get the pod ID based on your app label:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-bash"},"oc get pod --selector app= | xargs -I{} oc cp {}:\n")),(0,l.yg)("h3",{id:"copy-from-pod-to-local"},"Copy from pod to local"),(0,l.yg)("p",null,"Just do the inverse:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"oc cp : \n")),(0,l.yg)("h3",{id:"download-data-from-surfdrive"},"Download data from SURFdrive"),(0,l.yg)("p",null,"You can download data from your SURFdrive to your pod by creating a public link to the file:"),(0,l.yg)("ol",null,(0,l.yg)("li",{parentName:"ol"},"Go to the file in SURFdrive you'd like to share"),(0,l.yg)("li",{parentName:"ol"},"Click share and the create public link"),(0,l.yg)("li",{parentName:"ol"},"Fill in a name for the public link (like DSRI). The name does not matter much, but it can help you keep track of the goal of the public link."),(0,l.yg)("li",{parentName:"ol"},"Click copy to clipboard"),(0,l.yg)("li",{parentName:"ol"},"Visit link in browser and copy the direct URL displayed on that page."),(0,l.yg)("li",{parentName:"ol"},'Use the direct URL you just copied to download the file using either wget or curl (e.g. "wget ',(0,l.yg)("a",{parentName:"li",href:"https://surfdrive.surf.nl/files/index.php/s/5mFwyAKj4UexlJb/download%22"},'https://surfdrive.surf.nl/files/index.php/s/5mFwyAKj4UexlJb/download"'),")"),(0,l.yg)("li",{parentName:"ol"},"Revoke link in the SURFdrive portal")),(0,l.yg)("h2",{id:"synchronizes-files-with-oc-rsync"},"Synchronizes files with ",(0,l.yg)("inlineCode",{parentName:"h2"},"oc rsync")),(0,l.yg)("p",null,"If you have a lot of large files and/or they are updated regularly, you can use ",(0,l.yg)("inlineCode",{parentName:"p"},"rsync")," as it synchronizes the files if they already exist, preventing duplication and making synchronization faster. You can also see the progress with ",(0,l.yg)("inlineCode",{parentName:"p"},"rsync")," which you cannot with ",(0,l.yg)("inlineCode",{parentName:"p"},"cp"),". And if the upload is stopped for any reason ",(0,l.yg)("inlineCode",{parentName:"p"},"rsync")," should pick it up from where it stopped (instead of restarting from scratch like ",(0,l.yg)("inlineCode",{parentName:"p"},"oc cp")," does)"),(0,l.yg)("admonition",{type:"caution"},(0,l.yg)("p",{parentName:"admonition"},"Rsync does not work with symlinks (created with ",(0,l.yg)("inlineCode",{parentName:"p"},"ln -s"),")")),(0,l.yg)("h3",{id:"sync-local-to-pod"},"Sync local to pod"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"oc rsync --progress :\n")),(0,l.yg)("p",null,"You can also use this one-liner to automatically get the pod ID based on your app label:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-bash"},"oc get pod --selector app= | xargs -I{} oc rsync --progress {}:\n")),(0,l.yg)("h3",{id:"sync-pod-to-local"},"Sync pod to local"),(0,l.yg)("p",null,"Again, do the inverse:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"oc rsync --progress : \n")),(0,l.yg)("h3",{id:"more-options"},"More options"),(0,l.yg)("p",null,"You can use more options to improve the upload of large files:"),(0,l.yg)("table",null,(0,l.yg)("thead",{parentName:"table"},(0,l.yg)("tr",{parentName:"thead"},(0,l.yg)("th",{parentName:"tr",align:null},(0,l.yg)("inlineCode",{parentName:"th"},"--compress")),(0,l.yg)("th",{parentName:"tr",align:null},"compress file data during the transfer"))),(0,l.yg)("tbody",{parentName:"table"},(0,l.yg)("tr",{parentName:"tbody"},(0,l.yg)("td",{parentName:"tr",align:null},(0,l.yg)("inlineCode",{parentName:"td"},"--delete")),(0,l.yg)("td",{parentName:"tr",align:null},"delete files not present in source")),(0,l.yg)("tr",{parentName:"tbody"},(0,l.yg)("td",{parentName:"tr",align:null},(0,l.yg)("inlineCode",{parentName:"td"},"--watch")),(0,l.yg)("td",{parentName:"tr",align:null},"Watch directory for changes and resync automatically")))),(0,l.yg)("h2",{id:"one-liner"},"One-liner"))}u.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/de77a223.e6845580.js b/assets/js/de77a223.9fee0ddd.js similarity index 99% rename from assets/js/de77a223.e6845580.js rename to assets/js/de77a223.9fee0ddd.js index a1fb42002..8561c8bae 100644 --- a/assets/js/de77a223.e6845580.js +++ b/assets/js/de77a223.9fee0ddd.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[5416],{5680:(e,t,o)=>{o.d(t,{xA:()=>u,yg:()=>m});var a=o(6540);function n(e,t,o){return t in e?Object.defineProperty(e,t,{value:o,enumerable:!0,configurable:!0,writable:!0}):e[t]=o,e}function l(e,t){var o=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),o.push.apply(o,a)}return o}function r(e){for(var t=1;t=0||(n[o]=e[o]);return n}(e,t);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,o)&&(n[o]=e[o])}return n}var p=a.createContext({}),c=function(e){var t=a.useContext(p),o=t;return e&&(o="function"==typeof e?e(t):r(r({},t),e)),o},u=function(e){var t=c(e.components);return a.createElement(p.Provider,{value:t},e.children)},s={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},d=a.forwardRef((function(e,t){var o=e.components,n=e.mdxType,l=e.originalType,p=e.parentName,u=i(e,["components","mdxType","originalType","parentName"]),d=c(o),m=n,y=d["".concat(p,".").concat(m)]||d[m]||s[m]||l;return o?a.createElement(y,r(r({ref:t},u),{},{components:o})):a.createElement(y,r({ref:t},u))}));function m(e,t){var o=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var l=o.length,r=new Array(l);r[0]=d;var i={};for(var p in t)hasOwnProperty.call(t,p)&&(i[p]=t[p]);i.originalType=e,i.mdxType="string"==typeof e?e:n,r[1]=i;for(var c=2;c{o.r(t),o.d(t,{assets:()=>u,contentTitle:()=>p,default:()=>m,frontMatter:()=>i,metadata:()=>c,toc:()=>s});var a=o(9668),n=o(1367),l=(o(6540),o(5680)),r=["components"],i={id:"guide-dockerfile-to-openshift",title:"Deploy from a Dockerfile"},p=void 0,c={unversionedId:"guide-dockerfile-to-openshift",id:"guide-dockerfile-to-openshift",title:"Deploy from a Dockerfile",description:"Build from local Dockerfile",source:"@site/docs/guide-dockerfile-to-openshift.md",sourceDirName:".",slug:"/guide-dockerfile-to-openshift",permalink:"/docs/guide-dockerfile-to-openshift",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/guide-dockerfile-to-openshift.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"guide-dockerfile-to-openshift",title:"Deploy from a Dockerfile"},sidebar:"docs",previous:{title:"GPU applications",permalink:"/docs/deploy-on-gpu"},next:{title:"Deploy from a Docker image",permalink:"/docs/deploy-from-docker"}},u={},s=[{value:"Build from local Dockerfile",id:"build-from-local-dockerfile",level:2},{value:"Create new build configuration.",id:"create-new-build-configuration",level:3},{value:"Build the image",id:"build-the-image",level:3},{value:"Create your app",id:"create-your-app",level:3},{value:"Expose app",id:"expose-app",level:3},{value:"Delete the created build",id:"delete-the-created-build",level:3},{value:"Deploy from a local docker image",id:"deploy-from-a-local-docker-image",level:2},{value:"Deploy from a Git repository",id:"deploy-from-a-git-repository",level:2}],d={toc:s};function m(e){var t=e.components,o=(0,n.A)(e,r);return(0,l.yg)("wrapper",(0,a.A)({},d,o,{components:t,mdxType:"MDXLayout"}),(0,l.yg)("h2",{id:"build-from-local-dockerfile"},"Build from local Dockerfile"),(0,l.yg)("p",null,"This manual shows you an example of how to convert a dockerfile from your local machine to a running container on DSRI (openshift / okd). Start by cloning the example repository to your local machine."),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"git clone git@gitlab.maastrichtuniversity.nl:dsri-examples/dockerfile-to-okd.git\n")),(0,l.yg)("p",null,"After cloning you now have a local folder containing a Dockerfile and index.html file. Inspect both files."),(0,l.yg)("p",null,"Login with the openshift client:\n",(0,l.yg)("a",{parentName:"p",href:"/docs/openshift-install"},"Authenticate to the OpenShift cluster")," using ",(0,l.yg)("inlineCode",{parentName:"p"},"oc login")," ."),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"oc login --token=\n")),(0,l.yg)("p",null,"Create a new project if you don't have a project yet you can work with (change myproject to a project name of your choice:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"oc new-project myproject\n")),(0,l.yg)("hr",null),(0,l.yg)("h3",{id:"create-new-build-configuration"},"Create new build configuration."),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"oc new-build --name dockerfile-to-okd --binary\n")),(0,l.yg)("hr",null),(0,l.yg)("h3",{id:"build-the-image"},"Build the image"),(0,l.yg)("p",null,"Start a new build on the DSRI with the files provided:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"cd dockerfile-to-okd\noc start-build dockerfile-to-okd --from-dir=. --follow --wait\n")),(0,l.yg)("hr",null),(0,l.yg)("h3",{id:"create-your-app"},"Create your app"),(0,l.yg)("p",null,"Create a new app using the build we just created:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"oc new-app dockerfile-to-okd\n")),(0,l.yg)("p",null,"To properly deploy your app on OpenShift you will need to define a few more parameters:"),(0,l.yg)("ul",null,(0,l.yg)("li",{parentName:"ul"},"Enable root user access (with ",(0,l.yg)("inlineCode",{parentName:"li"},"serviceAccountName"),") by running this command:")),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-bash"},'oc patch deployment/dockerfile-to-okd --patch \'{"spec":{"template": {"spec":{"serviceAccountName": "anyuid"}}}}\'\n')),(0,l.yg)("ul",null,(0,l.yg)("li",{parentName:"ul"},(0,l.yg)("p",{parentName:"li"},"You can also add persistent storage (with ",(0,l.yg)("inlineCode",{parentName:"p"},"volumes")," and ",(0,l.yg)("inlineCode",{parentName:"p"},"containers: volumeMounts")," )"),(0,l.yg)("ul",{parentName:"li"},(0,l.yg)("li",{parentName:"ul"},(0,l.yg)("inlineCode",{parentName:"li"},"${STORAGE_NAME}"),": Name of your persistent volume claim in the ",(0,l.yg)("strong",{parentName:"li"},"Storage")," page of your project in the web UI"),(0,l.yg)("li",{parentName:"ul"},(0,l.yg)("inlineCode",{parentName:"li"},"${STORAGE_FOLDER}")," : Name of the folder inside the persistent volume claim to store the application data (so you can store multiple applications on the same persistent volume claim)")))),(0,l.yg)("p",null,"Open the configuration of the started app to fix its configuration:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"oc edit deployment/dockerfile-to-okd\n")),(0,l.yg)("p",null,"You can mount existing persistent volume this way (replace the variables, such as ",(0,l.yg)("inlineCode",{parentName:"p"},"${STORAGE_NAME}")," by your values):"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-yaml"},' template:\n spec:\n serviceAccountName: anyuid\n volumes:\n - name: data\n persistentVolumeClaim:\n claimName: "${STORAGE_NAME}"\n containers:\n - image: rstudio-root:latest\n volumeMounts:\n - name: data\n mountPath: "/home/rstudio"\n subPath: "${STORAGE_FOLDER}"\n')),(0,l.yg)("admonition",{title:"Generate deployment file in YAML",type:"info"},(0,l.yg)("p",{parentName:"admonition"},"You can also generate the app deployment in a YAML file to edit it before start:"),(0,l.yg)("pre",{parentName:"admonition"},(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"oc new-app dockerfile-to-okd -o yaml > myapp.yml\n# Edit myapp.yml\noc create -f myapp.yml\n"))),(0,l.yg)("hr",null),(0,l.yg)("h3",{id:"expose-app"},"Expose app"),(0,l.yg)("p",null,"Expose the application so you can reach it from your browser and check the route that was created"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"oc expose svc/dockerfile-to-okd\noc get route\n")),(0,l.yg)("p",null,"You can now visit the route shown in the HOST/PORT output of the ",(0,l.yg)("inlineCode",{parentName:"p"},"oc get route")," command and see if you have successfully converted the docker file. "),(0,l.yg)("p",null,"You can edit the created route to enable HTTPS with this command:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-bash"},'oc patch route/dockerfile-to-okd --patch \'{"spec":{"tls": {"termination": "edge", "insecureEdgeTerminationPolicy": "Redirect"}}}\'\n')),(0,l.yg)("hr",null),(0,l.yg)("h3",{id:"delete-the-created-build"},"Delete the created build"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"oc delete build dockerfile-to-okd\n")),(0,l.yg)("blockquote",null,(0,l.yg)("p",{parentName:"blockquote"},"See ",(0,l.yg)("a",{parentName:"p",href:"https://docs.openshift.com/enterprise/3.0/cli_reference/basic_cli_operations.html#application-modification-cli-operations"},"oc delete documentation"),".")),(0,l.yg)("hr",null),(0,l.yg)("h2",{id:"deploy-from-a-local-docker-image"},"Deploy from a local docker image"),(0,l.yg)("p",null,"You can also deploy a local docker image from your machine. "),(0,l.yg)("p",null,"First build the docker image:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"docker build -t my-docker-image:latest .\n")),(0,l.yg)("p",null,"Check you have the image locally on your system:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"docker images ls\n")),(0,l.yg)("p",null,"You should have a docker image for your application:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-bash"},"REPOSITORY TAG \nmy-docker-image latest\n")),(0,l.yg)("p",null,"You can then deploy providing the docker image name and the name of the application to be deployed:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-abash"},"oc new-app my-docker-image --name app-name-on-openshift\n")),(0,l.yg)("hr",null),(0,l.yg)("h2",{id:"deploy-from-a-git-repository"},"Deploy from a Git repository"),(0,l.yg)("p",null,"Go to ",(0,l.yg)("strong",{parentName:"p"},"+Add")," > ",(0,l.yg)("strong",{parentName:"p"},"From Git"),": ",(0,l.yg)("a",{parentName:"p",href:"https://console-openshift-console.apps.dsri2.unimaas.nl/import"},"https://console-openshift-console.apps.dsri2.unimaas.nl/import")),(0,l.yg)("p",null,"Follow the instructions given by the web UI: provide the URL to your git repository, the port on which the web interface will be deployed, you can also create a secret for git login if the repository is private."),(0,l.yg)("p",null,"Once the container has started you will need to make a small change to enable it running with any user ID (due to OpenShift security policies)."),(0,l.yg)("p",null,"You can do it with the command line (just change ",(0,l.yg)("inlineCode",{parentName:"p"},"your-app-name")," by your application name)"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-bash"},'oc patch deployment/your-app-name --patch \'{"spec":{"template": {"spec":{"serviceAccountName": "anyuid"}}}}\'\n')),(0,l.yg)("p",null,"Or through the web UI: click on your deployment, then ",(0,l.yg)("strong",{parentName:"p"},"Actions")," > ",(0,l.yg)("strong",{parentName:"p"},"Edit Deployment"),". And edit the YAML of your deployment to add ",(0,l.yg)("inlineCode",{parentName:"p"},"serviceAccountName: anyuid")," under ",(0,l.yg)("inlineCode",{parentName:"p"},"template.spec"),":"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-yaml"}," template:\n spec:\n serviceAccountName: anyuid\n containers:\n - [...]\n")))}m.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[5416],{5680:(e,t,o)=>{o.d(t,{xA:()=>u,yg:()=>m});var a=o(6540);function n(e,t,o){return t in e?Object.defineProperty(e,t,{value:o,enumerable:!0,configurable:!0,writable:!0}):e[t]=o,e}function l(e,t){var o=Object.keys(e);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);t&&(a=a.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),o.push.apply(o,a)}return o}function r(e){for(var t=1;t=0||(n[o]=e[o]);return n}(e,t);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);for(a=0;a=0||Object.prototype.propertyIsEnumerable.call(e,o)&&(n[o]=e[o])}return n}var p=a.createContext({}),c=function(e){var t=a.useContext(p),o=t;return e&&(o="function"==typeof e?e(t):r(r({},t),e)),o},u=function(e){var t=c(e.components);return a.createElement(p.Provider,{value:t},e.children)},s={inlineCode:"code",wrapper:function(e){var t=e.children;return a.createElement(a.Fragment,{},t)}},d=a.forwardRef((function(e,t){var o=e.components,n=e.mdxType,l=e.originalType,p=e.parentName,u=i(e,["components","mdxType","originalType","parentName"]),d=c(o),m=n,y=d["".concat(p,".").concat(m)]||d[m]||s[m]||l;return o?a.createElement(y,r(r({ref:t},u),{},{components:o})):a.createElement(y,r({ref:t},u))}));function m(e,t){var o=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var l=o.length,r=new Array(l);r[0]=d;var i={};for(var p in t)hasOwnProperty.call(t,p)&&(i[p]=t[p]);i.originalType=e,i.mdxType="string"==typeof e?e:n,r[1]=i;for(var c=2;c{o.r(t),o.d(t,{assets:()=>u,contentTitle:()=>p,default:()=>m,frontMatter:()=>i,metadata:()=>c,toc:()=>s});var a=o(9668),n=o(1367),l=(o(6540),o(5680)),r=["components"],i={id:"guide-dockerfile-to-openshift",title:"Deploy from a Dockerfile"},p=void 0,c={unversionedId:"guide-dockerfile-to-openshift",id:"guide-dockerfile-to-openshift",title:"Deploy from a Dockerfile",description:"Build from local Dockerfile",source:"@site/docs/guide-dockerfile-to-openshift.md",sourceDirName:".",slug:"/guide-dockerfile-to-openshift",permalink:"/docs/guide-dockerfile-to-openshift",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/guide-dockerfile-to-openshift.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"guide-dockerfile-to-openshift",title:"Deploy from a Dockerfile"},sidebar:"docs",previous:{title:"GPU applications",permalink:"/docs/deploy-on-gpu"},next:{title:"Deploy from a Docker image",permalink:"/docs/deploy-from-docker"}},u={},s=[{value:"Build from local Dockerfile",id:"build-from-local-dockerfile",level:2},{value:"Create new build configuration.",id:"create-new-build-configuration",level:3},{value:"Build the image",id:"build-the-image",level:3},{value:"Create your app",id:"create-your-app",level:3},{value:"Expose app",id:"expose-app",level:3},{value:"Delete the created build",id:"delete-the-created-build",level:3},{value:"Deploy from a local docker image",id:"deploy-from-a-local-docker-image",level:2},{value:"Deploy from a Git repository",id:"deploy-from-a-git-repository",level:2}],d={toc:s};function m(e){var t=e.components,o=(0,n.A)(e,r);return(0,l.yg)("wrapper",(0,a.A)({},d,o,{components:t,mdxType:"MDXLayout"}),(0,l.yg)("h2",{id:"build-from-local-dockerfile"},"Build from local Dockerfile"),(0,l.yg)("p",null,"This manual shows you an example of how to convert a dockerfile from your local machine to a running container on DSRI (openshift / okd). Start by cloning the example repository to your local machine."),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"git clone git@gitlab.maastrichtuniversity.nl:dsri-examples/dockerfile-to-okd.git\n")),(0,l.yg)("p",null,"After cloning you now have a local folder containing a Dockerfile and index.html file. Inspect both files."),(0,l.yg)("p",null,"Login with the openshift client:\n",(0,l.yg)("a",{parentName:"p",href:"/docs/openshift-install"},"Authenticate to the OpenShift cluster")," using ",(0,l.yg)("inlineCode",{parentName:"p"},"oc login")," ."),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"oc login --token=\n")),(0,l.yg)("p",null,"Create a new project if you don't have a project yet you can work with (change myproject to a project name of your choice:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"oc new-project myproject\n")),(0,l.yg)("hr",null),(0,l.yg)("h3",{id:"create-new-build-configuration"},"Create new build configuration."),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"oc new-build --name dockerfile-to-okd --binary\n")),(0,l.yg)("hr",null),(0,l.yg)("h3",{id:"build-the-image"},"Build the image"),(0,l.yg)("p",null,"Start a new build on the DSRI with the files provided:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"cd dockerfile-to-okd\noc start-build dockerfile-to-okd --from-dir=. --follow --wait\n")),(0,l.yg)("hr",null),(0,l.yg)("h3",{id:"create-your-app"},"Create your app"),(0,l.yg)("p",null,"Create a new app using the build we just created:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"oc new-app dockerfile-to-okd\n")),(0,l.yg)("p",null,"To properly deploy your app on OpenShift you will need to define a few more parameters:"),(0,l.yg)("ul",null,(0,l.yg)("li",{parentName:"ul"},"Enable root user access (with ",(0,l.yg)("inlineCode",{parentName:"li"},"serviceAccountName"),") by running this command:")),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-bash"},'oc patch deployment/dockerfile-to-okd --patch \'{"spec":{"template": {"spec":{"serviceAccountName": "anyuid"}}}}\'\n')),(0,l.yg)("ul",null,(0,l.yg)("li",{parentName:"ul"},(0,l.yg)("p",{parentName:"li"},"You can also add persistent storage (with ",(0,l.yg)("inlineCode",{parentName:"p"},"volumes")," and ",(0,l.yg)("inlineCode",{parentName:"p"},"containers: volumeMounts")," )"),(0,l.yg)("ul",{parentName:"li"},(0,l.yg)("li",{parentName:"ul"},(0,l.yg)("inlineCode",{parentName:"li"},"${STORAGE_NAME}"),": Name of your persistent volume claim in the ",(0,l.yg)("strong",{parentName:"li"},"Storage")," page of your project in the web UI"),(0,l.yg)("li",{parentName:"ul"},(0,l.yg)("inlineCode",{parentName:"li"},"${STORAGE_FOLDER}")," : Name of the folder inside the persistent volume claim to store the application data (so you can store multiple applications on the same persistent volume claim)")))),(0,l.yg)("p",null,"Open the configuration of the started app to fix its configuration:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"oc edit deployment/dockerfile-to-okd\n")),(0,l.yg)("p",null,"You can mount existing persistent volume this way (replace the variables, such as ",(0,l.yg)("inlineCode",{parentName:"p"},"${STORAGE_NAME}")," by your values):"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-yaml"},' template:\n spec:\n serviceAccountName: anyuid\n volumes:\n - name: data\n persistentVolumeClaim:\n claimName: "${STORAGE_NAME}"\n containers:\n - image: rstudio-root:latest\n volumeMounts:\n - name: data\n mountPath: "/home/rstudio"\n subPath: "${STORAGE_FOLDER}"\n')),(0,l.yg)("admonition",{title:"Generate deployment file in YAML",type:"info"},(0,l.yg)("p",{parentName:"admonition"},"You can also generate the app deployment in a YAML file to edit it before start:"),(0,l.yg)("pre",{parentName:"admonition"},(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"oc new-app dockerfile-to-okd -o yaml > myapp.yml\n# Edit myapp.yml\noc create -f myapp.yml\n"))),(0,l.yg)("hr",null),(0,l.yg)("h3",{id:"expose-app"},"Expose app"),(0,l.yg)("p",null,"Expose the application so you can reach it from your browser and check the route that was created"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"oc expose svc/dockerfile-to-okd\noc get route\n")),(0,l.yg)("p",null,"You can now visit the route shown in the HOST/PORT output of the ",(0,l.yg)("inlineCode",{parentName:"p"},"oc get route")," command and see if you have successfully converted the docker file. "),(0,l.yg)("p",null,"You can edit the created route to enable HTTPS with this command:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-bash"},'oc patch route/dockerfile-to-okd --patch \'{"spec":{"tls": {"termination": "edge", "insecureEdgeTerminationPolicy": "Redirect"}}}\'\n')),(0,l.yg)("hr",null),(0,l.yg)("h3",{id:"delete-the-created-build"},"Delete the created build"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"oc delete build dockerfile-to-okd\n")),(0,l.yg)("blockquote",null,(0,l.yg)("p",{parentName:"blockquote"},"See ",(0,l.yg)("a",{parentName:"p",href:"https://docs.openshift.com/enterprise/3.0/cli_reference/basic_cli_operations.html#application-modification-cli-operations"},"oc delete documentation"),".")),(0,l.yg)("hr",null),(0,l.yg)("h2",{id:"deploy-from-a-local-docker-image"},"Deploy from a local docker image"),(0,l.yg)("p",null,"You can also deploy a local docker image from your machine. "),(0,l.yg)("p",null,"First build the docker image:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"docker build -t my-docker-image:latest .\n")),(0,l.yg)("p",null,"Check you have the image locally on your system:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-shell"},"docker images ls\n")),(0,l.yg)("p",null,"You should have a docker image for your application:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-bash"},"REPOSITORY TAG \nmy-docker-image latest\n")),(0,l.yg)("p",null,"You can then deploy providing the docker image name and the name of the application to be deployed:"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-abash"},"oc new-app my-docker-image --name app-name-on-openshift\n")),(0,l.yg)("hr",null),(0,l.yg)("h2",{id:"deploy-from-a-git-repository"},"Deploy from a Git repository"),(0,l.yg)("p",null,"Go to ",(0,l.yg)("strong",{parentName:"p"},"+Add")," > ",(0,l.yg)("strong",{parentName:"p"},"From Git"),": ",(0,l.yg)("a",{parentName:"p",href:"https://console-openshift-console.apps.dsri2.unimaas.nl/import"},"https://console-openshift-console.apps.dsri2.unimaas.nl/import")),(0,l.yg)("p",null,"Follow the instructions given by the web UI: provide the URL to your git repository, the port on which the web interface will be deployed, you can also create a secret for git login if the repository is private."),(0,l.yg)("p",null,"Once the container has started you will need to make a small change to enable it running with any user ID (due to OpenShift security policies)."),(0,l.yg)("p",null,"You can do it with the command line (just change ",(0,l.yg)("inlineCode",{parentName:"p"},"your-app-name")," by your application name)"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-bash"},'oc patch deployment/your-app-name --patch \'{"spec":{"template": {"spec":{"serviceAccountName": "anyuid"}}}}\'\n')),(0,l.yg)("p",null,"Or through the web UI: click on your deployment, then ",(0,l.yg)("strong",{parentName:"p"},"Actions")," > ",(0,l.yg)("strong",{parentName:"p"},"Edit Deployment"),". And edit the YAML of your deployment to add ",(0,l.yg)("inlineCode",{parentName:"p"},"serviceAccountName: anyuid")," under ",(0,l.yg)("inlineCode",{parentName:"p"},"template.spec"),":"),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-yaml"}," template:\n spec:\n serviceAccountName: anyuid\n containers:\n - [...]\n")))}m.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/e53f82ff.a9867820.js b/assets/js/e53f82ff.a8289613.js similarity index 99% rename from assets/js/e53f82ff.a9867820.js rename to assets/js/e53f82ff.a8289613.js index 2755e159d..90d1a80dc 100644 --- a/assets/js/e53f82ff.a9867820.js +++ b/assets/js/e53f82ff.a8289613.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[7692],{5680:(e,t,o)=>{o.d(t,{xA:()=>d,yg:()=>g});var n=o(6540);function r(e,t,o){return t in e?Object.defineProperty(e,t,{value:o,enumerable:!0,configurable:!0,writable:!0}):e[t]=o,e}function a(e,t){var o=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),o.push.apply(o,n)}return o}function i(e){for(var t=1;t=0||(r[o]=e[o]);return r}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,o)&&(r[o]=e[o])}return r}var l=n.createContext({}),p=function(e){var t=n.useContext(l),o=t;return e&&(o="function"==typeof e?e(t):i(i({},t),e)),o},d=function(e){var t=p(e.components);return n.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},c=n.forwardRef((function(e,t){var o=e.components,r=e.mdxType,a=e.originalType,l=e.parentName,d=s(e,["components","mdxType","originalType","parentName"]),c=p(o),g=r,m=c["".concat(l,".").concat(g)]||c[g]||u[g]||a;return o?n.createElement(m,i(i({ref:t},d),{},{components:o})):n.createElement(m,i({ref:t},d))}));function g(e,t){var o=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=o.length,i=new Array(a);i[0]=c;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s.mdxType="string"==typeof e?e:r,i[1]=s;for(var p=2;p{o.r(t),o.d(t,{assets:()=>d,contentTitle:()=>l,default:()=>g,frontMatter:()=>s,metadata:()=>p,toc:()=>u});var n=o(9668),r=o(1367),a=(o(6540),o(5680)),i=["components"],s={id:"deploy-vscode",title:"VisualStudio Code"},l=void 0,p={unversionedId:"deploy-vscode",id:"deploy-vscode",title:"VisualStudio Code",description:"Start VisualStudio Code server",source:"@site/docs/deploy-vscode.md",sourceDirName:".",slug:"/deploy-vscode",permalink:"/docs/deploy-vscode",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/deploy-vscode.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"deploy-vscode",title:"VisualStudio Code"},sidebar:"docs",previous:{title:"RStudio",permalink:"/docs/deploy-rstudio"},next:{title:"Databases",permalink:"/docs/deploy-database"}},d={},u=[{value:"Start VisualStudio Code server",id:"start-visualstudio-code-server",level:2},{value:"Use Git in VSCode",id:"use-git-in-vscode",level:2},{value:"VSCode for GPU",id:"vscode-for-gpu",level:2}],c={toc:u};function g(e){var t=e.components,o=(0,r.A)(e,i);return(0,a.yg)("wrapper",(0,n.A)({},c,o,{components:t,mdxType:"MDXLayout"}),(0,a.yg)("h2",{id:"start-visualstudio-code-server"},"Start VisualStudio Code server"),(0,a.yg)("p",null,"Start a VisualStudio Code server with the ",(0,a.yg)("inlineCode",{parentName:"p"},"coder")," user, which has ",(0,a.yg)("inlineCode",{parentName:"p"},"sudo")," privileges."),(0,a.yg)("p",null,"You can deploy it using the ",(0,a.yg)("strong",{parentName:"p"},"VisualStudio Code server")," solution in the ",(0,a.yg)("a",{parentName:"p",href:"https://console-openshift-console.apps.dsri2.unimaas.nl/catalog"},"Catalog web UI")," (make sure the ",(0,a.yg)("strong",{parentName:"p"},"Templates")," checkbox is checked)"),(0,a.yg)("p",null,"Provide a few parameters, and instantiate the template. The DSRI will automatically create a persistent volume to store data you will put in the ",(0,a.yg)("inlineCode",{parentName:"p"},"/home/coder/project")," folder. You can find the persistent volumes in the DSRI web UI, go to the ",(0,a.yg)("strong",{parentName:"p"},"Administrator")," view > ",(0,a.yg)("strong",{parentName:"p"},"Storage")," > ",(0,a.yg)("strong",{parentName:"p"},"Persistent Volume Claims"),"."),(0,a.yg)("img",{src:"/img/screenshot-deploy-vscode.png",alt:"Deploy VSCode",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,a.yg)("h2",{id:"use-git-in-vscode"},"Use Git in VSCode"),(0,a.yg)("p",null,"The easiest way to login and clone a repository from GitHub is to use the built-in authentication system of VisualStudio Code, to do so click on ",(0,a.yg)("strong",{parentName:"p"},"clone repository...")," in the ",(0,a.yg)("strong",{parentName:"p"},"Welcome")," page, and follow the instructions in the top of the VisualStudio window."),(0,a.yg)("p",null,"If this solution does not work for you, you can use ",(0,a.yg)("inlineCode",{parentName:"p"},"git")," from the terminal to clone the git repository with ",(0,a.yg)("inlineCode",{parentName:"p"},"git clone"),". VisualStudio might ask you to login in the dialog box at the top of the page, enter your username and password when requested. For GitHub you might need to generate a token at ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/settings/tokens"},"https://github.com/settings/tokens")," to use as password."),(0,a.yg)("p",null,"Once the repository cloned, you can use git from the VSCode web UI to manage your ",(0,a.yg)("inlineCode",{parentName:"p"},"git")," repositories (add, commit, push changes), or in the terminal."),(0,a.yg)("p",null,"Before committing to GitHub or GitLab, you might need to configure you username and email in VSCode terminal:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},'git config --global user.name "Jean Dupont"\ngit config --global user.email jeandupont@gmail.com\n')),(0,a.yg)("admonition",{title:"Save your git password",type:"info"},(0,a.yg)("p",{parentName:"admonition"},"You can run this command to ask git to save your password for 15min:"),(0,a.yg)("pre",{parentName:"admonition"},(0,a.yg)("code",{parentName:"pre",className:"language-bash"},"git config credential.helper cache\n")),(0,a.yg)("p",{parentName:"admonition"},"Or store the password in a plain text file:"),(0,a.yg)("pre",{parentName:"admonition"},(0,a.yg)("code",{parentName:"pre",className:"language-bash"},"git config --global credential.helper 'store --file ~/.git-credentials'\n"))),(0,a.yg)("admonition",{title:"Git tip",type:"tip"},(0,a.yg)("p",{parentName:"admonition"},"We recommend to use SSH instead of HTTPS connection when possible, checkout ",(0,a.yg)("a",{parentName:"p",href:"https://docs.github.com/en/free-pro-team@latest/github/authenticating-to-github/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent"},"here")," how to generate SSH keys and use them with your GitHub account.")),(0,a.yg)("h2",{id:"vscode-for-gpu"},"VSCode for GPU"),(0,a.yg)("p",null,"See the ",(0,a.yg)("a",{parentName:"p",href:"/docs/deploy-on-gpu"},"Deploy on GPU")," page to deploy a VisualStudio Code server on GPU."))}g.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[7692],{5680:(e,t,o)=>{o.d(t,{xA:()=>d,yg:()=>g});var n=o(6540);function r(e,t,o){return t in e?Object.defineProperty(e,t,{value:o,enumerable:!0,configurable:!0,writable:!0}):e[t]=o,e}function a(e,t){var o=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),o.push.apply(o,n)}return o}function i(e){for(var t=1;t=0||(r[o]=e[o]);return r}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,o)&&(r[o]=e[o])}return r}var l=n.createContext({}),p=function(e){var t=n.useContext(l),o=t;return e&&(o="function"==typeof e?e(t):i(i({},t),e)),o},d=function(e){var t=p(e.components);return n.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},c=n.forwardRef((function(e,t){var o=e.components,r=e.mdxType,a=e.originalType,l=e.parentName,d=s(e,["components","mdxType","originalType","parentName"]),c=p(o),g=r,m=c["".concat(l,".").concat(g)]||c[g]||u[g]||a;return o?n.createElement(m,i(i({ref:t},d),{},{components:o})):n.createElement(m,i({ref:t},d))}));function g(e,t){var o=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=o.length,i=new Array(a);i[0]=c;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s.mdxType="string"==typeof e?e:r,i[1]=s;for(var p=2;p{o.r(t),o.d(t,{assets:()=>d,contentTitle:()=>l,default:()=>g,frontMatter:()=>s,metadata:()=>p,toc:()=>u});var n=o(9668),r=o(1367),a=(o(6540),o(5680)),i=["components"],s={id:"deploy-vscode",title:"VisualStudio Code"},l=void 0,p={unversionedId:"deploy-vscode",id:"deploy-vscode",title:"VisualStudio Code",description:"Start VisualStudio Code server",source:"@site/docs/deploy-vscode.md",sourceDirName:".",slug:"/deploy-vscode",permalink:"/docs/deploy-vscode",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/deploy-vscode.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"deploy-vscode",title:"VisualStudio Code"},sidebar:"docs",previous:{title:"RStudio",permalink:"/docs/deploy-rstudio"},next:{title:"Databases",permalink:"/docs/deploy-database"}},d={},u=[{value:"Start VisualStudio Code server",id:"start-visualstudio-code-server",level:2},{value:"Use Git in VSCode",id:"use-git-in-vscode",level:2},{value:"VSCode for GPU",id:"vscode-for-gpu",level:2}],c={toc:u};function g(e){var t=e.components,o=(0,r.A)(e,i);return(0,a.yg)("wrapper",(0,n.A)({},c,o,{components:t,mdxType:"MDXLayout"}),(0,a.yg)("h2",{id:"start-visualstudio-code-server"},"Start VisualStudio Code server"),(0,a.yg)("p",null,"Start a VisualStudio Code server with the ",(0,a.yg)("inlineCode",{parentName:"p"},"coder")," user, which has ",(0,a.yg)("inlineCode",{parentName:"p"},"sudo")," privileges."),(0,a.yg)("p",null,"You can deploy it using the ",(0,a.yg)("strong",{parentName:"p"},"VisualStudio Code server")," solution in the ",(0,a.yg)("a",{parentName:"p",href:"https://console-openshift-console.apps.dsri2.unimaas.nl/catalog"},"Catalog web UI")," (make sure the ",(0,a.yg)("strong",{parentName:"p"},"Templates")," checkbox is checked)"),(0,a.yg)("p",null,"Provide a few parameters, and instantiate the template. The DSRI will automatically create a persistent volume to store data you will put in the ",(0,a.yg)("inlineCode",{parentName:"p"},"/home/coder/project")," folder. You can find the persistent volumes in the DSRI web UI, go to the ",(0,a.yg)("strong",{parentName:"p"},"Administrator")," view > ",(0,a.yg)("strong",{parentName:"p"},"Storage")," > ",(0,a.yg)("strong",{parentName:"p"},"Persistent Volume Claims"),"."),(0,a.yg)("img",{src:"/img/screenshot-deploy-vscode.png",alt:"Deploy VSCode",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,a.yg)("h2",{id:"use-git-in-vscode"},"Use Git in VSCode"),(0,a.yg)("p",null,"The easiest way to login and clone a repository from GitHub is to use the built-in authentication system of VisualStudio Code, to do so click on ",(0,a.yg)("strong",{parentName:"p"},"clone repository...")," in the ",(0,a.yg)("strong",{parentName:"p"},"Welcome")," page, and follow the instructions in the top of the VisualStudio window."),(0,a.yg)("p",null,"If this solution does not work for you, you can use ",(0,a.yg)("inlineCode",{parentName:"p"},"git")," from the terminal to clone the git repository with ",(0,a.yg)("inlineCode",{parentName:"p"},"git clone"),". VisualStudio might ask you to login in the dialog box at the top of the page, enter your username and password when requested. For GitHub you might need to generate a token at ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/settings/tokens"},"https://github.com/settings/tokens")," to use as password."),(0,a.yg)("p",null,"Once the repository cloned, you can use git from the VSCode web UI to manage your ",(0,a.yg)("inlineCode",{parentName:"p"},"git")," repositories (add, commit, push changes), or in the terminal."),(0,a.yg)("p",null,"Before committing to GitHub or GitLab, you might need to configure you username and email in VSCode terminal:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},'git config --global user.name "Jean Dupont"\ngit config --global user.email jeandupont@gmail.com\n')),(0,a.yg)("admonition",{title:"Save your git password",type:"info"},(0,a.yg)("p",{parentName:"admonition"},"You can run this command to ask git to save your password for 15min:"),(0,a.yg)("pre",{parentName:"admonition"},(0,a.yg)("code",{parentName:"pre",className:"language-bash"},"git config credential.helper cache\n")),(0,a.yg)("p",{parentName:"admonition"},"Or store the password in a plain text file:"),(0,a.yg)("pre",{parentName:"admonition"},(0,a.yg)("code",{parentName:"pre",className:"language-bash"},"git config --global credential.helper 'store --file ~/.git-credentials'\n"))),(0,a.yg)("admonition",{title:"Git tip",type:"tip"},(0,a.yg)("p",{parentName:"admonition"},"We recommend to use SSH instead of HTTPS connection when possible, checkout ",(0,a.yg)("a",{parentName:"p",href:"https://docs.github.com/en/free-pro-team@latest/github/authenticating-to-github/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent"},"here")," how to generate SSH keys and use them with your GitHub account.")),(0,a.yg)("h2",{id:"vscode-for-gpu"},"VSCode for GPU"),(0,a.yg)("p",null,"See the ",(0,a.yg)("a",{parentName:"p",href:"/docs/deploy-on-gpu"},"Deploy on GPU")," page to deploy a VisualStudio Code server on GPU."))}g.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/e747ec83.4e79809f.js b/assets/js/e747ec83.7cb54f72.js similarity index 98% rename from assets/js/e747ec83.4e79809f.js rename to assets/js/e747ec83.7cb54f72.js index 9fc69ee29..e77cb41e1 100644 --- a/assets/js/e747ec83.4e79809f.js +++ b/assets/js/e747ec83.7cb54f72.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[7051],{5680:(e,t,n)=>{n.d(t,{xA:()=>c,yg:()=>y});var r=n(6540);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function i(e){for(var t=1;t=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var l=r.createContext({}),p=function(e){var t=r.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},c=function(e){var t=p(e.components);return r.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},d=r.forwardRef((function(e,t){var n=e.components,a=e.mdxType,o=e.originalType,l=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),d=p(n),y=a,g=d["".concat(l,".").concat(y)]||d[y]||u[y]||o;return n?r.createElement(g,i(i({ref:t},c),{},{components:n})):r.createElement(g,i({ref:t},c))}));function y(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=n.length,i=new Array(o);i[0]=d;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s.mdxType="string"==typeof e?e:a,i[1]=s;for(var p=2;p{n.r(t),n.d(t,{assets:()=>c,contentTitle:()=>l,default:()=>y,frontMatter:()=>s,metadata:()=>p,toc:()=>u});var r=n(9668),a=n(1367),o=(n(6540),n(5680)),i=["components"],s={id:"glossary",title:"Glossary"},l=void 0,p={unversionedId:"glossary",id:"glossary",title:"Glossary",description:"Docker",source:"@site/docs/glossary.md",sourceDirName:".",slug:"/glossary",permalink:"/docs/glossary",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/glossary.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"glossary",title:"Glossary"},sidebar:"docs",previous:{title:"Libraries for Machine Learning",permalink:"/docs/tools-machine-learning"},next:{title:"Introduction to workflows",permalink:"/docs/workflows-introduction"}},c={},u=[{value:"Docker",id:"docker",level:2},{value:"Kubernetes",id:"kubernetes",level:2},{value:"OpenShift",id:"openshift",level:2},{value:"OKD",id:"okd",level:2}],d={toc:u};function y(e){var t=e.components,n=(0,a.A)(e,i);return(0,o.yg)("wrapper",(0,r.A)({},d,n,{components:t,mdxType:"MDXLayout"}),(0,o.yg)("h2",{id:"docker"},"Docker"),(0,o.yg)("h2",{id:"kubernetes"},"Kubernetes"),(0,o.yg)("p",null,(0,o.yg)("a",{parentName:"p",href:"https://kubernetes.io/"},"Kubernetes")," is a portable, extensible, open-source platform for managing containerized workloads and services, that facilitates both declarative configuration and automation. It has a large, rapidly growing ecosystem. "),(0,o.yg)("p",null,"Kubernetes services, support, and tools are widely available."),(0,o.yg)("p",null,"Kubernetes, also known as K8s, is an open-source system for automating deployment, scaling, and management of containerized applications."),(0,o.yg)("img",{src:"/img/glossary_kubernetes.png",alt:"Kubernetes Architecture",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("p",null,"More Information: ",(0,o.yg)("a",{parentName:"p",href:"https://kubernetes.io/docs/concepts/overview/what-is-kubernetes/"},"https://kubernetes.io/docs/concepts/overview/what-is-kubernetes/")),(0,o.yg)("h2",{id:"openshift"},"OpenShift"),(0,o.yg)("p",null,(0,o.yg)("a",{parentName:"p",href:"https://www.openshift.com/"},"Red Hat OpenShift")," is a hybrid cloud, enterprise Kubernetes application platform, trusted by 2,000+ organizations."),(0,o.yg)("p",null,"It includes "),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},"Container host and runtime"),(0,o.yg)("li",{parentName:"ul"},"Enterprise Kubernetes"),(0,o.yg)("li",{parentName:"ul"},"Validated integrations"),(0,o.yg)("li",{parentName:"ul"},"Integrated container registry"),(0,o.yg)("li",{parentName:"ul"},"Developer workflows"),(0,o.yg)("li",{parentName:"ul"},"Easy access to services")),(0,o.yg)("img",{src:"/img/glossary_openshift.png",alt:"Red Hat Openshift",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("h2",{id:"okd"},"OKD"),(0,o.yg)("p",null,(0,o.yg)("a",{parentName:"p",href:"https://www.okd.io/"},"OKD")," is a ",(0,o.yg)("strong",{parentName:"p"},"distribution of Kubernetes")," optimized for continuous application development and multi-tenant deployment. OKD adds ",(0,o.yg)("strong",{parentName:"p"},"developer and operations-centric")," tools on top of Kubernetes to enable rapid application development, easy deployment and scaling, and long-term lifecycle maintenance for small and large teams. OKD is a ",(0,o.yg)("strong",{parentName:"p"},"sibling")," Kubernetes distribution to ",(0,o.yg)("strong",{parentName:"p"},"Red Hat OpenShift")),(0,o.yg)("p",null,(0,o.yg)("a",{parentName:"p",href:"https://docs.okd.io/latest/welcome/index.html"},"OKD 4 Documentation")))}y.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[7051],{5680:(e,t,n)=>{n.d(t,{xA:()=>c,yg:()=>y});var r=n(6540);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function i(e){for(var t=1;t=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var l=r.createContext({}),p=function(e){var t=r.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},c=function(e){var t=p(e.components);return r.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},d=r.forwardRef((function(e,t){var n=e.components,a=e.mdxType,o=e.originalType,l=e.parentName,c=s(e,["components","mdxType","originalType","parentName"]),d=p(n),y=a,g=d["".concat(l,".").concat(y)]||d[y]||u[y]||o;return n?r.createElement(g,i(i({ref:t},c),{},{components:n})):r.createElement(g,i({ref:t},c))}));function y(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=n.length,i=new Array(o);i[0]=d;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s.mdxType="string"==typeof e?e:a,i[1]=s;for(var p=2;p{n.r(t),n.d(t,{assets:()=>c,contentTitle:()=>l,default:()=>y,frontMatter:()=>s,metadata:()=>p,toc:()=>u});var r=n(9668),a=n(1367),o=(n(6540),n(5680)),i=["components"],s={id:"glossary",title:"Glossary"},l=void 0,p={unversionedId:"glossary",id:"glossary",title:"Glossary",description:"Docker",source:"@site/docs/glossary.md",sourceDirName:".",slug:"/glossary",permalink:"/docs/glossary",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/glossary.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"glossary",title:"Glossary"},sidebar:"docs",previous:{title:"Libraries for Machine Learning",permalink:"/docs/tools-machine-learning"},next:{title:"Introduction to workflows",permalink:"/docs/workflows-introduction"}},c={},u=[{value:"Docker",id:"docker",level:2},{value:"Kubernetes",id:"kubernetes",level:2},{value:"OpenShift",id:"openshift",level:2},{value:"OKD",id:"okd",level:2}],d={toc:u};function y(e){var t=e.components,n=(0,a.A)(e,i);return(0,o.yg)("wrapper",(0,r.A)({},d,n,{components:t,mdxType:"MDXLayout"}),(0,o.yg)("h2",{id:"docker"},"Docker"),(0,o.yg)("h2",{id:"kubernetes"},"Kubernetes"),(0,o.yg)("p",null,(0,o.yg)("a",{parentName:"p",href:"https://kubernetes.io/"},"Kubernetes")," is a portable, extensible, open-source platform for managing containerized workloads and services, that facilitates both declarative configuration and automation. It has a large, rapidly growing ecosystem. "),(0,o.yg)("p",null,"Kubernetes services, support, and tools are widely available."),(0,o.yg)("p",null,"Kubernetes, also known as K8s, is an open-source system for automating deployment, scaling, and management of containerized applications."),(0,o.yg)("img",{src:"/img/glossary_kubernetes.png",alt:"Kubernetes Architecture",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("p",null,"More Information: ",(0,o.yg)("a",{parentName:"p",href:"https://kubernetes.io/docs/concepts/overview/what-is-kubernetes/"},"https://kubernetes.io/docs/concepts/overview/what-is-kubernetes/")),(0,o.yg)("h2",{id:"openshift"},"OpenShift"),(0,o.yg)("p",null,(0,o.yg)("a",{parentName:"p",href:"https://www.openshift.com/"},"Red Hat OpenShift")," is a hybrid cloud, enterprise Kubernetes application platform, trusted by 2,000+ organizations."),(0,o.yg)("p",null,"It includes "),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},"Container host and runtime"),(0,o.yg)("li",{parentName:"ul"},"Enterprise Kubernetes"),(0,o.yg)("li",{parentName:"ul"},"Validated integrations"),(0,o.yg)("li",{parentName:"ul"},"Integrated container registry"),(0,o.yg)("li",{parentName:"ul"},"Developer workflows"),(0,o.yg)("li",{parentName:"ul"},"Easy access to services")),(0,o.yg)("img",{src:"/img/glossary_openshift.png",alt:"Red Hat Openshift",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("h2",{id:"okd"},"OKD"),(0,o.yg)("p",null,(0,o.yg)("a",{parentName:"p",href:"https://www.okd.io/"},"OKD")," is a ",(0,o.yg)("strong",{parentName:"p"},"distribution of Kubernetes")," optimized for continuous application development and multi-tenant deployment. OKD adds ",(0,o.yg)("strong",{parentName:"p"},"developer and operations-centric")," tools on top of Kubernetes to enable rapid application development, easy deployment and scaling, and long-term lifecycle maintenance for small and large teams. OKD is a ",(0,o.yg)("strong",{parentName:"p"},"sibling")," Kubernetes distribution to ",(0,o.yg)("strong",{parentName:"p"},"Red Hat OpenShift")),(0,o.yg)("p",null,(0,o.yg)("a",{parentName:"p",href:"https://docs.okd.io/latest/welcome/index.html"},"OKD 4 Documentation")))}y.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/e9a2555b.5f7c31bd.js b/assets/js/e9a2555b.edc4cc57.js similarity index 99% rename from assets/js/e9a2555b.5f7c31bd.js rename to assets/js/e9a2555b.edc4cc57.js index d373943b4..e71941c76 100644 --- a/assets/js/e9a2555b.5f7c31bd.js +++ b/assets/js/e9a2555b.edc4cc57.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[3807],{5680:(e,t,a)=>{a.d(t,{xA:()=>g,yg:()=>c});var n=a(6540);function o(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function r(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function i(e){for(var t=1;t=0||(o[a]=e[a]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(o[a]=e[a])}return o}var l=n.createContext({}),p=function(e){var t=n.useContext(l),a=t;return e&&(a="function"==typeof e?e(t):i(i({},t),e)),a},g=function(e){var t=p(e.components);return n.createElement(l.Provider,{value:t},e.children)},m={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},u=n.forwardRef((function(e,t){var a=e.components,o=e.mdxType,r=e.originalType,l=e.parentName,g=s(e,["components","mdxType","originalType","parentName"]),u=p(a),c=o,d=u["".concat(l,".").concat(c)]||u[c]||m[c]||r;return a?n.createElement(d,i(i({ref:t},g),{},{components:a})):n.createElement(d,i({ref:t},g))}));function c(e,t){var a=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=a.length,i=new Array(r);i[0]=u;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s.mdxType="string"==typeof e?e:o,i[1]=s;for(var p=2;p{a.r(t),a.d(t,{assets:()=>g,contentTitle:()=>l,default:()=>c,frontMatter:()=>s,metadata:()=>p,toc:()=>m});var n=a(9668),o=a(1367),r=(a(6540),a(5680)),i=["components"],s={id:"catalog-utilities",title:"Utilities"},l=void 0,p={unversionedId:"catalog-utilities",id:"catalog-utilities",title:"Utilities",description:"Feel free to propose new services using pull requests, or to request them by creating new issues.",source:"@site/docs/catalog-utilities.md",sourceDirName:".",slug:"/catalog-utilities",permalink:"/docs/catalog-utilities",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/catalog-utilities.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"catalog-utilities",title:"Utilities"},sidebar:"docs",previous:{title:"Imaging softwares",permalink:"/docs/catalog-imaging"},next:{title:"Access UM servers",permalink:"/docs/access-um-servers"}},g={},m=[{value:"Ubuntu",id:"ubuntu",level:2},{value:"With the terminal",id:"with-the-terminal",level:3},{value:"With a web UI",id:"with-a-web-ui",level:3},{value:"File browser",id:"file-browser",level:2},{value:"Creating or Connecting an Existing Persistent Storage",id:"creating-or-connecting-an-existing-persistent-storage",level:3}],u={toc:m};function c(e){var t=e.components,a=(0,o.A)(e,i);return(0,r.yg)("wrapper",(0,n.A)({},u,a,{components:t,mdxType:"MDXLayout"}),(0,r.yg)("p",null,"Feel free to propose new services using ",(0,r.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/pulls"},"pull requests"),", or to request them by creating ",(0,r.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/issues"},"new issues"),"."),(0,r.yg)("h2",{id:"ubuntu"},"Ubuntu"),(0,r.yg)("h3",{id:"with-the-terminal"},"With the terminal"),(0,r.yg)("p",null,"Start Ubuntu with the ",(0,r.yg)("inlineCode",{parentName:"p"},"root")," user which has ",(0,r.yg)("inlineCode",{parentName:"p"},"sudo")," permissions to install anything."),(0,r.yg)("p",null,"You can start the application using the ",(0,r.yg)("strong",{parentName:"p"},"Ubuntu")," template in the ",(0,r.yg)("a",{parentName:"p",href:"https://console-openshift-console.apps.dsri2.unimaas.nl/catalog"},"Catalog web UI")," (make sure the ",(0,r.yg)("strong",{parentName:"p"},"Templates")," checkbox is checked)"),(0,r.yg)("admonition",{title:"Login Credentials",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"Username: ",(0,r.yg)("strong",{parentName:"p"},"root")),(0,r.yg)("p",{parentName:"admonition"},"Password: ",(0,r.yg)("strong",{parentName:"p"},"Template creation password"))),(0,r.yg)("p",null,"This template uses the Ubuntu image hosted on DockerHub, see its documentation at ",(0,r.yg)("a",{parentName:"p",href:"https://hub.docker.com/r/ubuntu"},"https://hub.docker.com/r/ubuntu")),(0,r.yg)("admonition",{title:"Persistent data folder",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"\ud83d\udcc2 Use the ",(0,r.yg)("inlineCode",{parentName:"p"},"/root")," folder (home of the root user) to store your data in the existing persistent storage. You can find the persistent volumes in the DSRI web UI, go to the ",(0,r.yg)("strong",{parentName:"p"},"Administrator")," view > ",(0,r.yg)("strong",{parentName:"p"},"Storage")," > ",(0,r.yg)("strong",{parentName:"p"},"Persistent Volume Claims"),".")),(0,r.yg)("p",null,"We enabled the port ",(0,r.yg)("inlineCode",{parentName:"p"},"8080")," in the Ubuntu container if you need to deploy applications."),(0,r.yg)("p",null,"To quickly access it from the terminal you can use the ",(0,r.yg)("strong",{parentName:"p"},"Terminal")," tab in the pod page, or via your local terminal:"),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"Get the Ubuntu pod ID:"),(0,r.yg)("pre",{parentName:"li"},(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc get pods\n"))),(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"Connect to it:"),(0,r.yg)("pre",{parentName:"li"},(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc rsh POD_ID\n"))),(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"Enable Bash in the Ubuntu container (if it starts with the Shell)"),(0,r.yg)("pre",{parentName:"li"},(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"bash\n")))),(0,r.yg)("img",{src:"/img/screenshot-deploy-ubuntu.png",alt:"Deploy Ubuntu",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("h3",{id:"with-a-web-ui"},"With a web UI"),(0,r.yg)("p",null,"Start Ubuntu with a web UI accessible via a URL (using VNC). You will be the ",(0,r.yg)("inlineCode",{parentName:"p"},"root")," user which has elevated permissions to install anything via ",(0,r.yg)("inlineCode",{parentName:"p"},"apt install "),". Before you install a package run ",(0,r.yg)("inlineCode",{parentName:"p"},"apt update"),". This also solves ",(0,r.yg)("inlineCode",{parentName:"p"},"E: unable to locate package")," and ",(0,r.yg)("inlineCode",{parentName:"p"},"E: no installation candidate")," errors."),(0,r.yg)("p",null,"You can start the application using the ",(0,r.yg)("strong",{parentName:"p"},"Ubuntu with web UI")," template in the ",(0,r.yg)("a",{parentName:"p",href:"https://console-openshift-console.apps.dsri2.unimaas.nl/catalog"},"Catalog web UI")," (make sure the ",(0,r.yg)("strong",{parentName:"p"},"Templates")," checkbox is checked)"),(0,r.yg)("admonition",{title:"Login Credentials",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"Username: ",(0,r.yg)("strong",{parentName:"p"},"root")),(0,r.yg)("p",{parentName:"admonition"},"Password: ",(0,r.yg)("strong",{parentName:"p"},"Template creation password"))),(0,r.yg)("p",null,"This template uses the Docker image defined at ",(0,r.yg)("a",{parentName:"p",href:"https://github.com/fcwu/docker-ubuntu-vnc-desktop"},"https://github.com/fcwu/docker-ubuntu-vnc-desktop")),(0,r.yg)("admonition",{title:"Less stable than the official image",type:"caution"},(0,r.yg)("p",{parentName:"admonition"},"This image might be less stable than the original Ubuntu image. Let us know on Slack if you have any problem!")),(0,r.yg)("h2",{id:"file-browser"},"File browser"),(0,r.yg)("p",null,"Deploy a file browser on your persistent volume. This will provide a web UI to upload and download data to your DSRI persistent volume in case you need it (JupyterLab, RStudio and VisualStudio Code server already include a file browser)"),(0,r.yg)("p",null,"You can start a container using the ",(0,r.yg)("strong",{parentName:"p"},"File Browser for existing storage")," template in the ",(0,r.yg)("a",{parentName:"p",href:"https://console-openshift-console.apps.dsri2.unimaas.nl/catalog"},"Catalog web UI")," (make sure the ",(0,r.yg)("strong",{parentName:"p"},"Templates")," checkbox is checked)"),(0,r.yg)("img",{src:"/img/screenshot-deploy-filebrowser.png",alt:"Deploy File browser",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("p",null,"You can only deploy file browser on an existing Persistent Volume Claim, this enables you to add a web UI to access this storage."),(0,r.yg)("p",null,"The following parameters can be provided:"),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},"Provide a unique ",(0,r.yg)("strong",{parentName:"li"},"Application name"),". It will be used to generate the application URL."),(0,r.yg)("li",{parentName:"ol"},"Provide a ",(0,r.yg)("strong",{parentName:"li"},"Password"),", you will need to hash the password first for extra security, use this quick docker command to do it: ",(0,r.yg)("inlineCode",{parentName:"li"},"docker run filebrowser/filebrowser hash mypassword")),(0,r.yg)("li",{parentName:"ol"},"The ",(0,r.yg)("strong",{parentName:"li"},"Storage name")," of the Persistent Volume Claim (PVC) that will be exposed by the filebrowser."),(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("strong",{parentName:"li"},"Storage subpath")," in the the Persistent Volume Claim that will be exposed by the filebrowser. Let it empty to use the Root folder of the persistent volume.")),(0,r.yg)("p",null,"You can find the Storage name if you Go to the deployments page > Storage panel."),(0,r.yg)("h3",{id:"creating-or-connecting-an-existing-persistent-storage"},"Creating or Connecting an Existing Persistent Storage"),(0,r.yg)("p",null,"Find more details about the how to ",(0,r.yg)("a",{parentName:"p",href:"https://maastrichtu-ids.github.io/dsri-documentation/docs/openshift-storage/#create-the-persistent-storage"},"create persistent storage")),(0,r.yg)("img",{src:"/img/screenshot_pvc_storage.png",alt:"Create Persistent Storage",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("img",{src:"/img/screenshot_pvc_storage_create.png",alt:"Create Persistent Storage",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("admonition",{type:"info"},(0,r.yg)("p",{parentName:"admonition"},"The DSRI using the ",(0,r.yg)("a",{parentName:"p",href:"https://www.openshift.com/products/container-storage/"},(0,r.yg)("strong",{parentName:"a"},"Openshift Container Stroage"))," (",(0,r.yg)("inlineCode",{parentName:"p"},"OCS"),") which is based on ",(0,r.yg)("a",{parentName:"p",href:"https://ceph.io/ceph-storage/"},(0,r.yg)("strong",{parentName:"a"},"CEPH"))," offers ",(0,r.yg)("inlineCode",{parentName:"p"},"ReadWriteOnce")," and ",(0,r.yg)("inlineCode",{parentName:"p"},"ReadWriteMany")," access mode. "),(0,r.yg)("ul",{parentName:"admonition"},(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("inlineCode",{parentName:"li"},"ReadWriteOnce")," (",(0,r.yg)("a",{parentName:"li",href:"https://docs.openshift.com/container-platform/4.6/storage/understanding-persistent-storage.html"},(0,r.yg)("strong",{parentName:"a"},"RWO")),") volumes cannot be mounted on multiple nodes. Use the ",(0,r.yg)("inlineCode",{parentName:"li"},"ReadWriteMany")," (",(0,r.yg)("a",{parentName:"li",href:"https://docs.openshift.com/container-platform/4.6/storage/understanding-persistent-storage.html"},(0,r.yg)("strong",{parentName:"a"},"RWX")),") access mode when possible. If a node fails, the system does not allow the attached RWO volume to be mounted on a new node because it is already assigned to the failed node. If you encounter a multi-attach error message as a result, force delete the pod on a shut down or crashed node. "))),(0,r.yg)("p",null,"Find more details about the how to ",(0,r.yg)("a",{parentName:"p",href:"https://maastrichtu-ids.github.io/dsri-documentation/docs/openshift-storage/#connect-the-existing-persistent-storage"},"Connect the Existing persistent storage")),(0,r.yg)("img",{src:"/img/screenshot_existing_storage.png",alt:"Add Existing Persistent Storage",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("img",{src:"/img/screenshot_add_storage.png",alt:"Add Existing Persistent Storage",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("admonition",{type:"info"},(0,r.yg)("p",{parentName:"admonition"},"You can try above method if you want to connect ",(0,r.yg)("strong",{parentName:"p"},"more applications to the same storage"))),(0,r.yg)("p",null,"This deployment require to have root user enabled on your project. Contact the ",(0,r.yg)("a",{parentName:"p",href:"mailto:dsri-support-l@maastrichtuniversity.nl"},"DSRI support team")," or create a ",(0,r.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/issues"},"new issues")," to request root access or to create persistent volume for your project if you don't have them ."),(0,r.yg)("admonition",{title:"Credentials",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"Default credentials will be username ",(0,r.yg)("inlineCode",{parentName:"p"},"admin")," and password ",(0,r.yg)("inlineCode",{parentName:"p"},"admin"))),(0,r.yg)("admonition",{title:"Change password",type:"caution"},(0,r.yg)("p",{parentName:"admonition"},"Please ",(0,r.yg)("strong",{parentName:"p"},"change the password in the Filebrowser Web UI")," once it has been created.")))}c.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[3807],{5680:(e,t,a)=>{a.d(t,{xA:()=>g,yg:()=>c});var n=a(6540);function o(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function r(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function i(e){for(var t=1;t=0||(o[a]=e[a]);return o}(e,t);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(o[a]=e[a])}return o}var l=n.createContext({}),p=function(e){var t=n.useContext(l),a=t;return e&&(a="function"==typeof e?e(t):i(i({},t),e)),a},g=function(e){var t=p(e.components);return n.createElement(l.Provider,{value:t},e.children)},m={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},u=n.forwardRef((function(e,t){var a=e.components,o=e.mdxType,r=e.originalType,l=e.parentName,g=s(e,["components","mdxType","originalType","parentName"]),u=p(a),c=o,d=u["".concat(l,".").concat(c)]||u[c]||m[c]||r;return a?n.createElement(d,i(i({ref:t},g),{},{components:a})):n.createElement(d,i({ref:t},g))}));function c(e,t){var a=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var r=a.length,i=new Array(r);i[0]=u;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s.mdxType="string"==typeof e?e:o,i[1]=s;for(var p=2;p{a.r(t),a.d(t,{assets:()=>g,contentTitle:()=>l,default:()=>c,frontMatter:()=>s,metadata:()=>p,toc:()=>m});var n=a(9668),o=a(1367),r=(a(6540),a(5680)),i=["components"],s={id:"catalog-utilities",title:"Utilities"},l=void 0,p={unversionedId:"catalog-utilities",id:"catalog-utilities",title:"Utilities",description:"Feel free to propose new services using pull requests, or to request them by creating new issues.",source:"@site/docs/catalog-utilities.md",sourceDirName:".",slug:"/catalog-utilities",permalink:"/docs/catalog-utilities",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/catalog-utilities.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"catalog-utilities",title:"Utilities"},sidebar:"docs",previous:{title:"Imaging softwares",permalink:"/docs/catalog-imaging"},next:{title:"Access UM servers",permalink:"/docs/access-um-servers"}},g={},m=[{value:"Ubuntu",id:"ubuntu",level:2},{value:"With the terminal",id:"with-the-terminal",level:3},{value:"With a web UI",id:"with-a-web-ui",level:3},{value:"File browser",id:"file-browser",level:2},{value:"Creating or Connecting an Existing Persistent Storage",id:"creating-or-connecting-an-existing-persistent-storage",level:3}],u={toc:m};function c(e){var t=e.components,a=(0,o.A)(e,i);return(0,r.yg)("wrapper",(0,n.A)({},u,a,{components:t,mdxType:"MDXLayout"}),(0,r.yg)("p",null,"Feel free to propose new services using ",(0,r.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/pulls"},"pull requests"),", or to request them by creating ",(0,r.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/issues"},"new issues"),"."),(0,r.yg)("h2",{id:"ubuntu"},"Ubuntu"),(0,r.yg)("h3",{id:"with-the-terminal"},"With the terminal"),(0,r.yg)("p",null,"Start Ubuntu with the ",(0,r.yg)("inlineCode",{parentName:"p"},"root")," user which has ",(0,r.yg)("inlineCode",{parentName:"p"},"sudo")," permissions to install anything."),(0,r.yg)("p",null,"You can start the application using the ",(0,r.yg)("strong",{parentName:"p"},"Ubuntu")," template in the ",(0,r.yg)("a",{parentName:"p",href:"https://console-openshift-console.apps.dsri2.unimaas.nl/catalog"},"Catalog web UI")," (make sure the ",(0,r.yg)("strong",{parentName:"p"},"Templates")," checkbox is checked)"),(0,r.yg)("admonition",{title:"Login Credentials",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"Username: ",(0,r.yg)("strong",{parentName:"p"},"root")),(0,r.yg)("p",{parentName:"admonition"},"Password: ",(0,r.yg)("strong",{parentName:"p"},"Template creation password"))),(0,r.yg)("p",null,"This template uses the Ubuntu image hosted on DockerHub, see its documentation at ",(0,r.yg)("a",{parentName:"p",href:"https://hub.docker.com/r/ubuntu"},"https://hub.docker.com/r/ubuntu")),(0,r.yg)("admonition",{title:"Persistent data folder",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"\ud83d\udcc2 Use the ",(0,r.yg)("inlineCode",{parentName:"p"},"/root")," folder (home of the root user) to store your data in the existing persistent storage. You can find the persistent volumes in the DSRI web UI, go to the ",(0,r.yg)("strong",{parentName:"p"},"Administrator")," view > ",(0,r.yg)("strong",{parentName:"p"},"Storage")," > ",(0,r.yg)("strong",{parentName:"p"},"Persistent Volume Claims"),".")),(0,r.yg)("p",null,"We enabled the port ",(0,r.yg)("inlineCode",{parentName:"p"},"8080")," in the Ubuntu container if you need to deploy applications."),(0,r.yg)("p",null,"To quickly access it from the terminal you can use the ",(0,r.yg)("strong",{parentName:"p"},"Terminal")," tab in the pod page, or via your local terminal:"),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"Get the Ubuntu pod ID:"),(0,r.yg)("pre",{parentName:"li"},(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc get pods\n"))),(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"Connect to it:"),(0,r.yg)("pre",{parentName:"li"},(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"oc rsh POD_ID\n"))),(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("p",{parentName:"li"},"Enable Bash in the Ubuntu container (if it starts with the Shell)"),(0,r.yg)("pre",{parentName:"li"},(0,r.yg)("code",{parentName:"pre",className:"language-bash"},"bash\n")))),(0,r.yg)("img",{src:"/img/screenshot-deploy-ubuntu.png",alt:"Deploy Ubuntu",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("h3",{id:"with-a-web-ui"},"With a web UI"),(0,r.yg)("p",null,"Start Ubuntu with a web UI accessible via a URL (using VNC). You will be the ",(0,r.yg)("inlineCode",{parentName:"p"},"root")," user which has elevated permissions to install anything via ",(0,r.yg)("inlineCode",{parentName:"p"},"apt install "),". Before you install a package run ",(0,r.yg)("inlineCode",{parentName:"p"},"apt update"),". This also solves ",(0,r.yg)("inlineCode",{parentName:"p"},"E: unable to locate package")," and ",(0,r.yg)("inlineCode",{parentName:"p"},"E: no installation candidate")," errors."),(0,r.yg)("p",null,"You can start the application using the ",(0,r.yg)("strong",{parentName:"p"},"Ubuntu with web UI")," template in the ",(0,r.yg)("a",{parentName:"p",href:"https://console-openshift-console.apps.dsri2.unimaas.nl/catalog"},"Catalog web UI")," (make sure the ",(0,r.yg)("strong",{parentName:"p"},"Templates")," checkbox is checked)"),(0,r.yg)("admonition",{title:"Login Credentials",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"Username: ",(0,r.yg)("strong",{parentName:"p"},"root")),(0,r.yg)("p",{parentName:"admonition"},"Password: ",(0,r.yg)("strong",{parentName:"p"},"Template creation password"))),(0,r.yg)("p",null,"This template uses the Docker image defined at ",(0,r.yg)("a",{parentName:"p",href:"https://github.com/fcwu/docker-ubuntu-vnc-desktop"},"https://github.com/fcwu/docker-ubuntu-vnc-desktop")),(0,r.yg)("admonition",{title:"Less stable than the official image",type:"caution"},(0,r.yg)("p",{parentName:"admonition"},"This image might be less stable than the original Ubuntu image. Let us know on Slack if you have any problem!")),(0,r.yg)("h2",{id:"file-browser"},"File browser"),(0,r.yg)("p",null,"Deploy a file browser on your persistent volume. This will provide a web UI to upload and download data to your DSRI persistent volume in case you need it (JupyterLab, RStudio and VisualStudio Code server already include a file browser)"),(0,r.yg)("p",null,"You can start a container using the ",(0,r.yg)("strong",{parentName:"p"},"File Browser for existing storage")," template in the ",(0,r.yg)("a",{parentName:"p",href:"https://console-openshift-console.apps.dsri2.unimaas.nl/catalog"},"Catalog web UI")," (make sure the ",(0,r.yg)("strong",{parentName:"p"},"Templates")," checkbox is checked)"),(0,r.yg)("img",{src:"/img/screenshot-deploy-filebrowser.png",alt:"Deploy File browser",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("p",null,"You can only deploy file browser on an existing Persistent Volume Claim, this enables you to add a web UI to access this storage."),(0,r.yg)("p",null,"The following parameters can be provided:"),(0,r.yg)("ol",null,(0,r.yg)("li",{parentName:"ol"},"Provide a unique ",(0,r.yg)("strong",{parentName:"li"},"Application name"),". It will be used to generate the application URL."),(0,r.yg)("li",{parentName:"ol"},"Provide a ",(0,r.yg)("strong",{parentName:"li"},"Password"),", you will need to hash the password first for extra security, use this quick docker command to do it: ",(0,r.yg)("inlineCode",{parentName:"li"},"docker run filebrowser/filebrowser hash mypassword")),(0,r.yg)("li",{parentName:"ol"},"The ",(0,r.yg)("strong",{parentName:"li"},"Storage name")," of the Persistent Volume Claim (PVC) that will be exposed by the filebrowser."),(0,r.yg)("li",{parentName:"ol"},(0,r.yg)("strong",{parentName:"li"},"Storage subpath")," in the the Persistent Volume Claim that will be exposed by the filebrowser. Let it empty to use the Root folder of the persistent volume.")),(0,r.yg)("p",null,"You can find the Storage name if you Go to the deployments page > Storage panel."),(0,r.yg)("h3",{id:"creating-or-connecting-an-existing-persistent-storage"},"Creating or Connecting an Existing Persistent Storage"),(0,r.yg)("p",null,"Find more details about the how to ",(0,r.yg)("a",{parentName:"p",href:"https://maastrichtu-ids.github.io/dsri-documentation/docs/openshift-storage/#create-the-persistent-storage"},"create persistent storage")),(0,r.yg)("img",{src:"/img/screenshot_pvc_storage.png",alt:"Create Persistent Storage",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("img",{src:"/img/screenshot_pvc_storage_create.png",alt:"Create Persistent Storage",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("admonition",{type:"info"},(0,r.yg)("p",{parentName:"admonition"},"The DSRI using the ",(0,r.yg)("a",{parentName:"p",href:"https://www.openshift.com/products/container-storage/"},(0,r.yg)("strong",{parentName:"a"},"Openshift Container Stroage"))," (",(0,r.yg)("inlineCode",{parentName:"p"},"OCS"),") which is based on ",(0,r.yg)("a",{parentName:"p",href:"https://ceph.io/ceph-storage/"},(0,r.yg)("strong",{parentName:"a"},"CEPH"))," offers ",(0,r.yg)("inlineCode",{parentName:"p"},"ReadWriteOnce")," and ",(0,r.yg)("inlineCode",{parentName:"p"},"ReadWriteMany")," access mode. "),(0,r.yg)("ul",{parentName:"admonition"},(0,r.yg)("li",{parentName:"ul"},(0,r.yg)("inlineCode",{parentName:"li"},"ReadWriteOnce")," (",(0,r.yg)("a",{parentName:"li",href:"https://docs.openshift.com/container-platform/4.6/storage/understanding-persistent-storage.html"},(0,r.yg)("strong",{parentName:"a"},"RWO")),") volumes cannot be mounted on multiple nodes. Use the ",(0,r.yg)("inlineCode",{parentName:"li"},"ReadWriteMany")," (",(0,r.yg)("a",{parentName:"li",href:"https://docs.openshift.com/container-platform/4.6/storage/understanding-persistent-storage.html"},(0,r.yg)("strong",{parentName:"a"},"RWX")),") access mode when possible. If a node fails, the system does not allow the attached RWO volume to be mounted on a new node because it is already assigned to the failed node. If you encounter a multi-attach error message as a result, force delete the pod on a shut down or crashed node. "))),(0,r.yg)("p",null,"Find more details about the how to ",(0,r.yg)("a",{parentName:"p",href:"https://maastrichtu-ids.github.io/dsri-documentation/docs/openshift-storage/#connect-the-existing-persistent-storage"},"Connect the Existing persistent storage")),(0,r.yg)("img",{src:"/img/screenshot_existing_storage.png",alt:"Add Existing Persistent Storage",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("img",{src:"/img/screenshot_add_storage.png",alt:"Add Existing Persistent Storage",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,r.yg)("admonition",{type:"info"},(0,r.yg)("p",{parentName:"admonition"},"You can try above method if you want to connect ",(0,r.yg)("strong",{parentName:"p"},"more applications to the same storage"))),(0,r.yg)("p",null,"This deployment require to have root user enabled on your project. Contact the ",(0,r.yg)("a",{parentName:"p",href:"mailto:dsri-support-l@maastrichtuniversity.nl"},"DSRI support team")," or create a ",(0,r.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-documentation/issues"},"new issues")," to request root access or to create persistent volume for your project if you don't have them ."),(0,r.yg)("admonition",{title:"Credentials",type:"info"},(0,r.yg)("p",{parentName:"admonition"},"Default credentials will be username ",(0,r.yg)("inlineCode",{parentName:"p"},"admin")," and password ",(0,r.yg)("inlineCode",{parentName:"p"},"admin"))),(0,r.yg)("admonition",{title:"Change password",type:"caution"},(0,r.yg)("p",{parentName:"admonition"},"Please ",(0,r.yg)("strong",{parentName:"p"},"change the password in the Filebrowser Web UI")," once it has been created.")))}c.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/e9bae93a.b0c117fa.js b/assets/js/e9bae93a.69402e2c.js similarity index 98% rename from assets/js/e9bae93a.b0c117fa.js rename to assets/js/e9bae93a.69402e2c.js index 9298a0c49..257d0c8fa 100644 --- a/assets/js/e9bae93a.b0c117fa.js +++ b/assets/js/e9bae93a.69402e2c.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[7968],{5680:(e,t,r)=>{r.d(t,{xA:()=>d,yg:()=>m});var n=r(6540);function a(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function i(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function o(e){for(var t=1;t=0||(a[r]=e[r]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(a[r]=e[r])}return a}var l=n.createContext({}),c=function(e){var t=n.useContext(l),r=t;return e&&(r="function"==typeof e?e(t):o(o({},t),e)),r},d=function(e){var t=c(e.components);return n.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},p=n.forwardRef((function(e,t){var r=e.components,a=e.mdxType,i=e.originalType,l=e.parentName,d=s(e,["components","mdxType","originalType","parentName"]),p=c(r),m=a,y=p["".concat(l,".").concat(m)]||p[m]||u[m]||i;return r?n.createElement(y,o(o({ref:t},d),{},{components:r})):n.createElement(y,o({ref:t},d))}));function m(e,t){var r=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var i=r.length,o=new Array(i);o[0]=p;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s.mdxType="string"==typeof e?e:a,o[1]=s;for(var c=2;c{r.r(t),r.d(t,{assets:()=>d,contentTitle:()=>l,default:()=>m,frontMatter:()=>s,metadata:()=>c,toc:()=>u});var n=r(9668),a=r(1367),i=(r(6540),r(5680)),o=["components"],s={id:"sensible-data",title:"Working with sensible data"},l=void 0,c={unversionedId:"sensible-data",id:"sensible-data",title:"Working with sensible data",description:"Reminder: DSRI restrictions",source:"@site/docs/sensible-data.md",sourceDirName:".",slug:"/sensible-data",permalink:"/docs/sensible-data",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/sensible-data.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"sensible-data",title:"Working with sensible data"}},d={},u=[{value:"Reminder: DSRI restrictions",id:"reminder-dsri-restrictions",level:3},{value:"Disclaimer",id:"disclaimer",level:2}],p={toc:u};function m(e){var t=e.components,r=(0,a.A)(e,o);return(0,i.yg)("wrapper",(0,n.A)({},p,r,{components:t,mdxType:"MDXLayout"}),(0,i.yg)("h3",{id:"reminder-dsri-restrictions"},"Reminder: DSRI restrictions"),(0,i.yg)("ul",null,(0,i.yg)("li",{parentName:"ul"},"Since DSRI can only be accessed when on the physical UM network or using the ",(0,i.yg)("a",{parentName:"li",href:"https://vpn.maastrichtuniversity.nl/"},"UM VPN"),", deployed services will not be available on the public Internet \ud83d\udd12"),(0,i.yg)("li",{parentName:"ul"},"All activities must be legal in basis. You must closely examine and abide by the terms and conditions of any data, software, or web service that you use as part of your work \ud83d\udcdc")),(0,i.yg)("h2",{id:"disclaimer"},"Disclaimer"),(0,i.yg)("p",null,"The DSRI administration disclaims all responsibility in the misuse of sensible data processed on the DSRI "),(0,i.yg)("p",null,"We can guarantee you that only you, and 4 administrators are able to access the data (you might need to see with the data owner if that is not a problem)"),(0,i.yg)("p",null,"Feel to ask us more details"))}m.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[7968],{5680:(e,t,r)=>{r.d(t,{xA:()=>d,yg:()=>m});var n=r(6540);function a(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function i(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,n)}return r}function o(e){for(var t=1;t=0||(a[r]=e[r]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(a[r]=e[r])}return a}var l=n.createContext({}),c=function(e){var t=n.useContext(l),r=t;return e&&(r="function"==typeof e?e(t):o(o({},t),e)),r},d=function(e){var t=c(e.components);return n.createElement(l.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},p=n.forwardRef((function(e,t){var r=e.components,a=e.mdxType,i=e.originalType,l=e.parentName,d=s(e,["components","mdxType","originalType","parentName"]),p=c(r),m=a,y=p["".concat(l,".").concat(m)]||p[m]||u[m]||i;return r?n.createElement(y,o(o({ref:t},d),{},{components:r})):n.createElement(y,o({ref:t},d))}));function m(e,t){var r=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var i=r.length,o=new Array(i);o[0]=p;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s.mdxType="string"==typeof e?e:a,o[1]=s;for(var c=2;c{r.r(t),r.d(t,{assets:()=>d,contentTitle:()=>l,default:()=>m,frontMatter:()=>s,metadata:()=>c,toc:()=>u});var n=r(9668),a=r(1367),i=(r(6540),r(5680)),o=["components"],s={id:"sensible-data",title:"Working with sensible data"},l=void 0,c={unversionedId:"sensible-data",id:"sensible-data",title:"Working with sensible data",description:"Reminder: DSRI restrictions",source:"@site/docs/sensible-data.md",sourceDirName:".",slug:"/sensible-data",permalink:"/docs/sensible-data",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/sensible-data.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"sensible-data",title:"Working with sensible data"}},d={},u=[{value:"Reminder: DSRI restrictions",id:"reminder-dsri-restrictions",level:3},{value:"Disclaimer",id:"disclaimer",level:2}],p={toc:u};function m(e){var t=e.components,r=(0,a.A)(e,o);return(0,i.yg)("wrapper",(0,n.A)({},p,r,{components:t,mdxType:"MDXLayout"}),(0,i.yg)("h3",{id:"reminder-dsri-restrictions"},"Reminder: DSRI restrictions"),(0,i.yg)("ul",null,(0,i.yg)("li",{parentName:"ul"},"Since DSRI can only be accessed when on the physical UM network or using the ",(0,i.yg)("a",{parentName:"li",href:"https://vpn.maastrichtuniversity.nl/"},"UM VPN"),", deployed services will not be available on the public Internet \ud83d\udd12"),(0,i.yg)("li",{parentName:"ul"},"All activities must be legal in basis. You must closely examine and abide by the terms and conditions of any data, software, or web service that you use as part of your work \ud83d\udcdc")),(0,i.yg)("h2",{id:"disclaimer"},"Disclaimer"),(0,i.yg)("p",null,"The DSRI administration disclaims all responsibility in the misuse of sensible data processed on the DSRI "),(0,i.yg)("p",null,"We can guarantee you that only you, and 4 administrators are able to access the data (you might need to see with the data owner if that is not a problem)"),(0,i.yg)("p",null,"Feel to ask us more details"))}m.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/ecdc8e44.17da8799.js b/assets/js/ecdc8e44.2479a1fd.js similarity index 99% rename from assets/js/ecdc8e44.17da8799.js rename to assets/js/ecdc8e44.2479a1fd.js index 7f041b707..b0d622019 100644 --- a/assets/js/ecdc8e44.17da8799.js +++ b/assets/js/ecdc8e44.2479a1fd.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[7953],{5680:(e,t,a)=>{a.d(t,{xA:()=>c,yg:()=>g});var r=a(6540);function n(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function o(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,r)}return a}function p(e){for(var t=1;t=0||(n[a]=e[a]);return n}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(n[a]=e[a])}return n}var s=r.createContext({}),i=function(e){var t=r.useContext(s),a=t;return e&&(a="function"==typeof e?e(t):p(p({},t),e)),a},c=function(e){var t=i(e.components);return r.createElement(s.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},y=r.forwardRef((function(e,t){var a=e.components,n=e.mdxType,o=e.originalType,s=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),y=i(a),g=n,d=y["".concat(s,".").concat(g)]||y[g]||u[g]||o;return a?r.createElement(d,p(p({ref:t},c),{},{components:a})):r.createElement(d,p({ref:t},c))}));function g(e,t){var a=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var o=a.length,p=new Array(o);p[0]=y;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:n,p[1]=l;for(var i=2;i{a.r(t),a.d(t,{assets:()=>c,contentTitle:()=>s,default:()=>g,frontMatter:()=>l,metadata:()=>i,toc:()=>u});var r=a(9668),n=a(1367),o=(a(6540),a(5680)),p=["components"],l={id:"deploy-spark",title:"Spark cluster"},s=void 0,i={unversionedId:"deploy-spark",id:"deploy-spark",title:"Spark cluster",description:"To be able to deploy Spark you will need to ask the DSRI admins to enable the Spark Operator in your project. It will be done quickly, once enabled you will be able to start a Spark cluster in a few clicks.",source:"@site/docs/deploy-spark.md",sourceDirName:".",slug:"/deploy-spark",permalink:"/docs/deploy-spark",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/deploy-spark.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"deploy-spark",title:"Spark cluster"},sidebar:"docs",previous:{title:"Deploy Dask Cluster",permalink:"/docs/dask-cluster"},next:{title:"Run MPI jobs",permalink:"/docs/mpi-jobs"}},c={},u=[{value:"Deploy a Spark cluster",id:"deploy-a-spark-cluster",level:2},{value:"Deploy the cluster from the catalog",id:"deploy-the-cluster-from-the-catalog",level:3},{value:"Create a route to the Spark dashboard",id:"create-a-route-to-the-spark-dashboard",level:3},{value:"Run on Spark",id:"run-on-spark",level:2},{value:"Using PySpark",id:"using-pyspark",level:3},{value:"RDF analytics with SANSA and Zeppelin notebooks",id:"rdf-analytics-with-sansa-and-zeppelin-notebooks",level:3},{value:"Connect Spark to the persistent storage",id:"connect-spark-to-the-persistent-storage",level:2},{value:"Delete a running Spark cluster",id:"delete-a-running-spark-cluster",level:2}],y={toc:u};function g(e){var t=e.components,a=(0,n.A)(e,p);return(0,o.yg)("wrapper",(0,r.A)({},y,a,{components:t,mdxType:"MDXLayout"}),(0,o.yg)("admonition",{title:"Request access to the Spark Operator",type:"warning"},(0,o.yg)("p",{parentName:"admonition"},"To be able to deploy Spark you will need to ",(0,o.yg)("a",{parentName:"p",href:"/help"},"ask the DSRI admins")," to enable the Spark Operator in your project. It will be done quickly, once enabled you will be able to start a Spark cluster in a few clicks.")),(0,o.yg)("h2",{id:"deploy-a-spark-cluster"},"Deploy a Spark cluster"),(0,o.yg)("p",null,"Once the DSRI admins have enabled the Spark Operator your project, you should found a ",(0,o.yg)("strong",{parentName:"p"},"Spark Cluster")," entry in the Catalog (in the ",(0,o.yg)("strong",{parentName:"p"},"Operator Backed")," category)"),(0,o.yg)("h3",{id:"deploy-the-cluster-from-the-catalog"},"Deploy the cluster from the catalog"),(0,o.yg)("img",{src:"/img/screenshot-spark-operator1.png",alt:"Apache Spark in the Catalog",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("p",null,"Click on the ",(0,o.yg)("strong",{parentName:"p"},"Spark Cluster")," entry to deploy a Spark cluster."),(0,o.yg)("p",null,"You will be presented a form where you can provide the number of Spark workers in your cluster. "),(0,o.yg)("p",null,"Additionally you can provide a label which can be helpful later to manage or delete the cluster, use the name of your application and the label ",(0,o.yg)("inlineCode",{parentName:"p"},"app"),", e.g.: ",(0,o.yg)("inlineCode",{parentName:"p"},"app=my-spark-cluster")),(0,o.yg)("img",{src:"/img/screenshot-spark-operator2.png",alt:"Deploy a Apache Spark cluster",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("admonition",{title:"Change ",type:"tip"},(0,o.yg)("p",{parentName:"admonition"},"The number of Spark workers can be easily updated later in the Spark deployment YAML file.")),(0,o.yg)("h3",{id:"create-a-route-to-the-spark-dashboard"},"Create a route to the Spark dashboard"),(0,o.yg)("p",null,"Once the cluster has been started you can create a route to access the Spark web UI:"),(0,o.yg)("p",null,"Go to ",(0,o.yg)("strong",{parentName:"p"},"Search")," > Click on ",(0,o.yg)("strong",{parentName:"p"},"Resources")," and search for ",(0,o.yg)("strong",{parentName:"p"},"Route")," > Click on ",(0,o.yg)("strong",{parentName:"p"},"Route")),(0,o.yg)("p",null,"You should now see the routes deployed in your project. Click on the button ",(0,o.yg)("strong",{parentName:"p"},"Create Route")),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},"Give a short meaningful name to your route, e.g. ",(0,o.yg)("inlineCode",{parentName:"li"},"my-spark-ui")),(0,o.yg)("li",{parentName:"ul"},"Keep Hostname and Path as it is"),(0,o.yg)("li",{parentName:"ul"},"Select the ",(0,o.yg)("strong",{parentName:"li"},"Service")," corresponding your Spark cluster suffixed with ",(0,o.yg)("inlineCode",{parentName:"li"},"-ui"),", e.g. ",(0,o.yg)("inlineCode",{parentName:"li"},"my-spark-cluster-ui")),(0,o.yg)("li",{parentName:"ul"},"Select the ",(0,o.yg)("strong",{parentName:"li"},"Target Port")," of the route, it should be 8080")),(0,o.yg)("p",null,"You can now access the Spark web UI at the generated URL to see which jobs are running and the nodes in your cluster."),(0,o.yg)("h2",{id:"run-on-spark"},"Run on Spark"),(0,o.yg)("p",null,"You can now start a spark-enabled JupyterLab, or any other spark-enabled applications, to use the Spark cluster deployed."),(0,o.yg)("h3",{id:"using-pyspark"},"Using PySpark"),(0,o.yg)("p",null,"The easiest is to use a Spark-enabled JupyterLab image, such as ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/jupyter/docker-stacks/tree/master/pyspark-notebook"},"jupyter/pyspark-notebook")),(0,o.yg)("p",null,"But you can also use any image as long as you download the jar file, install all requirements, such as ",(0,o.yg)("inlineCode",{parentName:"p"},"pyspark"),", and set the right environment variable, such as ",(0,o.yg)("inlineCode",{parentName:"p"},"SPARK_HOME")),(0,o.yg)("p",null,"Connect to a Spark cluster deployed in the same project, replace ",(0,o.yg)("inlineCode",{parentName:"p"},"spark-cluster")," by your Spark cluster name:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},"from pyspark import SparkConf, SparkContext\nfrom pyspark.sql import SparkSession\n# Stop existing Spark Context\nspark = SparkSession.builder.master(\"spark://spark-cluster:7077\").getOrCreate()\nspark.sparkContext.stop()\n# Connect to the Spark cluster\nconf = SparkConf().setAppName('sansa').setMaster('spark://spark-cluster:7077') \nsc = SparkContext(conf=conf)\n\n# Run basic Spark test\nx = ['spark', 'rdd', 'example', 'sample', 'example'] \ny = sc.parallelize(x)\ny.collect()\n")),(0,o.yg)("h3",{id:"rdf-analytics-with-sansa-and-zeppelin-notebooks"},"RDF analytics with SANSA and Zeppelin notebooks"),(0,o.yg)("p",null,(0,o.yg)("a",{parentName:"p",href:"http://sansa-stack.net"},"SANSA")," is a big data engine for scalable processing of large-scale RDF data. SANSA uses Spark, or Flink, which offer fault-tolerant, highly available and scalable approaches to efficiently process massive sized datasets. SANSA provides the facilities for Semantic data representation, Querying, Inference, and Analytics."),(0,o.yg)("p",null,"Use the ",(0,o.yg)("strong",{parentName:"p"},"Zeppelin notebook for Spark")," template in the catalog to start a Spark-enabled Zeppelin notebook. You can find more information on the Zeppelin image at ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/rimolive/zeppelin-openshift"},"https://github.com/rimolive/zeppelin-openshift")),(0,o.yg)("p",null,"Connect and test Spark in a Zeppelin notebook, replace ",(0,o.yg)("inlineCode",{parentName:"p"},"spark-cluster")," by your Spark cluster name:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},"%pyspark\nfrom pyspark import SparkConf, SparkContext\nfrom pyspark.sql import SparkSession\n# Stop existing Spark Context\nspark = SparkSession.builder.master(\"spark://spark-cluster:7077\").getOrCreate()\nspark.sparkContext.stop()\n# Connect to the Spark cluster\nconf = SparkConf().setAppName('sansa').setMaster('spark://spark-cluster:7077') \nsc = SparkContext(conf=conf)\n\n# Run basic Spark test\nx = [1, 2, 3, 4, 5] \ny = sc.parallelize(x)\ny.collect()\n")),(0,o.yg)("p",null,"You should see the job running in the Spark web UI, kill the job with the ",(0,o.yg)("strong",{parentName:"p"},"kill")," button in the Spark dashboard."),(0,o.yg)("p",null,"You can now start to run your workload on the Spark cluster"),(0,o.yg)("admonition",{title:"Reset a Zeppelin notebook",type:"info"},(0,o.yg)("p",{parentName:"admonition"},"Click on the cranked wheel in the top right of the note: ",(0,o.yg)("strong",{parentName:"p"},"Interpreter binding"),", and reset the interpreter")),(0,o.yg)("p",null,"Use the official ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/SANSA-Stack/SANSA-Notebooks/tree/stack-merge/sansa-notebooks"},"SANSA notebooks examples")),(0,o.yg)("p",null,"See more examples:"),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("a",{parentName:"li",href:"https://github.com/rimolive/zeppelin-openshift"},"https://github.com/rimolive/zeppelin-openshift"))),(0,o.yg)("h2",{id:"connect-spark-to-the-persistent-storage"},"Connect Spark to the persistent storage"),(0,o.yg)("p",null,"Instructions available at ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/rimolive/ceph-spark-integration"},"https://github.com/rimolive/ceph-spark-integration")),(0,o.yg)("p",null,"Requirements:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"pip install boto\n")),(0,o.yg)("p",null,"Check the ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/rimolive/ceph-spark-integration/blob/master/notebooks/ceph-example.ipynb"},"example notebook for Ceph storage")),(0,o.yg)("h2",{id:"delete-a-running-spark-cluster"},"Delete a running Spark cluster"),(0,o.yg)("p",null,"Get all objects part of the Spark cluster, change ",(0,o.yg)("inlineCode",{parentName:"p"},"app=spark-cluster")," to match your Spark cluster name:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"oc get all,secret,configmaps --selector app=spark-cluster\n")),(0,o.yg)("p",null,"Then delete the Operator deployment from the OpenShift web UI overview."))}g.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[7953],{5680:(e,t,a)=>{a.d(t,{xA:()=>c,yg:()=>g});var r=a(6540);function n(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function o(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,r)}return a}function p(e){for(var t=1;t=0||(n[a]=e[a]);return n}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(n[a]=e[a])}return n}var s=r.createContext({}),i=function(e){var t=r.useContext(s),a=t;return e&&(a="function"==typeof e?e(t):p(p({},t),e)),a},c=function(e){var t=i(e.components);return r.createElement(s.Provider,{value:t},e.children)},u={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},y=r.forwardRef((function(e,t){var a=e.components,n=e.mdxType,o=e.originalType,s=e.parentName,c=l(e,["components","mdxType","originalType","parentName"]),y=i(a),g=n,d=y["".concat(s,".").concat(g)]||y[g]||u[g]||o;return a?r.createElement(d,p(p({ref:t},c),{},{components:a})):r.createElement(d,p({ref:t},c))}));function g(e,t){var a=arguments,n=t&&t.mdxType;if("string"==typeof e||n){var o=a.length,p=new Array(o);p[0]=y;var l={};for(var s in t)hasOwnProperty.call(t,s)&&(l[s]=t[s]);l.originalType=e,l.mdxType="string"==typeof e?e:n,p[1]=l;for(var i=2;i{a.r(t),a.d(t,{assets:()=>c,contentTitle:()=>s,default:()=>g,frontMatter:()=>l,metadata:()=>i,toc:()=>u});var r=a(9668),n=a(1367),o=(a(6540),a(5680)),p=["components"],l={id:"deploy-spark",title:"Spark cluster"},s=void 0,i={unversionedId:"deploy-spark",id:"deploy-spark",title:"Spark cluster",description:"To be able to deploy Spark you will need to ask the DSRI admins to enable the Spark Operator in your project. It will be done quickly, once enabled you will be able to start a Spark cluster in a few clicks.",source:"@site/docs/deploy-spark.md",sourceDirName:".",slug:"/deploy-spark",permalink:"/docs/deploy-spark",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/deploy-spark.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"deploy-spark",title:"Spark cluster"},sidebar:"docs",previous:{title:"Deploy Dask Cluster",permalink:"/docs/dask-cluster"},next:{title:"Run MPI jobs",permalink:"/docs/mpi-jobs"}},c={},u=[{value:"Deploy a Spark cluster",id:"deploy-a-spark-cluster",level:2},{value:"Deploy the cluster from the catalog",id:"deploy-the-cluster-from-the-catalog",level:3},{value:"Create a route to the Spark dashboard",id:"create-a-route-to-the-spark-dashboard",level:3},{value:"Run on Spark",id:"run-on-spark",level:2},{value:"Using PySpark",id:"using-pyspark",level:3},{value:"RDF analytics with SANSA and Zeppelin notebooks",id:"rdf-analytics-with-sansa-and-zeppelin-notebooks",level:3},{value:"Connect Spark to the persistent storage",id:"connect-spark-to-the-persistent-storage",level:2},{value:"Delete a running Spark cluster",id:"delete-a-running-spark-cluster",level:2}],y={toc:u};function g(e){var t=e.components,a=(0,n.A)(e,p);return(0,o.yg)("wrapper",(0,r.A)({},y,a,{components:t,mdxType:"MDXLayout"}),(0,o.yg)("admonition",{title:"Request access to the Spark Operator",type:"warning"},(0,o.yg)("p",{parentName:"admonition"},"To be able to deploy Spark you will need to ",(0,o.yg)("a",{parentName:"p",href:"/help"},"ask the DSRI admins")," to enable the Spark Operator in your project. It will be done quickly, once enabled you will be able to start a Spark cluster in a few clicks.")),(0,o.yg)("h2",{id:"deploy-a-spark-cluster"},"Deploy a Spark cluster"),(0,o.yg)("p",null,"Once the DSRI admins have enabled the Spark Operator your project, you should found a ",(0,o.yg)("strong",{parentName:"p"},"Spark Cluster")," entry in the Catalog (in the ",(0,o.yg)("strong",{parentName:"p"},"Operator Backed")," category)"),(0,o.yg)("h3",{id:"deploy-the-cluster-from-the-catalog"},"Deploy the cluster from the catalog"),(0,o.yg)("img",{src:"/img/screenshot-spark-operator1.png",alt:"Apache Spark in the Catalog",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("p",null,"Click on the ",(0,o.yg)("strong",{parentName:"p"},"Spark Cluster")," entry to deploy a Spark cluster."),(0,o.yg)("p",null,"You will be presented a form where you can provide the number of Spark workers in your cluster. "),(0,o.yg)("p",null,"Additionally you can provide a label which can be helpful later to manage or delete the cluster, use the name of your application and the label ",(0,o.yg)("inlineCode",{parentName:"p"},"app"),", e.g.: ",(0,o.yg)("inlineCode",{parentName:"p"},"app=my-spark-cluster")),(0,o.yg)("img",{src:"/img/screenshot-spark-operator2.png",alt:"Deploy a Apache Spark cluster",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,o.yg)("admonition",{title:"Change ",type:"tip"},(0,o.yg)("p",{parentName:"admonition"},"The number of Spark workers can be easily updated later in the Spark deployment YAML file.")),(0,o.yg)("h3",{id:"create-a-route-to-the-spark-dashboard"},"Create a route to the Spark dashboard"),(0,o.yg)("p",null,"Once the cluster has been started you can create a route to access the Spark web UI:"),(0,o.yg)("p",null,"Go to ",(0,o.yg)("strong",{parentName:"p"},"Search")," > Click on ",(0,o.yg)("strong",{parentName:"p"},"Resources")," and search for ",(0,o.yg)("strong",{parentName:"p"},"Route")," > Click on ",(0,o.yg)("strong",{parentName:"p"},"Route")),(0,o.yg)("p",null,"You should now see the routes deployed in your project. Click on the button ",(0,o.yg)("strong",{parentName:"p"},"Create Route")),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},"Give a short meaningful name to your route, e.g. ",(0,o.yg)("inlineCode",{parentName:"li"},"my-spark-ui")),(0,o.yg)("li",{parentName:"ul"},"Keep Hostname and Path as it is"),(0,o.yg)("li",{parentName:"ul"},"Select the ",(0,o.yg)("strong",{parentName:"li"},"Service")," corresponding your Spark cluster suffixed with ",(0,o.yg)("inlineCode",{parentName:"li"},"-ui"),", e.g. ",(0,o.yg)("inlineCode",{parentName:"li"},"my-spark-cluster-ui")),(0,o.yg)("li",{parentName:"ul"},"Select the ",(0,o.yg)("strong",{parentName:"li"},"Target Port")," of the route, it should be 8080")),(0,o.yg)("p",null,"You can now access the Spark web UI at the generated URL to see which jobs are running and the nodes in your cluster."),(0,o.yg)("h2",{id:"run-on-spark"},"Run on Spark"),(0,o.yg)("p",null,"You can now start a spark-enabled JupyterLab, or any other spark-enabled applications, to use the Spark cluster deployed."),(0,o.yg)("h3",{id:"using-pyspark"},"Using PySpark"),(0,o.yg)("p",null,"The easiest is to use a Spark-enabled JupyterLab image, such as ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/jupyter/docker-stacks/tree/master/pyspark-notebook"},"jupyter/pyspark-notebook")),(0,o.yg)("p",null,"But you can also use any image as long as you download the jar file, install all requirements, such as ",(0,o.yg)("inlineCode",{parentName:"p"},"pyspark"),", and set the right environment variable, such as ",(0,o.yg)("inlineCode",{parentName:"p"},"SPARK_HOME")),(0,o.yg)("p",null,"Connect to a Spark cluster deployed in the same project, replace ",(0,o.yg)("inlineCode",{parentName:"p"},"spark-cluster")," by your Spark cluster name:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},"from pyspark import SparkConf, SparkContext\nfrom pyspark.sql import SparkSession\n# Stop existing Spark Context\nspark = SparkSession.builder.master(\"spark://spark-cluster:7077\").getOrCreate()\nspark.sparkContext.stop()\n# Connect to the Spark cluster\nconf = SparkConf().setAppName('sansa').setMaster('spark://spark-cluster:7077') \nsc = SparkContext(conf=conf)\n\n# Run basic Spark test\nx = ['spark', 'rdd', 'example', 'sample', 'example'] \ny = sc.parallelize(x)\ny.collect()\n")),(0,o.yg)("h3",{id:"rdf-analytics-with-sansa-and-zeppelin-notebooks"},"RDF analytics with SANSA and Zeppelin notebooks"),(0,o.yg)("p",null,(0,o.yg)("a",{parentName:"p",href:"http://sansa-stack.net"},"SANSA")," is a big data engine for scalable processing of large-scale RDF data. SANSA uses Spark, or Flink, which offer fault-tolerant, highly available and scalable approaches to efficiently process massive sized datasets. SANSA provides the facilities for Semantic data representation, Querying, Inference, and Analytics."),(0,o.yg)("p",null,"Use the ",(0,o.yg)("strong",{parentName:"p"},"Zeppelin notebook for Spark")," template in the catalog to start a Spark-enabled Zeppelin notebook. You can find more information on the Zeppelin image at ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/rimolive/zeppelin-openshift"},"https://github.com/rimolive/zeppelin-openshift")),(0,o.yg)("p",null,"Connect and test Spark in a Zeppelin notebook, replace ",(0,o.yg)("inlineCode",{parentName:"p"},"spark-cluster")," by your Spark cluster name:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-python"},"%pyspark\nfrom pyspark import SparkConf, SparkContext\nfrom pyspark.sql import SparkSession\n# Stop existing Spark Context\nspark = SparkSession.builder.master(\"spark://spark-cluster:7077\").getOrCreate()\nspark.sparkContext.stop()\n# Connect to the Spark cluster\nconf = SparkConf().setAppName('sansa').setMaster('spark://spark-cluster:7077') \nsc = SparkContext(conf=conf)\n\n# Run basic Spark test\nx = [1, 2, 3, 4, 5] \ny = sc.parallelize(x)\ny.collect()\n")),(0,o.yg)("p",null,"You should see the job running in the Spark web UI, kill the job with the ",(0,o.yg)("strong",{parentName:"p"},"kill")," button in the Spark dashboard."),(0,o.yg)("p",null,"You can now start to run your workload on the Spark cluster"),(0,o.yg)("admonition",{title:"Reset a Zeppelin notebook",type:"info"},(0,o.yg)("p",{parentName:"admonition"},"Click on the cranked wheel in the top right of the note: ",(0,o.yg)("strong",{parentName:"p"},"Interpreter binding"),", and reset the interpreter")),(0,o.yg)("p",null,"Use the official ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/SANSA-Stack/SANSA-Notebooks/tree/stack-merge/sansa-notebooks"},"SANSA notebooks examples")),(0,o.yg)("p",null,"See more examples:"),(0,o.yg)("ul",null,(0,o.yg)("li",{parentName:"ul"},(0,o.yg)("a",{parentName:"li",href:"https://github.com/rimolive/zeppelin-openshift"},"https://github.com/rimolive/zeppelin-openshift"))),(0,o.yg)("h2",{id:"connect-spark-to-the-persistent-storage"},"Connect Spark to the persistent storage"),(0,o.yg)("p",null,"Instructions available at ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/rimolive/ceph-spark-integration"},"https://github.com/rimolive/ceph-spark-integration")),(0,o.yg)("p",null,"Requirements:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"pip install boto\n")),(0,o.yg)("p",null,"Check the ",(0,o.yg)("a",{parentName:"p",href:"https://github.com/rimolive/ceph-spark-integration/blob/master/notebooks/ceph-example.ipynb"},"example notebook for Ceph storage")),(0,o.yg)("h2",{id:"delete-a-running-spark-cluster"},"Delete a running Spark cluster"),(0,o.yg)("p",null,"Get all objects part of the Spark cluster, change ",(0,o.yg)("inlineCode",{parentName:"p"},"app=spark-cluster")," to match your Spark cluster name:"),(0,o.yg)("pre",null,(0,o.yg)("code",{parentName:"pre",className:"language-bash"},"oc get all,secret,configmaps --selector app=spark-cluster\n")),(0,o.yg)("p",null,"Then delete the Operator deployment from the OpenShift web UI overview."))}g.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/ed5ef82a.0153a628.js b/assets/js/ed5ef82a.cb577110.js similarity index 99% rename from assets/js/ed5ef82a.0153a628.js rename to assets/js/ed5ef82a.cb577110.js index 39fa14b63..a4046f468 100644 --- a/assets/js/ed5ef82a.0153a628.js +++ b/assets/js/ed5ef82a.cb577110.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[6683],{5680:(e,t,o)=>{o.d(t,{xA:()=>u,yg:()=>g});var n=o(6540);function r(e,t,o){return t in e?Object.defineProperty(e,t,{value:o,enumerable:!0,configurable:!0,writable:!0}):e[t]=o,e}function a(e,t){var o=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),o.push.apply(o,n)}return o}function i(e){for(var t=1;t=0||(r[o]=e[o]);return r}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,o)&&(r[o]=e[o])}return r}var l=n.createContext({}),p=function(e){var t=n.useContext(l),o=t;return e&&(o="function"==typeof e?e(t):i(i({},t),e)),o},u=function(e){var t=p(e.components);return n.createElement(l.Provider,{value:t},e.children)},c={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},d=n.forwardRef((function(e,t){var o=e.components,r=e.mdxType,a=e.originalType,l=e.parentName,u=s(e,["components","mdxType","originalType","parentName"]),d=p(o),g=r,m=d["".concat(l,".").concat(g)]||d[g]||c[g]||a;return o?n.createElement(m,i(i({ref:t},u),{},{components:o})):n.createElement(m,i({ref:t},u))}));function g(e,t){var o=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=o.length,i=new Array(a);i[0]=d;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s.mdxType="string"==typeof e?e:r,i[1]=s;for(var p=2;p{o.r(t),o.d(t,{assets:()=>u,contentTitle:()=>l,default:()=>g,frontMatter:()=>s,metadata:()=>p,toc:()=>c});var n=o(9668),r=o(1367),a=(o(6540),o(5680)),i=["components"],s={id:"deploy-rstudio",title:"RStudio"},l=void 0,p={unversionedId:"deploy-rstudio",id:"deploy-rstudio",title:"RStudio",description:"Start RStudio",source:"@site/docs/deploy-rstudio.md",sourceDirName:".",slug:"/deploy-rstudio",permalink:"/docs/deploy-rstudio",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/deploy-rstudio.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"deploy-rstudio",title:"RStudio"},sidebar:"docs",previous:{title:"Jupyter Notebooks",permalink:"/docs/deploy-jupyter"},next:{title:"VisualStudio Code",permalink:"/docs/deploy-vscode"}},u={},c=[{value:"Start RStudio",id:"start-rstudio",level:2},{value:"Use Git in RStudio",id:"use-git-in-rstudio",level:2},{value:"Run R jobs",id:"run-r-jobs",level:2}],d={toc:c};function g(e){var t=e.components,o=(0,r.A)(e,i);return(0,a.yg)("wrapper",(0,n.A)({},d,o,{components:t,mdxType:"MDXLayout"}),(0,a.yg)("h2",{id:"start-rstudio"},"Start RStudio"),(0,a.yg)("p",null,"Start a RStudio container based on ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/rocker-org/rocker/wiki/Using-the-RStudio-image"},"Rocker RStudio tidyverse images")," (debian), with ",(0,a.yg)("inlineCode",{parentName:"p"},"sudo")," privileges to install anything you need (e.g. pip or apt packages)"),(0,a.yg)("p",null,"You can start a container using the ",(0,a.yg)("strong",{parentName:"p"},"RStudio")," template in the ",(0,a.yg)("a",{parentName:"p",href:"https://console-openshift-console.apps.dsri2.unimaas.nl/catalog"},"Catalog web UI")," (make sure the ",(0,a.yg)("strong",{parentName:"p"},"Templates")," checkbox is checked)"),(0,a.yg)("p",null,"Provide a few parameters, and Instantiate the template. The ",(0,a.yg)("strong",{parentName:"p"},"username")," will be ",(0,a.yg)("inlineCode",{parentName:"p"},"root")," and the ",(0,a.yg)("strong",{parentName:"p"},"password")," will be what you configure yourself, the DSRI will automatically create a persistent volume in the project space which is the ",(0,a.yg)("inlineCode",{parentName:"p"},"/root")," folder. You can find the persistent volumes in the DSRI web UI, go to the ",(0,a.yg)("strong",{parentName:"p"},"Administrator")," view > ",(0,a.yg)("strong",{parentName:"p"},"Storage")," > ",(0,a.yg)("strong",{parentName:"p"},"Persistent Volume Claims"),"."),(0,a.yg)("img",{src:"/img/screenshot-deploy-rstudio.png",alt:"Deploy RStudio",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,a.yg)("admonition",{title:"Official image documentation",type:"info"},(0,a.yg)("p",{parentName:"admonition"},"See the ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/rocker-org/rocker/wiki/Using-the-RStudio-image"},"official Docker image documentation")," for more details about the container deployed.")),(0,a.yg)("h2",{id:"use-git-in-rstudio"},"Use Git in RStudio"),(0,a.yg)("p",null,"The fastest way to get started is to use ",(0,a.yg)("inlineCode",{parentName:"p"},"git")," from the terminal, for example to clone a git repository use ",(0,a.yg)("inlineCode",{parentName:"p"},"git clone")),(0,a.yg)("p",null,"You can also check how to enable Git integration in RStudio at ",(0,a.yg)("a",{parentName:"p",href:"https://support.rstudio.com/hc/en-us/articles/200532077"},"https://support.rstudio.com/hc/en-us/articles/200532077")),(0,a.yg)("p",null,"You can run this command to ask git to save your password for 15min:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},"git config credential.helper cache\n")),(0,a.yg)("p",null,"Or store the password/token in a plain text file:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},"git config --global credential.helper 'store --file ~/.git-credentials'\n")),(0,a.yg)("p",null,"Before pushing back to GitHub or GitLab, you will need to ",(0,a.yg)("strong",{parentName:"p"},"configure you username and email")," in the terminal:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},'git config --global user.name "Jean Dupont"\ngit config --global user.email jeandupont@gmail.com\n')),(0,a.yg)("admonition",{title:"Git tip",type:"tip"},(0,a.yg)("p",{parentName:"admonition"},"We recommend to use SSH instead of HTTPS connection when possible, checkout ",(0,a.yg)("a",{parentName:"p",href:"https://docs.github.com/en/free-pro-team@latest/github/authenticating-to-github/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent"},"here")," how to generate SSH keys and use them with your GitHub account.")),(0,a.yg)("h2",{id:"run-r-jobs"},"Run R jobs"),(0,a.yg)("p",null,"You can visit this folder that gives all resources and instructions to explain how to run a standalone R job on the DSRI: ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-demo/tree/main/r-job"},"https://github.com/MaastrichtU-IDS/dsri-demo/tree/main/r-job")),(0,a.yg)("p",null,"If you want to run jobs directly from RStudio, checkout this package to run chunks of R code as jobs directly through RStudio: ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/lindeloev/job"},"https://github.com/lindeloev/job")))}g.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[6683],{5680:(e,t,o)=>{o.d(t,{xA:()=>u,yg:()=>g});var n=o(6540);function r(e,t,o){return t in e?Object.defineProperty(e,t,{value:o,enumerable:!0,configurable:!0,writable:!0}):e[t]=o,e}function a(e,t){var o=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),o.push.apply(o,n)}return o}function i(e){for(var t=1;t=0||(r[o]=e[o]);return r}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,o)&&(r[o]=e[o])}return r}var l=n.createContext({}),p=function(e){var t=n.useContext(l),o=t;return e&&(o="function"==typeof e?e(t):i(i({},t),e)),o},u=function(e){var t=p(e.components);return n.createElement(l.Provider,{value:t},e.children)},c={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},d=n.forwardRef((function(e,t){var o=e.components,r=e.mdxType,a=e.originalType,l=e.parentName,u=s(e,["components","mdxType","originalType","parentName"]),d=p(o),g=r,m=d["".concat(l,".").concat(g)]||d[g]||c[g]||a;return o?n.createElement(m,i(i({ref:t},u),{},{components:o})):n.createElement(m,i({ref:t},u))}));function g(e,t){var o=arguments,r=t&&t.mdxType;if("string"==typeof e||r){var a=o.length,i=new Array(a);i[0]=d;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s.mdxType="string"==typeof e?e:r,i[1]=s;for(var p=2;p{o.r(t),o.d(t,{assets:()=>u,contentTitle:()=>l,default:()=>g,frontMatter:()=>s,metadata:()=>p,toc:()=>c});var n=o(9668),r=o(1367),a=(o(6540),o(5680)),i=["components"],s={id:"deploy-rstudio",title:"RStudio"},l=void 0,p={unversionedId:"deploy-rstudio",id:"deploy-rstudio",title:"RStudio",description:"Start RStudio",source:"@site/docs/deploy-rstudio.md",sourceDirName:".",slug:"/deploy-rstudio",permalink:"/docs/deploy-rstudio",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/deploy-rstudio.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"deploy-rstudio",title:"RStudio"},sidebar:"docs",previous:{title:"Jupyter Notebooks",permalink:"/docs/deploy-jupyter"},next:{title:"VisualStudio Code",permalink:"/docs/deploy-vscode"}},u={},c=[{value:"Start RStudio",id:"start-rstudio",level:2},{value:"Use Git in RStudio",id:"use-git-in-rstudio",level:2},{value:"Run R jobs",id:"run-r-jobs",level:2}],d={toc:c};function g(e){var t=e.components,o=(0,r.A)(e,i);return(0,a.yg)("wrapper",(0,n.A)({},d,o,{components:t,mdxType:"MDXLayout"}),(0,a.yg)("h2",{id:"start-rstudio"},"Start RStudio"),(0,a.yg)("p",null,"Start a RStudio container based on ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/rocker-org/rocker/wiki/Using-the-RStudio-image"},"Rocker RStudio tidyverse images")," (debian), with ",(0,a.yg)("inlineCode",{parentName:"p"},"sudo")," privileges to install anything you need (e.g. pip or apt packages)"),(0,a.yg)("p",null,"You can start a container using the ",(0,a.yg)("strong",{parentName:"p"},"RStudio")," template in the ",(0,a.yg)("a",{parentName:"p",href:"https://console-openshift-console.apps.dsri2.unimaas.nl/catalog"},"Catalog web UI")," (make sure the ",(0,a.yg)("strong",{parentName:"p"},"Templates")," checkbox is checked)"),(0,a.yg)("p",null,"Provide a few parameters, and Instantiate the template. The ",(0,a.yg)("strong",{parentName:"p"},"username")," will be ",(0,a.yg)("inlineCode",{parentName:"p"},"root")," and the ",(0,a.yg)("strong",{parentName:"p"},"password")," will be what you configure yourself, the DSRI will automatically create a persistent volume in the project space which is the ",(0,a.yg)("inlineCode",{parentName:"p"},"/root")," folder. You can find the persistent volumes in the DSRI web UI, go to the ",(0,a.yg)("strong",{parentName:"p"},"Administrator")," view > ",(0,a.yg)("strong",{parentName:"p"},"Storage")," > ",(0,a.yg)("strong",{parentName:"p"},"Persistent Volume Claims"),"."),(0,a.yg)("img",{src:"/img/screenshot-deploy-rstudio.png",alt:"Deploy RStudio",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,a.yg)("admonition",{title:"Official image documentation",type:"info"},(0,a.yg)("p",{parentName:"admonition"},"See the ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/rocker-org/rocker/wiki/Using-the-RStudio-image"},"official Docker image documentation")," for more details about the container deployed.")),(0,a.yg)("h2",{id:"use-git-in-rstudio"},"Use Git in RStudio"),(0,a.yg)("p",null,"The fastest way to get started is to use ",(0,a.yg)("inlineCode",{parentName:"p"},"git")," from the terminal, for example to clone a git repository use ",(0,a.yg)("inlineCode",{parentName:"p"},"git clone")),(0,a.yg)("p",null,"You can also check how to enable Git integration in RStudio at ",(0,a.yg)("a",{parentName:"p",href:"https://support.rstudio.com/hc/en-us/articles/200532077"},"https://support.rstudio.com/hc/en-us/articles/200532077")),(0,a.yg)("p",null,"You can run this command to ask git to save your password for 15min:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},"git config credential.helper cache\n")),(0,a.yg)("p",null,"Or store the password/token in a plain text file:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},"git config --global credential.helper 'store --file ~/.git-credentials'\n")),(0,a.yg)("p",null,"Before pushing back to GitHub or GitLab, you will need to ",(0,a.yg)("strong",{parentName:"p"},"configure you username and email")," in the terminal:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-bash"},'git config --global user.name "Jean Dupont"\ngit config --global user.email jeandupont@gmail.com\n')),(0,a.yg)("admonition",{title:"Git tip",type:"tip"},(0,a.yg)("p",{parentName:"admonition"},"We recommend to use SSH instead of HTTPS connection when possible, checkout ",(0,a.yg)("a",{parentName:"p",href:"https://docs.github.com/en/free-pro-team@latest/github/authenticating-to-github/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent"},"here")," how to generate SSH keys and use them with your GitHub account.")),(0,a.yg)("h2",{id:"run-r-jobs"},"Run R jobs"),(0,a.yg)("p",null,"You can visit this folder that gives all resources and instructions to explain how to run a standalone R job on the DSRI: ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-demo/tree/main/r-job"},"https://github.com/MaastrichtU-IDS/dsri-demo/tree/main/r-job")),(0,a.yg)("p",null,"If you want to run jobs directly from RStudio, checkout this package to run chunks of R code as jobs directly through RStudio: ",(0,a.yg)("a",{parentName:"p",href:"https://github.com/lindeloev/job"},"https://github.com/lindeloev/job")))}g.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/f704770b.3ac15f02.js b/assets/js/f704770b.51277459.js similarity index 99% rename from assets/js/f704770b.3ac15f02.js rename to assets/js/f704770b.51277459.js index fe3c2e927..e62c7b3d2 100644 --- a/assets/js/f704770b.3ac15f02.js +++ b/assets/js/f704770b.51277459.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[3870],{5680:(e,t,a)=>{a.d(t,{xA:()=>p,yg:()=>y});var n=a(6540);function o(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function l(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function r(e){for(var t=1;t=0||(o[a]=e[a]);return o}(e,t);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(o[a]=e[a])}return o}var s=n.createContext({}),u=function(e){var t=n.useContext(s),a=t;return e&&(a="function"==typeof e?e(t):r(r({},t),e)),a},p=function(e){var t=u(e.components);return n.createElement(s.Provider,{value:t},e.children)},c={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},m=n.forwardRef((function(e,t){var a=e.components,o=e.mdxType,l=e.originalType,s=e.parentName,p=i(e,["components","mdxType","originalType","parentName"]),m=u(a),y=o,d=m["".concat(s,".").concat(y)]||m[y]||c[y]||l;return a?n.createElement(d,r(r({ref:t},p),{},{components:a})):n.createElement(d,r({ref:t},p))}));function y(e,t){var a=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var l=a.length,r=new Array(l);r[0]=m;var i={};for(var s in t)hasOwnProperty.call(t,s)&&(i[s]=t[s]);i.originalType=e,i.mdxType="string"==typeof e?e:o,r[1]=i;for(var u=2;u{a.r(t),a.d(t,{assets:()=>p,contentTitle:()=>s,default:()=>y,frontMatter:()=>i,metadata:()=>u,toc:()=>c});var n=a(9668),o=a(1367),l=(a(6540),a(5680)),r=["components"],i={id:"deploy-matlab",title:"Matlab"},s=void 0,u={unversionedId:"deploy-matlab",id:"deploy-matlab",title:"Matlab",description:"Note that we are not expert in Matlab: feel free to contact Mathworks support directly if you are having any issues with their official Docker image. Because since it's closed source we cannot fix it ourselves.",source:"@site/docs/deploy-matlab.md",sourceDirName:".",slug:"/deploy-matlab",permalink:"/docs/deploy-matlab",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/deploy-matlab.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"deploy-matlab",title:"Matlab"},sidebar:"docs",previous:{title:"Databases",permalink:"/docs/deploy-database"},next:{title:"JupyterHub",permalink:"/docs/deploy-jupyterhub"}},p={},c=[{value:"Use the official Matlab image",id:"use-the-official-matlab-image",level:2},{value:"Use a stable Matlab image",id:"use-a-stable-matlab-image",level:2},{value:"Use Matlab in Jupyter",id:"use-matlab-in-jupyter",level:2},{value:"Deploy Matlab on GPU",id:"deploy-matlab-on-gpu",level:2},{value:"Build your own Matlab image",id:"build-your-own-matlab-image",level:2}],m={toc:c};function y(e){var t=e.components,a=(0,o.A)(e,r);return(0,l.yg)("wrapper",(0,n.A)({},m,a,{components:t,mdxType:"MDXLayout"}),(0,l.yg)("p",null,"Note that we are not expert in Matlab: feel free to contact Mathworks support directly if you are having any issues with their official Docker image. Because since it's closed source we cannot fix it ourselves."),(0,l.yg)("p",null,"You can request official support from Matlab at this address after login and connecting your account to the UM license: ",(0,l.yg)("a",{parentName:"p",href:"https://nl.mathworks.com/academia/tah-portal/maastricht-university-31574866.html#get"},"https://nl.mathworks.com/academia/tah-portal/maastricht-university-31574866.html#get")),(0,l.yg)("h2",{id:"use-the-official-matlab-image"},"Use the official Matlab image"),(0,l.yg)("p",null,"Start Matlab with a desktop UI accessible directly using your web browser at a URL automatically generated. "),(0,l.yg)("p",null,"Go to the ",(0,l.yg)("strong",{parentName:"p"},"Catalog"),", make sure ",(0,l.yg)("strong",{parentName:"p"},"Templates")," are displayed (box checked), and search for ",(0,l.yg)("strong",{parentName:"p"},"Matlab"),", and provide the right parameters:"),(0,l.yg)("ul",null,(0,l.yg)("li",{parentName:"ul"},"You will need to provide the password you will use to access the Matlab UI when filling the template. ",(0,l.yg)("strong",{parentName:"li"},"Ensure to use a password length greater than 6.")),(0,l.yg)("li",{parentName:"ul"},"You can also change the Matlab image version, see the latest version released in the ",(0,l.yg)("a",{parentName:"li",href:"https://hub.docker.com/r/mathworks/matlab"},"official Matlab Docker image documentation"))),(0,l.yg)("p",null,"Once Matlab start you can access it through 2 routes (URL), which can be accessed when clicking on the Matlab node in the ",(0,l.yg)("strong",{parentName:"p"},"Topology"),":"),(0,l.yg)("ul",null,(0,l.yg)("li",{parentName:"ul"},"The main ",(0,l.yg)("inlineCode",{parentName:"li"},"matlab")," route to access Matlab desktop UI directly in your web browser. It is recommended to use this route."),(0,l.yg)("li",{parentName:"ul"},"The ",(0,l.yg)("inlineCode",{parentName:"li"},"matlab-vnc")," route can be used to access Matlab using a VNC client (you will need to use the full URL to your Matlab VNC route). Only use it if you know what you're doing.")),(0,l.yg)("h2",{id:"use-a-stable-matlab-image"},"Use a stable Matlab image"),(0,l.yg)("p",null,"The official Matlab image is infamous for showing a black screening after a few hours of use, making it a bit cumbersome to be used trustfully."),(0,l.yg)("p",null,"We have a solution if you need to have a more stable Matlab image, that will require a bit more manual operations:"),(0,l.yg)("ul",null,(0,l.yg)("li",{parentName:"ul"},"Use the ",(0,l.yg)("strong",{parentName:"li"},"Ubuntu with GUI")," template to setup a Ubuntu pod on the DSRI with the image ",(0,l.yg)("a",{parentName:"li",href:"https://github.com/vemonet/docker-ubuntu-vnc-desktop"},(0,l.yg)("inlineCode",{parentName:"a"},"ghcr.io/vemonet/docker-ubuntu-vnc-desktop:latest"))),(0,l.yg)("li",{parentName:"ul"},"Start firefox and browse to ",(0,l.yg)("a",{parentName:"li",href:"https://nl.mathworks.com"},"https://nl.mathworks.com")),(0,l.yg)("li",{parentName:"ul"},"Login with your personal Matlab account, create one if you don\u2019t have it"),(0,l.yg)("li",{parentName:"ul"},"Choose ",(0,l.yg)("strong",{parentName:"li"},"get matlab")," and download, the linux matlab version"),(0,l.yg)("li",{parentName:"ul"},"Open a terminal window and run the following commands:")),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-bash"},"sudo apt-get update\nsudo apt-get install unzip\n# Unzip the previous downloaded matlab installation file\n# start the matlab installation with:\nsudo ./install\n")),(0,l.yg)("p",null,"You will then be prompted the Matlab installation process:"),(0,l.yg)("ul",null,(0,l.yg)("li",{parentName:"ul"},"Fill in your personal matlab account credentials"),(0,l.yg)("li",{parentName:"ul"},"\u26a0\ufe0f Fill in the username as used in the Ubuntu environment, in your case it will most probably be ",(0,l.yg)("strong",{parentName:"li"},"root")," (Matlab gives a license error if this is not correct, check with ",(0,l.yg)("inlineCode",{parentName:"li"},"whoami")," in the terminal when in doubt)"),(0,l.yg)("li",{parentName:"ul"},"Select every Matlab modules you want to be installed"),(0,l.yg)("li",{parentName:"ul"},'Check "symbolic link" and "Improve\u2026\u2026"')),(0,l.yg)("h2",{id:"use-matlab-in-jupyter"},"Use Matlab in Jupyter"),(0,l.yg)("p",null,(0,l.yg)("strong",{parentName:"p"},"For running Matlab in Jupyter, you need to first install the matlab on your Jupyter pod. if you need assistance ",(0,l.yg)("a",{parentName:"strong",href:"/help"},"ask the DSRI admins"))," "),(0,l.yg)("h2",{id:"deploy-matlab-on-gpu"},"Deploy Matlab on GPU"),(0,l.yg)("p",null,"We use the Matlab template in the DSRI catalog to deploy a pre-built ",(0,l.yg)("strong",{parentName:"p"},"Nvidia Matlab Deep Learning Container")," on CPU or GPU nodes. See the ",(0,l.yg)("a",{parentName:"p",href:"https://nl.mathworks.com/help/cloudcenter/ug/matlab-deep-learning-container-on-dgx.html"},"official documentation from MathWorks")," for more details about this image."),(0,l.yg)("admonition",{title:"Request access to Matlab",type:"caution"},(0,l.yg)("p",{parentName:"admonition"},"To be able to access the Matlab on GPU template you will need to ",(0,l.yg)("a",{parentName:"p",href:"/help"},"ask the DSRI admins")," to enable it in your project.")),(0,l.yg)("p",null,"2 options are available to connect to your running Matlab pod terminal:"),(0,l.yg)("ul",null,(0,l.yg)("li",{parentName:"ul"},"Go to the matlab pod page on the DSRI web UI "),(0,l.yg)("li",{parentName:"ul"},"Or connect from your terminal with ",(0,l.yg)("inlineCode",{parentName:"li"},"oc rsh MATLAB_POD_ID"))),(0,l.yg)("p",null,"Type ",(0,l.yg)("inlineCode",{parentName:"p"},"bash")," when first accessing to the terminal to have a better experience."),(0,l.yg)("p",null,"Type ",(0,l.yg)("inlineCode",{parentName:"p"},"cd /ContainerDeepLearningData")," to go in the persistent volume, and use this volume to store all data that should be preserved."),(0,l.yg)("p",null,"Type ",(0,l.yg)("inlineCode",{parentName:"p"},"matlab")," to access Matlab from the terminal"),(0,l.yg)("p",null,"It is possible to access the Matlab desktop UI through VNC and a web UI, but the script to start it in ",(0,l.yg)("inlineCode",{parentName:"p"},"/bin/run.sh")," seems to face some errors, let us know if you have any luck with this."),(0,l.yg)("p",null,"By default the image run with the ",(0,l.yg)("inlineCode",{parentName:"p"},"matlab")," user which does not have ",(0,l.yg)("inlineCode",{parentName:"p"},"sudo")," privilege, you can run the container as root if you need to install packages which require admin privileges. "),(0,l.yg)("h2",{id:"build-your-own-matlab-image"},"Build your own Matlab image"),(0,l.yg)("p",null,"Follow the instructions at: ",(0,l.yg)("a",{parentName:"p",href:"https://github.com/mathworks-ref-arch/matlab-dockerfile"},"https://github.com/mathworks-ref-arch/matlab-dockerfile")),(0,l.yg)("p",null,"This will require you to retrieve Matlab installation files to build your own container"),(0,l.yg)("p",null,"Once all the files have been properly placed in the folder and the license server URL has been set, you can start the build on DSRI by following the ",(0,l.yg)("a",{parentName:"p",href:"https://maastrichtu-ids.github.io/dsri-documentation/docs/guide-dockerfile-to-openshift#create-new-build-configuration"},"documentation to deploy from a ",(0,l.yg)("inlineCode",{parentName:"a"},"Dockerfile"))),(0,l.yg)("admonition",{title:"License server not available on your laptop",type:"caution"},(0,l.yg)("p",{parentName:"admonition"},"If you try to build Matlab directly on your laptop it will most probably fail as your machine might not have access to the license server. You will need to build the Matlab container directly on DSRI with ",(0,l.yg)("inlineCode",{parentName:"p"},"oc start-build"))),(0,l.yg)("p",null,"Once Matlab deployed, you will need to edit the matlab deployment YAML before it works."),(0,l.yg)("p",null,"Go to ",(0,l.yg)("strong",{parentName:"p"},"Topology"),", click on the Matlab node, click on the ",(0,l.yg)("strong",{parentName:"p"},"Actions")," button of the matlab details, and ",(0,l.yg)("strong",{parentName:"p"},"Edit deployment"),". In the deployment YAML search for ",(0,l.yg)("inlineCode",{parentName:"p"},"spec:")," which has a ",(0,l.yg)("inlineCode",{parentName:"p"},"containers:")," as child, and add the following under ",(0,l.yg)("inlineCode",{parentName:"p"},"spec:")),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-yaml"},"spec:\n serviceAccountName: anyuid\n containers: ...\n")),(0,l.yg)("p",null,"Your Matlab container should now be running!"),(0,l.yg)("p",null,"2 options are available to connect to your running Matlab pod terminal:"),(0,l.yg)("ul",null,(0,l.yg)("li",{parentName:"ul"},"Go to the matlab pod page on the DSRI web UI "),(0,l.yg)("li",{parentName:"ul"},"Or connect from your terminal with ",(0,l.yg)("inlineCode",{parentName:"li"},"oc rsh MATLAB_POD_ID"))),(0,l.yg)("p",null,"You can access Matlab from the terminal by running ",(0,l.yg)("inlineCode",{parentName:"p"},"matlab")),(0,l.yg)("p",null,"Unfortunately Matlab did not expected their users to need the graphical UI when using Matlab in containers. So only the command line is available by default. You can find more information to ",(0,l.yg)("a",{parentName:"p",href:"https://github.com/mathworks-ref-arch/matlab-dockerfile/issues/18"},"enable the Matlab UI in this issue"),"."))}y.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[3870],{5680:(e,t,a)=>{a.d(t,{xA:()=>p,yg:()=>y});var n=a(6540);function o(e,t,a){return t in e?Object.defineProperty(e,t,{value:a,enumerable:!0,configurable:!0,writable:!0}):e[t]=a,e}function l(e,t){var a=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);t&&(n=n.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),a.push.apply(a,n)}return a}function r(e){for(var t=1;t=0||(o[a]=e[a]);return o}(e,t);if(Object.getOwnPropertySymbols){var l=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,a)&&(o[a]=e[a])}return o}var s=n.createContext({}),u=function(e){var t=n.useContext(s),a=t;return e&&(a="function"==typeof e?e(t):r(r({},t),e)),a},p=function(e){var t=u(e.components);return n.createElement(s.Provider,{value:t},e.children)},c={inlineCode:"code",wrapper:function(e){var t=e.children;return n.createElement(n.Fragment,{},t)}},m=n.forwardRef((function(e,t){var a=e.components,o=e.mdxType,l=e.originalType,s=e.parentName,p=i(e,["components","mdxType","originalType","parentName"]),m=u(a),y=o,d=m["".concat(s,".").concat(y)]||m[y]||c[y]||l;return a?n.createElement(d,r(r({ref:t},p),{},{components:a})):n.createElement(d,r({ref:t},p))}));function y(e,t){var a=arguments,o=t&&t.mdxType;if("string"==typeof e||o){var l=a.length,r=new Array(l);r[0]=m;var i={};for(var s in t)hasOwnProperty.call(t,s)&&(i[s]=t[s]);i.originalType=e,i.mdxType="string"==typeof e?e:o,r[1]=i;for(var u=2;u{a.r(t),a.d(t,{assets:()=>p,contentTitle:()=>s,default:()=>y,frontMatter:()=>i,metadata:()=>u,toc:()=>c});var n=a(9668),o=a(1367),l=(a(6540),a(5680)),r=["components"],i={id:"deploy-matlab",title:"Matlab"},s=void 0,u={unversionedId:"deploy-matlab",id:"deploy-matlab",title:"Matlab",description:"Note that we are not expert in Matlab: feel free to contact Mathworks support directly if you are having any issues with their official Docker image. Because since it's closed source we cannot fix it ourselves.",source:"@site/docs/deploy-matlab.md",sourceDirName:".",slug:"/deploy-matlab",permalink:"/docs/deploy-matlab",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/deploy-matlab.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"deploy-matlab",title:"Matlab"},sidebar:"docs",previous:{title:"Databases",permalink:"/docs/deploy-database"},next:{title:"JupyterHub",permalink:"/docs/deploy-jupyterhub"}},p={},c=[{value:"Use the official Matlab image",id:"use-the-official-matlab-image",level:2},{value:"Use a stable Matlab image",id:"use-a-stable-matlab-image",level:2},{value:"Use Matlab in Jupyter",id:"use-matlab-in-jupyter",level:2},{value:"Deploy Matlab on GPU",id:"deploy-matlab-on-gpu",level:2},{value:"Build your own Matlab image",id:"build-your-own-matlab-image",level:2}],m={toc:c};function y(e){var t=e.components,a=(0,o.A)(e,r);return(0,l.yg)("wrapper",(0,n.A)({},m,a,{components:t,mdxType:"MDXLayout"}),(0,l.yg)("p",null,"Note that we are not expert in Matlab: feel free to contact Mathworks support directly if you are having any issues with their official Docker image. Because since it's closed source we cannot fix it ourselves."),(0,l.yg)("p",null,"You can request official support from Matlab at this address after login and connecting your account to the UM license: ",(0,l.yg)("a",{parentName:"p",href:"https://nl.mathworks.com/academia/tah-portal/maastricht-university-31574866.html#get"},"https://nl.mathworks.com/academia/tah-portal/maastricht-university-31574866.html#get")),(0,l.yg)("h2",{id:"use-the-official-matlab-image"},"Use the official Matlab image"),(0,l.yg)("p",null,"Start Matlab with a desktop UI accessible directly using your web browser at a URL automatically generated. "),(0,l.yg)("p",null,"Go to the ",(0,l.yg)("strong",{parentName:"p"},"Catalog"),", make sure ",(0,l.yg)("strong",{parentName:"p"},"Templates")," are displayed (box checked), and search for ",(0,l.yg)("strong",{parentName:"p"},"Matlab"),", and provide the right parameters:"),(0,l.yg)("ul",null,(0,l.yg)("li",{parentName:"ul"},"You will need to provide the password you will use to access the Matlab UI when filling the template. ",(0,l.yg)("strong",{parentName:"li"},"Ensure to use a password length greater than 6.")),(0,l.yg)("li",{parentName:"ul"},"You can also change the Matlab image version, see the latest version released in the ",(0,l.yg)("a",{parentName:"li",href:"https://hub.docker.com/r/mathworks/matlab"},"official Matlab Docker image documentation"))),(0,l.yg)("p",null,"Once Matlab start you can access it through 2 routes (URL), which can be accessed when clicking on the Matlab node in the ",(0,l.yg)("strong",{parentName:"p"},"Topology"),":"),(0,l.yg)("ul",null,(0,l.yg)("li",{parentName:"ul"},"The main ",(0,l.yg)("inlineCode",{parentName:"li"},"matlab")," route to access Matlab desktop UI directly in your web browser. It is recommended to use this route."),(0,l.yg)("li",{parentName:"ul"},"The ",(0,l.yg)("inlineCode",{parentName:"li"},"matlab-vnc")," route can be used to access Matlab using a VNC client (you will need to use the full URL to your Matlab VNC route). Only use it if you know what you're doing.")),(0,l.yg)("h2",{id:"use-a-stable-matlab-image"},"Use a stable Matlab image"),(0,l.yg)("p",null,"The official Matlab image is infamous for showing a black screening after a few hours of use, making it a bit cumbersome to be used trustfully."),(0,l.yg)("p",null,"We have a solution if you need to have a more stable Matlab image, that will require a bit more manual operations:"),(0,l.yg)("ul",null,(0,l.yg)("li",{parentName:"ul"},"Use the ",(0,l.yg)("strong",{parentName:"li"},"Ubuntu with GUI")," template to setup a Ubuntu pod on the DSRI with the image ",(0,l.yg)("a",{parentName:"li",href:"https://github.com/vemonet/docker-ubuntu-vnc-desktop"},(0,l.yg)("inlineCode",{parentName:"a"},"ghcr.io/vemonet/docker-ubuntu-vnc-desktop:latest"))),(0,l.yg)("li",{parentName:"ul"},"Start firefox and browse to ",(0,l.yg)("a",{parentName:"li",href:"https://nl.mathworks.com"},"https://nl.mathworks.com")),(0,l.yg)("li",{parentName:"ul"},"Login with your personal Matlab account, create one if you don\u2019t have it"),(0,l.yg)("li",{parentName:"ul"},"Choose ",(0,l.yg)("strong",{parentName:"li"},"get matlab")," and download, the linux matlab version"),(0,l.yg)("li",{parentName:"ul"},"Open a terminal window and run the following commands:")),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-bash"},"sudo apt-get update\nsudo apt-get install unzip\n# Unzip the previous downloaded matlab installation file\n# start the matlab installation with:\nsudo ./install\n")),(0,l.yg)("p",null,"You will then be prompted the Matlab installation process:"),(0,l.yg)("ul",null,(0,l.yg)("li",{parentName:"ul"},"Fill in your personal matlab account credentials"),(0,l.yg)("li",{parentName:"ul"},"\u26a0\ufe0f Fill in the username as used in the Ubuntu environment, in your case it will most probably be ",(0,l.yg)("strong",{parentName:"li"},"root")," (Matlab gives a license error if this is not correct, check with ",(0,l.yg)("inlineCode",{parentName:"li"},"whoami")," in the terminal when in doubt)"),(0,l.yg)("li",{parentName:"ul"},"Select every Matlab modules you want to be installed"),(0,l.yg)("li",{parentName:"ul"},'Check "symbolic link" and "Improve\u2026\u2026"')),(0,l.yg)("h2",{id:"use-matlab-in-jupyter"},"Use Matlab in Jupyter"),(0,l.yg)("p",null,(0,l.yg)("strong",{parentName:"p"},"For running Matlab in Jupyter, you need to first install the matlab on your Jupyter pod. if you need assistance ",(0,l.yg)("a",{parentName:"strong",href:"/help"},"ask the DSRI admins"))," "),(0,l.yg)("h2",{id:"deploy-matlab-on-gpu"},"Deploy Matlab on GPU"),(0,l.yg)("p",null,"We use the Matlab template in the DSRI catalog to deploy a pre-built ",(0,l.yg)("strong",{parentName:"p"},"Nvidia Matlab Deep Learning Container")," on CPU or GPU nodes. See the ",(0,l.yg)("a",{parentName:"p",href:"https://nl.mathworks.com/help/cloudcenter/ug/matlab-deep-learning-container-on-dgx.html"},"official documentation from MathWorks")," for more details about this image."),(0,l.yg)("admonition",{title:"Request access to Matlab",type:"caution"},(0,l.yg)("p",{parentName:"admonition"},"To be able to access the Matlab on GPU template you will need to ",(0,l.yg)("a",{parentName:"p",href:"/help"},"ask the DSRI admins")," to enable it in your project.")),(0,l.yg)("p",null,"2 options are available to connect to your running Matlab pod terminal:"),(0,l.yg)("ul",null,(0,l.yg)("li",{parentName:"ul"},"Go to the matlab pod page on the DSRI web UI "),(0,l.yg)("li",{parentName:"ul"},"Or connect from your terminal with ",(0,l.yg)("inlineCode",{parentName:"li"},"oc rsh MATLAB_POD_ID"))),(0,l.yg)("p",null,"Type ",(0,l.yg)("inlineCode",{parentName:"p"},"bash")," when first accessing to the terminal to have a better experience."),(0,l.yg)("p",null,"Type ",(0,l.yg)("inlineCode",{parentName:"p"},"cd /ContainerDeepLearningData")," to go in the persistent volume, and use this volume to store all data that should be preserved."),(0,l.yg)("p",null,"Type ",(0,l.yg)("inlineCode",{parentName:"p"},"matlab")," to access Matlab from the terminal"),(0,l.yg)("p",null,"It is possible to access the Matlab desktop UI through VNC and a web UI, but the script to start it in ",(0,l.yg)("inlineCode",{parentName:"p"},"/bin/run.sh")," seems to face some errors, let us know if you have any luck with this."),(0,l.yg)("p",null,"By default the image run with the ",(0,l.yg)("inlineCode",{parentName:"p"},"matlab")," user which does not have ",(0,l.yg)("inlineCode",{parentName:"p"},"sudo")," privilege, you can run the container as root if you need to install packages which require admin privileges. "),(0,l.yg)("h2",{id:"build-your-own-matlab-image"},"Build your own Matlab image"),(0,l.yg)("p",null,"Follow the instructions at: ",(0,l.yg)("a",{parentName:"p",href:"https://github.com/mathworks-ref-arch/matlab-dockerfile"},"https://github.com/mathworks-ref-arch/matlab-dockerfile")),(0,l.yg)("p",null,"This will require you to retrieve Matlab installation files to build your own container"),(0,l.yg)("p",null,"Once all the files have been properly placed in the folder and the license server URL has been set, you can start the build on DSRI by following the ",(0,l.yg)("a",{parentName:"p",href:"https://maastrichtu-ids.github.io/dsri-documentation/docs/guide-dockerfile-to-openshift#create-new-build-configuration"},"documentation to deploy from a ",(0,l.yg)("inlineCode",{parentName:"a"},"Dockerfile"))),(0,l.yg)("admonition",{title:"License server not available on your laptop",type:"caution"},(0,l.yg)("p",{parentName:"admonition"},"If you try to build Matlab directly on your laptop it will most probably fail as your machine might not have access to the license server. You will need to build the Matlab container directly on DSRI with ",(0,l.yg)("inlineCode",{parentName:"p"},"oc start-build"))),(0,l.yg)("p",null,"Once Matlab deployed, you will need to edit the matlab deployment YAML before it works."),(0,l.yg)("p",null,"Go to ",(0,l.yg)("strong",{parentName:"p"},"Topology"),", click on the Matlab node, click on the ",(0,l.yg)("strong",{parentName:"p"},"Actions")," button of the matlab details, and ",(0,l.yg)("strong",{parentName:"p"},"Edit deployment"),". In the deployment YAML search for ",(0,l.yg)("inlineCode",{parentName:"p"},"spec:")," which has a ",(0,l.yg)("inlineCode",{parentName:"p"},"containers:")," as child, and add the following under ",(0,l.yg)("inlineCode",{parentName:"p"},"spec:")),(0,l.yg)("pre",null,(0,l.yg)("code",{parentName:"pre",className:"language-yaml"},"spec:\n serviceAccountName: anyuid\n containers: ...\n")),(0,l.yg)("p",null,"Your Matlab container should now be running!"),(0,l.yg)("p",null,"2 options are available to connect to your running Matlab pod terminal:"),(0,l.yg)("ul",null,(0,l.yg)("li",{parentName:"ul"},"Go to the matlab pod page on the DSRI web UI "),(0,l.yg)("li",{parentName:"ul"},"Or connect from your terminal with ",(0,l.yg)("inlineCode",{parentName:"li"},"oc rsh MATLAB_POD_ID"))),(0,l.yg)("p",null,"You can access Matlab from the terminal by running ",(0,l.yg)("inlineCode",{parentName:"p"},"matlab")),(0,l.yg)("p",null,"Unfortunately Matlab did not expected their users to need the graphical UI when using Matlab in containers. So only the command line is available by default. You can find more information to ",(0,l.yg)("a",{parentName:"p",href:"https://github.com/mathworks-ref-arch/matlab-dockerfile/issues/18"},"enable the Matlab UI in this issue"),"."))}y.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/f8d48938.02663c46.js b/assets/js/f8d48938.840f5e24.js similarity index 99% rename from assets/js/f8d48938.02663c46.js rename to assets/js/f8d48938.840f5e24.js index 5c865a51c..de981b281 100644 --- a/assets/js/f8d48938.02663c46.js +++ b/assets/js/f8d48938.840f5e24.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[2526],{5680:(e,t,r)=>{r.d(t,{xA:()=>p,yg:()=>y});var o=r(6540);function a(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function i(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,o)}return r}function n(e){for(var t=1;t=0||(a[r]=e[r]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(o=0;o=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(a[r]=e[r])}return a}var u=o.createContext({}),l=function(e){var t=o.useContext(u),r=t;return e&&(r="function"==typeof e?e(t):n(n({},t),e)),r},p=function(e){var t=l(e.components);return o.createElement(u.Provider,{value:t},e.children)},c={inlineCode:"code",wrapper:function(e){var t=e.children;return o.createElement(o.Fragment,{},t)}},h=o.forwardRef((function(e,t){var r=e.components,a=e.mdxType,i=e.originalType,u=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),h=l(r),y=a,d=h["".concat(u,".").concat(y)]||h[y]||c[y]||i;return r?o.createElement(d,n(n({ref:t},p),{},{components:r})):o.createElement(d,n({ref:t},p))}));function y(e,t){var r=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var i=r.length,n=new Array(i);n[0]=h;var s={};for(var u in t)hasOwnProperty.call(t,u)&&(s[u]=t[u]);s.originalType=e,s.mdxType="string"==typeof e?e:a,n[1]=s;for(var l=2;l{r.r(t),r.d(t,{assets:()=>p,contentTitle:()=>u,default:()=>y,frontMatter:()=>s,metadata:()=>l,toc:()=>c});var o=r(9668),a=r(1367),i=(r(6540),r(5680)),n=["components"],s={id:"guide-workshop",title:"Prepare a workshop"},u=void 0,l={unversionedId:"guide-workshop",id:"guide-workshop",title:"Prepare a workshop",description:"The DSRI is a good platform to run a training or class within Maastricht University.",source:"@site/docs/guide-workshop.md",sourceDirName:".",slug:"/guide-workshop",permalink:"/docs/guide-workshop",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/guide-workshop.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"guide-workshop",title:"Prepare a workshop"},sidebar:"docs",previous:{title:"Install UM VPN",permalink:"/docs/guide-vpn"},next:{title:"Login to Docker registries",permalink:"/docs/login-docker-registry"}},p={},c=[{value:"Request VPN accounts for users",id:"request-vpn-accounts-for-users",level:2},{value:"Fill a form",id:"fill-a-form",level:2},{value:"Prepare you workshop",id:"prepare-you-workshop",level:2},{value:"Publish an image for your training",id:"publish-an-image-for-your-training",level:3},{value:"Show your users how to start a workspace",id:"show-your-users-how-to-start-a-workspace",level:3}],h={toc:c};function y(e){var t=e.components,r=(0,a.A)(e,n);return(0,i.yg)("wrapper",(0,o.A)({},h,r,{components:t,mdxType:"MDXLayout"}),(0,i.yg)("p",null,"The DSRI is a good platform to run a training or class within Maastricht University."),(0,i.yg)("h2",{id:"request-vpn-accounts-for-users"},"Request VPN accounts for users"),(0,i.yg)("p",null,"If the users are ",(0,i.yg)("strong",{parentName:"p"},"students")," from Maastricht University, ",(0,i.yg)("strong",{parentName:"p"},"or not")," from Maastricht University (without an email @maastrichtuniversity.nl, or @maastro.nl), you will need to contact the ICT support of your department to request the creation of accounts so that your users can connect to the UM VPN."),(0,i.yg)("p",null,"At FSE, you will need to send an email to ",(0,i.yg)("a",{parentName:"p",href:"mailto:lo-fse@maastrichtuniversity.nl"},"lo-fse@maastrichtuniversity.nl")," and ",(0,i.yg)("a",{parentName:"p",href:"mailto:DSRI-SUPPORT-L@maastrichtuniversity.nl"},"DSRI-SUPPORT-L@maastrichtuniversity.nl")," with the following information:"),(0,i.yg)("ul",null,(0,i.yg)("li",{parentName:"ul"},"Emails of the users"),(0,i.yg)("li",{parentName:"ul"},"Why they need access to the DSRI (provide the ID of the course at Maastricht University if it is for a course)"),(0,i.yg)("li",{parentName:"ul"},"Until which date the users will need those VPN accounts")),(0,i.yg)("h2",{id:"fill-a-form"},"Fill a form"),(0,i.yg)("p",null,"Fill this ",(0,i.yg)("a",{parentName:"p",href:"/register"},"form \ud83d\udcec")," to give us more details on your project (you don't need to do it if you have already filled it in the past)."),(0,i.yg)("h2",{id:"prepare-you-workshop"},"Prepare you workshop"),(0,i.yg)("p",null,"Use the ",(0,i.yg)("a",{parentName:"p",href:"/docs/access-dsri"},"DSRI documentation")," to explain to your users how to access the DSRI."),(0,i.yg)("h3",{id:"publish-an-image-for-your-training"},"Publish an image for your training"),(0,i.yg)("p",null,"Feel free to use the existing templates for JupyterLab, RStudio, or Visual Studio Code in the DSRI catalog."),(0,i.yg)("p",null,"You can easily reuse our images to adapt it to your training need and install all required dependencies:"),(0,i.yg)("ul",null,(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/jupyterlab"},"https://github.com/MaastrichtU-IDS/jupyterlab")),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/rstudio"},"https://github.com/MaastrichtU-IDS/rstudio")),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/code-server"},"https://github.com/MaastrichtU-IDS/code-server"))),(0,i.yg)("p",null,"Then you will just need to instruct your users to start an existing templates with your newly published image."),(0,i.yg)("p",null,"With the JupyterLab template you can also prepare a git repository to be cloned in the workspace as soon as they start it."),(0,i.yg)("p",null,"You can find some examples of python scripts with database to run on the DSRI in this repository: ",(0,i.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-demo"},"https://github.com/MaastrichtU-IDS/dsri-demo")),(0,i.yg)("h3",{id:"show-your-users-how-to-start-a-workspace"},"Show your users how to start a workspace"),(0,i.yg)("p",null,"You can use this video showing how to start a RStudio workspace, the process is similar for JupyterLab and VisualStudio Code: ",(0,i.yg)("a",{parentName:"p",href:"https://www.youtube.com/watch?v=Y0BjotH1LiE"},"https://www.youtube.com/watch?v=Y0BjotH1LiE")),(0,i.yg)("p",null,"Otherwise just do it directly with them."))}y.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[2526],{5680:(e,t,r)=>{r.d(t,{xA:()=>p,yg:()=>y});var o=r(6540);function a(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function i(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);t&&(o=o.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,o)}return r}function n(e){for(var t=1;t=0||(a[r]=e[r]);return a}(e,t);if(Object.getOwnPropertySymbols){var i=Object.getOwnPropertySymbols(e);for(o=0;o=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(a[r]=e[r])}return a}var u=o.createContext({}),l=function(e){var t=o.useContext(u),r=t;return e&&(r="function"==typeof e?e(t):n(n({},t),e)),r},p=function(e){var t=l(e.components);return o.createElement(u.Provider,{value:t},e.children)},c={inlineCode:"code",wrapper:function(e){var t=e.children;return o.createElement(o.Fragment,{},t)}},h=o.forwardRef((function(e,t){var r=e.components,a=e.mdxType,i=e.originalType,u=e.parentName,p=s(e,["components","mdxType","originalType","parentName"]),h=l(r),y=a,d=h["".concat(u,".").concat(y)]||h[y]||c[y]||i;return r?o.createElement(d,n(n({ref:t},p),{},{components:r})):o.createElement(d,n({ref:t},p))}));function y(e,t){var r=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var i=r.length,n=new Array(i);n[0]=h;var s={};for(var u in t)hasOwnProperty.call(t,u)&&(s[u]=t[u]);s.originalType=e,s.mdxType="string"==typeof e?e:a,n[1]=s;for(var l=2;l{r.r(t),r.d(t,{assets:()=>p,contentTitle:()=>u,default:()=>y,frontMatter:()=>s,metadata:()=>l,toc:()=>c});var o=r(9668),a=r(1367),i=(r(6540),r(5680)),n=["components"],s={id:"guide-workshop",title:"Prepare a workshop"},u=void 0,l={unversionedId:"guide-workshop",id:"guide-workshop",title:"Prepare a workshop",description:"The DSRI is a good platform to run a training or class within Maastricht University.",source:"@site/docs/guide-workshop.md",sourceDirName:".",slug:"/guide-workshop",permalink:"/docs/guide-workshop",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/guide-workshop.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"guide-workshop",title:"Prepare a workshop"},sidebar:"docs",previous:{title:"Install UM VPN",permalink:"/docs/guide-vpn"},next:{title:"Login to Docker registries",permalink:"/docs/login-docker-registry"}},p={},c=[{value:"Request VPN accounts for users",id:"request-vpn-accounts-for-users",level:2},{value:"Fill a form",id:"fill-a-form",level:2},{value:"Prepare you workshop",id:"prepare-you-workshop",level:2},{value:"Publish an image for your training",id:"publish-an-image-for-your-training",level:3},{value:"Show your users how to start a workspace",id:"show-your-users-how-to-start-a-workspace",level:3}],h={toc:c};function y(e){var t=e.components,r=(0,a.A)(e,n);return(0,i.yg)("wrapper",(0,o.A)({},h,r,{components:t,mdxType:"MDXLayout"}),(0,i.yg)("p",null,"The DSRI is a good platform to run a training or class within Maastricht University."),(0,i.yg)("h2",{id:"request-vpn-accounts-for-users"},"Request VPN accounts for users"),(0,i.yg)("p",null,"If the users are ",(0,i.yg)("strong",{parentName:"p"},"students")," from Maastricht University, ",(0,i.yg)("strong",{parentName:"p"},"or not")," from Maastricht University (without an email @maastrichtuniversity.nl, or @maastro.nl), you will need to contact the ICT support of your department to request the creation of accounts so that your users can connect to the UM VPN."),(0,i.yg)("p",null,"At FSE, you will need to send an email to ",(0,i.yg)("a",{parentName:"p",href:"mailto:lo-fse@maastrichtuniversity.nl"},"lo-fse@maastrichtuniversity.nl")," and ",(0,i.yg)("a",{parentName:"p",href:"mailto:DSRI-SUPPORT-L@maastrichtuniversity.nl"},"DSRI-SUPPORT-L@maastrichtuniversity.nl")," with the following information:"),(0,i.yg)("ul",null,(0,i.yg)("li",{parentName:"ul"},"Emails of the users"),(0,i.yg)("li",{parentName:"ul"},"Why they need access to the DSRI (provide the ID of the course at Maastricht University if it is for a course)"),(0,i.yg)("li",{parentName:"ul"},"Until which date the users will need those VPN accounts")),(0,i.yg)("h2",{id:"fill-a-form"},"Fill a form"),(0,i.yg)("p",null,"Fill this ",(0,i.yg)("a",{parentName:"p",href:"/register"},"form \ud83d\udcec")," to give us more details on your project (you don't need to do it if you have already filled it in the past)."),(0,i.yg)("h2",{id:"prepare-you-workshop"},"Prepare you workshop"),(0,i.yg)("p",null,"Use the ",(0,i.yg)("a",{parentName:"p",href:"/docs/access-dsri"},"DSRI documentation")," to explain to your users how to access the DSRI."),(0,i.yg)("h3",{id:"publish-an-image-for-your-training"},"Publish an image for your training"),(0,i.yg)("p",null,"Feel free to use the existing templates for JupyterLab, RStudio, or Visual Studio Code in the DSRI catalog."),(0,i.yg)("p",null,"You can easily reuse our images to adapt it to your training need and install all required dependencies:"),(0,i.yg)("ul",null,(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/jupyterlab"},"https://github.com/MaastrichtU-IDS/jupyterlab")),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/rstudio"},"https://github.com/MaastrichtU-IDS/rstudio")),(0,i.yg)("li",{parentName:"ul"},(0,i.yg)("a",{parentName:"li",href:"https://github.com/MaastrichtU-IDS/code-server"},"https://github.com/MaastrichtU-IDS/code-server"))),(0,i.yg)("p",null,"Then you will just need to instruct your users to start an existing templates with your newly published image."),(0,i.yg)("p",null,"With the JupyterLab template you can also prepare a git repository to be cloned in the workspace as soon as they start it."),(0,i.yg)("p",null,"You can find some examples of python scripts with database to run on the DSRI in this repository: ",(0,i.yg)("a",{parentName:"p",href:"https://github.com/MaastrichtU-IDS/dsri-demo"},"https://github.com/MaastrichtU-IDS/dsri-demo")),(0,i.yg)("h3",{id:"show-your-users-how-to-start-a-workspace"},"Show your users how to start a workspace"),(0,i.yg)("p",null,"You can use this video showing how to start a RStudio workspace, the process is similar for JupyterLab and VisualStudio Code: ",(0,i.yg)("a",{parentName:"p",href:"https://www.youtube.com/watch?v=Y0BjotH1LiE"},"https://www.youtube.com/watch?v=Y0BjotH1LiE")),(0,i.yg)("p",null,"Otherwise just do it directly with them."))}y.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/f931684d.76c8bdbe.js b/assets/js/f931684d.d8d3a33e.js similarity index 99% rename from assets/js/f931684d.76c8bdbe.js rename to assets/js/f931684d.d8d3a33e.js index 2a8034f6c..09e0c3385 100644 --- a/assets/js/f931684d.76c8bdbe.js +++ b/assets/js/f931684d.d8d3a33e.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[8850],{5680:(e,o,t)=>{t.d(o,{xA:()=>s,yg:()=>y});var r=t(6540);function n(e,o,t){return o in e?Object.defineProperty(e,o,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[o]=t,e}function a(e,o){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);o&&(r=r.filter((function(o){return Object.getOwnPropertyDescriptor(e,o).enumerable}))),t.push.apply(t,r)}return t}function i(e){for(var o=1;o=0||(n[t]=e[t]);return n}(e,o);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(n[t]=e[t])}return n}var p=r.createContext({}),c=function(e){var o=r.useContext(p),t=o;return e&&(t="function"==typeof e?e(o):i(i({},o),e)),t},s=function(e){var o=c(e.components);return r.createElement(p.Provider,{value:o},e.children)},u={inlineCode:"code",wrapper:function(e){var o=e.children;return r.createElement(r.Fragment,{},o)}},d=r.forwardRef((function(e,o){var t=e.components,n=e.mdxType,a=e.originalType,p=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),d=c(t),y=n,m=d["".concat(p,".").concat(y)]||d[y]||u[y]||a;return t?r.createElement(m,i(i({ref:o},s),{},{components:t})):r.createElement(m,i({ref:o},s))}));function y(e,o){var t=arguments,n=o&&o.mdxType;if("string"==typeof e||n){var a=t.length,i=new Array(a);i[0]=d;var l={};for(var p in o)hasOwnProperty.call(o,p)&&(l[p]=o[p]);l.originalType=e,l.mdxType="string"==typeof e?e:n,i[1]=l;for(var c=2;c{t.r(o),t.d(o,{assets:()=>s,contentTitle:()=>p,default:()=>y,frontMatter:()=>l,metadata:()=>c,toc:()=>u});var r=t(9668),n=t(1367),a=(t(6540),t(5680)),i=["components"],l={id:"deploy-from-docker",title:"Deploy from a Docker image"},p=void 0,c={unversionedId:"deploy-from-docker",id:"deploy-from-docker",title:"Deploy from a Docker image",description:"The DSRI is an OpenShift OKD cluster, based on Kubernetes. It uses Docker containers to deploy services and applications in pods.",source:"@site/docs/deploy-from-docker.md",sourceDirName:".",slug:"/deploy-from-docker",permalink:"/docs/deploy-from-docker",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/deploy-from-docker.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"deploy-from-docker",title:"Deploy from a Docker image"},sidebar:"docs",previous:{title:"Deploy from a Dockerfile",permalink:"/docs/guide-dockerfile-to-openshift"},next:{title:"Anatomy of a DSRI application",permalink:"/docs/anatomy-of-an-application"}},s={},u=[{value:"Find an image for your service",id:"find-an-image-for-your-service",level:2},{value:"Deploy the image on DSRI",id:"deploy-the-image-on-dsri",level:2},{value:"Build and push a new Docker image",id:"build-and-push-a-new-docker-image",level:2},{value:"Define a Dockerfile",id:"define-a-dockerfile",level:3},{value:"Build the image",id:"build-the-image",level:3},{value:"Push to DockerHub",id:"push-to-dockerhub",level:3}],d={toc:u};function y(e){var o=e.components,t=(0,n.A)(e,i);return(0,a.yg)("wrapper",(0,r.A)({},d,t,{components:o,mdxType:"MDXLayout"}),(0,a.yg)("p",null,"The DSRI is an ",(0,a.yg)("a",{parentName:"p",href:"https://www.okd.io/"},"OpenShift OKD")," cluster, based on ",(0,a.yg)("a",{parentName:"p",href:"https://kubernetes.io/"},"Kubernetes"),". It uses ",(0,a.yg)("a",{parentName:"p",href:"https://www.docker.com"},"Docker containers")," to deploy services and applications in ",(0,a.yg)("strong",{parentName:"p"},"pods"),"."),(0,a.yg)("p",null,"Any service or job can be run in a Docker container. If you want to run a service in Python for example, you will find Docker images for Python. "),(0,a.yg)("ul",null,(0,a.yg)("li",{parentName:"ul"},"You can find already existing images for the service you want to run on DockerHub"),(0,a.yg)("li",{parentName:"ul"},"or create a custom Docker image in a few minutes. ")),(0,a.yg)("h2",{id:"find-an-image-for-your-service"},"Find an image for your service"),(0,a.yg)("p",null,"The easiest way to deploy a service on the DSRI is to use a Docker image from ",(0,a.yg)("a",{parentName:"p",href:"https://hub.docker.com/"},"DockerHub \ud83d\udc33"),"."),(0,a.yg)("p",null,"Search for an image for your service published on ",(0,a.yg)("a",{parentName:"p",href:"https://hub.docker.com/"},"DockerHub")),(0,a.yg)("ul",null,(0,a.yg)("li",{parentName:"ul"},(0,a.yg)("a",{parentName:"li",href:"https://www.google.com/search?q=dockerhub+python"},'Google "dockerhub my_service_name"')),(0,a.yg)("li",{parentName:"ul"},"Sometimes multiple images can be found for your service. Take the official image when possible, or the one most relevant to your use-case.")),(0,a.yg)("admonition",{title:"Deploy from a Dockerfile",type:"info"},(0,a.yg)("p",{parentName:"admonition"},"If no suitable image can be found on ",(0,a.yg)("a",{parentName:"p",href:"https://hub.docker.com/"},"DockerHub"),", it can be ",(0,a.yg)("strong",{parentName:"p"},"deployed from a Dockerfile"),". See above to do so.")),(0,a.yg)("hr",null),(0,a.yg)("h2",{id:"deploy-the-image-on-dsri"},"Deploy the image on DSRI"),(0,a.yg)("p",null,"Once you have a Docker image for your application you can deploy it using the ",(0,a.yg)("a",{parentName:"p",href:"https://console-openshift-console.apps.dsri2.unimaas.nl/console/projects"},"DSRI web UI"),"."),(0,a.yg)("p",null,"Go to the ",(0,a.yg)("a",{parentName:"p",href:"https://console-openshift-console.apps.dsri2.unimaas.nl/console/projects"},"Overview page")," of your project."),(0,a.yg)("ul",null,(0,a.yg)("li",{parentName:"ul"},"Click the ",(0,a.yg)("strong",{parentName:"li"},"Add to Project")," button in top right corner > ",(0,a.yg)("strong",{parentName:"li"},"Deploy Image")),(0,a.yg)("li",{parentName:"ul"},"Select to deploy from ",(0,a.yg)("strong",{parentName:"li"},"Image Name"),(0,a.yg)("ul",{parentName:"li"},(0,a.yg)("li",{parentName:"ul"},"Provide your image name, e.g. ",(0,a.yg)("inlineCode",{parentName:"li"},"umdsri/freesurfer")),(0,a.yg)("li",{parentName:"ul"},"Eventually change the ",(0,a.yg)("strong",{parentName:"li"},"Name"),", it needs to be unique by project."),(0,a.yg)("li",{parentName:"ul"},"Click ",(0,a.yg)("strong",{parentName:"li"},"Deploy"),".")))),(0,a.yg)("img",{src:"/img/screenshot-deploy_image_from_ui.png",alt:"Deploy image from UI",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,a.yg)("admonition",{title:"Fix a common problem",type:"caution"},(0,a.yg)("p",{parentName:"admonition"},"Once the application is deployed it will most probably fail because it has not been optimized to work with OpenShift random user ID. You will need to add an entry to the deployment to enable your image to run using any user ID.")),(0,a.yg)("p",null,"Go to ",(0,a.yg)("strong",{parentName:"p"},"Topology"),", click on your application node, click on the ",(0,a.yg)("strong",{parentName:"p"},"Actions")," button of your application details, and ",(0,a.yg)("strong",{parentName:"p"},"Edit deployment"),". In the deployment YAML search for ",(0,a.yg)("inlineCode",{parentName:"p"},"spec:")," which has a ",(0,a.yg)("inlineCode",{parentName:"p"},"containers:")," as child, and add the following under ",(0,a.yg)("inlineCode",{parentName:"p"},"spec:")),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-yaml"},"spec:\n serviceAccountName: anyuid\n containers: ...\n")),(0,a.yg)("admonition",{title:"Access the application",type:"info"},(0,a.yg)("p",{parentName:"admonition"},"You should now see your pod deployed on the ",(0,a.yg)("a",{parentName:"p",href:"https://console-openshift-console.apps.dsri2.unimaas.nl/console/projects"},"Overview")," page of your project."),(0,a.yg)("p",{parentName:"admonition"},"You can expose routes to this pod in the ",(0,a.yg)("a",{parentName:"p",href:"https://console-openshift-console.apps.dsri2.unimaas.nl/console/projects"},"Overview")," page: ",(0,a.yg)("strong",{parentName:"p"},"Create route"))),(0,a.yg)("hr",null),(0,a.yg)("h2",{id:"build-and-push-a-new-docker-image"},"Build and push a new Docker image"),(0,a.yg)("p",null,"In case you there is no Docker image for your application you can build and push one."),(0,a.yg)("p",null,"To build and push a Docker image you will need to have ",(0,a.yg)("a",{parentName:"p",href:"https://docs.docker.com/get-docker/"},"Docker installed"),"."),(0,a.yg)("admonition",{title:"Install Docker",type:"info"},(0,a.yg)("p",{parentName:"admonition"},"See the ",(0,a.yg)("a",{parentName:"p",href:"https://docs.docker.com/get-docker/"},"official documentation to install Docker"),".")),(0,a.yg)("h3",{id:"define-a-dockerfile"},"Define a Dockerfile"),(0,a.yg)("p",null,"If no images are available on DockerHub, it is still possible that the developers created the ",(0,a.yg)("a",{parentName:"p",href:"https://docs.docker.com/engine/reference/builder/"},"Dockerfile to build the image")," without pushing it to DockerHub. Go to the GitHub/GitLab source code repository and search for a ",(0,a.yg)("inlineCode",{parentName:"p"},"Dockerfile"),", it can usually be found in"),(0,a.yg)("ul",null,(0,a.yg)("li",{parentName:"ul"},"the source code repository root folder"),(0,a.yg)("li",{parentName:"ul"},"a ",(0,a.yg)("inlineCode",{parentName:"li"},"docker")," subfolder"),(0,a.yg)("li",{parentName:"ul"},"as instructions in the ",(0,a.yg)("inlineCode",{parentName:"li"},"README.md"))),(0,a.yg)("p",null,"If no ",(0,a.yg)("inlineCode",{parentName:"p"},"Dockerfile")," are available we will need to define one. "),(0,a.yg)("admonition",{title:"Contact us",type:"info"},(0,a.yg)("p",{parentName:"admonition"},"Feel free to ",(0,a.yg)("a",{parentName:"p",href:"/help"},"contact us")," to get help with this, especially if you are unfamiliar with ",(0,a.yg)("a",{parentName:"p",href:"https://docs.docker.com/get-started/"},"Docker"),".")),(0,a.yg)("h3",{id:"build-the-image"},"Build the image"),(0,a.yg)("p",null,"Once a Dockerfile has been defined for the service you can build it by running the following command from the source code root folder, where the Dockerfile is:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-shell"},"docker build -t username/my-service .\n")),(0,a.yg)("p",null,"Arguments can be provided when starting the build, they need to be defined in the Dockerfile to be used."),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-shell"},"docker build -t username/my-service --build-args MY_ARG=my_value .\n")),(0,a.yg)("h3",{id:"push-to-dockerhub"},"Push to DockerHub"),(0,a.yg)("p",null,"Before pushing it to DockerHub you will need to create a repository. To do so, click on ",(0,a.yg)("strong",{parentName:"p"},(0,a.yg)("a",{parentName:"strong",href:"https://hub.docker.com/repository/create"},"Create Repository")),"."),(0,a.yg)("ul",null,(0,a.yg)("li",{parentName:"ul"},"DockerHub is free for public repositories"),(0,a.yg)("li",{parentName:"ul"},"Images can be published under your DockerHub user or an organization you belong to")),(0,a.yg)("p",null,"Login to DockerHub, if not already done:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-shell"},"docker login\n")),(0,a.yg)("p",null,"Push the image previously built to DockerHub:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-shell"},"docker push username/my-service\n")),(0,a.yg)("p",null,"You can link DockerHub to your source code repository and ask it to build the Docker image automatically (from the Dockerfile in the root folder). It should take between 10 and 30min for DockerHub to build your image"),(0,a.yg)("admonition",{title:"Deploy from a local Dockerfile",type:"tip"},(0,a.yg)("p",{parentName:"admonition"},"You can also deploy a service on the DSRI directly from a local ",(0,a.yg)("inlineCode",{parentName:"p"},"Dockerfile"),", to avoid using DockerHub. See ",(0,a.yg)("a",{parentName:"p",href:"/docs/guide-dockerfile-to-openshift"},"this page to deploy a service from a local Dockerfile")," for more instructions")))}y.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[8850],{5680:(e,o,t)=>{t.d(o,{xA:()=>s,yg:()=>y});var r=t(6540);function n(e,o,t){return o in e?Object.defineProperty(e,o,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[o]=t,e}function a(e,o){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);o&&(r=r.filter((function(o){return Object.getOwnPropertyDescriptor(e,o).enumerable}))),t.push.apply(t,r)}return t}function i(e){for(var o=1;o=0||(n[t]=e[t]);return n}(e,o);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(n[t]=e[t])}return n}var p=r.createContext({}),c=function(e){var o=r.useContext(p),t=o;return e&&(t="function"==typeof e?e(o):i(i({},o),e)),t},s=function(e){var o=c(e.components);return r.createElement(p.Provider,{value:o},e.children)},u={inlineCode:"code",wrapper:function(e){var o=e.children;return r.createElement(r.Fragment,{},o)}},d=r.forwardRef((function(e,o){var t=e.components,n=e.mdxType,a=e.originalType,p=e.parentName,s=l(e,["components","mdxType","originalType","parentName"]),d=c(t),y=n,m=d["".concat(p,".").concat(y)]||d[y]||u[y]||a;return t?r.createElement(m,i(i({ref:o},s),{},{components:t})):r.createElement(m,i({ref:o},s))}));function y(e,o){var t=arguments,n=o&&o.mdxType;if("string"==typeof e||n){var a=t.length,i=new Array(a);i[0]=d;var l={};for(var p in o)hasOwnProperty.call(o,p)&&(l[p]=o[p]);l.originalType=e,l.mdxType="string"==typeof e?e:n,i[1]=l;for(var c=2;c{t.r(o),t.d(o,{assets:()=>s,contentTitle:()=>p,default:()=>y,frontMatter:()=>l,metadata:()=>c,toc:()=>u});var r=t(9668),n=t(1367),a=(t(6540),t(5680)),i=["components"],l={id:"deploy-from-docker",title:"Deploy from a Docker image"},p=void 0,c={unversionedId:"deploy-from-docker",id:"deploy-from-docker",title:"Deploy from a Docker image",description:"The DSRI is an OpenShift OKD cluster, based on Kubernetes. It uses Docker containers to deploy services and applications in pods.",source:"@site/docs/deploy-from-docker.md",sourceDirName:".",slug:"/deploy-from-docker",permalink:"/docs/deploy-from-docker",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/deploy-from-docker.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"deploy-from-docker",title:"Deploy from a Docker image"},sidebar:"docs",previous:{title:"Deploy from a Dockerfile",permalink:"/docs/guide-dockerfile-to-openshift"},next:{title:"Anatomy of a DSRI application",permalink:"/docs/anatomy-of-an-application"}},s={},u=[{value:"Find an image for your service",id:"find-an-image-for-your-service",level:2},{value:"Deploy the image on DSRI",id:"deploy-the-image-on-dsri",level:2},{value:"Build and push a new Docker image",id:"build-and-push-a-new-docker-image",level:2},{value:"Define a Dockerfile",id:"define-a-dockerfile",level:3},{value:"Build the image",id:"build-the-image",level:3},{value:"Push to DockerHub",id:"push-to-dockerhub",level:3}],d={toc:u};function y(e){var o=e.components,t=(0,n.A)(e,i);return(0,a.yg)("wrapper",(0,r.A)({},d,t,{components:o,mdxType:"MDXLayout"}),(0,a.yg)("p",null,"The DSRI is an ",(0,a.yg)("a",{parentName:"p",href:"https://www.okd.io/"},"OpenShift OKD")," cluster, based on ",(0,a.yg)("a",{parentName:"p",href:"https://kubernetes.io/"},"Kubernetes"),". It uses ",(0,a.yg)("a",{parentName:"p",href:"https://www.docker.com"},"Docker containers")," to deploy services and applications in ",(0,a.yg)("strong",{parentName:"p"},"pods"),"."),(0,a.yg)("p",null,"Any service or job can be run in a Docker container. If you want to run a service in Python for example, you will find Docker images for Python. "),(0,a.yg)("ul",null,(0,a.yg)("li",{parentName:"ul"},"You can find already existing images for the service you want to run on DockerHub"),(0,a.yg)("li",{parentName:"ul"},"or create a custom Docker image in a few minutes. ")),(0,a.yg)("h2",{id:"find-an-image-for-your-service"},"Find an image for your service"),(0,a.yg)("p",null,"The easiest way to deploy a service on the DSRI is to use a Docker image from ",(0,a.yg)("a",{parentName:"p",href:"https://hub.docker.com/"},"DockerHub \ud83d\udc33"),"."),(0,a.yg)("p",null,"Search for an image for your service published on ",(0,a.yg)("a",{parentName:"p",href:"https://hub.docker.com/"},"DockerHub")),(0,a.yg)("ul",null,(0,a.yg)("li",{parentName:"ul"},(0,a.yg)("a",{parentName:"li",href:"https://www.google.com/search?q=dockerhub+python"},'Google "dockerhub my_service_name"')),(0,a.yg)("li",{parentName:"ul"},"Sometimes multiple images can be found for your service. Take the official image when possible, or the one most relevant to your use-case.")),(0,a.yg)("admonition",{title:"Deploy from a Dockerfile",type:"info"},(0,a.yg)("p",{parentName:"admonition"},"If no suitable image can be found on ",(0,a.yg)("a",{parentName:"p",href:"https://hub.docker.com/"},"DockerHub"),", it can be ",(0,a.yg)("strong",{parentName:"p"},"deployed from a Dockerfile"),". See above to do so.")),(0,a.yg)("hr",null),(0,a.yg)("h2",{id:"deploy-the-image-on-dsri"},"Deploy the image on DSRI"),(0,a.yg)("p",null,"Once you have a Docker image for your application you can deploy it using the ",(0,a.yg)("a",{parentName:"p",href:"https://console-openshift-console.apps.dsri2.unimaas.nl/console/projects"},"DSRI web UI"),"."),(0,a.yg)("p",null,"Go to the ",(0,a.yg)("a",{parentName:"p",href:"https://console-openshift-console.apps.dsri2.unimaas.nl/console/projects"},"Overview page")," of your project."),(0,a.yg)("ul",null,(0,a.yg)("li",{parentName:"ul"},"Click the ",(0,a.yg)("strong",{parentName:"li"},"Add to Project")," button in top right corner > ",(0,a.yg)("strong",{parentName:"li"},"Deploy Image")),(0,a.yg)("li",{parentName:"ul"},"Select to deploy from ",(0,a.yg)("strong",{parentName:"li"},"Image Name"),(0,a.yg)("ul",{parentName:"li"},(0,a.yg)("li",{parentName:"ul"},"Provide your image name, e.g. ",(0,a.yg)("inlineCode",{parentName:"li"},"umdsri/freesurfer")),(0,a.yg)("li",{parentName:"ul"},"Eventually change the ",(0,a.yg)("strong",{parentName:"li"},"Name"),", it needs to be unique by project."),(0,a.yg)("li",{parentName:"ul"},"Click ",(0,a.yg)("strong",{parentName:"li"},"Deploy"),".")))),(0,a.yg)("img",{src:"/img/screenshot-deploy_image_from_ui.png",alt:"Deploy image from UI",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,a.yg)("admonition",{title:"Fix a common problem",type:"caution"},(0,a.yg)("p",{parentName:"admonition"},"Once the application is deployed it will most probably fail because it has not been optimized to work with OpenShift random user ID. You will need to add an entry to the deployment to enable your image to run using any user ID.")),(0,a.yg)("p",null,"Go to ",(0,a.yg)("strong",{parentName:"p"},"Topology"),", click on your application node, click on the ",(0,a.yg)("strong",{parentName:"p"},"Actions")," button of your application details, and ",(0,a.yg)("strong",{parentName:"p"},"Edit deployment"),". In the deployment YAML search for ",(0,a.yg)("inlineCode",{parentName:"p"},"spec:")," which has a ",(0,a.yg)("inlineCode",{parentName:"p"},"containers:")," as child, and add the following under ",(0,a.yg)("inlineCode",{parentName:"p"},"spec:")),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-yaml"},"spec:\n serviceAccountName: anyuid\n containers: ...\n")),(0,a.yg)("admonition",{title:"Access the application",type:"info"},(0,a.yg)("p",{parentName:"admonition"},"You should now see your pod deployed on the ",(0,a.yg)("a",{parentName:"p",href:"https://console-openshift-console.apps.dsri2.unimaas.nl/console/projects"},"Overview")," page of your project."),(0,a.yg)("p",{parentName:"admonition"},"You can expose routes to this pod in the ",(0,a.yg)("a",{parentName:"p",href:"https://console-openshift-console.apps.dsri2.unimaas.nl/console/projects"},"Overview")," page: ",(0,a.yg)("strong",{parentName:"p"},"Create route"))),(0,a.yg)("hr",null),(0,a.yg)("h2",{id:"build-and-push-a-new-docker-image"},"Build and push a new Docker image"),(0,a.yg)("p",null,"In case you there is no Docker image for your application you can build and push one."),(0,a.yg)("p",null,"To build and push a Docker image you will need to have ",(0,a.yg)("a",{parentName:"p",href:"https://docs.docker.com/get-docker/"},"Docker installed"),"."),(0,a.yg)("admonition",{title:"Install Docker",type:"info"},(0,a.yg)("p",{parentName:"admonition"},"See the ",(0,a.yg)("a",{parentName:"p",href:"https://docs.docker.com/get-docker/"},"official documentation to install Docker"),".")),(0,a.yg)("h3",{id:"define-a-dockerfile"},"Define a Dockerfile"),(0,a.yg)("p",null,"If no images are available on DockerHub, it is still possible that the developers created the ",(0,a.yg)("a",{parentName:"p",href:"https://docs.docker.com/engine/reference/builder/"},"Dockerfile to build the image")," without pushing it to DockerHub. Go to the GitHub/GitLab source code repository and search for a ",(0,a.yg)("inlineCode",{parentName:"p"},"Dockerfile"),", it can usually be found in"),(0,a.yg)("ul",null,(0,a.yg)("li",{parentName:"ul"},"the source code repository root folder"),(0,a.yg)("li",{parentName:"ul"},"a ",(0,a.yg)("inlineCode",{parentName:"li"},"docker")," subfolder"),(0,a.yg)("li",{parentName:"ul"},"as instructions in the ",(0,a.yg)("inlineCode",{parentName:"li"},"README.md"))),(0,a.yg)("p",null,"If no ",(0,a.yg)("inlineCode",{parentName:"p"},"Dockerfile")," are available we will need to define one. "),(0,a.yg)("admonition",{title:"Contact us",type:"info"},(0,a.yg)("p",{parentName:"admonition"},"Feel free to ",(0,a.yg)("a",{parentName:"p",href:"/help"},"contact us")," to get help with this, especially if you are unfamiliar with ",(0,a.yg)("a",{parentName:"p",href:"https://docs.docker.com/get-started/"},"Docker"),".")),(0,a.yg)("h3",{id:"build-the-image"},"Build the image"),(0,a.yg)("p",null,"Once a Dockerfile has been defined for the service you can build it by running the following command from the source code root folder, where the Dockerfile is:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-shell"},"docker build -t username/my-service .\n")),(0,a.yg)("p",null,"Arguments can be provided when starting the build, they need to be defined in the Dockerfile to be used."),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-shell"},"docker build -t username/my-service --build-args MY_ARG=my_value .\n")),(0,a.yg)("h3",{id:"push-to-dockerhub"},"Push to DockerHub"),(0,a.yg)("p",null,"Before pushing it to DockerHub you will need to create a repository. To do so, click on ",(0,a.yg)("strong",{parentName:"p"},(0,a.yg)("a",{parentName:"strong",href:"https://hub.docker.com/repository/create"},"Create Repository")),"."),(0,a.yg)("ul",null,(0,a.yg)("li",{parentName:"ul"},"DockerHub is free for public repositories"),(0,a.yg)("li",{parentName:"ul"},"Images can be published under your DockerHub user or an organization you belong to")),(0,a.yg)("p",null,"Login to DockerHub, if not already done:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-shell"},"docker login\n")),(0,a.yg)("p",null,"Push the image previously built to DockerHub:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-shell"},"docker push username/my-service\n")),(0,a.yg)("p",null,"You can link DockerHub to your source code repository and ask it to build the Docker image automatically (from the Dockerfile in the root folder). It should take between 10 and 30min for DockerHub to build your image"),(0,a.yg)("admonition",{title:"Deploy from a local Dockerfile",type:"tip"},(0,a.yg)("p",{parentName:"admonition"},"You can also deploy a service on the DSRI directly from a local ",(0,a.yg)("inlineCode",{parentName:"p"},"Dockerfile"),", to avoid using DockerHub. See ",(0,a.yg)("a",{parentName:"p",href:"/docs/guide-dockerfile-to-openshift"},"this page to deploy a service from a local Dockerfile")," for more instructions")))}y.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/f97cc2c7.39969b67.js b/assets/js/f97cc2c7.709e45df.js similarity index 99% rename from assets/js/f97cc2c7.39969b67.js rename to assets/js/f97cc2c7.709e45df.js index 43462df9a..90efafba6 100644 --- a/assets/js/f97cc2c7.39969b67.js +++ b/assets/js/f97cc2c7.709e45df.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[5167],{5680:(e,n,t)=>{t.d(n,{xA:()=>p,yg:()=>d});var o=t(6540);function i(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function a(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);n&&(o=o.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,o)}return t}function r(e){for(var n=1;n=0||(i[t]=e[t]);return i}(e,n);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(o=0;o=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(i[t]=e[t])}return i}var c=o.createContext({}),s=function(e){var n=o.useContext(c),t=n;return e&&(t="function"==typeof e?e(n):r(r({},n),e)),t},p=function(e){var n=s(e.components);return o.createElement(c.Provider,{value:n},e.children)},h={inlineCode:"code",wrapper:function(e){var n=e.children;return o.createElement(o.Fragment,{},n)}},m=o.forwardRef((function(e,n){var t=e.components,i=e.mdxType,a=e.originalType,c=e.parentName,p=l(e,["components","mdxType","originalType","parentName"]),m=s(t),d=i,u=m["".concat(c,".").concat(d)]||m[d]||h[d]||a;return t?o.createElement(u,r(r({ref:n},p),{},{components:t})):o.createElement(u,r({ref:n},p))}));function d(e,n){var t=arguments,i=n&&n.mdxType;if("string"==typeof e||i){var a=t.length,r=new Array(a);r[0]=m;var l={};for(var c in n)hasOwnProperty.call(n,c)&&(l[c]=n[c]);l.originalType=e,l.mdxType="string"==typeof e?e:i,r[1]=l;for(var s=2;s{t.r(n),t.d(n,{assets:()=>p,contentTitle:()=>c,default:()=>d,frontMatter:()=>l,metadata:()=>s,toc:()=>h});var o=t(9668),i=t(1367),a=(t(6540),t(5680)),r=["components"],l={id:"checkpointing-ml-training",title:"Checkpointing Machine Learning Training"},c=void 0,s={unversionedId:"checkpointing-ml-training",id:"checkpointing-ml-training",title:"Checkpointing Machine Learning Training",description:"What is Checkpointing?",source:"@site/docs/checkpointing-ml-training-models.md",sourceDirName:".",slug:"/checkpointing-ml-training",permalink:"/docs/checkpointing-ml-training",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/checkpointing-ml-training-models.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"checkpointing-ml-training",title:"Checkpointing Machine Learning Training"},sidebar:"docs",previous:{title:"Access UM servers",permalink:"/docs/access-um-servers"},next:{title:"Parallelization using Dask",permalink:"/docs/dask-tutorial"}},p={},h=[{value:"What is Checkpointing?",id:"what-is-checkpointing",level:2},{value:"Checkpointing fequency?",id:"checkpointing-fequency",level:2},{value:"Support for Checkpointing in Tensorflow/Keras and PyTorch ?",id:"support-for-checkpointing-in-tensorflowkeras-and-pytorch-",level:2},{value:"Example of Tensorflow/Keras based checkpointing:",id:"example-of-tensorflowkeras-based-checkpointing",level:2},{value:"Example of PyTorch based checkpointing:",id:"example-of-pytorch-based-checkpointing",level:2},{value:"External Resources",id:"external-resources",level:2}],m={toc:h};function d(e){var n=e.components,t=(0,i.A)(e,r);return(0,a.yg)("wrapper",(0,o.A)({},m,t,{components:n,mdxType:"MDXLayout"}),(0,a.yg)("h2",{id:"what-is-checkpointing"},"What is Checkpointing?"),(0,a.yg)("p",null,"Checkpointing is periodically saving the learned model parameters and current hyperparameter values during training. It helps to resume training of a model where you left off, instead of restarting the training from the beginning."),(0,a.yg)("p",null,"On shared DSRI cluster, you might have access to a GPU node for a limited number of time in one stretch, for example, maybe for 24 hours.\nTherefore, whenever the training job fails (due to timelimit expiry or otherwise), many hours of training can be lost. This problem is mitigated by a frequent checkpoint saving. When the training is resumed it'll continue from the last checkpoint saved. If the failure occurred 12 hours after the last checkpoint has been saved, 12 hours of training is lost and needs to be re-done. This can be very expensive."),(0,a.yg)("h2",{id:"checkpointing-fequency"},"Checkpointing fequency?"),(0,a.yg)("p",null,"In theory one could save a checkpoint every 10 minutes and only ever lose 10 minutes of training time, but this too would dramatically delay the reaching of the finish line because large models can't be saved quickly and if the saving time starts to create a bottleneck for the training this approach becomes counterproductive."),(0,a.yg)("p",null,"Depending on your checkpointing methodology and the speed of your IO storage partition the saving of a large model can take from dozens of seconds to several minutes. Therefore, the optimal approach to saving frequency lies somewhere in the middle."),(0,a.yg)("p",null,"The math is quite simple - measure the amount of time it takes to save the checkpoint, multiply it by how many times you'd want to save it and see how much of an additional delay the checkpoint saving will contribute to the total training time."),(0,a.yg)("p",null,"For instance, Let suppose, "),(0,a.yg)("p",null,"1) Training Time (TT), i.e. allocated time on cluster : x days\n2) Time needed to save every checkpoint: y seconds\n3) Checkpoint fequencty: every z hours"),(0,a.yg)("p",null,"=> Then, Total Number of Checkpoints during the complete training time (NCP) = (x *24)/ z"),(0,a.yg)("p",null,"=> Total Time Spent on Checkpointing (TTSC) ","[in hours]"," = NCP * y/3600 "),(0,a.yg)("p",null,"=> % of Training time spent on checkpointing = (TTSC/TT",(0,a.yg)("em",{parentName:"p"},"24) ")," 100"),(0,a.yg)("p",null,"------------------Example calculations------------------------------------"),(0,a.yg)("p",null,"Training Time (TT or x): 7 days"),(0,a.yg)("p",null,"Time needed to save every checkpoint (y): 20 secs"),(0,a.yg)("p",null,"Checkpoint fequency (z): every 30 minutes, i.e., 0.5 hours"),(0,a.yg)("p",null,"Then, "),(0,a.yg)("p",null,"NCP = 7*24/0.5 = 336"),(0,a.yg)("p",null,"TTSC = 336* 20/3600 = 1.87 hours"),(0,a.yg)("p",null,"% of Training time spent on checkpointing = (1.87/7",(0,a.yg)("em",{parentName:"p"},"24)"),"100 ~ 1.2 % "),(0,a.yg)("h2",{id:"support-for-checkpointing-in-tensorflowkeras-and-pytorch-"},"Support for Checkpointing in Tensorflow/Keras and PyTorch ?"),(0,a.yg)("p",null,"Both PyTorch and TensorFlow/Keras support checkpointing. The follwoing sections provide an example of how Checkpointing can be done in these libraries."),(0,a.yg)("h2",{id:"example-of-tensorflowkeras-based-checkpointing"},"Example of Tensorflow/Keras based checkpointing:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-python"},"import tensorflow as tf\n\n#Imports the ModelCheckpoint class\nfrom tensorflow.keras.callbacks import ModelCheckpoint\n\n# Create your model as you normally would and compile it:\nmodel = tf.keras.models.Sequential([\n tf.keras.layers.Dense(64, activation='relu', input_shape=(32,)),\n tf.keras.layers.Dense(10, activation='softmax')\n])\nmodel.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n\n# Create a Checkpoint Callback\ncheckpoint_callback = ModelCheckpoint(\n#filepath should be a path to your persistent volume. Example, /home/jovyan path in your JupyterLab pod.\n filepath='model_checkpoint.h5', # You can use formats like .hdf5 or .ckpt. \n save_best_only=True,\n monitor='val_loss',\n mode='min',\n verbose=1\n)\n\n# Train the Model with the Checkpoint Callback\nhistory = model.fit(\n x_train, y_train,\n validation_data=(x_val, y_val),\n epochs=10,\n callbacks=[checkpoint_callback]\n)\n\n# Loading a Saved Checkpoint\n# Load the model architecture + weights if you saved the full model\nmodel = tf.keras.models.load_model('model_checkpoint.h5')\n\n# If you saved only the weights, you would need to create the model architecture first, then load weights:\nmodel.load_weights('model_checkpoint.h5')\n\n# Optional Parameters for Checkpointing, Example with Custom Save Intervals\ncheckpoint_callback = ModelCheckpoint(\n filepath='model_checkpoint_epoch_{epoch:02d}.h5',\n save_freq='epoch',\n save_weights_only=True,\n verbose=1\n)\n\n\n")),(0,a.yg)("h2",{id:"example-of-pytorch-based-checkpointing"},"Example of PyTorch based checkpointing:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-python"},"import torch\n\n# Example model\nmodel = torch.nn.Linear(10, 2)\n\n# Save the entire model\ntorch.save(model, 'model.pth')\n\n# Loading the Entire Model\nmodel = torch.load('model.pth')\n\n# Saving and Loading Optimizer State, i.e., To continue training exactly as before, you may want to save the optimizer state as well.\n\noptimizer = torch.optim.SGD(model.parameters(), lr=0.01)\n\n# Save model and optimizer state_dicts\ncheckpoint = {\n 'epoch': 5,\n 'model_state_dict': model.state_dict(),\n 'optimizer_state_dict': optimizer.state_dict(),\n 'loss': 0.5,\n}\ntorch.save(checkpoint, 'checkpoint.pth')\n\n# Load checkpoint\ncheckpoint = torch.load('checkpoint.pth')\nmodel.load_state_dict(checkpoint['model_state_dict'])\noptimizer.load_state_dict(checkpoint['optimizer_state_dict'])\nepoch = checkpoint['epoch']\nloss = checkpoint['loss']\nmodel.train() # Ensure model is in training mode if needed\n\n\n\n\n\n")),(0,a.yg)("h2",{id:"external-resources"},"External Resources"),(0,a.yg)("ul",null,(0,a.yg)("li",{parentName:"ul"},(0,a.yg)("p",{parentName:"li"},"PyTorch Documentation: ",(0,a.yg)("a",{parentName:"p",href:"https://pytorch.org/tutorials/beginner/saving_loading_models.html#save-on-gpu-load-on-cpu"},"https://pytorch.org/tutorials/beginner/saving_loading_models.html#save-on-gpu-load-on-cpu"))),(0,a.yg)("li",{parentName:"ul"},(0,a.yg)("p",{parentName:"li"},"Tensorflow/Keras Documentation:"),(0,a.yg)("p",{parentName:"li"},(0,a.yg)("a",{parentName:"p",href:"https://www.digitalocean.com/community/tutorials/checkpointing-in-tensorflow"},"https://www.digitalocean.com/community/tutorials/checkpointing-in-tensorflow")),(0,a.yg)("p",{parentName:"li"},(0,a.yg)("a",{parentName:"p",href:"https://keras.io/api/callbacks/model_checkpoint/"},"https://keras.io/api/callbacks/model_checkpoint/"))),(0,a.yg)("li",{parentName:"ul"},(0,a.yg)("p",{parentName:"li"},"Machine Learning Engineering by stas bekman:\n",(0,a.yg)("a",{parentName:"p",href:"https://stasosphere.com/machine-learning/"},"https://stasosphere.com/machine-learning/")))))}d.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[5167],{5680:(e,n,t)=>{t.d(n,{xA:()=>p,yg:()=>d});var o=t(6540);function i(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function a(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);n&&(o=o.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,o)}return t}function r(e){for(var n=1;n=0||(i[t]=e[t]);return i}(e,n);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(o=0;o=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(i[t]=e[t])}return i}var c=o.createContext({}),s=function(e){var n=o.useContext(c),t=n;return e&&(t="function"==typeof e?e(n):r(r({},n),e)),t},p=function(e){var n=s(e.components);return o.createElement(c.Provider,{value:n},e.children)},h={inlineCode:"code",wrapper:function(e){var n=e.children;return o.createElement(o.Fragment,{},n)}},m=o.forwardRef((function(e,n){var t=e.components,i=e.mdxType,a=e.originalType,c=e.parentName,p=l(e,["components","mdxType","originalType","parentName"]),m=s(t),d=i,u=m["".concat(c,".").concat(d)]||m[d]||h[d]||a;return t?o.createElement(u,r(r({ref:n},p),{},{components:t})):o.createElement(u,r({ref:n},p))}));function d(e,n){var t=arguments,i=n&&n.mdxType;if("string"==typeof e||i){var a=t.length,r=new Array(a);r[0]=m;var l={};for(var c in n)hasOwnProperty.call(n,c)&&(l[c]=n[c]);l.originalType=e,l.mdxType="string"==typeof e?e:i,r[1]=l;for(var s=2;s{t.r(n),t.d(n,{assets:()=>p,contentTitle:()=>c,default:()=>d,frontMatter:()=>l,metadata:()=>s,toc:()=>h});var o=t(9668),i=t(1367),a=(t(6540),t(5680)),r=["components"],l={id:"checkpointing-ml-training",title:"Checkpointing Machine Learning Training"},c=void 0,s={unversionedId:"checkpointing-ml-training",id:"checkpointing-ml-training",title:"Checkpointing Machine Learning Training",description:"What is Checkpointing?",source:"@site/docs/checkpointing-ml-training-models.md",sourceDirName:".",slug:"/checkpointing-ml-training",permalink:"/docs/checkpointing-ml-training",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/checkpointing-ml-training-models.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"checkpointing-ml-training",title:"Checkpointing Machine Learning Training"},sidebar:"docs",previous:{title:"Access UM servers",permalink:"/docs/access-um-servers"},next:{title:"Parallelization using Dask",permalink:"/docs/dask-tutorial"}},p={},h=[{value:"What is Checkpointing?",id:"what-is-checkpointing",level:2},{value:"Checkpointing fequency?",id:"checkpointing-fequency",level:2},{value:"Support for Checkpointing in Tensorflow/Keras and PyTorch ?",id:"support-for-checkpointing-in-tensorflowkeras-and-pytorch-",level:2},{value:"Example of Tensorflow/Keras based checkpointing:",id:"example-of-tensorflowkeras-based-checkpointing",level:2},{value:"Example of PyTorch based checkpointing:",id:"example-of-pytorch-based-checkpointing",level:2},{value:"External Resources",id:"external-resources",level:2}],m={toc:h};function d(e){var n=e.components,t=(0,i.A)(e,r);return(0,a.yg)("wrapper",(0,o.A)({},m,t,{components:n,mdxType:"MDXLayout"}),(0,a.yg)("h2",{id:"what-is-checkpointing"},"What is Checkpointing?"),(0,a.yg)("p",null,"Checkpointing is periodically saving the learned model parameters and current hyperparameter values during training. It helps to resume training of a model where you left off, instead of restarting the training from the beginning."),(0,a.yg)("p",null,"On shared DSRI cluster, you might have access to a GPU node for a limited number of time in one stretch, for example, maybe for 24 hours.\nTherefore, whenever the training job fails (due to timelimit expiry or otherwise), many hours of training can be lost. This problem is mitigated by a frequent checkpoint saving. When the training is resumed it'll continue from the last checkpoint saved. If the failure occurred 12 hours after the last checkpoint has been saved, 12 hours of training is lost and needs to be re-done. This can be very expensive."),(0,a.yg)("h2",{id:"checkpointing-fequency"},"Checkpointing fequency?"),(0,a.yg)("p",null,"In theory one could save a checkpoint every 10 minutes and only ever lose 10 minutes of training time, but this too would dramatically delay the reaching of the finish line because large models can't be saved quickly and if the saving time starts to create a bottleneck for the training this approach becomes counterproductive."),(0,a.yg)("p",null,"Depending on your checkpointing methodology and the speed of your IO storage partition the saving of a large model can take from dozens of seconds to several minutes. Therefore, the optimal approach to saving frequency lies somewhere in the middle."),(0,a.yg)("p",null,"The math is quite simple - measure the amount of time it takes to save the checkpoint, multiply it by how many times you'd want to save it and see how much of an additional delay the checkpoint saving will contribute to the total training time."),(0,a.yg)("p",null,"For instance, Let suppose, "),(0,a.yg)("p",null,"1) Training Time (TT), i.e. allocated time on cluster : x days\n2) Time needed to save every checkpoint: y seconds\n3) Checkpoint fequencty: every z hours"),(0,a.yg)("p",null,"=> Then, Total Number of Checkpoints during the complete training time (NCP) = (x *24)/ z"),(0,a.yg)("p",null,"=> Total Time Spent on Checkpointing (TTSC) ","[in hours]"," = NCP * y/3600 "),(0,a.yg)("p",null,"=> % of Training time spent on checkpointing = (TTSC/TT",(0,a.yg)("em",{parentName:"p"},"24) ")," 100"),(0,a.yg)("p",null,"------------------Example calculations------------------------------------"),(0,a.yg)("p",null,"Training Time (TT or x): 7 days"),(0,a.yg)("p",null,"Time needed to save every checkpoint (y): 20 secs"),(0,a.yg)("p",null,"Checkpoint fequency (z): every 30 minutes, i.e., 0.5 hours"),(0,a.yg)("p",null,"Then, "),(0,a.yg)("p",null,"NCP = 7*24/0.5 = 336"),(0,a.yg)("p",null,"TTSC = 336* 20/3600 = 1.87 hours"),(0,a.yg)("p",null,"% of Training time spent on checkpointing = (1.87/7",(0,a.yg)("em",{parentName:"p"},"24)"),"100 ~ 1.2 % "),(0,a.yg)("h2",{id:"support-for-checkpointing-in-tensorflowkeras-and-pytorch-"},"Support for Checkpointing in Tensorflow/Keras and PyTorch ?"),(0,a.yg)("p",null,"Both PyTorch and TensorFlow/Keras support checkpointing. The follwoing sections provide an example of how Checkpointing can be done in these libraries."),(0,a.yg)("h2",{id:"example-of-tensorflowkeras-based-checkpointing"},"Example of Tensorflow/Keras based checkpointing:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-python"},"import tensorflow as tf\n\n#Imports the ModelCheckpoint class\nfrom tensorflow.keras.callbacks import ModelCheckpoint\n\n# Create your model as you normally would and compile it:\nmodel = tf.keras.models.Sequential([\n tf.keras.layers.Dense(64, activation='relu', input_shape=(32,)),\n tf.keras.layers.Dense(10, activation='softmax')\n])\nmodel.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\n\n# Create a Checkpoint Callback\ncheckpoint_callback = ModelCheckpoint(\n#filepath should be a path to your persistent volume. Example, /home/jovyan path in your JupyterLab pod.\n filepath='model_checkpoint.h5', # You can use formats like .hdf5 or .ckpt. \n save_best_only=True,\n monitor='val_loss',\n mode='min',\n verbose=1\n)\n\n# Train the Model with the Checkpoint Callback\nhistory = model.fit(\n x_train, y_train,\n validation_data=(x_val, y_val),\n epochs=10,\n callbacks=[checkpoint_callback]\n)\n\n# Loading a Saved Checkpoint\n# Load the model architecture + weights if you saved the full model\nmodel = tf.keras.models.load_model('model_checkpoint.h5')\n\n# If you saved only the weights, you would need to create the model architecture first, then load weights:\nmodel.load_weights('model_checkpoint.h5')\n\n# Optional Parameters for Checkpointing, Example with Custom Save Intervals\ncheckpoint_callback = ModelCheckpoint(\n filepath='model_checkpoint_epoch_{epoch:02d}.h5',\n save_freq='epoch',\n save_weights_only=True,\n verbose=1\n)\n\n\n")),(0,a.yg)("h2",{id:"example-of-pytorch-based-checkpointing"},"Example of PyTorch based checkpointing:"),(0,a.yg)("pre",null,(0,a.yg)("code",{parentName:"pre",className:"language-python"},"import torch\n\n# Example model\nmodel = torch.nn.Linear(10, 2)\n\n# Save the entire model\ntorch.save(model, 'model.pth')\n\n# Loading the Entire Model\nmodel = torch.load('model.pth')\n\n# Saving and Loading Optimizer State, i.e., To continue training exactly as before, you may want to save the optimizer state as well.\n\noptimizer = torch.optim.SGD(model.parameters(), lr=0.01)\n\n# Save model and optimizer state_dicts\ncheckpoint = {\n 'epoch': 5,\n 'model_state_dict': model.state_dict(),\n 'optimizer_state_dict': optimizer.state_dict(),\n 'loss': 0.5,\n}\ntorch.save(checkpoint, 'checkpoint.pth')\n\n# Load checkpoint\ncheckpoint = torch.load('checkpoint.pth')\nmodel.load_state_dict(checkpoint['model_state_dict'])\noptimizer.load_state_dict(checkpoint['optimizer_state_dict'])\nepoch = checkpoint['epoch']\nloss = checkpoint['loss']\nmodel.train() # Ensure model is in training mode if needed\n\n\n\n\n\n")),(0,a.yg)("h2",{id:"external-resources"},"External Resources"),(0,a.yg)("ul",null,(0,a.yg)("li",{parentName:"ul"},(0,a.yg)("p",{parentName:"li"},"PyTorch Documentation: ",(0,a.yg)("a",{parentName:"p",href:"https://pytorch.org/tutorials/beginner/saving_loading_models.html#save-on-gpu-load-on-cpu"},"https://pytorch.org/tutorials/beginner/saving_loading_models.html#save-on-gpu-load-on-cpu"))),(0,a.yg)("li",{parentName:"ul"},(0,a.yg)("p",{parentName:"li"},"Tensorflow/Keras Documentation:"),(0,a.yg)("p",{parentName:"li"},(0,a.yg)("a",{parentName:"p",href:"https://www.digitalocean.com/community/tutorials/checkpointing-in-tensorflow"},"https://www.digitalocean.com/community/tutorials/checkpointing-in-tensorflow")),(0,a.yg)("p",{parentName:"li"},(0,a.yg)("a",{parentName:"p",href:"https://keras.io/api/callbacks/model_checkpoint/"},"https://keras.io/api/callbacks/model_checkpoint/"))),(0,a.yg)("li",{parentName:"ul"},(0,a.yg)("p",{parentName:"li"},"Machine Learning Engineering by stas bekman:\n",(0,a.yg)("a",{parentName:"p",href:"https://stasosphere.com/machine-learning/"},"https://stasosphere.com/machine-learning/")))))}d.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/fdea7475.cfd523fe.js b/assets/js/fdea7475.466759ce.js similarity index 99% rename from assets/js/fdea7475.cfd523fe.js rename to assets/js/fdea7475.466759ce.js index 670458d09..61b45d4c5 100644 --- a/assets/js/fdea7475.cfd523fe.js +++ b/assets/js/fdea7475.466759ce.js @@ -1 +1 @@ -"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[918],{5680:(e,t,r)=>{r.d(t,{xA:()=>u,yg:()=>f});var s=r(6540);function a(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function n(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var s=Object.getOwnPropertySymbols(e);t&&(s=s.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,s)}return r}function o(e){for(var t=1;t=0||(a[r]=e[r]);return a}(e,t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);for(s=0;s=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(a[r]=e[r])}return a}var l=s.createContext({}),c=function(e){var t=s.useContext(l),r=t;return e&&(r="function"==typeof e?e(t):o(o({},t),e)),r},u=function(e){var t=c(e.components);return s.createElement(l.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return s.createElement(s.Fragment,{},t)}},p=s.forwardRef((function(e,t){var r=e.components,a=e.mdxType,n=e.originalType,l=e.parentName,u=i(e,["components","mdxType","originalType","parentName"]),p=c(r),f=a,h=p["".concat(l,".").concat(f)]||p[f]||d[f]||n;return r?s.createElement(h,o(o({ref:t},u),{},{components:r})):s.createElement(h,o({ref:t},u))}));function f(e,t){var r=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var n=r.length,o=new Array(n);o[0]=p;var i={};for(var l in t)hasOwnProperty.call(t,l)&&(i[l]=t[l]);i.originalType=e,i.mdxType="string"==typeof e?e:a,o[1]=i;for(var c=2;c{r.r(t),r.d(t,{assets:()=>u,contentTitle:()=>l,default:()=>f,frontMatter:()=>i,metadata:()=>c,toc:()=>d});var s=r(9668),a=r(1367),n=(r(6540),r(5680)),o=["components"],i={id:"surf-offerings",title:"SURF Offerings"},l=void 0,c={unversionedId:"surf-offerings",id:"surf-offerings",title:"SURF Offerings",description:"SURF's Digital Services for Research and Development",source:"@site/docs/surf-offerings.md",sourceDirName:".",slug:"/surf-offerings",permalink:"/docs/surf-offerings",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/surf-offerings.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736860945,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"surf-offerings",title:"SURF Offerings"},sidebar:"docs",previous:{title:"Tensorflow Optimization",permalink:"/docs/speeding-tensorflow-dl"}},u={},d=[{value:"SURF's Digital Services for Research and Development",id:"surfs-digital-services-for-research-and-development",level:2},{value:"What is SURF?",id:"what-is-surf",level:2},{value:"What is a cluster computer?",id:"what-is-a-cluster-computer",level:2},{value:"Different types of Services provided by SURF:",id:"different-types-of-services-provided-by-surf",level:2},{value:"How to Get Started with SURF Services?",id:"how-to-get-started-with-surf-services",level:2},{value:"External Resources and references",id:"external-resources-and-references",level:2}],p={toc:d};function f(e){var t=e.components,r=(0,a.A)(e,o);return(0,n.yg)("wrapper",(0,s.A)({},p,r,{components:t,mdxType:"MDXLayout"}),(0,n.yg)("h2",{id:"surfs-digital-services-for-research-and-development"},"SURF's Digital Services for Research and Development"),(0,n.yg)("h2",{id:"what-is-surf"},"What is SURF?"),(0,n.yg)("p",null,"SURF is the ICT cooperative for Dutch education and research institutions. As a collaborative organization, SURF\u2019s members\u2014its owners\u2014work together to deliver top-tier digital services, address complex innovation challenges, and exchange valuable knowledge."),(0,n.yg)("p",null,"Computing and storage infrastructure are essential for cutting-edge research. SURF supports researchers with a diverse range of computing and storage services. But before diving into these services, let\u2019s briefly explore what a cluster computer is."),(0,n.yg)("h2",{id:"what-is-a-cluster-computer"},"What is a cluster computer?"),(0,n.yg)("p",null,"A cluster computer is essentially a group of interconnected computers, called nodes, working together as a unified system. Each node has its own CPU, memory, and disk space, along with access to a shared file system. Imagine these nodes connected by network cables, like those in your home or office."),(0,n.yg)("p",null,"Cluster computers are designed for high-performance workloads, allowing users to run hundreds of computational tasks simultaneously."),(0,n.yg)("img",{src:"/img/screenshot_cluster.png",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,n.yg)("h2",{id:"different-types-of-services-provided-by-surf"},"Different types of Services provided by SURF:"),(0,n.yg)("p",null,"Some of the computing and storage solution provided by SURF are:"),(0,n.yg)("p",null,"1) Spider Cluster - High-performance Data Processing (DP) platform:"),(0,n.yg)("p",null," Spider is a versatile DP platform aimed at processing large structured data sets. Spider is an in house compute cluster built on top of SURF\u2019s in-house elastic Cloud. This allows for scalable processing of many terabytes or even petabytes of data, utilizing many hundreds of cores simultaneously, in exceedingly short timespans. Superb network throughput ensures connectivity to external data storage systems. Spider is used for large scale multi-year data intensive projects, for users to actively process their data, such are large static data sets or continuously growing data sets. Examples include genomics data, astronomic telescope data, physics detector data and satellite earth observations."),(0,n.yg)("p",null,"2) Snellius Cluster - the Dutch National supercomputer:"),(0,n.yg)("p",null," Snellius is the Dutch National supercomputer hosted at SURF. The system facilitates scientific research carried out in many Universities, independent research institutes, governmental organizations, and private companies in the Netherlands. Snellius is a cluster of heterogeneous nodes built by Lenovo, containing predominantly AMD technology, with capabilities for high performance computing (parallel, symmetric multiprocessing). The system also has several system-specific storage resources, that are geared towards supporting the various types of computing."),(0,n.yg)("p",null,"3) SURF Research Cloud (SRC):"),(0,n.yg)("p",null,' SURF Research Cloud is a service to facilitate scientists\u2019 collaborative work. The central idea in SRC is collaborative workspace. A workspcae translates directly to a "Virtual Machine".\nThese hosted workspaces aka virtual machines can be used for conducting research and development individually or together with your team/project members. '),(0,n.yg)("p",null,"4) Research Data Storage Services:"),(0,n.yg)("p",null," 4.1) Data Archive : The SURF Data Archive allows users to safely archive up to petabytes of valuable research data to ensure the long term accessibility and reproducibility of their work. The Data Archive is also connected to SURF\u2019s compute infrastructure, via a fast network connection, allowing for the seamless depositing and retrieval of data."),(0,n.yg)("p",null," 4.2) Data Repository : The Data Repository service is a web-based data publication and archiving platform that allows researchers to store, annotate and publish research data to ensure long-term preservation and availability of their datasets. All published datasets get their own DOI and Handle, while every file gets its own independent Handle to allow persistent reference on all levels."),(0,n.yg)("p",null," 4.3) dCache : dCache is scalable storage system. It contains more than 50 petabytes of scientific data, accessible through several authentication methods and protocols. It consists of\nmagnetic tape storage and hard disk storage and both are addressed by a common file system."),(0,n.yg)("p",null," 4.4) Object Store : Object storage is ideal for storing unstructured data that can grow without bound. Object storage does not have a directory-type structure like a normal file system has\nbut it organises its data in so-called containers that contain objects. There is no tree-like structure with files and directories. There are only containers with objects in them. SURF Object Store service is based on Ceph RGW and provides access using the S3 protocol, which is the defacto standard for addressing object storage."),(0,n.yg)("h2",{id:"how-to-get-started-with-surf-services"},"How to Get Started with SURF Services?"),(0,n.yg)("p",null,"The DSRI team is here to help you navigate SURF\u2019s services, including:"),(0,n.yg)("p",null,"1) Grant Applications:",(0,n.yg)("br",{parentName:"p"}),"\n","We assist researchers in applying for SURF grants. For instance:"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre"},"* Small applications: Up to 1 million System Billing Units (SBU) on Snellius and/or 100 TB of dCache storage.(https://www.surf.nl/en/small-compute-applications-nwo)\n* Large applications: Customized resource allocations based on project needs.\n")),(0,n.yg)("p",null,"2) Resource Estimation:",(0,n.yg)("br",{parentName:"p"}),"\n","Unsure about your computing and storage requirements? We help estimate your needs in terms of SURF\u2019s billing units."),(0,n.yg)("p",null,"3) Use Case Analysis:",(0,n.yg)("br",{parentName:"p"}),"\n","We assess whether your research project is a good fit for SURF\u2019s services."),(0,n.yg)("h2",{id:"external-resources-and-references"},"External Resources and references"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"SURF: ",(0,n.yg)("a",{parentName:"li",href:"https://www.surf.nl/en"},"https://www.surf.nl/en")),(0,n.yg)("li",{parentName:"ul"},"Deep Learning Tutorials by UvA: ",(0,n.yg)("a",{parentName:"li",href:"https://uvadlc-notebooks.readthedocs.io/en/latest/index.html"},"https://uvadlc-notebooks.readthedocs.io/en/latest/index.html"))))}f.isMDXComponent=!0}}]); \ No newline at end of file +"use strict";(self.webpackChunk=self.webpackChunk||[]).push([[918],{5680:(e,t,r)=>{r.d(t,{xA:()=>u,yg:()=>f});var s=r(6540);function a(e,t,r){return t in e?Object.defineProperty(e,t,{value:r,enumerable:!0,configurable:!0,writable:!0}):e[t]=r,e}function n(e,t){var r=Object.keys(e);if(Object.getOwnPropertySymbols){var s=Object.getOwnPropertySymbols(e);t&&(s=s.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),r.push.apply(r,s)}return r}function o(e){for(var t=1;t=0||(a[r]=e[r]);return a}(e,t);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);for(s=0;s=0||Object.prototype.propertyIsEnumerable.call(e,r)&&(a[r]=e[r])}return a}var l=s.createContext({}),c=function(e){var t=s.useContext(l),r=t;return e&&(r="function"==typeof e?e(t):o(o({},t),e)),r},u=function(e){var t=c(e.components);return s.createElement(l.Provider,{value:t},e.children)},d={inlineCode:"code",wrapper:function(e){var t=e.children;return s.createElement(s.Fragment,{},t)}},p=s.forwardRef((function(e,t){var r=e.components,a=e.mdxType,n=e.originalType,l=e.parentName,u=i(e,["components","mdxType","originalType","parentName"]),p=c(r),f=a,h=p["".concat(l,".").concat(f)]||p[f]||d[f]||n;return r?s.createElement(h,o(o({ref:t},u),{},{components:r})):s.createElement(h,o({ref:t},u))}));function f(e,t){var r=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var n=r.length,o=new Array(n);o[0]=p;var i={};for(var l in t)hasOwnProperty.call(t,l)&&(i[l]=t[l]);i.originalType=e,i.mdxType="string"==typeof e?e:a,o[1]=i;for(var c=2;c{r.r(t),r.d(t,{assets:()=>u,contentTitle:()=>l,default:()=>f,frontMatter:()=>i,metadata:()=>c,toc:()=>d});var s=r(9668),a=r(1367),n=(r(6540),r(5680)),o=["components"],i={id:"surf-offerings",title:"SURF Offerings"},l=void 0,c={unversionedId:"surf-offerings",id:"surf-offerings",title:"SURF Offerings",description:"SURF's Digital Services for Research and Development",source:"@site/docs/surf-offerings.md",sourceDirName:".",slug:"/surf-offerings",permalink:"/docs/surf-offerings",draft:!1,editUrl:"https://github.com/MaastrichtU-IDS/dsri-documentation/edit/master/website/docs/surf-offerings.md",tags:[],version:"current",lastUpdatedBy:"Adekunle Onaopepo",lastUpdatedAt:1736869254,formattedLastUpdatedAt:"Jan 14, 2025",frontMatter:{id:"surf-offerings",title:"SURF Offerings"},sidebar:"docs",previous:{title:"Tensorflow Optimization",permalink:"/docs/speeding-tensorflow-dl"}},u={},d=[{value:"SURF's Digital Services for Research and Development",id:"surfs-digital-services-for-research-and-development",level:2},{value:"What is SURF?",id:"what-is-surf",level:2},{value:"What is a cluster computer?",id:"what-is-a-cluster-computer",level:2},{value:"Different types of Services provided by SURF:",id:"different-types-of-services-provided-by-surf",level:2},{value:"How to Get Started with SURF Services?",id:"how-to-get-started-with-surf-services",level:2},{value:"External Resources and references",id:"external-resources-and-references",level:2}],p={toc:d};function f(e){var t=e.components,r=(0,a.A)(e,o);return(0,n.yg)("wrapper",(0,s.A)({},p,r,{components:t,mdxType:"MDXLayout"}),(0,n.yg)("h2",{id:"surfs-digital-services-for-research-and-development"},"SURF's Digital Services for Research and Development"),(0,n.yg)("h2",{id:"what-is-surf"},"What is SURF?"),(0,n.yg)("p",null,"SURF is the ICT cooperative for Dutch education and research institutions. As a collaborative organization, SURF\u2019s members\u2014its owners\u2014work together to deliver top-tier digital services, address complex innovation challenges, and exchange valuable knowledge."),(0,n.yg)("p",null,"Computing and storage infrastructure are essential for cutting-edge research. SURF supports researchers with a diverse range of computing and storage services. But before diving into these services, let\u2019s briefly explore what a cluster computer is."),(0,n.yg)("h2",{id:"what-is-a-cluster-computer"},"What is a cluster computer?"),(0,n.yg)("p",null,"A cluster computer is essentially a group of interconnected computers, called nodes, working together as a unified system. Each node has its own CPU, memory, and disk space, along with access to a shared file system. Imagine these nodes connected by network cables, like those in your home or office."),(0,n.yg)("p",null,"Cluster computers are designed for high-performance workloads, allowing users to run hundreds of computational tasks simultaneously."),(0,n.yg)("img",{src:"/img/screenshot_cluster.png",style:{maxWidth:"100%",maxHeight:"100%"}}),(0,n.yg)("h2",{id:"different-types-of-services-provided-by-surf"},"Different types of Services provided by SURF:"),(0,n.yg)("p",null,"Some of the computing and storage solution provided by SURF are:"),(0,n.yg)("p",null,"1) Spider Cluster - High-performance Data Processing (DP) platform:"),(0,n.yg)("p",null," Spider is a versatile DP platform aimed at processing large structured data sets. Spider is an in house compute cluster built on top of SURF\u2019s in-house elastic Cloud. This allows for scalable processing of many terabytes or even petabytes of data, utilizing many hundreds of cores simultaneously, in exceedingly short timespans. Superb network throughput ensures connectivity to external data storage systems. Spider is used for large scale multi-year data intensive projects, for users to actively process their data, such are large static data sets or continuously growing data sets. Examples include genomics data, astronomic telescope data, physics detector data and satellite earth observations."),(0,n.yg)("p",null,"2) Snellius Cluster - the Dutch National supercomputer:"),(0,n.yg)("p",null," Snellius is the Dutch National supercomputer hosted at SURF. The system facilitates scientific research carried out in many Universities, independent research institutes, governmental organizations, and private companies in the Netherlands. Snellius is a cluster of heterogeneous nodes built by Lenovo, containing predominantly AMD technology, with capabilities for high performance computing (parallel, symmetric multiprocessing). The system also has several system-specific storage resources, that are geared towards supporting the various types of computing."),(0,n.yg)("p",null,"3) SURF Research Cloud (SRC):"),(0,n.yg)("p",null,' SURF Research Cloud is a service to facilitate scientists\u2019 collaborative work. The central idea in SRC is collaborative workspace. A workspcae translates directly to a "Virtual Machine".\nThese hosted workspaces aka virtual machines can be used for conducting research and development individually or together with your team/project members. '),(0,n.yg)("p",null,"4) Research Data Storage Services:"),(0,n.yg)("p",null," 4.1) Data Archive : The SURF Data Archive allows users to safely archive up to petabytes of valuable research data to ensure the long term accessibility and reproducibility of their work. The Data Archive is also connected to SURF\u2019s compute infrastructure, via a fast network connection, allowing for the seamless depositing and retrieval of data."),(0,n.yg)("p",null," 4.2) Data Repository : The Data Repository service is a web-based data publication and archiving platform that allows researchers to store, annotate and publish research data to ensure long-term preservation and availability of their datasets. All published datasets get their own DOI and Handle, while every file gets its own independent Handle to allow persistent reference on all levels."),(0,n.yg)("p",null," 4.3) dCache : dCache is scalable storage system. It contains more than 50 petabytes of scientific data, accessible through several authentication methods and protocols. It consists of\nmagnetic tape storage and hard disk storage and both are addressed by a common file system."),(0,n.yg)("p",null," 4.4) Object Store : Object storage is ideal for storing unstructured data that can grow without bound. Object storage does not have a directory-type structure like a normal file system has\nbut it organises its data in so-called containers that contain objects. There is no tree-like structure with files and directories. There are only containers with objects in them. SURF Object Store service is based on Ceph RGW and provides access using the S3 protocol, which is the defacto standard for addressing object storage."),(0,n.yg)("h2",{id:"how-to-get-started-with-surf-services"},"How to Get Started with SURF Services?"),(0,n.yg)("p",null,"The DSRI team is here to help you navigate SURF\u2019s services, including:"),(0,n.yg)("p",null,"1) Grant Applications:",(0,n.yg)("br",{parentName:"p"}),"\n","We assist researchers in applying for SURF grants. For instance:"),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre"},"* Small applications: Up to 1 million System Billing Units (SBU) on Snellius and/or 100 TB of dCache storage.(https://www.surf.nl/en/small-compute-applications-nwo)\n* Large applications: Customized resource allocations based on project needs.\n")),(0,n.yg)("p",null,"2) Resource Estimation:",(0,n.yg)("br",{parentName:"p"}),"\n","Unsure about your computing and storage requirements? We help estimate your needs in terms of SURF\u2019s billing units."),(0,n.yg)("p",null,"3) Use Case Analysis:",(0,n.yg)("br",{parentName:"p"}),"\n","We assess whether your research project is a good fit for SURF\u2019s services."),(0,n.yg)("h2",{id:"external-resources-and-references"},"External Resources and references"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"SURF: ",(0,n.yg)("a",{parentName:"li",href:"https://www.surf.nl/en"},"https://www.surf.nl/en")),(0,n.yg)("li",{parentName:"ul"},"Deep Learning Tutorials by UvA: ",(0,n.yg)("a",{parentName:"li",href:"https://uvadlc-notebooks.readthedocs.io/en/latest/index.html"},"https://uvadlc-notebooks.readthedocs.io/en/latest/index.html"))))}f.isMDXComponent=!0}}]); \ No newline at end of file diff --git a/assets/js/main.4bfa7dab.js b/assets/js/main.8bb96046.js similarity index 99% rename from assets/js/main.4bfa7dab.js rename to assets/js/main.8bb96046.js index e0ad43897..d35bd9bc6 100644 --- a/assets/js/main.4bfa7dab.js +++ b/assets/js/main.8bb96046.js @@ -1 +1 @@ -(self.webpackChunk=self.webpackChunk||[]).push([[8792],{870:(e,t,n)=>{"use strict";function r(e){var t,n,o="";if("string"==typeof e||"number"==typeof e)o+=e;else if("object"==typeof e)if(Array.isArray(e))for(t=0;to});const o=function(){for(var e,t,n=0,o="";n{"use strict";function r(e){var t,n,o="";if("string"==typeof e||"number"==typeof e)o+=e;else if("object"==typeof e)if(Array.isArray(e))for(t=0;to});const o=function(){for(var e,t,n=0,o="";n{"use strict";n.d(t,{A:()=>p});var r=n(6540),o=n(9668),a=n(3259),i=n.n(a),l=n(4054);const s={"02589645":[function(){return n.e(8926).then(n.bind(n,2763))},"@site/blog/2016-03-11-blog-post.md?truncated=true",2763],"09722083":[function(){return n.e(9676).then(n.bind(n,4005))},"@site/docs/catalog-genomics.md",4005],"09f56f30":[function(){return n.e(3360).then(n.bind(n,7366))},"@site/docs/anatomy-of-an-application.md",7366],"0e4359fd":[function(){return n.e(2957).then(n.bind(n,7780))},"@site/docs/helm.md",7780],"10a2e8c0":[function(){return n.e(6083).then(n.bind(n,7622))},"@site/blog/2017-10-24-new-version-1.0.0.md?truncated=true",7622],"11da9ee4":[function(){return n.e(4809).then(n.bind(n,8655))},"@site/docs/catalog-opendatahub.md",8655],17896441:[function(){return Promise.all([n.e(1869),n.e(6685),n.e(8401)]).then(n.bind(n,3597))},"@theme/DocItem",3597],"1a119cf5":[function(){return n.e(7160).then(n.bind(n,9802))},"@site/src/pages/acknowledgement.md",9802],"1a4e3797":[function(){return Promise.all([n.e(1869),n.e(2138)]).then(n.bind(n,8988))},"@theme/SearchPage",8988],"1be78505":[function(){return Promise.all([n.e(1869),n.e(8714)]).then(n.bind(n,10))},"@theme/DocPage",10],"1c23ce5e":[function(){return n.e(3432).then(n.t.bind(n,2945,19))},"/home/runner/work/dsri-documentation/dsri-documentation/website/.docusaurus/docusaurus-plugin-content-blog/default/plugin-route-context-module-100.json",2945],"1dc85e61":[function(){return n.e(9328).then(n.bind(n,6334))},"@site/docs/deploy-gitlab-runner.md",6334],"1df93b7f":[function(){return Promise.all([n.e(1869),n.e(6046),n.e(4991),n.e(4583)]).then(n.bind(n,6866))},"@site/src/pages/index.tsx",6866],"1f391b9e":[function(){return Promise.all([n.e(1869),n.e(6685),n.e(6061)]).then(n.bind(n,7973))},"@theme/MDXPage",7973],"21dd1498":[function(){return n.e(4896).then(n.bind(n,6022))},"@site/docs/openshift-storage.md",6022],"2406662c":[function(){return n.e(9030).then(n.bind(n,4664))},"@site/docs/prepare-project-for-dsri.md",4664],"2b1adfae":[function(){return n.e(8793).then(n.bind(n,4170))},"@site/docs/increase-process-speed.md",4170],"2b7d82ba":[function(){return n.e(5447).then(n.bind(n,9284))},"@site/docs/workflows-github-actions.md",9284],"2e2e5152":[function(){return n.e(2412).then(n.bind(n,6325))},"@site/docs/guide-known-issues.md",6325],"36f9137d":[function(){return n.e(5210).then(n.bind(n,9271))},"@site/docs/deploy-jupyter.md",9271],"389e2b0f":[function(){return n.e(610).then(n.bind(n,4361))},"@site/docs/openshift-commands.md",4361],"391576c4":[function(){return n.e(2567).then(n.bind(n,1106))},"@site/docs/enabling-vpn-wsl.md",1106],"3985f3fe":[function(){return n.e(4572).then(n.bind(n,3419))},"@site/blog/2017-04-10-blog-post-two.md?truncated=true",3419],"3a93ea81":[function(){return n.e(3684).then(n.bind(n,691))},"@site/docs/jupyterhub-spark.md",691],"3cfdff65":[function(){return n.e(4792).then(n.bind(n,4100))},"@site/docs/guide-publish-image.md",4100],"3f4d8b80":[function(){return n.e(4190).then(n.bind(n,9980))},"@site/docs/deploy-jupyterhub.md",9980],"47a4a695":[function(){return n.e(3836).then(n.bind(n,8550))},"@site/docs/login-docker-registry.md",8550],"4ba7e5a3":[function(){return n.e(9591).then(n.bind(n,4247))},"@site/docs/contribute.md",4247],"4d7bd50c":[function(){return Promise.all([n.e(1869),n.e(6046),n.e(3570),n.e(3892),n.e(1590)]).then(n.bind(n,8458))},"@site/src/pages/gpu-booking.tsx",8458],"4d826c8d":[function(){return n.e(3348).then(n.bind(n,6583))},"@site/blog/2017-09-25-testing-rss.md",6583],"517aa66b":[function(){return n.e(6364).then(n.bind(n,8097))},"@site/blog/2017-09-26-adding-rss.md?truncated=true",8097],"521adc3d":[function(){return n.e(7234).then(n.bind(n,8289))},"@site/docs/create-dsri-project.md",8289],"5514662e":[function(){return n.e(2865).then(n.bind(n,4255))},"@site/docs/catalog-data-streaming.md",4255],"5890eb41":[function(){return n.e(8603).then(n.bind(n,7363))},"@site/docs/guide-vpn.md",7363],"5db33872":[function(){return n.e(2479).then(n.bind(n,9076))},"@site/docs/openshift-install.md",9076],"5e9f5e1a":[function(){return Promise.resolve().then(n.bind(n,4784))},"@generated/docusaurus.config",4784],"613f0a4f":[function(){return n.e(6108).then(n.bind(n,984))},"@site/docs/dask-tutorial.md",984],"63528e5f":[function(){return n.e(5803).then(n.bind(n,2333))},"@site/docs/openshift-delete-objects.md",2333],"66c10cbb":[function(){return n.e(1332).then(n.bind(n,9886))},"@site/src/pages/contact.md",9886],"682467b2":[function(){return n.e(6746).then(n.bind(n,315))},"@site/docs/profile-pytorch-code.md",315],"6b741ffd":[function(){return n.e(93).then(n.bind(n,819))},"@site/docs/workflows-cwl.md",819],"707d3f57":[function(){return n.e(248).then(n.bind(n,8954))},"@site/docs/deploy-on-gpu.md",8954],"7a61fedb":[function(){return n.e(5217).then(n.bind(n,2864))},"@site/docs/workflows-argo.md",2864],"7d87cf11":[function(){return n.e(6268).then(n.bind(n,4043))},"@site/src/pages/help.md",4043],"814f3328":[function(){return n.e(7472).then(n.t.bind(n,5513,19))},"~blog/default/blog-post-list-prop-default.json",5513],"8cf96c0d":[function(){return n.e(4062).then(n.bind(n,2489))},"@site/docs/workflows-airflow.md",2489],"935f2afb":[function(){return n.e(8581).then(n.t.bind(n,5610,19))},"~docs/default/version-current-metadata-prop-751.json",5610],"94db8302":[function(){return n.e(695).then(n.bind(n,5669))},"@site/docs/deploy-neurodocker.md",5669],"97d64f0a":[function(){return n.e(6970).then(n.bind(n,1582))},"@site/docs/start-workspace.md",1582],"99b0f44e":[function(){return n.e(7668).then(n.bind(n,5249))},"@site/docs/workflows-introduction.md",5249],"9bfda053":[function(){return n.e(8151).then(n.bind(n,9962))},"@site/docs/dask-cluster.md",9962],"9e298cf7":[function(){return n.e(7512).then(n.bind(n,766))},"@site/docs/catalog-imaging.md",766],"9e4087bc":[function(){return n.e(2711).then(n.bind(n,9331))},"@theme/BlogArchivePage",9331],"9f389101":[function(){return n.e(5020).then(n.bind(n,512))},"@site/docs/workflows-nextflow.md",512],a09c2993:[function(){return n.e(5899).then(n.bind(n,6740))},"@site/docs/introduction.md",6740],a1c3d222:[function(){return n.e(8433).then(n.bind(n,1395))},"@site/docs/access-dsri.md",1395],a51ad1ed:[function(){return Promise.all([n.e(1869),n.e(6046),n.e(3570),n.e(3143),n.e(893)]).then(n.bind(n,1903))},"@site/src/pages/register.tsx",1903],a6aa9e1f:[function(){return Promise.all([n.e(1869),n.e(6685),n.e(9176),n.e(7643)]).then(n.bind(n,2667))},"@theme/BlogListPage",2667],ad7efe8c:[function(){return n.e(8796).then(n.bind(n,9439))},"@site/blog/2017-09-26-adding-rss.md",9439],b18de7ec:[function(){return n.e(7340).then(n.t.bind(n,1142,19))},"/home/runner/work/dsri-documentation/dsri-documentation/website/.docusaurus/@easyops-cn/docusaurus-search-local/default/plugin-route-context-module-100.json",1142],b258fc9c:[function(){return n.e(9725).then(n.bind(n,5326))},"@site/docs/guide-local-install.md",5326],b2b675dd:[function(){return n.e(1991).then(n.t.bind(n,9775,19))},"~blog/default/blog-c06.json",9775],b2f554cd:[function(){return n.e(5894).then(n.t.bind(n,6042,19))},"~blog/default/blog-archive-80c.json",6042],b6e2013e:[function(){return n.e(9759).then(n.bind(n,7704))},"@site/docs/access-um-servers.md",7704],bbb095bf:[function(){return n.e(6251).then(n.bind(n,4690))},"@site/blog/2017-10-24-new-version-1.0.0.md",4690],bbb26d62:[function(){return n.e(4969).then(n.bind(n,7080))},"@site/docs/speeding-tensorflow-dl.md",7080],bd7c3f6d:[function(){return n.e(3762).then(n.bind(n,5865))},"@site/docs/openshift-delete-services.md",5865],bf9e930a:[function(){return n.e(8424).then(n.bind(n,6617))},"@site/blog/2016-03-11-blog-post.md",6617],c7a22f1b:[function(){return n.e(3265).then(n.bind(n,2329))},"@site/blog/2017-09-25-testing-rss.md?truncated=true",2329],ccc49370:[function(){return Promise.all([n.e(1869),n.e(6685),n.e(9176),n.e(3249)]).then(n.bind(n,4029))},"@theme/BlogPostPage",4029],cdd0b013:[function(){return n.e(6236).then(n.bind(n,4496))},"@site/docs/jupyterhub-workspace.md",4496],cfec30f8:[function(){return n.e(9215).then(n.bind(n,7853))},"@site/docs/deploy-database.md",7853],d013d563:[function(){return n.e(5299).then(n.bind(n,3128))},"@site/docs/tools-machine-learning.md",3128],d306cda8:[function(){return n.e(4100).then(n.bind(n,2525))},"@site/docs/guide-monitoring.md",2525],d6e8f7a3:[function(){return n.e(747).then(n.bind(n,4418))},"@site/src/pages/training.md",4418],d8f096f7:[function(){return n.e(145).then(n.bind(n,6007))},"@site/docs/mpi-jobs.md",6007],d8f7a64b:[function(){return n.e(6669).then(n.t.bind(n,1966,19))},"/home/runner/work/dsri-documentation/dsri-documentation/website/.docusaurus/docusaurus-plugin-content-docs/default/plugin-route-context-module-100.json",1966],d9be6bc9:[function(){return n.e(4989).then(n.bind(n,4136))},"@site/src/pages/ticket.tsx",4136],dbeba2b5:[function(){return n.e(4070).then(n.bind(n,2535))},"@site/docs/operators.md",2535],dca73612:[function(){return n.e(6690).then(n.bind(n,1680))},"@site/docs/openshift-load-data.md",1680],de77a223:[function(){return n.e(5416).then(n.bind(n,6826))},"@site/docs/guide-dockerfile-to-openshift.md",6826],dfadc8c2:[function(){return n.e(1052).then(n.bind(n,9289))},"@site/blog/2017-04-10-blog-post-two.md",9289],e0dd0623:[function(){return n.e(1107).then(n.t.bind(n,4061,19))},"/home/runner/work/dsri-documentation/dsri-documentation/website/.docusaurus/docusaurus-plugin-content-pages/default/plugin-route-context-module-100.json",4061],e53f82ff:[function(){return n.e(7692).then(n.bind(n,8654))},"@site/docs/deploy-vscode.md",8654],e747ec83:[function(){return n.e(7051).then(n.bind(n,1192))},"@site/docs/glossary.md",1192],e9a2555b:[function(){return n.e(3807).then(n.bind(n,4924))},"@site/docs/catalog-utilities.md",4924],e9bae93a:[function(){return n.e(7968).then(n.bind(n,6912))},"@site/docs/sensible-data.md",6912],ecdc8e44:[function(){return n.e(7953).then(n.bind(n,5899))},"@site/docs/deploy-spark.md",5899],ed5ef82a:[function(){return n.e(6683).then(n.bind(n,6038))},"@site/docs/deploy-rstudio.md",6038],f704770b:[function(){return n.e(3870).then(n.bind(n,7881))},"@site/docs/deploy-matlab.md",7881],f8d48938:[function(){return n.e(2526).then(n.bind(n,8026))},"@site/docs/guide-workshop.md",8026],f931684d:[function(){return n.e(8850).then(n.bind(n,8689))},"@site/docs/deploy-from-docker.md",8689],f97cc2c7:[function(){return n.e(5167).then(n.bind(n,9342))},"@site/docs/checkpointing-ml-training-models.md",9342],fdea7475:[function(){return n.e(918).then(n.bind(n,7718))},"@site/docs/surf-offerings.md",7718]};function u(e){var t=e.error,n=e.retry,o=e.pastDelay;return t?r.createElement("div",{style:{textAlign:"center",color:"#fff",backgroundColor:"#fa383e",borderColor:"#fa383e",borderStyle:"solid",borderRadius:"0.25rem",borderWidth:"1px",boxSizing:"border-box",display:"block",padding:"1rem",flex:"0 0 50%",marginLeft:"25%",marginRight:"25%",marginTop:"5rem",maxWidth:"50%",width:"100%"}},r.createElement("p",null,String(t)),r.createElement("div",null,r.createElement("button",{type:"button",onClick:n},"Retry"))):o?r.createElement("div",{style:{display:"flex",justifyContent:"center",alignItems:"center",height:"100vh"}},r.createElement("svg",{id:"loader",style:{width:128,height:110,position:"absolute",top:"calc(100vh - 64%)"},viewBox:"0 0 45 45",xmlns:"http://www.w3.org/2000/svg",stroke:"#61dafb"},r.createElement("g",{fill:"none",fillRule:"evenodd",transform:"translate(1 1)",strokeWidth:"2"},r.createElement("circle",{cx:"22",cy:"22",r:"6",strokeOpacity:"0"},r.createElement("animate",{attributeName:"r",begin:"1.5s",dur:"3s",values:"6;22",calcMode:"linear",repeatCount:"indefinite"}),r.createElement("animate",{attributeName:"stroke-opacity",begin:"1.5s",dur:"3s",values:"1;0",calcMode:"linear",repeatCount:"indefinite"}),r.createElement("animate",{attributeName:"stroke-width",begin:"1.5s",dur:"3s",values:"2;0",calcMode:"linear",repeatCount:"indefinite"})),r.createElement("circle",{cx:"22",cy:"22",r:"6",strokeOpacity:"0"},r.createElement("animate",{attributeName:"r",begin:"3s",dur:"3s",values:"6;22",calcMode:"linear",repeatCount:"indefinite"}),r.createElement("animate",{attributeName:"stroke-opacity",begin:"3s",dur:"3s",values:"1;0",calcMode:"linear",repeatCount:"indefinite"}),r.createElement("animate",{attributeName:"stroke-width",begin:"3s",dur:"3s",values:"2;0",calcMode:"linear",repeatCount:"indefinite"})),r.createElement("circle",{cx:"22",cy:"22",r:"8"},r.createElement("animate",{attributeName:"r",begin:"0s",dur:"1.5s",values:"6;1;2;3;4;5;6",calcMode:"linear",repeatCount:"indefinite"}))))):null}var c=n(6921),d=n(3102);function f(e,t){if("*"===e)return i()({loading:u,loader:function(){return n.e(1774).then(n.bind(n,1774))},modules:["@theme/NotFound"],webpack:function(){return[1774]},render:function(e,t){var n=e.default;return r.createElement(d.W,{value:{plugin:{name:"native",id:"default"}}},r.createElement(n,t))}});var a=l[e+"-"+t],f={},p=[],m=[],h=(0,c.A)(a);return Object.entries(h).forEach((function(e){var t=e[0],n=e[1],r=s[n];r&&(f[t]=r[0],p.push(r[1]),m.push(r[2]))})),i().Map({loading:u,loader:f,modules:p,webpack:function(){return m},render:function(t,n){var i=JSON.parse(JSON.stringify(a));Object.entries(t).forEach((function(t){var n=t[0],r=t[1],o=r.default;if(!o)throw new Error("The page component at "+e+" doesn't have a default export. This makes it impossible to render anything. Consider default-exporting a React component.");"object"!=typeof o&&"function"!=typeof o||Object.keys(r).filter((function(e){return"default"!==e})).forEach((function(e){o[e]=r[e]}));var a=i,l=n.split(".");l.slice(0,-1).forEach((function(e){a=a[e]})),a[l[l.length-1]]=o}));var l=i.__comp;delete i.__comp;var s=i.__context;return delete i.__context,r.createElement(d.W,{value:s},r.createElement(l,(0,o.A)({},i,n)))}})}const p=[{path:"/acknowledgement",component:f("/acknowledgement","02e"),exact:!0},{path:"/blog",component:f("/blog","9de"),exact:!0},{path:"/blog/2016/03/11/blog-post",component:f("/blog/2016/03/11/blog-post","463"),exact:!0},{path:"/blog/2017/04/10/blog-post-two",component:f("/blog/2017/04/10/blog-post-two","95d"),exact:!0},{path:"/blog/2017/09/25/testing-rss",component:f("/blog/2017/09/25/testing-rss","f9a"),exact:!0},{path:"/blog/2017/09/26/adding-rss",component:f("/blog/2017/09/26/adding-rss","b18"),exact:!0},{path:"/blog/2017/10/24/new-version-1.0.0",component:f("/blog/2017/10/24/new-version-1.0.0","dca"),exact:!0},{path:"/blog/archive",component:f("/blog/archive","788"),exact:!0},{path:"/contact",component:f("/contact","1ab"),exact:!0},{path:"/gpu-booking",component:f("/gpu-booking","0f8"),exact:!0},{path:"/help",component:f("/help","8e9"),exact:!0},{path:"/register",component:f("/register","ab8"),exact:!0},{path:"/search",component:f("/search","70e"),exact:!0},{path:"/ticket",component:f("/ticket","f9a"),exact:!0},{path:"/training",component:f("/training","22d"),exact:!0},{path:"/docs",component:f("/docs","f28"),routes:[{path:"/docs/",component:f("/docs/","06f"),exact:!0,sidebar:"docs"},{path:"/docs/access-dsri",component:f("/docs/access-dsri","1b2"),exact:!0,sidebar:"docs"},{path:"/docs/access-um-servers",component:f("/docs/access-um-servers","515"),exact:!0,sidebar:"docs"},{path:"/docs/anatomy-of-an-application",component:f("/docs/anatomy-of-an-application","b5e"),exact:!0,sidebar:"docs"},{path:"/docs/catalog-data-streaming",component:f("/docs/catalog-data-streaming","ac6"),exact:!0},{path:"/docs/catalog-genomics",component:f("/docs/catalog-genomics","fb0"),exact:!0,sidebar:"docs"},{path:"/docs/catalog-imaging",component:f("/docs/catalog-imaging","446"),exact:!0,sidebar:"docs"},{path:"/docs/catalog-opendatahub",component:f("/docs/catalog-opendatahub","579"),exact:!0},{path:"/docs/catalog-utilities",component:f("/docs/catalog-utilities","9a5"),exact:!0,sidebar:"docs"},{path:"/docs/checkpointing-ml-training",component:f("/docs/checkpointing-ml-training","0ce"),exact:!0,sidebar:"docs"},{path:"/docs/contribute",component:f("/docs/contribute","727"),exact:!0,sidebar:"docs"},{path:"/docs/dask-cluster",component:f("/docs/dask-cluster","005"),exact:!0,sidebar:"docs"},{path:"/docs/dask-tutorial",component:f("/docs/dask-tutorial","9bd"),exact:!0,sidebar:"docs"},{path:"/docs/deploy-database",component:f("/docs/deploy-database","aba"),exact:!0,sidebar:"docs"},{path:"/docs/deploy-from-docker",component:f("/docs/deploy-from-docker","ede"),exact:!0,sidebar:"docs"},{path:"/docs/deploy-gitlab-runner",component:f("/docs/deploy-gitlab-runner","69b"),exact:!0},{path:"/docs/deploy-jupyter",component:f("/docs/deploy-jupyter","7d6"),exact:!0,sidebar:"docs"},{path:"/docs/deploy-jupyterhub",component:f("/docs/deploy-jupyterhub","8f0"),exact:!0,sidebar:"docs"},{path:"/docs/deploy-matlab",component:f("/docs/deploy-matlab","6d0"),exact:!0,sidebar:"docs"},{path:"/docs/deploy-on-gpu",component:f("/docs/deploy-on-gpu","677"),exact:!0,sidebar:"docs"},{path:"/docs/deploy-rstudio",component:f("/docs/deploy-rstudio","a7b"),exact:!0,sidebar:"docs"},{path:"/docs/deploy-spark",component:f("/docs/deploy-spark","44f"),exact:!0,sidebar:"docs"},{path:"/docs/deploy-vscode",component:f("/docs/deploy-vscode","8cd"),exact:!0,sidebar:"docs"},{path:"/docs/enabling-vpn-wsl",component:f("/docs/enabling-vpn-wsl","b96"),exact:!0,sidebar:"docs"},{path:"/docs/glossary",component:f("/docs/glossary","949"),exact:!0,sidebar:"docs"},{path:"/docs/guide-dockerfile-to-openshift",component:f("/docs/guide-dockerfile-to-openshift","561"),exact:!0,sidebar:"docs"},{path:"/docs/guide-known-issues",component:f("/docs/guide-known-issues","e53"),exact:!0,sidebar:"docs"},{path:"/docs/guide-local-install",component:f("/docs/guide-local-install","f50"),exact:!0},{path:"/docs/guide-monitoring",component:f("/docs/guide-monitoring","8af"),exact:!0,sidebar:"docs"},{path:"/docs/guide-publish-image",component:f("/docs/guide-publish-image","e82"),exact:!0,sidebar:"docs"},{path:"/docs/guide-vpn",component:f("/docs/guide-vpn","972"),exact:!0,sidebar:"docs"},{path:"/docs/guide-workshop",component:f("/docs/guide-workshop","233"),exact:!0,sidebar:"docs"},{path:"/docs/helm",component:f("/docs/helm","a35"),exact:!0,sidebar:"docs"},{path:"/docs/increase-process-speed",component:f("/docs/increase-process-speed","8dd"),exact:!0,sidebar:"docs"},{path:"/docs/jupyterhub-spark",component:f("/docs/jupyterhub-spark","ca8"),exact:!0},{path:"/docs/jupyterhub-workspace",component:f("/docs/jupyterhub-workspace","4d3"),exact:!0},{path:"/docs/login-docker-registry",component:f("/docs/login-docker-registry","a37"),exact:!0,sidebar:"docs"},{path:"/docs/mpi-jobs",component:f("/docs/mpi-jobs","842"),exact:!0,sidebar:"docs"},{path:"/docs/neuroscience",component:f("/docs/neuroscience","9b3"),exact:!0,sidebar:"docs"},{path:"/docs/openshift-commands",component:f("/docs/openshift-commands","ba7"),exact:!0,sidebar:"docs"},{path:"/docs/openshift-delete-objects",component:f("/docs/openshift-delete-objects","1d9"),exact:!0,sidebar:"docs"},{path:"/docs/openshift-delete-services",component:f("/docs/openshift-delete-services","df1"),exact:!0,sidebar:"docs"},{path:"/docs/openshift-install",component:f("/docs/openshift-install","a51"),exact:!0,sidebar:"docs"},{path:"/docs/openshift-load-data",component:f("/docs/openshift-load-data","826"),exact:!0,sidebar:"docs"},{path:"/docs/openshift-storage",component:f("/docs/openshift-storage","4fc"),exact:!0,sidebar:"docs"},{path:"/docs/operators",component:f("/docs/operators","107"),exact:!0,sidebar:"docs"},{path:"/docs/prepare-project-for-dsri",component:f("/docs/prepare-project-for-dsri","c81"),exact:!0,sidebar:"docs"},{path:"/docs/profile-pytorch-code",component:f("/docs/profile-pytorch-code","bae"),exact:!0,sidebar:"docs"},{path:"/docs/project-management",component:f("/docs/project-management","3d7"),exact:!0,sidebar:"docs"},{path:"/docs/sensible-data",component:f("/docs/sensible-data","c80"),exact:!0},{path:"/docs/speeding-tensorflow-dl",component:f("/docs/speeding-tensorflow-dl","2c0"),exact:!0,sidebar:"docs"},{path:"/docs/start-workspace",component:f("/docs/start-workspace","159"),exact:!0,sidebar:"docs"},{path:"/docs/surf-offerings",component:f("/docs/surf-offerings","8b2"),exact:!0,sidebar:"docs"},{path:"/docs/tools-machine-learning",component:f("/docs/tools-machine-learning","725"),exact:!0,sidebar:"docs"},{path:"/docs/workflows-airflow",component:f("/docs/workflows-airflow","cd0"),exact:!0,sidebar:"docs"},{path:"/docs/workflows-argo",component:f("/docs/workflows-argo","5c6"),exact:!0,sidebar:"docs"},{path:"/docs/workflows-cwl",component:f("/docs/workflows-cwl","7a0"),exact:!0,sidebar:"docs"},{path:"/docs/workflows-github-actions",component:f("/docs/workflows-github-actions","8e7"),exact:!0,sidebar:"docs"},{path:"/docs/workflows-introduction",component:f("/docs/workflows-introduction","a7a"),exact:!0,sidebar:"docs"},{path:"/docs/workflows-nextflow",component:f("/docs/workflows-nextflow","073"),exact:!0,sidebar:"docs"}]},{path:"/",component:f("/","927"),exact:!0},{path:"*",component:f("*")}]},6125:(e,t,n)=>{"use strict";n.d(t,{o:()=>o,x:()=>a});var r=n(6540),o=r.createContext(!1);function a(e){var t=e.children,n=(0,r.useState)(!1),a=n[0],i=n[1];return(0,r.useEffect)((function(){i(!0)}),[]),r.createElement(o.Provider,{value:a},t)}},5660:(e,t,n)=>{"use strict";var r=n(6540),o=n(961),a=n(4625),i=n(545),l=n(8193);const s=[n(119),n(6491),n(6294),n(1916),n(5509)];var u=n(8328),c=n(6347),d=n(2831);function f(e){var t=e.children;return r.createElement(r.Fragment,null,t)}var p=n(9668),m=n(5260),h=n(4586),g=n(6025),v=n(6342),b=n(1003),y=n(2131),w=n(4090),E=n(2967),k=n(1463);function x(){var e=(0,h.A)().i18n,t=e.defaultLocale,n=e.localeConfigs,o=(0,y.o)();return r.createElement(m.A,null,Object.entries(n).map((function(e){var t=e[0],n=e[1].htmlLang;return r.createElement("link",{key:t,rel:"alternate",href:o.createUrl({locale:t,fullyQualified:!0}),hrefLang:n})})),r.createElement("link",{rel:"alternate",href:o.createUrl({locale:t,fullyQualified:!0}),hrefLang:"x-default"}))}function S(e){var t=e.permalink,n=(0,h.A)().siteConfig.url,o=function(){var e=(0,h.A)().siteConfig.url,t=(0,c.zy)().pathname;return e+(0,g.A)(t)}(),a=t?""+n+t:o;return r.createElement(m.A,null,r.createElement("meta",{property:"og:url",content:a}),r.createElement("link",{rel:"canonical",href:a}))}function T(){var e=(0,h.A)().i18n.currentLocale,t=(0,v.p)(),n=t.metadata,o=t.image;return r.createElement(r.Fragment,null,r.createElement(m.A,null,r.createElement("meta",{name:"twitter:card",content:"summary_large_image"}),r.createElement("body",{className:w.w})),o&&r.createElement(b.be,{image:o}),r.createElement(S,null),r.createElement(x,null),r.createElement(k.A,{tag:E.Cy,locale:e}),r.createElement(m.A,null,n.map((function(e,t){return r.createElement("meta",(0,p.A)({key:t},e))}))))}var _=new Map;function A(e){if(_.has(e.pathname))return Object.assign({},e,{pathname:_.get(e.pathname)});if((0,d.u)(u.A,e.pathname).some((function(e){return!0===e.route.exact})))return _.set(e.pathname,e.pathname),e;var t=e.pathname.trim().replace(/(?:\/index)?\.html$/,"")||"/";return _.set(e.pathname,t),Object.assign({},e,{pathname:t})}var C=n(6125),P=n(6988),O=n(6711);function L(e){for(var t=arguments.length,n=new Array(t>1?t-1:0),r=1;r\n

Your Docusaurus site did not load properly.

\n

A very common reason is a wrong site baseUrl configuration.

\n

Current configured baseUrl = '+e+" "+("/"===e?" (default value)":"")+'

\n

We suggest trying baseUrl =

\n\n'}(e)).replace(/{"use strict";n.d(t,{o:()=>c,l:()=>d});var r=n(6540),o=n(4784);const a=JSON.parse('{"docusaurus-plugin-content-docs":{"default":{"path":"/docs","versions":[{"name":"current","label":"Next","isLast":true,"path":"/docs","mainDocId":"introduction","docs":[{"id":"access-dsri","path":"/docs/access-dsri","sidebar":"docs"},{"id":"access-um-servers","path":"/docs/access-um-servers","sidebar":"docs"},{"id":"anatomy-of-an-application","path":"/docs/anatomy-of-an-application","sidebar":"docs"},{"id":"catalog-data-streaming","path":"/docs/catalog-data-streaming"},{"id":"catalog-genomics","path":"/docs/catalog-genomics","sidebar":"docs"},{"id":"catalog-imaging","path":"/docs/catalog-imaging","sidebar":"docs"},{"id":"catalog-opendatahub","path":"/docs/catalog-opendatahub"},{"id":"catalog-utilities","path":"/docs/catalog-utilities","sidebar":"docs"},{"id":"checkpointing-ml-training","path":"/docs/checkpointing-ml-training","sidebar":"docs"},{"id":"contribute","path":"/docs/contribute","sidebar":"docs"},{"id":"dask-cluster","path":"/docs/dask-cluster","sidebar":"docs"},{"id":"dask-tutorial","path":"/docs/dask-tutorial","sidebar":"docs"},{"id":"deploy-database","path":"/docs/deploy-database","sidebar":"docs"},{"id":"deploy-from-docker","path":"/docs/deploy-from-docker","sidebar":"docs"},{"id":"deploy-gitlab-runner","path":"/docs/deploy-gitlab-runner"},{"id":"deploy-jupyter","path":"/docs/deploy-jupyter","sidebar":"docs"},{"id":"deploy-jupyterhub","path":"/docs/deploy-jupyterhub","sidebar":"docs"},{"id":"deploy-matlab","path":"/docs/deploy-matlab","sidebar":"docs"},{"id":"deploy-on-gpu","path":"/docs/deploy-on-gpu","sidebar":"docs"},{"id":"deploy-rstudio","path":"/docs/deploy-rstudio","sidebar":"docs"},{"id":"deploy-spark","path":"/docs/deploy-spark","sidebar":"docs"},{"id":"deploy-vscode","path":"/docs/deploy-vscode","sidebar":"docs"},{"id":"enabling-vpn-wsl","path":"/docs/enabling-vpn-wsl","sidebar":"docs"},{"id":"glossary","path":"/docs/glossary","sidebar":"docs"},{"id":"guide-dockerfile-to-openshift","path":"/docs/guide-dockerfile-to-openshift","sidebar":"docs"},{"id":"guide-known-issues","path":"/docs/guide-known-issues","sidebar":"docs"},{"id":"guide-local-install","path":"/docs/guide-local-install"},{"id":"guide-monitoring","path":"/docs/guide-monitoring","sidebar":"docs"},{"id":"guide-publish-image","path":"/docs/guide-publish-image","sidebar":"docs"},{"id":"guide-vpn","path":"/docs/guide-vpn","sidebar":"docs"},{"id":"guide-workshop","path":"/docs/guide-workshop","sidebar":"docs"},{"id":"helm","path":"/docs/helm","sidebar":"docs"},{"id":"increase-process-speed","path":"/docs/increase-process-speed","sidebar":"docs"},{"id":"introduction","path":"/docs/","sidebar":"docs"},{"id":"jupyterhub-spark","path":"/docs/jupyterhub-spark"},{"id":"jupyterhub-workspace","path":"/docs/jupyterhub-workspace"},{"id":"login-docker-registry","path":"/docs/login-docker-registry","sidebar":"docs"},{"id":"mpi-jobs","path":"/docs/mpi-jobs","sidebar":"docs"},{"id":"neuroscience","path":"/docs/neuroscience","sidebar":"docs"},{"id":"openshift-commands","path":"/docs/openshift-commands","sidebar":"docs"},{"id":"openshift-delete-objects","path":"/docs/openshift-delete-objects","sidebar":"docs"},{"id":"openshift-delete-services","path":"/docs/openshift-delete-services","sidebar":"docs"},{"id":"openshift-install","path":"/docs/openshift-install","sidebar":"docs"},{"id":"openshift-load-data","path":"/docs/openshift-load-data","sidebar":"docs"},{"id":"openshift-storage","path":"/docs/openshift-storage","sidebar":"docs"},{"id":"operators","path":"/docs/operators","sidebar":"docs"},{"id":"prepare-project-for-dsri","path":"/docs/prepare-project-for-dsri","sidebar":"docs"},{"id":"profile-pytorch-code","path":"/docs/profile-pytorch-code","sidebar":"docs"},{"id":"project-management","path":"/docs/project-management","sidebar":"docs"},{"id":"sensible-data","path":"/docs/sensible-data"},{"id":"speeding-tensorflow-dl","path":"/docs/speeding-tensorflow-dl","sidebar":"docs"},{"id":"start-workspace","path":"/docs/start-workspace","sidebar":"docs"},{"id":"surf-offerings","path":"/docs/surf-offerings","sidebar":"docs"},{"id":"tools-machine-learning","path":"/docs/tools-machine-learning","sidebar":"docs"},{"id":"workflows-airflow","path":"/docs/workflows-airflow","sidebar":"docs"},{"id":"workflows-argo","path":"/docs/workflows-argo","sidebar":"docs"},{"id":"workflows-cwl","path":"/docs/workflows-cwl","sidebar":"docs"},{"id":"workflows-github-actions","path":"/docs/workflows-github-actions","sidebar":"docs"},{"id":"workflows-introduction","path":"/docs/workflows-introduction","sidebar":"docs"},{"id":"workflows-nextflow","path":"/docs/workflows-nextflow","sidebar":"docs"}],"draftIds":[],"sidebars":{"docs":{"link":{"path":"/docs/","label":"introduction"}}}}],"breadcrumbs":true}}}'),i=JSON.parse('{"defaultLocale":"en","locales":["en"],"path":"i18n","currentLocale":"en","localeConfigs":{"en":{"label":"English","direction":"ltr","htmlLang":"en","calendar":"gregory","path":"en"}}}');var l=n(2654);const s=JSON.parse('{"docusaurusVersion":"2.1.0","pluginVersions":{"docusaurus-plugin-content-docs":{"type":"package","name":"@docusaurus/plugin-content-docs","version":"2.1.0"},"docusaurus-plugin-content-blog":{"type":"package","name":"@docusaurus/plugin-content-blog","version":"2.1.0"},"docusaurus-plugin-content-pages":{"type":"package","name":"@docusaurus/plugin-content-pages","version":"2.1.0"},"docusaurus-plugin-sitemap":{"type":"package","name":"@docusaurus/plugin-sitemap","version":"2.1.0"},"docusaurus-theme-classic":{"type":"package","name":"@docusaurus/theme-classic","version":"2.1.0"},"@easyops-cn/docusaurus-search-local":{"type":"package","name":"@easyops-cn/docusaurus-search-local","version":"0.23.0"},"docusaurus-plugin-image-zoom":{"type":"package","name":"docusaurus-plugin-image-zoom","version":"0.0.2"},"docusaurus2-dotenv":{"type":"package","name":"docusaurus2-dotenv","version":"1.4.0"}}}');var u={siteConfig:o.default,siteMetadata:s,globalData:a,i18n:i,codeTranslations:l},c=r.createContext(u);function d(e){var t=e.children;return r.createElement(c.Provider,{value:u},t)}},7489:(e,t,n)=>{"use strict";n.d(t,{A:()=>d});var r=n(6711),o=n(6540),a=n(8193),i=n(5260),l=n(6750);function s(e){var t=e.error,n=e.tryAgain;return o.createElement("div",{style:{display:"flex",flexDirection:"column",justifyContent:"center",alignItems:"center",height:"50vh",width:"100%",fontSize:"20px"}},o.createElement("h1",null,"This page crashed."),o.createElement("p",null,t.message),o.createElement("button",{type:"button",onClick:n},"Try again"))}function u(e){var t=e.error,n=e.tryAgain;return o.createElement(d,{fallback:function(){return o.createElement(s,{error:t,tryAgain:n})}},o.createElement(i.A,null,o.createElement("title",null,"Page Error")),o.createElement(l.A,null,o.createElement(s,{error:t,tryAgain:n})))}var c=function(e){return o.createElement(u,e)},d=function(e){function t(t){var n;return(n=e.call(this,t)||this).state={error:null},n}(0,r.A)(t,e);var n=t.prototype;return n.componentDidCatch=function(e){a.A.canUseDOM&&this.setState({error:e})},n.render=function(){var e=this,t=this.props.children,n=this.state.error;if(n){var r,o={error:n,tryAgain:function(){return e.setState({error:null})}};return(null!=(r=this.props.fallback)?r:c)(o)}return null!=t?t:null},t}(o.Component)},8193:(e,t,n)=>{"use strict";n.d(t,{A:()=>o});var r="undefined"!=typeof window&&"document"in window&&"createElement"in window.document;const o={canUseDOM:r,canUseEventListeners:r&&("addEventListener"in window||"attachEvent"in window),canUseIntersectionObserver:r&&"IntersectionObserver"in window,canUseViewport:r&&"screen"in window}},5260:(e,t,n)=>{"use strict";n.d(t,{A:()=>a});var r=n(6540),o=n(545);function a(e){return r.createElement(o.mg,e)}},5489:(e,t,n)=>{"use strict";n.d(t,{A:()=>h});var r=n(9668),o=n(1367),a=n(6540),i=n(4625),l=n(440),s=n(4586),u=n(6654),c=n(8193),d=a.createContext({collectLink:function(){}});var f=n(6025),p=["isNavLink","to","href","activeClassName","isActive","data-noBrokenLinkCheck","autoAddBaseUrl"];function m(e,t){var n,m,h=e.isNavLink,g=e.to,v=e.href,b=e.activeClassName,y=e.isActive,w=e["data-noBrokenLinkCheck"],E=e.autoAddBaseUrl,k=void 0===E||E,x=(0,o.A)(e,p),S=(0,s.A)().siteConfig,T=S.trailingSlash,_=S.baseUrl,A=(0,f.h)().withBaseUrl,C=(0,a.useContext)(d),P=(0,a.useRef)(null);(0,a.useImperativeHandle)(t,(function(){return P.current}));var O=g||v;var L,N=(0,u.A)(O),D=null==O?void 0:O.replace("pathname://",""),I=void 0!==D?(L=D,k&&function(e){return e.startsWith("/")}(L)?A(L):L):void 0;I&&N&&(I=(0,l.applyTrailingSlash)(I,{trailingSlash:T,baseUrl:_}));var R=(0,a.useRef)(!1),F=h?i.k2:i.N_,M=c.A.canUseIntersectionObserver,z=(0,a.useRef)();(0,a.useEffect)((function(){return!M&&N&&null!=I&&window.docusaurus.prefetch(I),function(){M&&z.current&&z.current.disconnect()}}),[z,I,M,N]);var j=null!=(n=null==(m=I)?void 0:m.startsWith("#"))&&n,B=!I||!N||j;return B||w||C.collectLink(I),B?a.createElement("a",(0,r.A)({ref:P,href:I},O&&!N&&{target:"_blank",rel:"noopener noreferrer"},x)):a.createElement(F,(0,r.A)({},x,{onMouseEnter:function(){R.current||null==I||(window.docusaurus.preload(I),R.current=!0)},innerRef:function(e){P.current=e,M&&e&&N&&(z.current=new window.IntersectionObserver((function(t){t.forEach((function(t){e===t.target&&(t.isIntersecting||t.intersectionRatio>0)&&(z.current.unobserve(e),z.current.disconnect(),null!=I&&window.docusaurus.prefetch(I))}))})),z.current.observe(e))},to:I},h&&{isActive:y,activeClassName:b}))}const h=a.forwardRef(m)},1312:(e,t,n)=>{"use strict";n.d(t,{A:()=>s,T:()=>l});var r=n(6540);function o(e,t){var n=e.split(/(\{\w+\})/).map((function(e,n){if(n%2==1){var r=null==t?void 0:t[e.slice(1,-1)];if(void 0!==r)return r}return e}));return n.some((function(e){return(0,r.isValidElement)(e)}))?n.map((function(e,t){return(0,r.isValidElement)(e)?r.cloneElement(e,{key:t}):e})).filter((function(e){return""!==e})):n.join("")}var a=n(2654);function i(e){var t,n,r=e.id,o=e.message;if(void 0===r&&void 0===o)throw new Error("Docusaurus translation declarations must have at least a translation id or a default translation message");return null!=(t=null!=(n=a[null!=r?r:o])?n:o)?t:r}function l(e,t){return o(i({message:e.message,id:e.id}),t)}function s(e){var t=e.children,n=e.id,a=e.values;if(t&&"string"!=typeof t)throw console.warn("Illegal children",t),new Error("The Docusaurus component only accept simple string values");var l=i({message:t,id:n});return r.createElement(r.Fragment,null,o(l,a))}},7065:(e,t,n)=>{"use strict";n.d(t,{W:()=>r});var r="default"},6654:(e,t,n)=>{"use strict";function r(e){return/^(?:\w*:|\/\/)/.test(e)}function o(e){return void 0!==e&&!r(e)}n.d(t,{A:()=>o,z:()=>r})},6025:(e,t,n)=>{"use strict";n.d(t,{A:()=>i,h:()=>a});var r=n(4586),o=n(6654);function a(){var e=(0,r.A)().siteConfig,t=e.baseUrl,n=e.url;return{withBaseUrl:function(e,r){return function(e,t,n,r){var a=void 0===r?{}:r,i=a.forcePrependBaseUrl,l=void 0!==i&&i,s=a.absolute,u=void 0!==s&&s;if(!n||n.startsWith("#")||(0,o.z)(n))return n;if(l)return t+n.replace(/^\//,"");if(n===t.replace(/\/$/,""))return t;var c=!n.startsWith(t)?t+n.replace(/^\//,""):n;return u?e+c:c}(n,t,e,r)}}}function i(e,t){return void 0===t&&(t={}),(0,a().withBaseUrl)(e,t)}},4586:(e,t,n)=>{"use strict";n.d(t,{A:()=>a});var r=n(6540),o=n(6988);function a(){return(0,r.useContext)(o.o)}},2303:(e,t,n)=>{"use strict";n.d(t,{A:()=>a});var r=n(6540),o=n(6125);function a(){return(0,r.useContext)(o.o)}},6921:(e,t,n)=>{"use strict";n.d(t,{A:()=>r});function r(e){var t={};return function e(n,r){Object.entries(n).forEach((function(n){var o,a=n[0],i=n[1],l=r?r+"."+a:a;"object"==typeof(o=i)&&o&&Object.keys(o).length>0?e(i,l):t[l]=i}))}(e),t}},3102:(e,t,n)=>{"use strict";n.d(t,{W:()=>a,o:()=>o});var r=n(6540),o=r.createContext(null);function a(e){var t=e.children,n=e.value,a=r.useContext(o),i=(0,r.useMemo)((function(){return function(e){var t=e.parent,n=e.value;if(!t){if(!n)throw new Error("Unexpected: no Docusaurus route context found");if(!("plugin"in n))throw new Error("Unexpected: Docusaurus topmost route context has no `plugin` attribute");return n}var r=Object.assign({},t.data,null==n?void 0:n.data);return{plugin:t.plugin,data:r}}({parent:a,value:n})}),[a,n]);return r.createElement(o.Provider,{value:i},t)}},4070:(e,t,n)=>{"use strict";n.d(t,{zK:()=>h,vT:()=>f,Gy:()=>c,HW:()=>g,ht:()=>d,r7:()=>m,jh:()=>p});var r=n(6347),o=n(4586),a=n(7065);function i(e,t){void 0===t&&(t={});var n=(0,o.A)().globalData[e];if(!n&&t.failfast)throw new Error('Docusaurus plugin global data not found for "'+e+'" plugin.');return n}var l=function(e){return e.versions.find((function(e){return e.isLast}))};function s(e,t){var n,o,a=function(e,t){var n=l(e);return[].concat(e.versions.filter((function(e){return e!==n})),[n]).find((function(e){return!!(0,r.B6)(t,{path:e.path,exact:!1,strict:!1})}))}(e,t),i=null==a?void 0:a.docs.find((function(e){return!!(0,r.B6)(t,{path:e.path,exact:!0,strict:!1})}));return{activeVersion:a,activeDoc:i,alternateDocVersions:i?(n=i.id,o={},e.versions.forEach((function(e){e.docs.forEach((function(t){t.id===n&&(o[e.name]=t)}))})),o):{}}}var u={},c=function(){var e;return null!=(e=i("docusaurus-plugin-content-docs"))?e:u},d=function(e){return function(e,t,n){void 0===t&&(t=a.W),void 0===n&&(n={});var r=i(e),o=null==r?void 0:r[t];if(!o&&n.failfast)throw new Error('Docusaurus plugin global data not found for "'+e+'" plugin with id "'+t+'".');return o}("docusaurus-plugin-content-docs",e,{failfast:!0})};function f(e){return void 0===e&&(e={}),function(e,t,n){void 0===n&&(n={});var o=Object.entries(e).sort((function(e,t){return t[1].path.localeCompare(e[1].path)})).find((function(e){var n=e[1];return!!(0,r.B6)(t,{path:n.path,exact:!1,strict:!1})})),a=o?{pluginId:o[0],pluginData:o[1]}:void 0;if(!a&&n.failfast)throw new Error("Can't find active docs plugin for \""+t+'" pathname, while it was expected to be found. Maybe you tried to use a docs feature that can only be used on a docs-related page? Existing docs plugin paths are: '+Object.values(e).map((function(e){return e.path})).join(", "));return a}(c(),(0,r.zy)().pathname,e)}function p(e){return d(e).versions}function m(e){var t=d(e);return l(t)}function h(e){return s(d(e),(0,r.zy)().pathname)}function g(e){return function(e,t){var n=l(e);return{latestDocSuggestion:s(e,t).alternateDocVersions[n.name],latestVersionSuggestion:n}}(d(e),(0,r.zy)().pathname)}},6294:(e,t,n)=>{"use strict";n.r(t),n.d(t,{default:()=>a});var r=n(5947),o=n.n(r);o().configure({showSpinner:!1});const a={onRouteUpdate:function(e){var t=e.location,n=e.previousLocation;if(n&&t.pathname!==n.pathname){var r=window.setTimeout((function(){o().start()}),200);return function(){return window.clearTimeout(r)}}},onRouteDidUpdate:function(){o().done()}}},6491:(e,t,n)=>{"use strict";var r=n(1258),o=n(8193),a=n(4784);(function(e){if(o.A.canUseDOM){var t=a.default.themeConfig.prism,r=(t=void 0===t?{}:t).additionalLanguages,i=void 0===r?[]:r;window.Prism=e,i.forEach((function(e){n(2433)("./prism-"+e)})),delete window.Prism}})(r.A)},3186:(e,t,n)=>{"use strict";n.d(t,{A:()=>a});var r=n(6540);const o="iconExternalLink_nPIU";function a(e){var t=e.width,n=void 0===t?13.5:t,a=e.height,i=void 0===a?13.5:a;return r.createElement("svg",{width:n,height:i,"aria-hidden":"true",viewBox:"0 0 24 24",className:o},r.createElement("path",{fill:"currentColor",d:"M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"}))}},6750:(e,t,n)=>{"use strict";n.d(t,{A:()=>Kt});var r=n(6540),o=n(870),a=n(7489),i=n(1003),l=n(7559),s=n(4090),u=n(1312),c=n(6347),d=n(5062);function f(e){e.setAttribute("tabindex","-1"),e.focus(),e.removeAttribute("tabindex")}const p="skipToContent_fXgn";function m(){var e=function(){var e=(0,r.useRef)(null),t=(0,c.W6)().action,n=(0,r.useCallback)((function(e){var t;e.preventDefault();var n=null!=(t=document.querySelector("main:first-of-type"))?t:document.querySelector("."+l.G.wrapper.main);n&&f(n)}),[]);return(0,d.$)((function(n){var r=n.location;e.current&&!r.hash&&"PUSH"===t&&f(e.current)})),{containerRef:e,handleSkip:n}}(),t=e.containerRef,n=e.handleSkip;return r.createElement("div",{ref:t,role:"region","aria-label":(0,u.T)({id:"theme.common.skipToMainContent"})},r.createElement("a",{href:"#",className:p,onClick:n},r.createElement(u.A,{id:"theme.common.skipToMainContent",description:"The skip to content label used for accessibility, allowing to rapidly navigate to main content with keyboard tab/enter navigation"},"Skip to main content")))}var h=n(6342),g=n(5041),v=n(9668),b=n(1367),y=["width","height","color","strokeWidth","className"];function w(e){var t=e.width,n=void 0===t?21:t,o=e.height,a=void 0===o?21:o,i=e.color,l=void 0===i?"currentColor":i,s=e.strokeWidth,u=void 0===s?1.2:s,c=(e.className,(0,b.A)(e,y));return r.createElement("svg",(0,v.A)({viewBox:"0 0 15 15",width:n,height:a},c),r.createElement("g",{stroke:l,strokeWidth:u},r.createElement("path",{d:"M.75.75l13.5 13.5M14.25.75L.75 14.25"})))}const E="closeButton_CVFx";function k(e){return r.createElement("button",(0,v.A)({type:"button","aria-label":(0,u.T)({id:"theme.AnnouncementBar.closeButtonAriaLabel",message:"Close",description:"The ARIA label for close button of announcement bar"})},e,{className:(0,o.A)("clean-btn close",E,e.className)}),r.createElement(w,{width:14,height:14,strokeWidth:3.1}))}const x="content_knG7";function S(e){var t=(0,h.p)().announcementBar.content;return r.createElement("div",(0,v.A)({},e,{className:(0,o.A)(x,e.className),dangerouslySetInnerHTML:{__html:t}}))}const T="announcementBar_mb4j",_="announcementBarPlaceholder_vyr4",A="announcementBarClose_gvF7",C="announcementBarContent_xLdY";function P(){var e=(0,h.p)().announcementBar,t=(0,g.Mj)(),n=t.isActive,o=t.close;if(!n)return null;var a=e.backgroundColor,i=e.textColor,l=e.isCloseable;return r.createElement("div",{className:T,style:{backgroundColor:a,color:i},role:"banner"},l&&r.createElement("div",{className:_}),r.createElement(S,{className:C}),l&&r.createElement(k,{onClick:o,className:A}))}var O=n(9876),L=n(3104);var N=n(2021),D=n(5600),I=r.createContext(null);function R(e){var t,n,o,a,i,l,s,u=e.children,c=(t=(0,O.M)(),n=(0,D.YL)(),o=(0,r.useState)(!1),a=o[0],i=o[1],l=null!==n.component,s=(0,N.ZC)(l),(0,r.useEffect)((function(){l&&!s&&i(!0)}),[l,s]),(0,r.useEffect)((function(){l?t.shown||i(!0):i(!1)}),[t.shown,l]),(0,r.useMemo)((function(){return[a,i]}),[a]));return r.createElement(I.Provider,{value:c},u)}function F(e){if(e.component){var t=e.component;return r.createElement(t,e.props)}}function M(){var e=(0,r.useContext)(I);if(!e)throw new N.dV("NavbarSecondaryMenuDisplayProvider");var t=e[0],n=e[1],o=(0,r.useCallback)((function(){return n(!1)}),[n]),a=(0,D.YL)();return(0,r.useMemo)((function(){return{shown:t,hide:o,content:F(a)}}),[o,a,t])}function z(e){var t=e.header,n=e.primaryMenu,a=e.secondaryMenu,i=M().shown;return r.createElement("div",{className:"navbar-sidebar"},t,r.createElement("div",{className:(0,o.A)("navbar-sidebar__items",{"navbar-sidebar__items--show-secondary":i})},r.createElement("div",{className:"navbar-sidebar__item menu"},n),r.createElement("div",{className:"navbar-sidebar__item menu"},a)))}var j=n(5293),B=n(2303);function U(e){return r.createElement("svg",(0,v.A)({viewBox:"0 0 24 24",width:24,height:24},e),r.createElement("path",{fill:"currentColor",d:"M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"}))}function $(e){return r.createElement("svg",(0,v.A)({viewBox:"0 0 24 24",width:24,height:24},e),r.createElement("path",{fill:"currentColor",d:"M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"}))}const H={toggle:"toggle_vylO",toggleButton:"toggleButton_gllP",darkToggleIcon:"darkToggleIcon_wfgR",lightToggleIcon:"lightToggleIcon_pyhR",toggleButtonDisabled:"toggleButtonDisabled_aARS"};function Q(e){var t=e.className,n=e.value,a=e.onChange,i=(0,B.A)(),l=(0,u.T)({message:"Switch between dark and light mode (currently {mode})",id:"theme.colorToggle.ariaLabel",description:"The ARIA label for the navbar color mode toggle"},{mode:"dark"===n?(0,u.T)({message:"dark mode",id:"theme.colorToggle.ariaLabel.mode.dark",description:"The name for the dark color mode"}):(0,u.T)({message:"light mode",id:"theme.colorToggle.ariaLabel.mode.light",description:"The name for the light color mode"})});return r.createElement("div",{className:(0,o.A)(H.toggle,t)},r.createElement("button",{className:(0,o.A)("clean-btn",H.toggleButton,!i&&H.toggleButtonDisabled),type:"button",onClick:function(){return a("dark"===n?"light":"dark")},disabled:!i,title:l,"aria-label":l},r.createElement(U,{className:(0,o.A)(H.toggleIcon,H.lightToggleIcon)}),r.createElement($,{className:(0,o.A)(H.toggleIcon,H.darkToggleIcon)})))}const V=r.memo(Q);function W(e){var t=e.className,n=(0,h.p)().colorMode.disableSwitch,o=(0,j.G)(),a=o.colorMode,i=o.setColorMode;return n?null:r.createElement(V,{className:t,value:a,onChange:i})}var G=n(110);function q(){return r.createElement(G.A,{className:"navbar__brand",imageClassName:"navbar__logo",titleClassName:"navbar__title text--truncate"})}function K(){var e=(0,O.M)();return r.createElement("button",{type:"button",className:"clean-btn navbar-sidebar__close",onClick:function(){return e.toggle()}},r.createElement(w,{color:"var(--ifm-color-emphasis-600)"}))}function Y(){return r.createElement("div",{className:"navbar-sidebar__brand"},r.createElement(q,null),r.createElement(W,{className:"margin-right--md"}),r.createElement(K,null))}var X=n(5489),Z=n(6025),J=n(6654);function ee(e,t){return void 0!==e&&void 0!==t&&new RegExp(e,"gi").test(t)}var te=n(3186),ne=["activeBasePath","activeBaseRegex","to","href","label","html","isDropdownLink","prependBaseUrlToHref"];function re(e){var t=e.activeBasePath,n=e.activeBaseRegex,o=e.to,a=e.href,i=e.label,l=e.html,s=e.isDropdownLink,u=e.prependBaseUrlToHref,c=(0,b.A)(e,ne),d=(0,Z.A)(o),f=(0,Z.A)(t),p=(0,Z.A)(a,{forcePrependBaseUrl:!0}),m=i&&a&&!(0,J.A)(a),h=l?{dangerouslySetInnerHTML:{__html:l}}:{children:r.createElement(r.Fragment,null,i,m&&r.createElement(te.A,s&&{width:12,height:12}))};return a?r.createElement(X.A,(0,v.A)({href:u?p:a},c,h)):r.createElement(X.A,(0,v.A)({to:d,isNavLink:!0},(t||n)&&{isActive:function(e,t){return n?ee(n,t.pathname):t.pathname.startsWith(f)}},c,h))}var oe=["className","isDropdownItem"],ae=["className","isDropdownItem"],ie=["mobile","position"];function le(e){var t=e.className,n=e.isDropdownItem,a=void 0!==n&&n,i=(0,b.A)(e,oe),l=r.createElement(re,(0,v.A)({className:(0,o.A)(a?"dropdown__link":"navbar__item navbar__link",t),isDropdownLink:a},i));return a?r.createElement("li",null,l):l}function se(e){var t=e.className,n=(e.isDropdownItem,(0,b.A)(e,ae));return r.createElement("li",{className:"menu__list-item"},r.createElement(re,(0,v.A)({className:(0,o.A)("menu__link",t)},n)))}function ue(e){var t,n=e.mobile,o=void 0!==n&&n,a=(e.position,(0,b.A)(e,ie)),i=o?se:le;return r.createElement(i,(0,v.A)({},a,{activeClassName:null!=(t=a.activeClassName)?t:o?"menu__link--active":"navbar__link--active"}))}var ce=n(1422),de=n(9169),fe=n(4586);var pe=["items","position","className","onClick"],me=["items","className","position","onClick"],he=["mobile"];function ge(e,t){return e.some((function(e){return function(e,t){return!!(0,de.ys)(e.to,t)||(!!ee(e.activeBaseRegex,t)||!(!e.activeBasePath||!t.startsWith(e.activeBasePath)))}(e,t)}))}function ve(e){var t,n=e.items,a=e.position,i=e.className,l=(e.onClick,(0,b.A)(e,pe)),s=(0,r.useRef)(null),u=(0,r.useState)(!1),c=u[0],d=u[1];return(0,r.useEffect)((function(){var e=function(e){s.current&&!s.current.contains(e.target)&&d(!1)};return document.addEventListener("mousedown",e),document.addEventListener("touchstart",e),function(){document.removeEventListener("mousedown",e),document.removeEventListener("touchstart",e)}}),[s]),r.createElement("div",{ref:s,className:(0,o.A)("navbar__item","dropdown","dropdown--hoverable",{"dropdown--right":"right"===a,"dropdown--show":c})},r.createElement(re,(0,v.A)({"aria-haspopup":"true","aria-expanded":c,role:"button",href:l.to?void 0:"#",className:(0,o.A)("navbar__link",i)},l,{onClick:l.to?void 0:function(e){return e.preventDefault()},onKeyDown:function(e){"Enter"===e.key&&(e.preventDefault(),d(!c))}}),null!=(t=l.children)?t:l.label),r.createElement("ul",{className:"dropdown__menu"},n.map((function(e,t){return r.createElement(xt,(0,v.A)({isDropdownItem:!0,onKeyDown:function(e){if(t===n.length-1&&"Tab"===e.key){e.preventDefault(),d(!1);var r=s.current.nextElementSibling;if(r)(r instanceof HTMLAnchorElement?r:r.querySelector("a")).focus()}},activeClassName:"dropdown__link--active"},e,{key:t}))}))))}function be(e){var t,n,a=e.items,i=e.className,l=(e.position,e.onClick),s=(0,b.A)(e,me),u=(n=(0,fe.A)().siteConfig.baseUrl,(0,c.zy)().pathname.replace(n,"/")),d=ge(a,u),f=(0,ce.u)({initialState:function(){return!d}}),p=f.collapsed,m=f.toggleCollapsed,h=f.setCollapsed;return(0,r.useEffect)((function(){d&&h(!d)}),[u,d,h]),r.createElement("li",{className:(0,o.A)("menu__list-item",{"menu__list-item--collapsed":p})},r.createElement(re,(0,v.A)({role:"button",className:(0,o.A)("menu__link menu__link--sublist menu__link--sublist-caret",i)},s,{onClick:function(e){e.preventDefault(),m()}}),null!=(t=s.children)?t:s.label),r.createElement(ce.N,{lazy:!0,as:"ul",className:"menu__list",collapsed:p},a.map((function(e,t){return r.createElement(xt,(0,v.A)({mobile:!0,isDropdownItem:!0,onClick:l,activeClassName:"menu__link--active"},e,{key:t}))}))))}function ye(e){var t=e.mobile,n=void 0!==t&&t,o=(0,b.A)(e,he),a=n?be:ve;return r.createElement(a,o)}var we=n(2131),Ee=["width","height"];function ke(e){var t=e.width,n=void 0===t?20:t,o=e.height,a=void 0===o?20:o,i=(0,b.A)(e,Ee);return r.createElement("svg",(0,v.A)({viewBox:"0 0 24 24",width:n,height:a,"aria-hidden":!0},i),r.createElement("path",{fill:"currentColor",d:"M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"}))}const xe="iconLanguage_nlXk";var Se=["mobile","dropdownItemsBefore","dropdownItemsAfter"];var Te=n(1088),_e=n(6751),Ae=n(9593),Ce=n(7591),Pe=n(53),Oe=n(8193),Le=n(5891),Ne=n(2384),De=n(6841),Ie=n(3810),Re=n(7674);const Fe="searchBar_RVTs",Me="dropdownMenu_qbY6",ze="suggestion_fB_2",je="cursor_eG29",Be="hitTree_kk6K",Ue="hitIcon_a7Zy",$e="hitPath_ieM4",He="noResultsIcon_EBY5",Qe="hitFooter_E9YW",Ve="hitWrapper_sAK8",We="hitTitle_vyVt",Ge="hitAction_NqkB",qe="noResults_l6Q3",Ke="searchBarContainer_NW3z",Ye="searchBarLoadingRing_YnHq",Xe="searchIndexLoading_EJ1f",Ze="searchHintContainer_Pkmr",Je="searchHint_iIMx",et="input_FOTf",tt="hint_URu1",nt="suggestions_X8XU",rt="dataset_QiCy",ot="empty_eITn";function at(e){var t=e.document,n=e.type,r=e.page,o=e.metadata,a=e.tokens,i=e.isInterOfTree,l=e.isLastOfTree,s=0===n,u=1===n,c=[];i?c.push(''):l&&c.push('');var d=c.map((function(e){return''+e+""})),f=''+(s?'':u?'':'')+"",p=[''+(0,Ie.C)(t.t,(0,Re.g)(o,"t"),a)+""];return s||p.push(''+(0,De.Z)(r.t||(t.u.startsWith("/docs/api-reference/")?"API Reference":""),a)+""),[].concat(d,[f,''],p,["",'']).join("")}function it(){return''+Te.P9.no_results+""}var lt=n(2849);function st(){return ut.apply(this,arguments)}function ut(){return(ut=(0,Ce.A)((0,_e.A)().mark((function e(){var t,r;return(0,_e.A)().wrap((function(e){for(;;)switch(e.prev=e.next){case 0:return e.next=2,Promise.all([n.e(489),n.e(5741)]).then(n.t.bind(n,489,23));case 2:return t=e.sent,(r=t.default).noConflict?r.noConflict():t.noConflict&&t.noConflict(),e.abrupt("return",r);case 6:case"end":return e.stop()}}),e)})))).apply(this,arguments)}const ct=function(e){var t,n,o,a=e.handleSearchBarToggle,i=(0,fe.A)().siteConfig.baseUrl,l=(0,c.W6)(),s=(0,c.zy)(),u=(0,r.useRef)(null),d=(0,r.useRef)("empty"),f=(0,r.useRef)(!1),p=(0,r.useState)(!1),m=p[0],h=p[1],g=(0,r.useState)(!1),v=g[0],b=g[1],y=(0,r.useCallback)((0,Ce.A)((0,_e.A)().mark((function e(){var t,n,r,o,a,s,c;return(0,_e.A)().wrap((function(e){for(;;)switch(e.prev=e.next){case 0:if("empty"===d.current){e.next=2;break}return e.abrupt("return");case 2:return d.current="loading",h(!0),e.next=6,Promise.all([(0,Le.Z)(i),st()]);case 6:t=e.sent,n=t[0],r=n.wrappedIndexes,o=n.zhDictionary,a=t[1],s=a(u.current,{hint:!1,autoselect:!0,openOnFocus:!0,cssClasses:{root:Fe,noPrefix:!0,dropdownMenu:Me,input:et,hint:tt,suggestions:nt,suggestion:ze,cursor:je,dataset:rt,empty:ot}},[{source:(0,Ne.m)(r,o,Te.AT),templates:{suggestion:at,empty:it,footer:function(e){var t=e.query;if(!e.isEmpty){var n=document.createElement("a"),r=i+"search?q="+encodeURIComponent(t);n.href=r,n.textContent=Te.P9.see_all_results,n.addEventListener("click",(function(e){e.ctrlKey||e.metaKey||(e.preventDefault(),s.autocomplete.close(),l.push(r))}));var o=document.createElement("div");return o.className=Qe,o.appendChild(n),o}}}}]).on("autocomplete:selected",(function(e,t){var n,r=t.document,o=r.u,a=r.h,i=t.tokens;null==(n=u.current)||n.blur();var s=o;if(Te.CU&&i.length>0){for(var c,d=new URLSearchParams,f=(0,Ae.A)(i);!(c=f()).done;){var p=c.value;d.append("_highlight",p)}s+="?"+d.toString()}a&&(s+=a),l.push(s)})).on("autocomplete:closed",(function(){var e;null==(e=u.current)||e.blur()})),d.current="done",h(!1),f.current&&((c=u.current).value&&s.autocomplete.open(),c.focus());case 15:case"end":return e.stop()}}),e)}))),[i,l]);(0,r.useEffect)((function(){if(Te.CU){var e=Oe.A.canUseDOM?new URLSearchParams(s.search).getAll("_highlight"):[];0!==e.length&&setTimeout((function(){var t=document.querySelector("article");if(t){var n=new Te.CU(t);n.unmark(),n.mark(e)}}))}}),[s.search]);var w=(0,r.useCallback)((function(){f.current=!0,y(),null==a||a(!0)}),[a,y]),E=(0,r.useCallback)((function(){null==a||a(!1)}),[a]),k=(0,r.useCallback)((function(){y()}),[y]),x=(0,r.useCallback)((function(e){e.target.value&&b(!0)}),[]),S=!!Oe.A.canUseDOM&&/mac/i.test(null!=(t=null==(n=navigator.userAgentData)?void 0:n.platform)?t:navigator.platform);return(0,r.useEffect)((function(){function e(e){var t;(S?e.metaKey:e.ctrlKey)&&"KeyK"===e.code&&(e.preventDefault(),null==(t=u.current)||t.focus(),w())}return document.addEventListener("keydown",e),function(){document.removeEventListener("keydown",e)}}),[S,w]),r.createElement("div",{className:(0,Pe.A)("navbar__search",Ke,(o={},o[Xe]=m&&v,o))},r.createElement("input",{placeholder:Te.P9.search_placeholder,"aria-label":"Search",className:"navbar__search-input",onMouseEnter:k,onFocus:w,onBlur:E,onChange:x,ref:u}),r.createElement(lt.A,{className:Ye}),r.createElement("div",{className:Ze},r.createElement("kbd",{className:Je},S?"\u2318":"ctrl"),r.createElement("kbd",{className:Je},"K")))},dt="searchBox_ZlJk";function ft(e){var t=e.children,n=e.className;return r.createElement("div",{className:(0,o.A)(n,dt)},t)}var pt=n(4070),mt=n(1754),ht=["docId","label","docsPluginId"];var gt=["sidebarId","label","docsPluginId"];var vt=["label","to","docsPluginId"];var bt=n(5597),yt=["mobile","docsPluginId","dropdownActiveClassDisabled","dropdownItemsBefore","dropdownItemsAfter"],wt=function(e){return e.docs.find((function(t){return t.id===e.mainDocId}))};const Et={default:ue,localeDropdown:function(e){var t=e.mobile,n=e.dropdownItemsBefore,o=e.dropdownItemsAfter,a=(0,b.A)(e,Se),i=(0,fe.A)().i18n,l=i.currentLocale,s=i.locales,c=i.localeConfigs,d=(0,we.o)(),f=s.map((function(e){var n="pathname://"+d.createUrl({locale:e,fullyQualified:!1});return{label:c[e].label,lang:c[e].htmlLang,to:n,target:"_self",autoAddBaseUrl:!1,className:e===l?t?"menu__link--active":"dropdown__link--active":""}})),p=[].concat(n,f,o),m=t?(0,u.T)({message:"Languages",id:"theme.navbar.mobileLanguageDropdown.label",description:"The label for the mobile language switcher dropdown"}):c[l].label;return r.createElement(ye,(0,v.A)({},a,{mobile:t,label:r.createElement(r.Fragment,null,r.createElement(ke,{className:xe}),m),items:p}))},search:function(e){var t=e.mobile,n=e.className;return t?null:r.createElement(ft,{className:n},r.createElement(ct,null))},dropdown:ye,html:function(e){var t=e.value,n=e.className,a=e.mobile,i=void 0!==a&&a,l=e.isDropdownItem,s=void 0!==l&&l,u=s?"li":"div";return r.createElement(u,{className:(0,o.A)({navbar__item:!i&&!s,"menu__list-item":i},n),dangerouslySetInnerHTML:{__html:t}})},doc:function(e){var t=e.docId,n=e.label,o=e.docsPluginId,a=(0,b.A)(e,ht),i=(0,pt.zK)(o).activeDoc,l=(0,mt.QB)(t,o);return null===l?null:r.createElement(ue,(0,v.A)({exact:!0},a,{isActive:function(){return(null==i?void 0:i.path)===l.path||!(null==i||!i.sidebar)&&i.sidebar===l.sidebar},label:null!=n?n:l.id,to:l.path}))},docSidebar:function(e){var t=e.sidebarId,n=e.label,o=e.docsPluginId,a=(0,b.A)(e,gt),i=(0,pt.zK)(o).activeDoc,l=(0,mt.fW)(t,o).link;if(!l)throw new Error('DocSidebarNavbarItem: Sidebar with ID "'+t+"\" doesn't have anything to be linked to.");return r.createElement(ue,(0,v.A)({exact:!0},a,{isActive:function(){return(null==i?void 0:i.sidebar)===t},label:null!=n?n:l.label,to:l.path}))},docsVersion:function(e){var t=e.label,n=e.to,o=e.docsPluginId,a=(0,b.A)(e,vt),i=(0,mt.Vd)(o)[0],l=null!=t?t:i.label,s=null!=n?n:function(e){return e.docs.find((function(t){return t.id===e.mainDocId}))}(i).path;return r.createElement(ue,(0,v.A)({},a,{label:l,to:s}))},docsVersionDropdown:function(e){var t=e.mobile,n=e.docsPluginId,o=e.dropdownActiveClassDisabled,a=e.dropdownItemsBefore,i=e.dropdownItemsAfter,l=(0,b.A)(e,yt),s=(0,pt.zK)(n),c=(0,pt.jh)(n),d=(0,bt.g1)(n).savePreferredVersionName,f=c.map((function(e){var t,n=null!=(t=s.alternateDocVersions[e.name])?t:wt(e);return{label:e.label,to:n.path,isActive:function(){return e===s.activeVersion},onClick:function(){return d(e.name)}}})),p=[].concat(a,f,i),m=(0,mt.Vd)(n)[0],h=t&&p.length>1?(0,u.T)({id:"theme.navbar.mobileVersionsDropdown.label",message:"Versions",description:"The label for the navbar versions dropdown on mobile view"}):m.label,g=t&&p.length>1?void 0:wt(m).path;return p.length<=1?r.createElement(ue,(0,v.A)({},l,{mobile:t,label:h,to:g,isActive:o?function(){return!1}:void 0})):r.createElement(ye,(0,v.A)({},l,{mobile:t,label:h,to:g,items:p,isActive:o?function(){return!1}:void 0}))}};var kt=["type"];function xt(e){var t=e.type,n=(0,b.A)(e,kt),o=function(e,t){return e&&"default"!==e?e:"items"in t?"dropdown":"default"}(t,n),a=Et[o];if(!a)throw new Error('No NavbarItem component found for type "'+t+'".');return r.createElement(a,n)}function St(){var e=(0,O.M)(),t=(0,h.p)().navbar.items;return r.createElement("ul",{className:"menu__list"},t.map((function(t,n){return r.createElement(xt,(0,v.A)({mobile:!0},t,{onClick:function(){return e.toggle()},key:n}))})))}function Tt(e){return r.createElement("button",(0,v.A)({},e,{type:"button",className:"clean-btn navbar-sidebar__back"}),r.createElement(u.A,{id:"theme.navbar.mobileSidebarSecondaryMenu.backButtonLabel",description:"The label of the back button to return to main menu, inside the mobile navbar sidebar secondary menu (notably used to display the docs sidebar)"},"\u2190 Back to main menu"))}function _t(){var e=0===(0,h.p)().navbar.items.length,t=M();return r.createElement(r.Fragment,null,!e&&r.createElement(Tt,{onClick:function(){return t.hide()}}),t.content)}function At(){var e,t=(0,O.M)();return void 0===(e=t.shown)&&(e=!0),(0,r.useEffect)((function(){return document.body.style.overflow=e?"hidden":"visible",function(){document.body.style.overflow="visible"}}),[e]),t.shouldRender?r.createElement(z,{header:r.createElement(Y,null),primaryMenu:r.createElement(St,null),secondaryMenu:r.createElement(_t,null)}):null}const Ct="navbarHideable_m1mJ",Pt="navbarHidden_jGov";function Ot(e){return r.createElement("div",(0,v.A)({role:"presentation"},e,{className:(0,o.A)("navbar-sidebar__backdrop",e.className)}))}function Lt(e){var t=e.children,n=(0,h.p)().navbar,a=n.hideOnScroll,i=n.style,l=(0,O.M)(),s=function(e){var t=(0,r.useState)(e),n=t[0],o=t[1],a=(0,r.useRef)(!1),i=(0,r.useRef)(0),l=(0,r.useCallback)((function(e){null!==e&&(i.current=e.getBoundingClientRect().height)}),[]);return(0,L.Mq)((function(t,n){var r=t.scrollY;if(e)if(r=l?o(!1):r+u0&&r.createElement("div",{className:"row footer__links"},a.map((function(e,t){return r.createElement("div",{key:t,className:"col footer__col"},null!=e.title?r.createElement("h4",{className:"footer__title"},e.title):null,null!=e.items&&Array.isArray(e.items)&&e.items.length>0?r.createElement("ul",{className:"footer__items"},e.items.map((function(e,t){return e.html?r.createElement("li",{key:t,className:"footer__item",dangerouslySetInnerHTML:{__html:e.html}}):r.createElement("li",{key:e.href||e.to,className:"footer__item"},r.createElement(Ut,e))}))):null)}))),(l||n)&&r.createElement("div",{className:"text--center"},r.createElement("div",null,r.createElement("a",{href:"https://creativecommons.org/licenses/by/4.0/",title:"Creative Commons Attribution 4.0 International license",target:"_blank",rel:"noopener noreferrer"},r.createElement("img",{src:"/img/cc-by.svg",alt:"cc by license"}))),r.createElement("div",{style:{marginBottom:"10px"},dangerouslySetInnerHTML:{__html:n}})))):null};var Ht=n(9466),Qt=r.createContext(void 0);var Vt=(0,N.fM)([j.a,g.oq,function(e){var t=e.children,n=function(){var e=(0,r.useState)({}),t=e[0],n=e[1],o=(0,r.useCallback)((function(e,t){(0,Ht.W)("docusaurus.tab."+e).set(t)}),[]);(0,r.useEffect)((function(){try{var e={};(0,Ht.E)().forEach((function(t){if(t.startsWith("docusaurus.tab.")){var n=t.substring("docusaurus.tab.".length);e[n]=(0,Ht.W)(t).get()}})),n(e)}catch(t){console.error(t)}}),[]);var a=(0,r.useCallback)((function(e,t){n((function(n){var r;return Object.assign({},n,((r={})[e]=t,r))})),o(e,t)}),[o]);return(0,r.useMemo)((function(){return{tabGroupChoices:t,setTabGroupChoices:a}}),[t,a])}();return r.createElement(Qt.Provider,{value:n},t)},L.Tv,bt.VQ,i.Jx,function(e){var t=e.children;return r.createElement(D.y_,null,r.createElement(O.e,null,r.createElement(R,null,t)))}]);function Wt(e){var t=e.children;return r.createElement(Vt,null,t)}function Gt(e){var t=e.error,n=e.tryAgain;return r.createElement("main",{className:"container margin-vert--xl"},r.createElement("div",{className:"row"},r.createElement("div",{className:"col col--6 col--offset-3"},r.createElement("h1",{className:"hero__title"},r.createElement(u.A,{id:"theme.ErrorPageContent.title",description:"The title of the fallback page when the page crashed"},"This page crashed.")),r.createElement("p",null,t.message),r.createElement("div",null,r.createElement("button",{type:"button",onClick:n},r.createElement(u.A,{id:"theme.ErrorPageContent.tryAgain",description:"The label of the button to try again when the page crashed"},"Try again"))))))}const qt="mainWrapper_z2l0";function Kt(e){var t=e.children,n=e.noFooter,u=e.wrapperClassName,c=e.title,d=e.description;return(0,s.J)(),r.createElement(Wt,null,r.createElement(i.be,{title:c,description:d}),r.createElement(m,null),r.createElement(P,null),r.createElement(jt,null),r.createElement("div",{className:(0,o.A)(l.G.wrapper.main,qt,u)},r.createElement(a.A,{fallback:function(e){return r.createElement(Gt,e)}},t)),!n&&r.createElement($t,null))}},110:(e,t,n)=>{"use strict";n.d(t,{A:()=>b});var r=n(9668),o=n(1367),a=n(6540),i=n(5489),l=n(6025),s=n(4586),u=n(6342),c=n(870),d=n(2303),f=n(5293);const p={themedImage:"themedImage_ToTc","themedImage--light":"themedImage--light_HNdA","themedImage--dark":"themedImage--dark_i4oU"};var m=["sources","className","alt"];function h(e){var t=(0,d.A)(),n=(0,f.G)().colorMode,i=e.sources,l=e.className,s=e.alt,u=(0,o.A)(e,m),h=t?"dark"===n?["dark"]:["light"]:["light","dark"];return a.createElement(a.Fragment,null,h.map((function(e){return a.createElement("img",(0,r.A)({key:e,src:i[e],alt:s,className:(0,c.A)(p.themedImage,p["themedImage--"+e],l)},u))})))}var g=["imageClassName","titleClassName"];function v(e){var t=e.logo,n=e.alt,r=e.imageClassName,o={light:(0,l.A)(t.src),dark:(0,l.A)(t.srcDark||t.src)},i=a.createElement(h,{className:t.className,sources:o,height:t.height,width:t.width,alt:n,style:t.style});return r?a.createElement("div",{className:r},i):i}function b(e){var t,n=(0,s.A)().siteConfig.title,c=(0,u.p)().navbar,d=c.title,f=c.logo,p=e.imageClassName,m=e.titleClassName,h=(0,o.A)(e,g),b=(0,l.A)((null==f?void 0:f.href)||"/"),y=d?"":n,w=null!=(t=null==f?void 0:f.alt)?t:y;return a.createElement(i.A,(0,r.A)({to:b},h,(null==f?void 0:f.target)&&{target:f.target}),f&&a.createElement(v,{logo:f,alt:w,imageClassName:p}),null!=d&&a.createElement("b",{className:m},d))}},1463:(e,t,n)=>{"use strict";n.d(t,{A:()=>a});var r=n(6540),o=n(5260);function a(e){var t=e.locale,n=e.version,a=e.tag,i=t;return r.createElement(o.A,null,t&&r.createElement("meta",{name:"docusaurus_locale",content:t}),n&&r.createElement("meta",{name:"docusaurus_version",content:n}),a&&r.createElement("meta",{name:"docusaurus_tag",content:a}),i&&r.createElement("meta",{name:"docsearch:language",content:i}),n&&r.createElement("meta",{name:"docsearch:version",content:n}),a&&r.createElement("meta",{name:"docsearch:docusaurus_tag",content:a}))}},1422:(e,t,n)=>{"use strict";n.d(t,{N:()=>v,u:()=>u});var r=n(9668),o=n(1367),a=n(6540),i=n(8193),l=["collapsed"],s=["lazy"];function u(e){var t=e.initialState,n=(0,a.useState)(null!=t&&t),r=n[0],o=n[1],i=(0,a.useCallback)((function(){o((function(e){return!e}))}),[]);return{collapsed:r,setCollapsed:o,toggleCollapsed:i}}var c={display:"none",overflow:"hidden",height:"0px"},d={display:"block",overflow:"visible",height:"auto"};function f(e,t){var n=t?c:d;e.style.display=n.display,e.style.overflow=n.overflow,e.style.height=n.height}function p(e){var t=e.collapsibleRef,n=e.collapsed,r=e.animation,o=(0,a.useRef)(!1);(0,a.useEffect)((function(){var e,a=t.current;function i(){var e,t,n=a.scrollHeight;return{transition:"height "+(null!=(e=null==r?void 0:r.duration)?e:function(e){var t=e/36;return Math.round(10*(4+15*Math.pow(t,.25)+t/5))}(n))+"ms "+(null!=(t=null==r?void 0:r.easing)?t:"ease-in-out"),height:n+"px"}}function l(){var e=i();a.style.transition=e.transition,a.style.height=e.height}if(!o.current)return f(a,n),void(o.current=!0);return a.style.willChange="height",e=requestAnimationFrame((function(){n?(l(),requestAnimationFrame((function(){a.style.height=c.height,a.style.overflow=c.overflow}))):(a.style.display="block",requestAnimationFrame((function(){l()})))})),function(){return cancelAnimationFrame(e)}}),[t,n,r])}function m(e){if(!i.A.canUseDOM)return e?c:d}function h(e){var t=e.as,n=void 0===t?"div":t,r=e.collapsed,o=e.children,i=e.animation,l=e.onCollapseTransitionEnd,s=e.className,u=e.disableSSRStyle,c=(0,a.useRef)(null);return p({collapsibleRef:c,collapsed:r,animation:i}),a.createElement(n,{ref:c,style:u?void 0:m(r),onTransitionEnd:function(e){"height"===e.propertyName&&(f(c.current,r),null==l||l(r))},className:s},o)}function g(e){var t=e.collapsed,n=(0,o.A)(e,l),i=(0,a.useState)(!t),s=i[0],u=i[1],c=(0,a.useState)(t),d=c[0],f=c[1];return(0,a.useLayoutEffect)((function(){t||u(!0)}),[t]),(0,a.useLayoutEffect)((function(){s&&f(t)}),[s,t]),s?a.createElement(h,(0,r.A)({},n,{collapsed:d})):null}function v(e){var t=e.lazy,n=(0,o.A)(e,s),r=t?g:h;return a.createElement(r,n)}},5041:(e,t,n)=>{"use strict";n.d(t,{Mj:()=>m,oq:()=>p});var r=n(6540),o=n(2303),a=n(9466),i=n(2021),l=n(6342),s=(0,a.W)("docusaurus.announcement.dismiss"),u=(0,a.W)("docusaurus.announcement.id"),c=function(){return"true"===s.get()},d=function(e){return s.set(String(e))},f=r.createContext(null);function p(e){var t=e.children,n=function(){var e=(0,l.p)().announcementBar,t=(0,o.A)(),n=(0,r.useState)((function(){return!!t&&c()})),a=n[0],i=n[1];(0,r.useEffect)((function(){i(c())}),[]);var s=(0,r.useCallback)((function(){d(!0),i(!0)}),[]);return(0,r.useEffect)((function(){if(e){var t=e.id,n=u.get();"annoucement-bar"===n&&(n="announcement-bar");var r=t!==n;u.set(t),r&&d(!1),!r&&c()||i(!1)}}),[e]),(0,r.useMemo)((function(){return{isActive:!!e&&!a,close:s}}),[e,a,s])}();return r.createElement(f.Provider,{value:n},t)}function m(){var e=(0,r.useContext)(f);if(!e)throw new i.dV("AnnouncementBarProvider");return e}},5293:(e,t,n)=>{"use strict";n.d(t,{G:()=>h,a:()=>m});var r=n(6540),o=n(8193),a=n(2021),i=n(9466),l=n(6342),s=r.createContext(void 0),u=(0,i.W)("theme"),c="light",d="dark",f=function(e){return e===d?d:c};function p(){var e=(0,l.p)().colorMode,t=e.defaultMode,n=e.disableSwitch,a=e.respectPrefersColorScheme,i=(0,r.useState)(function(e){return o.A.canUseDOM?f(document.documentElement.getAttribute("data-theme")):f(e)}(t)),s=i[0],p=i[1];(0,r.useEffect)((function(){n&&u.del()}),[n]);var m=(0,r.useCallback)((function(e,n){void 0===n&&(n={});var r=n.persist,o=void 0===r||r;e?(p(e),o&&function(e){u.set(f(e))}(e)):(p(a?window.matchMedia("(prefers-color-scheme: dark)").matches?d:c:t),u.del())}),[a,t]);(0,r.useEffect)((function(){document.documentElement.setAttribute("data-theme",f(s))}),[s]),(0,r.useEffect)((function(){if(!n){var e=function(e){if("theme"===e.key){var t=u.get();null!==t&&m(f(t))}};return window.addEventListener("storage",e),function(){return window.removeEventListener("storage",e)}}}),[n,m]);var h=(0,r.useRef)(!1);return(0,r.useEffect)((function(){if(!n||a){var e=window.matchMedia("(prefers-color-scheme: dark)"),t=function(){window.matchMedia("print").matches||h.current?h.current=window.matchMedia("print").matches:m(null)};return e.addListener(t),function(){return e.removeListener(t)}}}),[m,n,a]),(0,r.useMemo)((function(){return{colorMode:s,setColorMode:m,get isDarkTheme(){return s===d},setLightTheme:function(){m(c)},setDarkTheme:function(){m(d)}}}),[s,m])}function m(e){var t=e.children,n=p();return r.createElement(s.Provider,{value:n},t)}function h(){var e=(0,r.useContext)(s);if(null==e)throw new a.dV("ColorModeProvider","Please see https://docusaurus.io/docs/api/themes/configuration#use-color-mode.");return e}},5597:(e,t,n)=>{"use strict";n.d(t,{VQ:()=>v,g1:()=>y});var r=n(6540),o=n(4070),a=n(7065),i=n(6342),l=n(1754),s=n(2021),u=n(9466),c=function(e){return"docs-preferred-version-"+e},d=function(e,t,n){(0,u.W)(c(e),{persistence:t}).set(n)},f=function(e,t){return(0,u.W)(c(e),{persistence:t}).get()},p=function(e,t){(0,u.W)(c(e),{persistence:t}).del()};var m=r.createContext(null);function h(){var e=(0,o.Gy)(),t=(0,i.p)().docs.versionPersistence,n=(0,r.useMemo)((function(){return Object.keys(e)}),[e]),a=(0,r.useState)((function(){return function(e){return Object.fromEntries(e.map((function(e){return[e,{preferredVersionName:null}]})))}(n)})),l=a[0],s=a[1];return(0,r.useEffect)((function(){s(function(e){var t=e.pluginIds,n=e.versionPersistence,r=e.allDocsData;return Object.fromEntries(t.map((function(e){return[e,(t=e,o=f(t,n),r[t].versions.some((function(e){return e.name===o}))?{preferredVersionName:o}:(p(t,n),{preferredVersionName:null}))];var t,o})))}({allDocsData:e,versionPersistence:t,pluginIds:n}))}),[e,t,n]),[l,(0,r.useMemo)((function(){return{savePreferredVersion:function(e,n){d(e,t,n),s((function(t){var r;return Object.assign({},t,((r={})[e]={preferredVersionName:n},r))}))}}}),[t])]}function g(e){var t=e.children,n=h();return r.createElement(m.Provider,{value:n},t)}function v(e){var t=e.children;return l.C5?r.createElement(g,null,t):r.createElement(r.Fragment,null,t)}function b(){var e=(0,r.useContext)(m);if(!e)throw new s.dV("DocsPreferredVersionContextProvider");return e}function y(e){var t;void 0===e&&(e=a.W);var n=(0,o.ht)(e),i=b(),l=i[0],s=i[1],u=l[e].preferredVersionName;return{preferredVersion:null!=(t=n.versions.find((function(e){return e.name===u})))?t:null,savePreferredVersionName:(0,r.useCallback)((function(t){s.savePreferredVersion(e,t)}),[s,e])}}},6588:(e,t,n)=>{"use strict";n.d(t,{V:()=>l,t:()=>s});var r=n(6540),o=n(2021),a=Symbol("EmptyContext"),i=r.createContext(a);function l(e){var t=e.children,n=e.name,o=e.items,a=(0,r.useMemo)((function(){return n&&o?{name:n,items:o}:null}),[n,o]);return r.createElement(i.Provider,{value:a},t)}function s(){var e=(0,r.useContext)(i);if(e===a)throw new o.dV("DocsSidebarProvider");return e}},9876:(e,t,n)=>{"use strict";n.d(t,{e:()=>f,M:()=>p});var r=n(6540),o=n(5600),a=n(4581),i=n(6347),l=n(2021);function s(e){!function(e){var t=(0,i.W6)(),n=(0,l._q)(e);(0,r.useEffect)((function(){return t.block((function(e,t){return n(e,t)}))}),[t,n])}((function(t,n){if("POP"===n)return e(t,n)}))}var u=n(6342),c=r.createContext(void 0);function d(){var e,t=(e=(0,o.YL)(),0===(0,u.p)().navbar.items.length&&!e.component),n=(0,a.l)(),i=!t&&"mobile"===n,l=(0,r.useState)(!1),c=l[0],d=l[1];s((function(){if(c)return d(!1),!1}));var f=(0,r.useCallback)((function(){d((function(e){return!e}))}),[]);return(0,r.useEffect)((function(){"desktop"===n&&d(!1)}),[n]),(0,r.useMemo)((function(){return{disabled:t,shouldRender:i,toggle:f,shown:c}}),[t,i,f,c])}function f(e){var t=e.children,n=d();return r.createElement(c.Provider,{value:n},t)}function p(){var e=r.useContext(c);if(void 0===e)throw new l.dV("NavbarMobileSidebarProvider");return e}},5600:(e,t,n)=>{"use strict";n.d(t,{GX:()=>s,YL:()=>l,y_:()=>i});var r=n(6540),o=n(2021),a=r.createContext(null);function i(e){var t=e.children,n=(0,r.useState)({component:null,props:null});return r.createElement(a.Provider,{value:n},t)}function l(){var e=(0,r.useContext)(a);if(!e)throw new o.dV("NavbarSecondaryMenuContentProvider");return e[0]}function s(e){var t=e.component,n=e.props,i=(0,r.useContext)(a);if(!i)throw new o.dV("NavbarSecondaryMenuContentProvider");var l=i[1],s=(0,o.Be)(n);return(0,r.useEffect)((function(){l({component:t,props:s})}),[l,t,s]),(0,r.useEffect)((function(){return function(){return l({component:null,props:null})}}),[l]),null}},4090:(e,t,n)=>{"use strict";n.d(t,{w:()=>o,J:()=>a});var r=n(6540),o="navigation-with-keyboard";function a(){(0,r.useEffect)((function(){function e(e){"keydown"===e.type&&"Tab"===e.key&&document.body.classList.add(o),"mousedown"===e.type&&document.body.classList.remove(o)}return document.addEventListener("keydown",e),document.addEventListener("mousedown",e),function(){document.body.classList.remove(o),document.removeEventListener("keydown",e),document.removeEventListener("mousedown",e)}}),[])}},4581:(e,t,n)=>{"use strict";n.d(t,{l:()=>u});var r=n(6540),o=n(8193),a="desktop",i="mobile",l="ssr";function s(){return o.A.canUseDOM?window.innerWidth>996?a:i:l}function u(){var e=(0,r.useState)((function(){return s()})),t=e[0],n=e[1];return(0,r.useEffect)((function(){function e(){n(s())}return window.addEventListener("resize",e),function(){window.removeEventListener("resize",e),clearTimeout(void 0)}}),[]),t}},7559:(e,t,n)=>{"use strict";n.d(t,{G:()=>r});var r={page:{blogListPage:"blog-list-page",blogPostPage:"blog-post-page",blogTagsListPage:"blog-tags-list-page",blogTagPostListPage:"blog-tags-post-list-page",docsDocPage:"docs-doc-page",docsTagsListPage:"docs-tags-list-page",docsTagDocListPage:"docs-tags-doc-list-page",mdxPage:"mdx-page"},wrapper:{main:"main-wrapper",blogPages:"blog-wrapper",docsPages:"docs-wrapper",mdxPages:"mdx-wrapper"},common:{editThisPage:"theme-edit-this-page",lastUpdated:"theme-last-updated",backToTopButton:"theme-back-to-top-button",codeBlock:"theme-code-block",admonition:"theme-admonition",admonitionType:function(e){return"theme-admonition-"+e}},layout:{},docs:{docVersionBanner:"theme-doc-version-banner",docVersionBadge:"theme-doc-version-badge",docBreadcrumbs:"theme-doc-breadcrumbs",docMarkdown:"theme-doc-markdown",docTocMobile:"theme-doc-toc-mobile",docTocDesktop:"theme-doc-toc-desktop",docFooter:"theme-doc-footer",docFooterTagsRow:"theme-doc-footer-tags-row",docFooterEditMetaRow:"theme-doc-footer-edit-meta-row",docSidebarContainer:"theme-doc-sidebar-container",docSidebarMenu:"theme-doc-sidebar-menu",docSidebarItemCategory:"theme-doc-sidebar-item-category",docSidebarItemLink:"theme-doc-sidebar-item-link",docSidebarItemCategoryLevel:function(e){return"theme-doc-sidebar-item-category-level-"+e},docSidebarItemLinkLevel:function(e){return"theme-doc-sidebar-item-link-level-"+e}},blog:{}}},1754:(e,t,n)=>{"use strict";n.d(t,{_o:()=>p,w8:()=>h,C5:()=>f,mz:()=>E,Vd:()=>b,QB:()=>w,fW:()=>y,OF:()=>v});var r=n(9593),o=n(6540),a=n(6347),i=n(2831),l=n(4070),s=n(5597),u=n(6588);function c(e){return Array.from(new Set(e))}var d=n(9169),f=!!l.Gy;function p(e){if(e.href)return e.href;for(var t,n=(0,r.A)(e.items);!(t=n()).done;){var o=t.value;if("link"===o.type)return o.href;if("category"===o.type){var a=p(o);if(a)return a}}}var m=function(e,t){return void 0!==e&&(0,d.ys)(e,t)};function h(e,t){return"link"===e.type?m(e.href,t):"category"===e.type&&(m(e.href,t)||function(e,t){return e.some((function(e){return h(e,t)}))}(e.items,t))}function g(e){var t=e.sidebarItems,n=e.pathname,o=e.onlyCategories,a=void 0!==o&&o,i=[];return function e(t){for(var o,l=(0,r.A)(t);!(o=l()).done;){var s=o.value;if("category"===s.type&&((0,d.ys)(s.href,n)||e(s.items))||"link"===s.type&&(0,d.ys)(s.href,n))return a&&"category"!==s.type||i.unshift(s),!0}return!1}(t),i}function v(){var e,t=(0,u.t)(),n=(0,a.zy)().pathname;return!1!==(null==(e=(0,l.vT)())?void 0:e.pluginData.breadcrumbs)&&t?g({sidebarItems:t.items,pathname:n}):null}function b(e){var t=(0,l.zK)(e).activeVersion,n=(0,s.g1)(e).preferredVersion,r=(0,l.r7)(e);return(0,o.useMemo)((function(){return c([t,n,r].filter(Boolean))}),[t,n,r])}function y(e,t){var n=b(t);return(0,o.useMemo)((function(){var t=n.flatMap((function(e){return e.sidebars?Object.entries(e.sidebars):[]})),r=t.find((function(t){return t[0]===e}));if(!r)throw new Error("Can't find any sidebar with id \""+e+'" in version'+(n.length>1?"s":"")+" "+n.map((function(e){return e.name})).join(", ")+'".\n Available sidebar ids are:\n - '+Object.keys(t).join("\n- "));return r[1]}),[e,n])}function w(e,t){var n=b(t);return(0,o.useMemo)((function(){var t=n.flatMap((function(e){return e.docs})),r=t.find((function(t){return t.id===e}));if(!r){if(n.flatMap((function(e){return e.draftIds})).includes(e))return null;throw new Error("DocNavbarItem: couldn't find any doc with id \""+e+'" in version'+(n.length>1?"s":"")+" "+n.map((function(e){return e.name})).join(", ")+'".\nAvailable doc ids are:\n- '+c(t.map((function(e){return e.id}))).join("\n- "))}return r}),[e,n])}function E(e){var t=e.route,n=e.versionMetadata,r=(0,a.zy)(),o=t.routes,l=o.find((function(e){return(0,a.B6)(r.pathname,e)}));if(!l)return null;var s=l.sidebar,u=s?n.docsSidebars[s]:void 0;return{docElement:(0,i.v)(o),sidebarName:s,sidebarItems:u}}},1003:(e,t,n)=>{"use strict";n.d(t,{e3:()=>f,be:()=>c,Jx:()=>p});var r=n(6540),o=n(9423),a=n(5260),i=n(3102);function l(){var e=r.useContext(i.o);if(!e)throw new Error("Unexpected: no Docusaurus route context found");return e}var s=n(6025),u=n(4586);function c(e){var t=e.title,n=e.description,o=e.keywords,i=e.image,l=e.children,c=function(e){var t=(0,u.A)().siteConfig,n=t.title,r=t.titleDelimiter;return null!=e&&e.trim().length?e.trim()+" "+r+" "+n:n}(t),d=(0,s.h)().withBaseUrl,f=i?d(i,{absolute:!0}):void 0;return r.createElement(a.A,null,t&&r.createElement("title",null,c),t&&r.createElement("meta",{property:"og:title",content:c}),n&&r.createElement("meta",{name:"description",content:n}),n&&r.createElement("meta",{property:"og:description",content:n}),o&&r.createElement("meta",{name:"keywords",content:Array.isArray(o)?o.join(","):o}),f&&r.createElement("meta",{property:"og:image",content:f}),f&&r.createElement("meta",{name:"twitter:image",content:f}),l)}var d=r.createContext(void 0);function f(e){var t=e.className,n=e.children,i=r.useContext(d),l=(0,o.A)(i,t);return r.createElement(d.Provider,{value:l},r.createElement(a.A,null,r.createElement("html",{className:l})),n)}function p(e){var t=e.children,n=l(),a="plugin-"+n.plugin.name.replace(/docusaurus-(?:plugin|theme)-(?:content-)?/gi,""),i="plugin-id-"+n.plugin.id;return r.createElement(f,{className:(0,o.A)(a,i)},t)}},2021:(e,t,n)=>{"use strict";n.d(t,{dV:()=>m,fM:()=>g,_q:()=>f,ZC:()=>p,Be:()=>h});var r=n(17),o=n(6711);function a(e){return(a=Object.setPrototypeOf?Object.getPrototypeOf.bind():function(e){return e.__proto__||Object.getPrototypeOf(e)})(e)}var i=n(7778);function l(){if("undefined"==typeof Reflect||!Reflect.construct)return!1;if(Reflect.construct.sham)return!1;if("function"==typeof Proxy)return!0;try{return Boolean.prototype.valueOf.call(Reflect.construct(Boolean,[],(function(){}))),!0}catch(e){return!1}}function s(e,t,n){return(s=l()?Reflect.construct.bind():function(e,t,n){var r=[null];r.push.apply(r,t);var o=new(Function.bind.apply(e,r));return n&&(0,i.A)(o,n.prototype),o}).apply(null,arguments)}function u(e){var t="function"==typeof Map?new Map:void 0;return(u=function(e){if(null===e||(n=e,-1===Function.toString.call(n).indexOf("[native code]")))return e;var n;if("function"!=typeof e)throw new TypeError("Super expression must either be null or a function");if(void 0!==t){if(t.has(e))return t.get(e);t.set(e,r)}function r(){return s(e,arguments,a(this).constructor)}return r.prototype=Object.create(e.prototype,{constructor:{value:r,enumerable:!1,writable:!0,configurable:!0}}),(0,i.A)(r,e)})(e)}var c=n(6540),d=n(8193).A.canUseDOM?c.useLayoutEffect:c.useEffect;function f(e){var t=(0,c.useRef)(e);return d((function(){t.current=e}),[e]),(0,c.useCallback)((function(){return t.current.apply(t,arguments)}),[])}function p(e){var t=(0,c.useRef)();return d((function(){t.current=e})),t.current}var m=function(e){function t(t,n){var o,a,i,l,s;return(s=e.call(this)||this).name="ReactContextError",s.message="Hook "+(null!=(o=null==(a=s.stack)||null==(i=a.split("\n")[1])||null==(l=i.match((0,r.A)(/at (?:\w+\.)?(\w+)/,{name:1})))?void 0:l.groups.name)?o:"")+" is called outside the <"+t+">. "+(null!=n?n:""),s}return(0,o.A)(t,e),t}(u(Error));function h(e){var t=Object.entries(e);return t.sort((function(e,t){return e[0].localeCompare(t[0])})),(0,c.useMemo)((function(){return e}),t.flat())}function g(e){return function(t){var n=t.children;return c.createElement(c.Fragment,null,e.reduceRight((function(e,t){return c.createElement(t,null,e)}),n))}}},9169:(e,t,n)=>{"use strict";n.d(t,{Dt:()=>l,ys:()=>i});var r=n(6540),o=n(8328),a=n(4586);function i(e,t){var n=function(e){var t;return null==(t=!e||e.endsWith("/")?e:e+"/")?void 0:t.toLowerCase()};return n(e)===n(t)}function l(){var e=(0,a.A)().siteConfig.baseUrl;return(0,r.useMemo)((function(){return function(e){var t=e.baseUrl;function n(e){return e.path===t&&!0===e.exact}function r(e){return e.path===t&&!e.exact}return function e(t){if(0!==t.length){var o=t.find(n);return o||e(t.filter(r).flatMap((function(e){var t;return null!=(t=e.routes)?t:[]})))}}(e.routes)}({routes:o.A,baseUrl:e})}),[e])}},3104:(e,t,n)=>{"use strict";n.d(t,{Mq:()=>d,Tv:()=>s,gk:()=>f});var r=n(6540),o=n(8193),a=n(2303),i=n(2021);var l=r.createContext(void 0);function s(e){var t,n=e.children,o=(t=(0,r.useRef)(!0),(0,r.useMemo)((function(){return{scrollEventsEnabledRef:t,enableScrollEvents:function(){t.current=!0},disableScrollEvents:function(){t.current=!1}}}),[]));return r.createElement(l.Provider,{value:o},n)}function u(){var e=(0,r.useContext)(l);if(null==e)throw new i.dV("ScrollControllerProvider");return e}var c=function(){return o.A.canUseDOM?{scrollX:window.pageXOffset,scrollY:window.pageYOffset}:null};function d(e,t){void 0===t&&(t=[]);var n=u().scrollEventsEnabledRef,o=(0,r.useRef)(c()),a=(0,i._q)(e);(0,r.useEffect)((function(){var e=function(){if(n.current){var e=c();a(e,o.current),o.current=e}},t={passive:!0};return e(),window.addEventListener("scroll",e,t),function(){return window.removeEventListener("scroll",e,t)}}),[a,n].concat(t))}function f(){var e=(0,r.useRef)(null),t=(0,a.A)()&&"smooth"===getComputedStyle(document.documentElement).scrollBehavior;return{startScroll:function(n){e.current=t?function(e){return window.scrollTo({top:e,behavior:"smooth"}),function(){}}(n):function(e){var t=null,n=document.documentElement.scrollTop>e;return function r(){var o=document.documentElement.scrollTop;(n&&o>e||!n&&o{"use strict";n.d(t,{Cy:()=>r,tU:()=>o});n(4586);var r="default";function o(e,t){return"docs-"+e+"-"+t}},9466:(e,t,n)=>{"use strict";n.d(t,{E:()=>l,W:()=>i});function r(e){if(void 0===e&&(e="localStorage"),"undefined"==typeof window)throw new Error("Browser storage is not available on Node.js/Docusaurus SSR process.");if("none"===e)return null;try{return window[e]}catch(n){return t=n,o||(console.warn("Docusaurus browser storage is not available.\nPossible reasons: running Docusaurus in an iframe, in an incognito browser session, or using too strict browser privacy settings.",t),o=!0),null}var t}var o=!1;var a={get:function(){return null},set:function(){},del:function(){}};function i(e,t){if("undefined"==typeof window)return function(e){function t(){throw new Error('Illegal storage API usage for storage key "'+e+'".\nDocusaurus storage APIs are not supposed to be called on the server-rendering process.\nPlease only call storage APIs in effects and event handlers.')}return{get:t,set:t,del:t}}(e);var n=r(null==t?void 0:t.persistence);return null===n?a:{get:function(){try{return n.getItem(e)}catch(t){return console.error("Docusaurus storage error, can't get key="+e,t),null}},set:function(t){try{n.setItem(e,t)}catch(r){console.error("Docusaurus storage error, can't set "+e+"="+t,r)}},del:function(){try{n.removeItem(e)}catch(t){console.error("Docusaurus storage error, can't delete key="+e,t)}}}}function l(e){void 0===e&&(e="localStorage");var t=r(e);if(!t)return[];for(var n=[],o=0;o{"use strict";n.d(t,{o:()=>a});var r=n(4586),o=n(6347);function a(){var e=(0,r.A)(),t=e.siteConfig,n=t.baseUrl,a=t.url,i=e.i18n,l=i.defaultLocale,s=i.currentLocale,u=(0,o.zy)().pathname,c=s===l?n:n.replace("/"+s+"/","/"),d=u.replace(n,"");return{createUrl:function(e){var t=e.locale;return""+(e.fullyQualified?a:"")+function(e){return e===l?""+c:""+c+e+"/"}(t)+d}}}},5062:(e,t,n)=>{"use strict";n.d(t,{$:()=>i});var r=n(6540),o=n(6347),a=n(2021);function i(e){var t=(0,o.zy)(),n=(0,a.ZC)(t),i=(0,a._q)(e);(0,r.useEffect)((function(){n&&t!==n&&i({location:t,previousLocation:n})}),[i,t,n])}},6342:(e,t,n)=>{"use strict";n.d(t,{p:()=>o});var r=n(4586);function o(){return(0,r.A)().siteConfig.themeConfig}},2983:(e,t)=>{"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.default=function(e,t){var n=t.trailingSlash,r=t.baseUrl;if(e.startsWith("#"))return e;if(void 0===n)return e;var o,a=e.split(/[#?]/)[0],i="/"===a||a===r?a:(o=a,n?function(e){return e.endsWith("/")?e:e+"/"}(o):function(e){return e.endsWith("/")?e.slice(0,-1):e}(o));return e.replace(a,i)}},440:function(e,t,n){"use strict";var r=this&&this.__importDefault||function(e){return e&&e.__esModule?e:{default:e}};Object.defineProperty(t,"__esModule",{value:!0}),t.applyTrailingSlash=t.blogPostContainerID=void 0,t.blogPostContainerID="post-content";var o=n(2983);Object.defineProperty(t,"applyTrailingSlash",{enumerable:!0,get:function(){return r(o).default}})},2849:(e,t,n)=>{"use strict";n.d(t,{A:()=>i});var r=n(6540),o=n(53);const a="loadingRing_RJI3";function i(e){var t=e.className;return r.createElement("div",{className:(0,o.A)(a,t)},r.createElement("div",null),r.createElement("div",null),r.createElement("div",null),r.createElement("div",null))}},5891:(e,t,n)=>{"use strict";n.d(t,{Z:()=>u});var r=n(6751),o=n(9593),a=n(7591),i=n(8291),l=n.n(i),s=n(1088);function u(e){return c.apply(this,arguments)}function c(){return(c=(0,a.A)((0,r.A)().mark((function e(t){var n,a,i;return(0,r.A)().wrap((function(e){for(;;)switch(e.prev=e.next){case 0:return e.next=3,fetch(t+"search-index.json?_="+s.Xi);case 3:return e.next=5,e.sent.json();case 5:return n=e.sent,a=n.map((function(e,t){var n=e.documents,r=e.index;return{type:t,documents:n,index:l().Index.load(r)}})),i=n.reduce((function(e,t){for(var n,r=(0,o.A)(t.index.invertedIndex);!(n=r()).done;){var a=n.value;/(?:[\u3400-\u4DBF\u4E00-\u9FFF\uFA0E\uFA0F\uFA11\uFA13\uFA14\uFA1F\uFA21\uFA23\uFA24\uFA27-\uFA29]|[\uD840-\uD868\uD86A-\uD86C\uD86F-\uD872\uD874-\uD879\uD880-\uD883\uD885-\uD887][\uDC00-\uDFFF]|\uD869[\uDC00-\uDEDF\uDF00-\uDFFF]|\uD86D[\uDC00-\uDF39\uDF40-\uDFFF]|\uD86E[\uDC00-\uDC1D\uDC20-\uDFFF]|\uD873[\uDC00-\uDEA1\uDEB0-\uDFFF]|\uD87A[\uDC00-\uDFE0]|\uD884[\uDC00-\uDF4A\uDF50-\uDFFF]|\uD888[\uDC00-\uDFAF])/.test(a[0][0])&&e.add(a[0])}return e}),new Set),e.abrupt("return",{wrappedIndexes:a,zhDictionary:Array.from(i)});case 9:return e.abrupt("return",{wrappedIndexes:[],zhDictionary:[]});case 10:case"end":return e.stop()}}),e)})))).apply(this,arguments)}},2384:(e,t,n)=>{"use strict";n.d(t,{m:()=>c});var r=n(9593),o=n(8291),a=n.n(o);function i(e,t){var n=[];return function e(o,a){if(0!==o.length){var i=o[0];if(/(?:[\u3400-\u4DBF\u4E00-\u9FFF\uFA0E\uFA0F\uFA11\uFA13\uFA14\uFA1F\uFA21\uFA23\uFA24\uFA27-\uFA29]|[\uD840-\uD868\uD86A-\uD86C\uD86F-\uD872\uD874-\uD879\uD880-\uD883\uD885-\uD887][\uDC00-\uDFFF]|\uD869[\uDC00-\uDEDF\uDF00-\uDFFF]|\uD86D[\uDC00-\uDF39\uDF40-\uDFFF]|\uD86E[\uDC00-\uDC1D\uDC20-\uDFFF]|\uD873[\uDC00-\uDEA1\uDEB0-\uDFFF]|\uD87A[\uDC00-\uDFE0]|\uD884[\uDC00-\uDF4A\uDF50-\uDFFF]|\uD888[\uDC00-\uDFAF])/.test(i))for(var l,s=function(e,t){var n=[];return function e(o,a){for(var i,l=0,s=!1,u=(0,r.A)(t);!(i=u()).done;){var c=i.value;if(o.substr(0,c.length)===c){var d={missed:a.missed,term:a.term.concat({value:c})};o.length>c.length?e(o.substr(c.length),d):n.push(d),s=!0}else for(var f=c.length-1;f>l;f-=1){var p=c.substr(0,f);if(o.substr(0,f)===p){l=f;var m={missed:a.missed,term:a.term.concat({value:p,trailing:!0})};o.length>f?e(o.substr(f),m):n.push(m),s=!0;break}}}s||(o.length>0?e(o.substr(1),{missed:a.missed+1,term:a.term}):a.term.length>0&&n.push(a))}(e,{missed:0,term:[]}),n.sort((function(e,t){var n=e.missed>0?1:0,r=t.missed>0?1:0;return n!==r?n-r:e.term.length-t.term.length})).map((function(e){return e.term}))}(i,t),u=(0,r.A)(s);!(l=u()).done;){var c=l.value,d=a.concat.apply(a,c);e(o.slice(1),d)}else{var f=a.concat({value:i});e(o.slice(1),f)}}else n.push(a)}(e,[]),n}var l=n(1088);function s(e){return u(e).concat(u(e.filter((function(e){var t=e[e.length-1];return!t.trailing&&t.maybeTyping})),!0))}function u(e,t){return e.map((function(e){return{tokens:e.map((function(e){return e.value})),term:e.map((function(e){return{value:e.value,presence:a().Query.presence.REQUIRED,wildcard:(t?e.trailing||e.maybeTyping:e.trailing)?a().Query.wildcard.TRAILING:a().Query.wildcard.NONE}}))}}))}function c(e,t,n){return function(o,u){var c=function(e,t){if(1===t.length&&["ja","jp","th"].includes(t[0]))return a()[t[0]].tokenizer(e).map((function(e){return e.toString()}));var n=/[^-\s]+/g;return t.includes("zh")&&(n=/[0-9A-Z_a-z]+|(?:[\u3400-\u4DBF\u4E00-\u9FFF\uFA0E\uFA0F\uFA11\uFA13\uFA14\uFA1F\uFA21\uFA23\uFA24\uFA27-\uFA29]|[\uD840-\uD868\uD86A-\uD86C\uD86F-\uD872\uD874-\uD879\uD880-\uD883\uD885-\uD887][\uDC00-\uDFFF]|\uD869[\uDC00-\uDEDF\uDF00-\uDFFF]|\uD86D[\uDC00-\uDF39\uDF40-\uDFFF]|\uD86E[\uDC00-\uDC1D\uDC20-\uDFFF]|\uD873[\uDC00-\uDEA1\uDEB0-\uDFFF]|\uD87A[\uDC00-\uDFE0]|\uD884[\uDC00-\uDF4A\uDF50-\uDFFF]|\uD888[\uDC00-\uDFAF])+/g),e.toLowerCase().match(n)||[]}(o,l.BH);if(0!==c.length){var d=function(e,t){var n=i(e,t);if(0===n.length)return[{tokens:e,term:e.map((function(e){return{value:e,presence:a().Query.presence.REQUIRED,wildcard:a().Query.wildcard.LEADING|a().Query.wildcard.TRAILING}}))}];for(var o,u=(0,r.A)(n);!(o=u()).done;){var c=o.value;c[c.length-1].maybeTyping=!0}for(var d,f,p=[],m=(0,r.A)(l.BH);!(d=m()).done;){var h=d.value;if("en"===h)l.sx||p.unshift(a().stopWordFilter);else{var g=a()[h];g.stopWordFilter&&p.unshift(g.stopWordFilter)}}if(p.length>0){var v=function(e){return p.reduce((function(e,t){return e.filter((function(e){return t(e.value)}))}),e)};f=[];for(var b,y=[],w=(0,r.A)(n);!(b=w()).done;){var E=b.value,k=v(E);f.push(k),k.length0&&y.push(k)}n.push.apply(n,y)}else f=n.slice();for(var x,S=[],T=(0,r.A)(f);!(x=T()).done;){var _=x.value;if(_.length>2)for(var A=_.length-1;A>=0;A-=1)S.push(_.slice(0,A).concat(_.slice(A+1)))}return s(n).concat(s(S))}(c,t),f=[],p=function(){for(var t,o=m.value,a=o.term,i=o.tokens,l=function(){var o=t.value,l=o.documents,s=o.index,u=o.type;if(f.push.apply(f,s.query((function(e){for(var t,n=(0,r.A)(a);!(t=n()).done;){var o=t.value;e.term(o.value,{wildcard:o.wildcard,presence:o.presence})}})).slice(0,n).filter((function(e){return!f.some((function(t){return t.document.i.toString()===e.ref}))})).slice(0,n-f.length).map((function(t){var n=l.find((function(e){return e.i.toString()===t.ref}));return{document:n,type:u,page:0!==u&&e[0].documents.find((function(e){return e.i===n.p})),metadata:t.matchData.metadata,tokens:i,score:t.score}}))),f.length>=n)return{v:"break|search"}},s=(0,r.A)(e);!(t=s()).done;){var u=l();if("object"==typeof u)return u.v}};e:for(var m,h=(0,r.A)(d);!(m=h()).done;){if("break|search"===p())break e}!function(e){e.forEach((function(e,t){e.index=t})),e.sort((function(t,n){var r=t.type>0&&t.page?e.findIndex((function(e){return e.document===t.page})):t.index,o=n.type>0&&n.page?e.findIndex((function(e){return e.document===n.page})):n.index;return-1===r&&(r=t.index),-1===o&&(o=n.index),r===o?0===t.type?-1:0===n.type?1:t.index-n.index:r-o}))}(f),function(e){e.forEach((function(t,n){n>0&&t.page&&e.some((function(e){return e.document===t.page}))&&(n{"use strict";function r(e){return e.replace(/&/g,"&").replace(//g,">").replace(/"/g,""").replace(/'/g,"'")}n.d(t,{Z:()=>r})},7674:(e,t,n)=>{"use strict";function r(e,t){for(var n=[],r=0,o=Object.values(e);rr})},6841:(e,t,n)=>{"use strict";n.d(t,{Z:()=>a});var r=n(9593),o=n(3103);function a(e,t,n){for(var i,l=[],s=(0,r.A)(t);!(i=s()).done;){var u=i.value,c=e.toLowerCase().indexOf(u);if(c>=0){c>0&&l.push(a(e.substr(0,c),t)),l.push(""+(0,o.Z)(e.substr(c,u.length))+"");var d=c+u.length;d"+(0,o.Z)(e)+"":(0,o.Z)(e):l.join("")}},3810:(e,t,n)=>{"use strict";n.d(t,{C:()=>s});var r=n(3103),o=n(6841),a=/[0-9A-Z_a-z]+|(?:[\u3400-\u4DBF\u4E00-\u9FFF\uFA0E\uFA0F\uFA11\uFA13\uFA14\uFA1F\uFA21\uFA23\uFA24\uFA27-\uFA29]|[\uD840-\uD868\uD86A-\uD86C\uD86F-\uD872\uD874-\uD879\uD880-\uD883\uD885-\uD887][\uDC00-\uDFFF]|\uD869[\uDC00-\uDEDF\uDF00-\uDFFF]|\uD86D[\uDC00-\uDF39\uDF40-\uDFFF]|\uD86E[\uDC00-\uDC1D\uDC20-\uDFFF]|\uD873[\uDC00-\uDEA1\uDEB0-\uDFFF]|\uD87A[\uDC00-\uDFE0]|\uD884[\uDC00-\uDF4A\uDF50-\uDFFF]|\uD888[\uDC00-\uDFAF])/;function i(e){for(var t=[],n=0,r=e;r.length>0;){var o=r.match(a);if(!o){t.push(r);break}o.index>0&&t.push(r.substr(0,o.index)),t.push(o[0]),n+=o.index+o[0].length,r=e.substr(n)}return t}var l=n(1088);function s(e,t,n,a){void 0===a&&(a=l.rG);for(var s={chunkIndex:-1},u=function e(t,n,a,l,s,u){var c=[],d=n[l],f=d[0],p=d[1];if(fs&&c.push.apply(c,i(t.substring(s,f)).map((function(e){return{html:(0,r.Z)(e),textLength:e.length}}))),u&&(u.chunkIndex=c.length),c.push({html:(0,o.Z)(t.substr(f,p),a,!0),textLength:p});var m=f+p;(l+=1)0){var y=c.pop();m+y.textLength<=a?(f.unshift(y.html),h+=y.textLength,m+=y.textLength):(v=!0,c.length=0)}else{if(!(p.length>0))break;var w=p.shift();m+w.textLength<=a?(f.push(w.html),g+=w.textLength,m+=w.textLength):(b=!0,p.length=0)}return(v||c.length>0)&&f.unshift("\u2026"),(b||p.length>0)&&f.push("\u2026"),f.join("")}},1088:(e,t,n)=>{"use strict";n.d(t,{CU:()=>a,Xi:()=>i,BH:()=>r,sx:()=>o,rG:()=>s,AT:()=>l,P9:()=>u});n(8291);var r=["en"],o=!1,a=null,i="a1d0dac0",l=8,s=50,u={search_placeholder:"Search",see_all_results:"See all results",no_results:"No results.",search_results_for:'Search results for "{{ keyword }}"',search_the_documentation:"Search the documentation",count_documents_found_plural:"{{ count }} documents found",count_documents_found:"{{ count }} document found",no_documents_were_found:"No documents were found"}},5509:(e,t,n)=>{"use strict";n.r(t),n.d(t,{default:()=>h});var r=n(4784),o=Object.assign||function(e){for(var t=1;t1&&void 0!==arguments[1]?arguments[1]:{},r=window.Promise||function(e){function t(){}e(t,t)},a=function(e){var t=e.target;t!==N?-1!==_.indexOf(t)&&k({target:t}):E()},f=function(){if(!C&&L.original){var e=window.pageYOffset||document.documentElement.scrollTop||document.body.scrollTop||0;Math.abs(P-e)>O.scrollOffset&&setTimeout(E,150)}},p=function(e){var t=e.key||e.keyCode;"Escape"!==t&&"Esc"!==t&&27!==t||E()},m=function(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},t=e;if(e.background&&(N.style.background=e.background),e.container&&e.container instanceof Object&&(t.container=o({},O.container,e.container)),e.template){var n=i(e.template)?e.template:document.querySelector(e.template);t.template=n}return O=o({},O,t),_.forEach((function(e){e.dispatchEvent(d("medium-zoom:update",{detail:{zoom:D}}))})),D},h=function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{};return e(o({},O,t))},g=function(){for(var e=arguments.length,t=Array(e),n=0;n0?t.reduce((function(e,t){return[].concat(e,s(t))}),[]):_;return r.forEach((function(e){e.classList.remove("medium-zoom-image"),e.dispatchEvent(d("medium-zoom:detach",{detail:{zoom:D}}))})),_=_.filter((function(e){return-1===r.indexOf(e)})),D},b=function(e,t){var n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:{};return _.forEach((function(r){r.addEventListener("medium-zoom:"+e,t,n)})),A.push({type:"medium-zoom:"+e,listener:t,options:n}),D},y=function(e,t){var n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:{};return _.forEach((function(r){r.removeEventListener("medium-zoom:"+e,t,n)})),A=A.filter((function(n){return!(n.type==="medium-zoom:"+e&&n.listener.toString()===t.toString())})),D},w=function(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},t=e.target,n=function(){var e={width:document.documentElement.clientWidth,height:document.documentElement.clientHeight,left:0,top:0,right:0,bottom:0},t=void 0,n=void 0;if(O.container)if(O.container instanceof Object)t=(e=o({},e,O.container)).width-e.left-e.right-2*O.margin,n=e.height-e.top-e.bottom-2*O.margin;else{var r=(i(O.container)?O.container:document.querySelector(O.container)).getBoundingClientRect(),a=r.width,s=r.height,u=r.left,c=r.top;e=o({},e,{width:a,height:s,left:u,top:c})}t=t||e.width-2*O.margin,n=n||e.height-2*O.margin;var d=L.zoomedHd||L.original,f=l(d)?t:d.naturalWidth||t,p=l(d)?n:d.naturalHeight||n,m=d.getBoundingClientRect(),h=m.top,g=m.left,v=m.width,b=m.height,y=Math.min(f,t)/v,w=Math.min(p,n)/b,E=Math.min(y,w),k="scale("+E+") translate3d("+((t-v)/2-g+O.margin+e.left)/E+"px, "+((n-b)/2-h+O.margin+e.top)/E+"px, 0)";L.zoomed.style.transform=k,L.zoomedHd&&(L.zoomedHd.style.transform=k)};return new r((function(e){if(t&&-1===_.indexOf(t))e(D);else{if(L.zoomed)e(D);else{if(t)L.original=t;else{if(!(_.length>0))return void e(D);var r=_;L.original=r[0]}if(L.original.dispatchEvent(d("medium-zoom:open",{detail:{zoom:D}})),P=window.pageYOffset||document.documentElement.scrollTop||document.body.scrollTop||0,C=!0,L.zoomed=c(L.original),document.body.appendChild(N),O.template){var o=i(O.template)?O.template:document.querySelector(O.template);L.template=document.createElement("div"),L.template.appendChild(o.content.cloneNode(!0)),document.body.appendChild(L.template)}if(document.body.appendChild(L.zoomed),window.requestAnimationFrame((function(){document.body.classList.add("medium-zoom--opened")})),L.original.classList.add("medium-zoom-image--hidden"),L.zoomed.classList.add("medium-zoom-image--opened"),L.zoomed.addEventListener("click",E),L.zoomed.addEventListener("transitionend",(function t(){C=!1,L.zoomed.removeEventListener("transitionend",t),L.original.dispatchEvent(d("medium-zoom:opened",{detail:{zoom:D}})),e(D)})),L.original.getAttribute("data-zoom-src")){L.zoomedHd=L.zoomed.cloneNode(),L.zoomedHd.removeAttribute("srcset"),L.zoomedHd.removeAttribute("sizes"),L.zoomedHd.src=L.zoomed.getAttribute("data-zoom-src"),L.zoomedHd.onerror=function(){clearInterval(a),console.warn("Unable to reach the zoom image target "+L.zoomedHd.src),L.zoomedHd=null,n()};var a=setInterval((function(){L.zoomedHd.complete&&(clearInterval(a),L.zoomedHd.classList.add("medium-zoom-image--opened"),L.zoomedHd.addEventListener("click",E),document.body.appendChild(L.zoomedHd),n())}),10)}else if(L.original.hasAttribute("srcset")){L.zoomedHd=L.zoomed.cloneNode(),L.zoomedHd.removeAttribute("sizes"),L.zoomedHd.removeAttribute("loading");var l=L.zoomedHd.addEventListener("load",(function(){L.zoomedHd.removeEventListener("load",l),L.zoomedHd.classList.add("medium-zoom-image--opened"),L.zoomedHd.addEventListener("click",E),document.body.appendChild(L.zoomedHd),n()}))}else n()}}}))},E=function(){return new r((function(e){if(!C&&L.original){C=!0,document.body.classList.remove("medium-zoom--opened"),L.zoomed.style.transform="",L.zoomedHd&&(L.zoomedHd.style.transform=""),L.template&&(L.template.style.transition="opacity 150ms",L.template.style.opacity=0),L.original.dispatchEvent(d("medium-zoom:close",{detail:{zoom:D}})),L.zoomed.addEventListener("transitionend",(function t(){L.original.classList.remove("medium-zoom-image--hidden"),document.body.removeChild(L.zoomed),L.zoomedHd&&document.body.removeChild(L.zoomedHd),document.body.removeChild(N),L.zoomed.classList.remove("medium-zoom-image--opened"),L.template&&document.body.removeChild(L.template),C=!1,L.zoomed.removeEventListener("transitionend",t),L.original.dispatchEvent(d("medium-zoom:closed",{detail:{zoom:D}})),L.original=null,L.zoomed=null,L.zoomedHd=null,L.template=null,e(D)}))}else e(D)}))},k=function(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},t=e.target;return L.original?E():w({target:t})},x=function(){return O},S=function(){return _},T=function(){return L.original},_=[],A=[],C=!1,P=0,O=n,L={original:null,zoomed:null,zoomedHd:null,template:null};"[object Object]"===Object.prototype.toString.call(t)?O=t:(t||"string"==typeof t)&&g(t),O=o({margin:0,background:"#fff",scrollOffset:40,container:null,template:null},O);var N=u(O.background);document.addEventListener("click",a),document.addEventListener("keyup",p),document.addEventListener("scroll",f),window.addEventListener("resize",E);var D={open:w,close:E,toggle:k,update:m,clone:h,attach:g,detach:v,on:b,off:y,getOptions:x,getImages:S,getZoomedImage:T};return D};var p=r.default.themeConfig;function m(e){var t,n;return document.querySelector('html[data-theme="dark"]')?(null==(t=e.background)?void 0:t.dark)||"rgb(50, 50, 50)":(null==(n=e.background)?void 0:n.light)||"rgb(255, 255, 255)"}const h=function(){if("undefined"==typeof window)return null;var e,t=p.zoom,n=t||{},r=n.selector,o=void 0===r?".markdown img":r,a=n.config,i=void 0===a?{}:a;return i.background=m(t),new MutationObserver((function(n){e&&e.update({background:m(t)})})).observe(document.querySelector("html"),{attributes:!0,attributeFilter:["data-theme"]}),setTimeout((function(){e=f(o,i)}),1e3),{onRouteUpdate:function(){setTimeout((function(){e=f(o,i)}),1e3)}}}()},53:(e,t,n)=>{"use strict";function r(e){var t,n,o="";if("string"==typeof e||"number"==typeof e)o+=e;else if("object"==typeof e)if(Array.isArray(e))for(t=0;to})},1513:(e,t,n)=>{"use strict";n.d(t,{zR:()=>b,TM:()=>x,yJ:()=>p,sC:()=>T,AO:()=>f});var r=n(9668);function o(e){return"/"===e.charAt(0)}function a(e,t){for(var n=t,r=n+1,o=e.length;r=0;f--){var p=i[f];"."===p?a(i,f):".."===p?(a(i,f),d++):d&&(a(i,f),d--)}if(!u)for(;d--;d)i.unshift("..");!u||""===i[0]||i[0]&&o(i[0])||i.unshift("");var m=i.join("/");return n&&"/"!==m.substr(-1)&&(m+="/"),m};var l=n(8997);function s(e){return"/"===e.charAt(0)?e:"/"+e}function u(e){return"/"===e.charAt(0)?e.substr(1):e}function c(e,t){return function(e,t){return 0===e.toLowerCase().indexOf(t.toLowerCase())&&-1!=="/?#".indexOf(e.charAt(t.length))}(e,t)?e.substr(t.length):e}function d(e){return"/"===e.charAt(e.length-1)?e.slice(0,-1):e}function f(e){var t=e.pathname,n=e.search,r=e.hash,o=t||"/";return n&&"?"!==n&&(o+="?"===n.charAt(0)?n:"?"+n),r&&"#"!==r&&(o+="#"===r.charAt(0)?r:"#"+r),o}function p(e,t,n,o){var a;"string"==typeof e?(a=function(e){var t=e||"/",n="",r="",o=t.indexOf("#");-1!==o&&(r=t.substr(o),t=t.substr(0,o));var a=t.indexOf("?");return-1!==a&&(n=t.substr(a),t=t.substr(0,a)),{pathname:t,search:"?"===n?"":n,hash:"#"===r?"":r}}(e)).state=t:(void 0===(a=(0,r.A)({},e)).pathname&&(a.pathname=""),a.search?"?"!==a.search.charAt(0)&&(a.search="?"+a.search):a.search="",a.hash?"#"!==a.hash.charAt(0)&&(a.hash="#"+a.hash):a.hash="",void 0!==t&&void 0===a.state&&(a.state=t));try{a.pathname=decodeURI(a.pathname)}catch(l){throw l instanceof URIError?new URIError('Pathname "'+a.pathname+'" could not be decoded. This is likely caused by an invalid percent-encoding.'):l}return n&&(a.key=n),o?a.pathname?"/"!==a.pathname.charAt(0)&&(a.pathname=i(a.pathname,o.pathname)):a.pathname=o.pathname:a.pathname||(a.pathname="/"),a}function m(){var e=null;var t=[];return{setPrompt:function(t){return e=t,function(){e===t&&(e=null)}},confirmTransitionTo:function(t,n,r,o){if(null!=e){var a="function"==typeof e?e(t,n):e;"string"==typeof a?"function"==typeof r?r(a,o):o(!0):o(!1!==a)}else o(!0)},appendListener:function(e){var n=!0;function r(){n&&e.apply(void 0,arguments)}return t.push(r),function(){n=!1,t=t.filter((function(e){return e!==r}))}},notifyListeners:function(){for(var e=arguments.length,n=new Array(e),r=0;rt?n.splice(t,n.length-t,r):n.push(r),d({action:"PUSH",location:r,index:t,entries:n})}}))},replace:function(e,t){var r=p(e,t,h(),w.location);c.confirmTransitionTo(r,"REPLACE",n,(function(e){e&&(w.entries[w.index]=r,d({action:"REPLACE",location:r}))}))},go:y,goBack:function(){y(-1)},goForward:function(){y(1)},canGo:function(e){var t=w.index+e;return t>=0&&t{"use strict";var r=n(3404),o={childContextTypes:!0,contextType:!0,contextTypes:!0,defaultProps:!0,displayName:!0,getDefaultProps:!0,getDerivedStateFromError:!0,getDerivedStateFromProps:!0,mixins:!0,propTypes:!0,type:!0},a={name:!0,length:!0,prototype:!0,caller:!0,callee:!0,arguments:!0,arity:!0},i={$$typeof:!0,compare:!0,defaultProps:!0,displayName:!0,propTypes:!0,type:!0},l={};function s(e){return r.isMemo(e)?i:l[e.$$typeof]||o}l[r.ForwardRef]={$$typeof:!0,render:!0,defaultProps:!0,displayName:!0,propTypes:!0},l[r.Memo]=i;var u=Object.defineProperty,c=Object.getOwnPropertyNames,d=Object.getOwnPropertySymbols,f=Object.getOwnPropertyDescriptor,p=Object.getPrototypeOf,m=Object.prototype;e.exports=function e(t,n,r){if("string"!=typeof n){if(m){var o=p(n);o&&o!==m&&e(t,o,r)}var i=c(n);d&&(i=i.concat(d(n)));for(var l=s(t),h=s(n),g=0;g{"use strict";var n="function"==typeof Symbol&&Symbol.for,r=n?Symbol.for("react.element"):60103,o=n?Symbol.for("react.portal"):60106,a=n?Symbol.for("react.fragment"):60107,i=n?Symbol.for("react.strict_mode"):60108,l=n?Symbol.for("react.profiler"):60114,s=n?Symbol.for("react.provider"):60109,u=n?Symbol.for("react.context"):60110,c=n?Symbol.for("react.async_mode"):60111,d=n?Symbol.for("react.concurrent_mode"):60111,f=n?Symbol.for("react.forward_ref"):60112,p=n?Symbol.for("react.suspense"):60113,m=n?Symbol.for("react.suspense_list"):60120,h=n?Symbol.for("react.memo"):60115,g=n?Symbol.for("react.lazy"):60116,v=n?Symbol.for("react.block"):60121,b=n?Symbol.for("react.fundamental"):60117,y=n?Symbol.for("react.responder"):60118,w=n?Symbol.for("react.scope"):60119;function E(e){if("object"==typeof e&&null!==e){var t=e.$$typeof;switch(t){case r:switch(e=e.type){case c:case d:case a:case l:case i:case p:return e;default:switch(e=e&&e.$$typeof){case u:case f:case g:case h:case s:return e;default:return t}}case o:return t}}}function k(e){return E(e)===d}t.AsyncMode=c,t.ConcurrentMode=d,t.ContextConsumer=u,t.ContextProvider=s,t.Element=r,t.ForwardRef=f,t.Fragment=a,t.Lazy=g,t.Memo=h,t.Portal=o,t.Profiler=l,t.StrictMode=i,t.Suspense=p,t.isAsyncMode=function(e){return k(e)||E(e)===c},t.isConcurrentMode=k,t.isContextConsumer=function(e){return E(e)===u},t.isContextProvider=function(e){return E(e)===s},t.isElement=function(e){return"object"==typeof e&&null!==e&&e.$$typeof===r},t.isForwardRef=function(e){return E(e)===f},t.isFragment=function(e){return E(e)===a},t.isLazy=function(e){return E(e)===g},t.isMemo=function(e){return E(e)===h},t.isPortal=function(e){return E(e)===o},t.isProfiler=function(e){return E(e)===l},t.isStrictMode=function(e){return E(e)===i},t.isSuspense=function(e){return E(e)===p},t.isValidElementType=function(e){return"string"==typeof e||"function"==typeof e||e===a||e===d||e===l||e===i||e===p||e===m||"object"==typeof e&&null!==e&&(e.$$typeof===g||e.$$typeof===h||e.$$typeof===s||e.$$typeof===u||e.$$typeof===f||e.$$typeof===b||e.$$typeof===y||e.$$typeof===w||e.$$typeof===v)},t.typeOf=E},3404:(e,t,n)=>{"use strict";e.exports=n(3072)},311:e=>{"use strict";e.exports=function(e,t,n,r,o,a,i,l){if(!e){var s;if(void 0===t)s=new Error("Minified exception occurred; use the non-minified dev environment for the full error message and additional helpful warnings.");else{var u=[n,r,o,a,i,l],c=0;(s=new Error(t.replace(/%s/g,(function(){return u[c++]})))).name="Invariant Violation"}throw s.framesToPop=1,s}}},8291:(e,t,n)=>{var r,o;!function(){var a,i,l,s,u,c,d,f,p,m,h,g,v,b,y,w,E,k,x,S,T,_,A,C,P,O,L=function(e){var t=new L.Builder;return t.pipeline.add(L.trimmer,L.stopWordFilter,L.stemmer),t.searchPipeline.add(L.stemmer),e.call(t,t),t.build()};L.version="2.3.9",L.utils={},L.utils.warn=(a=this,function(e){a.console&&console.warn&&console.warn(e)}),L.utils.asString=function(e){return null==e?"":e.toString()},L.utils.clone=function(e){if(null==e)return e;for(var t=Object.create(null),n=Object.keys(e),r=0;r0){var s=L.utils.clone(t)||{};s.position=[i,l],s.index=o.length,o.push(new L.Token(n.slice(i,a),s))}i=a+1}}return o},L.tokenizer.separator=/[\s\-]+/,L.Pipeline=function(){this._stack=[]},L.Pipeline.registeredFunctions=Object.create(null),L.Pipeline.registerFunction=function(e,t){t in this.registeredFunctions&&L.utils.warn("Overwriting existing registered function: "+t),e.label=t,L.Pipeline.registeredFunctions[e.label]=e},L.Pipeline.warnIfFunctionNotRegistered=function(e){e.label&&e.label in this.registeredFunctions||L.utils.warn("Function is not registered with pipeline. This may cause problems when serialising the index.\n",e)},L.Pipeline.load=function(e){var t=new L.Pipeline;return e.forEach((function(e){var n=L.Pipeline.registeredFunctions[e];if(!n)throw new Error("Cannot load unregistered function: "+e);t.add(n)})),t},L.Pipeline.prototype.add=function(){var e=Array.prototype.slice.call(arguments);e.forEach((function(e){L.Pipeline.warnIfFunctionNotRegistered(e),this._stack.push(e)}),this)},L.Pipeline.prototype.after=function(e,t){L.Pipeline.warnIfFunctionNotRegistered(t);var n=this._stack.indexOf(e);if(-1==n)throw new Error("Cannot find existingFn");n+=1,this._stack.splice(n,0,t)},L.Pipeline.prototype.before=function(e,t){L.Pipeline.warnIfFunctionNotRegistered(t);var n=this._stack.indexOf(e);if(-1==n)throw new Error("Cannot find existingFn");this._stack.splice(n,0,t)},L.Pipeline.prototype.remove=function(e){var t=this._stack.indexOf(e);-1!=t&&this._stack.splice(t,1)},L.Pipeline.prototype.run=function(e){for(var t=this._stack.length,n=0;n1&&(ae&&(n=o),a!=e);)r=n-t,o=t+Math.floor(r/2),a=this.elements[2*o];return a==e||a>e?2*o:al?u+=2:i==l&&(t+=n[s+1]*r[u+1],s+=2,u+=2);return t},L.Vector.prototype.similarity=function(e){return this.dot(e)/this.magnitude()||0},L.Vector.prototype.toArray=function(){for(var e=new Array(this.elements.length/2),t=1,n=0;t0){var a,i=o.str.charAt(0);i in o.node.edges?a=o.node.edges[i]:(a=new L.TokenSet,o.node.edges[i]=a),1==o.str.length&&(a.final=!0),r.push({node:a,editsRemaining:o.editsRemaining,str:o.str.slice(1)})}if(0!=o.editsRemaining){if("*"in o.node.edges)var l=o.node.edges["*"];else{l=new L.TokenSet;o.node.edges["*"]=l}if(0==o.str.length&&(l.final=!0),r.push({node:l,editsRemaining:o.editsRemaining-1,str:o.str}),o.str.length>1&&r.push({node:o.node,editsRemaining:o.editsRemaining-1,str:o.str.slice(1)}),1==o.str.length&&(o.node.final=!0),o.str.length>=1){if("*"in o.node.edges)var s=o.node.edges["*"];else{s=new L.TokenSet;o.node.edges["*"]=s}1==o.str.length&&(s.final=!0),r.push({node:s,editsRemaining:o.editsRemaining-1,str:o.str.slice(1)})}if(o.str.length>1){var u,c=o.str.charAt(0),d=o.str.charAt(1);d in o.node.edges?u=o.node.edges[d]:(u=new L.TokenSet,o.node.edges[d]=u),1==o.str.length&&(u.final=!0),r.push({node:u,editsRemaining:o.editsRemaining-1,str:c+o.str.slice(2)})}}}return n},L.TokenSet.fromString=function(e){for(var t=new L.TokenSet,n=t,r=0,o=e.length;r=e;t--){var n=this.uncheckedNodes[t],r=n.child.toString();r in this.minimizedNodes?n.parent.edges[n.char]=this.minimizedNodes[r]:(n.child._str=r,this.minimizedNodes[r]=n.child),this.uncheckedNodes.pop()}},L.Index=function(e){this.invertedIndex=e.invertedIndex,this.fieldVectors=e.fieldVectors,this.tokenSet=e.tokenSet,this.fields=e.fields,this.pipeline=e.pipeline},L.Index.prototype.search=function(e){return this.query((function(t){new L.QueryParser(e,t).parse()}))},L.Index.prototype.query=function(e){for(var t=new L.Query(this.fields),n=Object.create(null),r=Object.create(null),o=Object.create(null),a=Object.create(null),i=Object.create(null),l=0;l1?1:e},L.Builder.prototype.k1=function(e){this._k1=e},L.Builder.prototype.add=function(e,t){var n=e[this._ref],r=Object.keys(this._fields);this._documents[n]=t||{},this.documentCount+=1;for(var o=0;o=this.length)return L.QueryLexer.EOS;var e=this.str.charAt(this.pos);return this.pos+=1,e},L.QueryLexer.prototype.width=function(){return this.pos-this.start},L.QueryLexer.prototype.ignore=function(){this.start==this.pos&&(this.pos+=1),this.start=this.pos},L.QueryLexer.prototype.backup=function(){this.pos-=1},L.QueryLexer.prototype.acceptDigitRun=function(){var e,t;do{t=(e=this.next()).charCodeAt(0)}while(t>47&&t<58);e!=L.QueryLexer.EOS&&this.backup()},L.QueryLexer.prototype.more=function(){return this.pos1&&(e.backup(),e.emit(L.QueryLexer.TERM)),e.ignore(),e.more())return L.QueryLexer.lexText},L.QueryLexer.lexEditDistance=function(e){return e.ignore(),e.acceptDigitRun(),e.emit(L.QueryLexer.EDIT_DISTANCE),L.QueryLexer.lexText},L.QueryLexer.lexBoost=function(e){return e.ignore(),e.acceptDigitRun(),e.emit(L.QueryLexer.BOOST),L.QueryLexer.lexText},L.QueryLexer.lexEOS=function(e){e.width()>0&&e.emit(L.QueryLexer.TERM)},L.QueryLexer.termSeparator=L.tokenizer.separator,L.QueryLexer.lexText=function(e){for(;;){var t=e.next();if(t==L.QueryLexer.EOS)return L.QueryLexer.lexEOS;if(92!=t.charCodeAt(0)){if(":"==t)return L.QueryLexer.lexField;if("~"==t)return e.backup(),e.width()>0&&e.emit(L.QueryLexer.TERM),L.QueryLexer.lexEditDistance;if("^"==t)return e.backup(),e.width()>0&&e.emit(L.QueryLexer.TERM),L.QueryLexer.lexBoost;if("+"==t&&1===e.width())return e.emit(L.QueryLexer.PRESENCE),L.QueryLexer.lexText;if("-"==t&&1===e.width())return e.emit(L.QueryLexer.PRESENCE),L.QueryLexer.lexText;if(t.match(L.QueryLexer.termSeparator))return L.QueryLexer.lexTerm}else e.escapeCharacter()}},L.QueryParser=function(e,t){this.lexer=new L.QueryLexer(e),this.query=t,this.currentClause={},this.lexemeIdx=0},L.QueryParser.prototype.parse=function(){this.lexer.run(),this.lexemes=this.lexer.lexemes;for(var e=L.QueryParser.parseClause;e;)e=e(this);return this.query},L.QueryParser.prototype.peekLexeme=function(){return this.lexemes[this.lexemeIdx]},L.QueryParser.prototype.consumeLexeme=function(){var e=this.peekLexeme();return this.lexemeIdx+=1,e},L.QueryParser.prototype.nextClause=function(){var e=this.currentClause;this.query.clause(e),this.currentClause={}},L.QueryParser.parseClause=function(e){var t=e.peekLexeme();if(null!=t)switch(t.type){case L.QueryLexer.PRESENCE:return L.QueryParser.parsePresence;case L.QueryLexer.FIELD:return L.QueryParser.parseField;case L.QueryLexer.TERM:return L.QueryParser.parseTerm;default:var n="expected either a field or a term, found "+t.type;throw t.str.length>=1&&(n+=" with value '"+t.str+"'"),new L.QueryParseError(n,t.start,t.end)}},L.QueryParser.parsePresence=function(e){var t=e.consumeLexeme();if(null!=t){switch(t.str){case"-":e.currentClause.presence=L.Query.presence.PROHIBITED;break;case"+":e.currentClause.presence=L.Query.presence.REQUIRED;break;default:var n="unrecognised presence operator'"+t.str+"'";throw new L.QueryParseError(n,t.start,t.end)}var r=e.peekLexeme();if(null==r){n="expecting term or field, found nothing";throw new L.QueryParseError(n,t.start,t.end)}switch(r.type){case L.QueryLexer.FIELD:return L.QueryParser.parseField;case L.QueryLexer.TERM:return L.QueryParser.parseTerm;default:n="expecting term or field, found '"+r.type+"'";throw new L.QueryParseError(n,r.start,r.end)}}},L.QueryParser.parseField=function(e){var t=e.consumeLexeme();if(null!=t){if(-1==e.query.allFields.indexOf(t.str)){var n=e.query.allFields.map((function(e){return"'"+e+"'"})).join(", "),r="unrecognised field '"+t.str+"', possible fields: "+n;throw new L.QueryParseError(r,t.start,t.end)}e.currentClause.fields=[t.str];var o=e.peekLexeme();if(null==o){r="expecting term, found nothing";throw new L.QueryParseError(r,t.start,t.end)}switch(o.type){case L.QueryLexer.TERM:return L.QueryParser.parseTerm;default:r="expecting term, found '"+o.type+"'";throw new L.QueryParseError(r,o.start,o.end)}}},L.QueryParser.parseTerm=function(e){var t=e.consumeLexeme();if(null!=t){e.currentClause.term=t.str.toLowerCase(),-1!=t.str.indexOf("*")&&(e.currentClause.usePipeline=!1);var n=e.peekLexeme();if(null!=n)switch(n.type){case L.QueryLexer.TERM:return e.nextClause(),L.QueryParser.parseTerm;case L.QueryLexer.FIELD:return e.nextClause(),L.QueryParser.parseField;case L.QueryLexer.EDIT_DISTANCE:return L.QueryParser.parseEditDistance;case L.QueryLexer.BOOST:return L.QueryParser.parseBoost;case L.QueryLexer.PRESENCE:return e.nextClause(),L.QueryParser.parsePresence;default:var r="Unexpected lexeme type '"+n.type+"'";throw new L.QueryParseError(r,n.start,n.end)}else e.nextClause()}},L.QueryParser.parseEditDistance=function(e){var t=e.consumeLexeme();if(null!=t){var n=parseInt(t.str,10);if(isNaN(n)){var r="edit distance must be numeric";throw new L.QueryParseError(r,t.start,t.end)}e.currentClause.editDistance=n;var o=e.peekLexeme();if(null!=o)switch(o.type){case L.QueryLexer.TERM:return e.nextClause(),L.QueryParser.parseTerm;case L.QueryLexer.FIELD:return e.nextClause(),L.QueryParser.parseField;case L.QueryLexer.EDIT_DISTANCE:return L.QueryParser.parseEditDistance;case L.QueryLexer.BOOST:return L.QueryParser.parseBoost;case L.QueryLexer.PRESENCE:return e.nextClause(),L.QueryParser.parsePresence;default:r="Unexpected lexeme type '"+o.type+"'";throw new L.QueryParseError(r,o.start,o.end)}else e.nextClause()}},L.QueryParser.parseBoost=function(e){var t=e.consumeLexeme();if(null!=t){var n=parseInt(t.str,10);if(isNaN(n)){var r="boost must be numeric";throw new L.QueryParseError(r,t.start,t.end)}e.currentClause.boost=n;var o=e.peekLexeme();if(null!=o)switch(o.type){case L.QueryLexer.TERM:return e.nextClause(),L.QueryParser.parseTerm;case L.QueryLexer.FIELD:return e.nextClause(),L.QueryParser.parseField;case L.QueryLexer.EDIT_DISTANCE:return L.QueryParser.parseEditDistance;case L.QueryLexer.BOOST:return L.QueryParser.parseBoost;case L.QueryLexer.PRESENCE:return e.nextClause(),L.QueryParser.parsePresence;default:r="Unexpected lexeme type '"+o.type+"'";throw new L.QueryParseError(r,o.start,o.end)}else e.nextClause()}},void 0===(o="function"==typeof(r=function(){return L})?r.call(t,n,t,e):r)||(e.exports=o)}()},119:(e,t,n)=>{"use strict";n.r(t)},1916:(e,t,n)=>{"use strict";n.r(t)},5947:function(e,t,n){var r,o;void 0===(o="function"==typeof(r=function(){var e,t,n={version:"0.2.0"},r=n.settings={minimum:.08,easing:"ease",positionUsing:"",speed:200,trickle:!0,trickleRate:.02,trickleSpeed:800,showSpinner:!0,barSelector:'[role="bar"]',spinnerSelector:'[role="spinner"]',parent:"body",template:'
'};function o(e,t,n){return en?n:e}function a(e){return 100*(-1+e)}n.configure=function(e){var t,n;for(t in e)void 0!==(n=e[t])&&e.hasOwnProperty(t)&&(r[t]=n);return this},n.status=null,n.set=function(e){var t=n.isStarted();e=o(e,r.minimum,1),n.status=1===e?null:e;var s=n.render(!t),u=s.querySelector(r.barSelector),c=r.speed,d=r.easing;return s.offsetWidth,i((function(t){""===r.positionUsing&&(r.positionUsing=n.getPositioningCSS()),l(u,function(e,t,n){var o;return(o="translate3d"===r.positionUsing?{transform:"translate3d("+a(e)+"%,0,0)"}:"translate"===r.positionUsing?{transform:"translate("+a(e)+"%,0)"}:{"margin-left":a(e)+"%"}).transition="all "+t+"ms "+n,o}(e,c,d)),1===e?(l(s,{transition:"none",opacity:1}),s.offsetWidth,setTimeout((function(){l(s,{transition:"all "+c+"ms linear",opacity:0}),setTimeout((function(){n.remove(),t()}),c)}),c)):setTimeout(t,c)})),this},n.isStarted=function(){return"number"==typeof n.status},n.start=function(){n.status||n.set(0);var e=function(){setTimeout((function(){n.status&&(n.trickle(),e())}),r.trickleSpeed)};return r.trickle&&e(),this},n.done=function(e){return e||n.status?n.inc(.3+.5*Math.random()).set(1):this},n.inc=function(e){var t=n.status;return t?("number"!=typeof e&&(e=(1-t)*o(Math.random()*t,.1,.95)),t=o(t+e,0,.994),n.set(t)):n.start()},n.trickle=function(){return n.inc(Math.random()*r.trickleRate)},e=0,t=0,n.promise=function(r){return r&&"resolved"!==r.state()?(0===t&&n.start(),e++,t++,r.always((function(){0==--t?(e=0,n.done()):n.set((e-t)/e)})),this):this},n.render=function(e){if(n.isRendered())return document.getElementById("nprogress");u(document.documentElement,"nprogress-busy");var t=document.createElement("div");t.id="nprogress",t.innerHTML=r.template;var o,i=t.querySelector(r.barSelector),s=e?"-100":a(n.status||0),c=document.querySelector(r.parent);return l(i,{transition:"all 0 linear",transform:"translate3d("+s+"%,0,0)"}),r.showSpinner||(o=t.querySelector(r.spinnerSelector))&&f(o),c!=document.body&&u(c,"nprogress-custom-parent"),c.appendChild(t),t},n.remove=function(){c(document.documentElement,"nprogress-busy"),c(document.querySelector(r.parent),"nprogress-custom-parent");var e=document.getElementById("nprogress");e&&f(e)},n.isRendered=function(){return!!document.getElementById("nprogress")},n.getPositioningCSS=function(){var e=document.body.style,t="WebkitTransform"in e?"Webkit":"MozTransform"in e?"Moz":"msTransform"in e?"ms":"OTransform"in e?"O":"";return t+"Perspective"in e?"translate3d":t+"Transform"in e?"translate":"margin"};var i=function(){var e=[];function t(){var n=e.shift();n&&n(t)}return function(n){e.push(n),1==e.length&&t()}}(),l=function(){var e=["Webkit","O","Moz","ms"],t={};function n(n){return n=n.replace(/^-ms-/,"ms-").replace(/-([\da-z])/gi,(function(e,t){return t.toUpperCase()})),t[n]||(t[n]=function(t){var n=document.body.style;if(t in n)return t;for(var r,o=e.length,a=t.charAt(0).toUpperCase()+t.slice(1);o--;)if((r=e[o]+a)in n)return r;return t}(n))}function r(e,t,r){t=n(t),e.style[t]=r}return function(e,t){var n,o,a=arguments;if(2==a.length)for(n in t)void 0!==(o=t[n])&&t.hasOwnProperty(n)&&r(e,n,o);else r(e,a[1],a[2])}}();function s(e,t){return("string"==typeof e?e:d(e)).indexOf(" "+t+" ")>=0}function u(e,t){var n=d(e),r=n+t;s(n,t)||(e.className=r.substring(1))}function c(e,t){var n,r=d(e);s(e,t)&&(n=r.replace(" "+t+" "," "),e.className=n.substring(1,n.length-1))}function d(e){return(" "+(e.className||"")+" ").replace(/\s+/gi," ")}function f(e){e&&e.parentNode&&e.parentNode.removeChild(e)}return n})?r.call(t,n,t,e):r)||(e.exports=o)},5228:e=>{"use strict";var t=Object.getOwnPropertySymbols,n=Object.prototype.hasOwnProperty,r=Object.prototype.propertyIsEnumerable;function o(e){if(null==e)throw new TypeError("Object.assign cannot be called with null or undefined");return Object(e)}e.exports=function(){try{if(!Object.assign)return!1;var e=new String("abc");if(e[5]="de","5"===Object.getOwnPropertyNames(e)[0])return!1;for(var t={},n=0;n<10;n++)t["_"+String.fromCharCode(n)]=n;if("0123456789"!==Object.getOwnPropertyNames(t).map((function(e){return t[e]})).join(""))return!1;var r={};return"abcdefghijklmnopqrst".split("").forEach((function(e){r[e]=e})),"abcdefghijklmnopqrst"===Object.keys(Object.assign({},r)).join("")}catch(o){return!1}}()?Object.assign:function(e,a){for(var i,l,s=o(e),u=1;u{"use strict";n.d(t,{A:()=>a});var r=function(){var e=/(?:^|\s)lang(?:uage)?-([\w-]+)(?=\s|$)/i,t=0,n={},r={util:{encode:function e(t){return t instanceof o?new o(t.type,e(t.content),t.alias):Array.isArray(t)?t.map(e):t.replace(/&/g,"&").replace(/=d.reach);x+=k.value.length,k=k.next){var S=k.value;if(n.length>t.length)return;if(!(S instanceof o)){var T,_=1;if(b){if(!(T=a(E,x,t,v))||T.index>=t.length)break;var A=T.index,C=T.index+T[0].length,P=x;for(P+=k.value.length;A>=P;)k=k.next,P+=k.value.length;if(P-=k.value.length,x=P,k.value instanceof o)continue;for(var O=k;O!==n.tail&&(Pd.reach&&(d.reach=I);var R=k.prev;N&&(R=l(n,R,N),x+=N.length),s(n,R,_);var F=new o(f,g?r.tokenize(L,g):L,y,L);if(k=l(n,R,F),D&&l(n,k,D),_>1){var M={cause:f+","+m,reach:I};e(t,n,i,k.prev,x,M),d&&M.reach>d.reach&&(d.reach=M.reach)}}}}}}(e,c,t,c.head,0),function(e){var t=[],n=e.head.next;for(;n!==e.tail;)t.push(n.value),n=n.next;return t}(c)},hooks:{all:{},add:function(e,t){var n=r.hooks.all;n[e]=n[e]||[],n[e].push(t)},run:function(e,t){var n=r.hooks.all[e];if(n&&n.length)for(var o,a=0;o=n[a++];)o(t)}},Token:o};function o(e,t,n,r){this.type=e,this.content=t,this.alias=n,this.length=0|(r||"").length}function a(e,t,n,r){e.lastIndex=t;var o=e.exec(n);if(o&&r&&o[1]){var a=o[1].length;o.index+=a,o[0]=o[0].slice(a)}return o}function i(){var e={value:null,prev:null,next:null},t={value:null,prev:e,next:null};e.next=t,this.head=e,this.tail=t,this.length=0}function l(e,t,n){var r=t.next,o={value:n,prev:t,next:r};return t.next=o,r.prev=o,e.length++,o}function s(e,t,n){for(var r=t.next,o=0;o"+a.content+""},r}(),o=r;r.default=r,o.languages.markup={comment:{pattern://,greedy:!0},prolog:{pattern:/<\?[\s\S]+?\?>/,greedy:!0},doctype:{pattern:/"'[\]]|"[^"]*"|'[^']*')+(?:\[(?:[^<"'\]]|"[^"]*"|'[^']*'|<(?!!--)|)*\]\s*)?>/i,greedy:!0,inside:{"internal-subset":{pattern:/(^[^\[]*\[)[\s\S]+(?=\]>$)/,lookbehind:!0,greedy:!0,inside:null},string:{pattern:/"[^"]*"|'[^']*'/,greedy:!0},punctuation:/^$|[[\]]/,"doctype-tag":/^DOCTYPE/i,name:/[^\s<>'"]+/}},cdata:{pattern://i,greedy:!0},tag:{pattern:/<\/?(?!\d)[^\s>\/=$<%]+(?:\s(?:\s*[^\s>\/=]+(?:\s*=\s*(?:"[^"]*"|'[^']*'|[^\s'">=]+(?=[\s>]))|(?=[\s/>])))+)?\s*\/?>/,greedy:!0,inside:{tag:{pattern:/^<\/?[^\s>\/]+/,inside:{punctuation:/^<\/?/,namespace:/^[^\s>\/:]+:/}},"special-attr":[],"attr-value":{pattern:/=\s*(?:"[^"]*"|'[^']*'|[^\s'">=]+)/,inside:{punctuation:[{pattern:/^=/,alias:"attr-equals"},/"|'/]}},punctuation:/\/?>/,"attr-name":{pattern:/[^\s>\/]+/,inside:{namespace:/^[^\s>\/:]+:/}}}},entity:[{pattern:/&[\da-z]{1,8};/i,alias:"named-entity"},/&#x?[\da-f]{1,8};/i]},o.languages.markup.tag.inside["attr-value"].inside.entity=o.languages.markup.entity,o.languages.markup.doctype.inside["internal-subset"].inside=o.languages.markup,o.hooks.add("wrap",(function(e){"entity"===e.type&&(e.attributes.title=e.content.replace(/&/,"&"))})),Object.defineProperty(o.languages.markup.tag,"addInlined",{value:function(e,t){var n={};n["language-"+t]={pattern:/(^$)/i,lookbehind:!0,inside:o.languages[t]},n.cdata=/^$/i;var r={"included-cdata":{pattern://i,inside:n}};r["language-"+t]={pattern:/[\s\S]+/,inside:o.languages[t]};var a={};a[e]={pattern:RegExp(/(<__[^>]*>)(?:))*\]\]>|(?!)/.source.replace(/__/g,(function(){return e})),"i"),lookbehind:!0,greedy:!0,inside:r},o.languages.insertBefore("markup","cdata",a)}}),Object.defineProperty(o.languages.markup.tag,"addAttribute",{value:function(e,t){o.languages.markup.tag.inside["special-attr"].push({pattern:RegExp(/(^|["'\s])/.source+"(?:"+e+")"+/\s*=\s*(?:"[^"]*"|'[^']*'|[^\s'">=]+(?=[\s>]))/.source,"i"),lookbehind:!0,inside:{"attr-name":/^[^\s=]+/,"attr-value":{pattern:/=[\s\S]+/,inside:{value:{pattern:/(^=\s*(["']|(?!["'])))\S[\s\S]*(?=\2$)/,lookbehind:!0,alias:[t,"language-"+t],inside:o.languages[t]},punctuation:[{pattern:/^=/,alias:"attr-equals"},/"|'/]}}}})}}),o.languages.html=o.languages.markup,o.languages.mathml=o.languages.markup,o.languages.svg=o.languages.markup,o.languages.xml=o.languages.extend("markup",{}),o.languages.ssml=o.languages.xml,o.languages.atom=o.languages.xml,o.languages.rss=o.languages.xml,function(e){var t="\\b(?:BASH|BASHOPTS|BASH_ALIASES|BASH_ARGC|BASH_ARGV|BASH_CMDS|BASH_COMPLETION_COMPAT_DIR|BASH_LINENO|BASH_REMATCH|BASH_SOURCE|BASH_VERSINFO|BASH_VERSION|COLORTERM|COLUMNS|COMP_WORDBREAKS|DBUS_SESSION_BUS_ADDRESS|DEFAULTS_PATH|DESKTOP_SESSION|DIRSTACK|DISPLAY|EUID|GDMSESSION|GDM_LANG|GNOME_KEYRING_CONTROL|GNOME_KEYRING_PID|GPG_AGENT_INFO|GROUPS|HISTCONTROL|HISTFILE|HISTFILESIZE|HISTSIZE|HOME|HOSTNAME|HOSTTYPE|IFS|INSTANCE|JOB|LANG|LANGUAGE|LC_ADDRESS|LC_ALL|LC_IDENTIFICATION|LC_MEASUREMENT|LC_MONETARY|LC_NAME|LC_NUMERIC|LC_PAPER|LC_TELEPHONE|LC_TIME|LESSCLOSE|LESSOPEN|LINES|LOGNAME|LS_COLORS|MACHTYPE|MAILCHECK|MANDATORY_PATH|NO_AT_BRIDGE|OLDPWD|OPTERR|OPTIND|ORBIT_SOCKETDIR|OSTYPE|PAPERSIZE|PATH|PIPESTATUS|PPID|PS1|PS2|PS3|PS4|PWD|RANDOM|REPLY|SECONDS|SELINUX_INIT|SESSION|SESSIONTYPE|SESSION_MANAGER|SHELL|SHELLOPTS|SHLVL|SSH_AUTH_SOCK|TERM|UID|UPSTART_EVENTS|UPSTART_INSTANCE|UPSTART_JOB|UPSTART_SESSION|USER|WINDOWID|XAUTHORITY|XDG_CONFIG_DIRS|XDG_CURRENT_DESKTOP|XDG_DATA_DIRS|XDG_GREETER_DATA_DIR|XDG_MENU_PREFIX|XDG_RUNTIME_DIR|XDG_SEAT|XDG_SEAT_PATH|XDG_SESSION_DESKTOP|XDG_SESSION_ID|XDG_SESSION_PATH|XDG_SESSION_TYPE|XDG_VTNR|XMODIFIERS)\\b",n={pattern:/(^(["']?)\w+\2)[ \t]+\S.*/,lookbehind:!0,alias:"punctuation",inside:null},r={bash:n,environment:{pattern:RegExp("\\$"+t),alias:"constant"},variable:[{pattern:/\$?\(\([\s\S]+?\)\)/,greedy:!0,inside:{variable:[{pattern:/(^\$\(\([\s\S]+)\)\)/,lookbehind:!0},/^\$\(\(/],number:/\b0x[\dA-Fa-f]+\b|(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:[Ee]-?\d+)?/,operator:/--|\+\+|\*\*=?|<<=?|>>=?|&&|\|\||[=!+\-*/%<>^&|]=?|[?~:]/,punctuation:/\(\(?|\)\)?|,|;/}},{pattern:/\$\((?:\([^)]+\)|[^()])+\)|`[^`]+`/,greedy:!0,inside:{variable:/^\$\(|^`|\)$|`$/}},{pattern:/\$\{[^}]+\}/,greedy:!0,inside:{operator:/:[-=?+]?|[!\/]|##?|%%?|\^\^?|,,?/,punctuation:/[\[\]]/,environment:{pattern:RegExp("(\\{)"+t),lookbehind:!0,alias:"constant"}}},/\$(?:\w+|[#?*!@$])/],entity:/\\(?:[abceEfnrtv\\"]|O?[0-7]{1,3}|U[0-9a-fA-F]{8}|u[0-9a-fA-F]{4}|x[0-9a-fA-F]{1,2})/};e.languages.bash={shebang:{pattern:/^#!\s*\/.*/,alias:"important"},comment:{pattern:/(^|[^"{\\$])#.*/,lookbehind:!0},"function-name":[{pattern:/(\bfunction\s+)[\w-]+(?=(?:\s*\(?:\s*\))?\s*\{)/,lookbehind:!0,alias:"function"},{pattern:/\b[\w-]+(?=\s*\(\s*\)\s*\{)/,alias:"function"}],"for-or-select":{pattern:/(\b(?:for|select)\s+)\w+(?=\s+in\s)/,alias:"variable",lookbehind:!0},"assign-left":{pattern:/(^|[\s;|&]|[<>]\()\w+(?=\+?=)/,inside:{environment:{pattern:RegExp("(^|[\\s;|&]|[<>]\\()"+t),lookbehind:!0,alias:"constant"}},alias:"variable",lookbehind:!0},string:[{pattern:/((?:^|[^<])<<-?\s*)(\w+)\s[\s\S]*?(?:\r?\n|\r)\2/,lookbehind:!0,greedy:!0,inside:r},{pattern:/((?:^|[^<])<<-?\s*)(["'])(\w+)\2\s[\s\S]*?(?:\r?\n|\r)\3/,lookbehind:!0,greedy:!0,inside:{bash:n}},{pattern:/(^|[^\\](?:\\\\)*)"(?:\\[\s\S]|\$\([^)]+\)|\$(?!\()|`[^`]+`|[^"\\`$])*"/,lookbehind:!0,greedy:!0,inside:r},{pattern:/(^|[^$\\])'[^']*'/,lookbehind:!0,greedy:!0},{pattern:/\$'(?:[^'\\]|\\[\s\S])*'/,greedy:!0,inside:{entity:r.entity}}],environment:{pattern:RegExp("\\$?"+t),alias:"constant"},variable:r.variable,function:{pattern:/(^|[\s;|&]|[<>]\()(?:add|apropos|apt|apt-cache|apt-get|aptitude|aspell|automysqlbackup|awk|basename|bash|bc|bconsole|bg|bzip2|cal|cat|cfdisk|chgrp|chkconfig|chmod|chown|chroot|cksum|clear|cmp|column|comm|composer|cp|cron|crontab|csplit|curl|cut|date|dc|dd|ddrescue|debootstrap|df|diff|diff3|dig|dir|dircolors|dirname|dirs|dmesg|docker|docker-compose|du|egrep|eject|env|ethtool|expand|expect|expr|fdformat|fdisk|fg|fgrep|file|find|fmt|fold|format|free|fsck|ftp|fuser|gawk|git|gparted|grep|groupadd|groupdel|groupmod|groups|grub-mkconfig|gzip|halt|head|hg|history|host|hostname|htop|iconv|id|ifconfig|ifdown|ifup|import|install|ip|jobs|join|kill|killall|less|link|ln|locate|logname|logrotate|look|lpc|lpr|lprint|lprintd|lprintq|lprm|ls|lsof|lynx|make|man|mc|mdadm|mkconfig|mkdir|mke2fs|mkfifo|mkfs|mkisofs|mknod|mkswap|mmv|more|most|mount|mtools|mtr|mutt|mv|nano|nc|netstat|nice|nl|node|nohup|notify-send|npm|nslookup|op|open|parted|passwd|paste|pathchk|ping|pkill|pnpm|podman|podman-compose|popd|pr|printcap|printenv|ps|pushd|pv|quota|quotacheck|quotactl|ram|rar|rcp|reboot|remsync|rename|renice|rev|rm|rmdir|rpm|rsync|scp|screen|sdiff|sed|sendmail|seq|service|sftp|sh|shellcheck|shuf|shutdown|sleep|slocate|sort|split|ssh|stat|strace|su|sudo|sum|suspend|swapon|sync|tac|tail|tar|tee|time|timeout|top|touch|tr|traceroute|tsort|tty|umount|uname|unexpand|uniq|units|unrar|unshar|unzip|update-grub|uptime|useradd|userdel|usermod|users|uudecode|uuencode|v|vcpkg|vdir|vi|vim|virsh|vmstat|wait|watch|wc|wget|whereis|which|who|whoami|write|xargs|xdg-open|yarn|yes|zenity|zip|zsh|zypper)(?=$|[)\s;|&])/,lookbehind:!0},keyword:{pattern:/(^|[\s;|&]|[<>]\()(?:case|do|done|elif|else|esac|fi|for|function|if|in|select|then|until|while)(?=$|[)\s;|&])/,lookbehind:!0},builtin:{pattern:/(^|[\s;|&]|[<>]\()(?:\.|:|alias|bind|break|builtin|caller|cd|command|continue|declare|echo|enable|eval|exec|exit|export|getopts|hash|help|let|local|logout|mapfile|printf|pwd|read|readarray|readonly|return|set|shift|shopt|source|test|times|trap|type|typeset|ulimit|umask|unalias|unset)(?=$|[)\s;|&])/,lookbehind:!0,alias:"class-name"},boolean:{pattern:/(^|[\s;|&]|[<>]\()(?:false|true)(?=$|[)\s;|&])/,lookbehind:!0},"file-descriptor":{pattern:/\B&\d\b/,alias:"important"},operator:{pattern:/\d?<>|>\||\+=|=[=~]?|!=?|<<[<-]?|[&\d]?>>|\d[<>]&?|[<>][&=]?|&[>&]?|\|[&|]?/,inside:{"file-descriptor":{pattern:/^\d/,alias:"important"}}},punctuation:/\$?\(\(?|\)\)?|\.\.|[{}[\];\\]/,number:{pattern:/(^|\s)(?:[1-9]\d*|0)(?:[.,]\d+)?\b/,lookbehind:!0}},n.inside=e.languages.bash;for(var o=["comment","function-name","for-or-select","assign-left","string","environment","function","keyword","builtin","boolean","file-descriptor","operator","punctuation","number"],a=r.variable[1].inside,i=0;i]=?|[!=]=?=?|--?|\+\+?|&&?|\|\|?|[?*/~^%]/,punctuation:/[{}[\];(),.:]/},o.languages.c=o.languages.extend("clike",{comment:{pattern:/\/\/(?:[^\r\n\\]|\\(?:\r\n?|\n|(?![\r\n])))*|\/\*[\s\S]*?(?:\*\/|$)/,greedy:!0},string:{pattern:/"(?:\\(?:\r\n|[\s\S])|[^"\\\r\n])*"/,greedy:!0},"class-name":{pattern:/(\b(?:enum|struct)\s+(?:__attribute__\s*\(\([\s\S]*?\)\)\s*)?)\w+|\b[a-z]\w*_t\b/,lookbehind:!0},keyword:/\b(?:_Alignas|_Alignof|_Atomic|_Bool|_Complex|_Generic|_Imaginary|_Noreturn|_Static_assert|_Thread_local|__attribute__|asm|auto|break|case|char|const|continue|default|do|double|else|enum|extern|float|for|goto|if|inline|int|long|register|return|short|signed|sizeof|static|struct|switch|typedef|typeof|union|unsigned|void|volatile|while)\b/,function:/\b[a-z_]\w*(?=\s*\()/i,number:/(?:\b0x(?:[\da-f]+(?:\.[\da-f]*)?|\.[\da-f]+)(?:p[+-]?\d+)?|(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:e[+-]?\d+)?)[ful]{0,4}/i,operator:/>>=?|<<=?|->|([-+&|:])\1|[?:~]|[-+*/%&|^!=<>]=?/}),o.languages.insertBefore("c","string",{char:{pattern:/'(?:\\(?:\r\n|[\s\S])|[^'\\\r\n]){0,32}'/,greedy:!0}}),o.languages.insertBefore("c","string",{macro:{pattern:/(^[\t ]*)#\s*[a-z](?:[^\r\n\\/]|\/(?!\*)|\/\*(?:[^*]|\*(?!\/))*\*\/|\\(?:\r\n|[\s\S]))*/im,lookbehind:!0,greedy:!0,alias:"property",inside:{string:[{pattern:/^(#\s*include\s*)<[^>]+>/,lookbehind:!0},o.languages.c.string],char:o.languages.c.char,comment:o.languages.c.comment,"macro-name":[{pattern:/(^#\s*define\s+)\w+\b(?!\()/i,lookbehind:!0},{pattern:/(^#\s*define\s+)\w+\b(?=\()/i,lookbehind:!0,alias:"function"}],directive:{pattern:/^(#\s*)[a-z]+/,lookbehind:!0,alias:"keyword"},"directive-hash":/^#/,punctuation:/##|\\(?=[\r\n])/,expression:{pattern:/\S[\s\S]*/,inside:o.languages.c}}}}),o.languages.insertBefore("c","function",{constant:/\b(?:EOF|NULL|SEEK_CUR|SEEK_END|SEEK_SET|__DATE__|__FILE__|__LINE__|__TIMESTAMP__|__TIME__|__func__|stderr|stdin|stdout)\b/}),delete o.languages.c.boolean,function(e){var t=/\b(?:alignas|alignof|asm|auto|bool|break|case|catch|char|char16_t|char32_t|char8_t|class|co_await|co_return|co_yield|compl|concept|const|const_cast|consteval|constexpr|constinit|continue|decltype|default|delete|do|double|dynamic_cast|else|enum|explicit|export|extern|final|float|for|friend|goto|if|import|inline|int|int16_t|int32_t|int64_t|int8_t|long|module|mutable|namespace|new|noexcept|nullptr|operator|override|private|protected|public|register|reinterpret_cast|requires|return|short|signed|sizeof|static|static_assert|static_cast|struct|switch|template|this|thread_local|throw|try|typedef|typeid|typename|uint16_t|uint32_t|uint64_t|uint8_t|union|unsigned|using|virtual|void|volatile|wchar_t|while)\b/,n=/\b(?!)\w+(?:\s*\.\s*\w+)*\b/.source.replace(//g,(function(){return t.source}));e.languages.cpp=e.languages.extend("c",{"class-name":[{pattern:RegExp(/(\b(?:class|concept|enum|struct|typename)\s+)(?!)\w+/.source.replace(//g,(function(){return t.source}))),lookbehind:!0},/\b[A-Z]\w*(?=\s*::\s*\w+\s*\()/,/\b[A-Z_]\w*(?=\s*::\s*~\w+\s*\()/i,/\b\w+(?=\s*<(?:[^<>]|<(?:[^<>]|<[^<>]*>)*>)*>\s*::\s*\w+\s*\()/],keyword:t,number:{pattern:/(?:\b0b[01']+|\b0x(?:[\da-f']+(?:\.[\da-f']*)?|\.[\da-f']+)(?:p[+-]?[\d']+)?|(?:\b[\d']+(?:\.[\d']*)?|\B\.[\d']+)(?:e[+-]?[\d']+)?)[ful]{0,4}/i,greedy:!0},operator:/>>=?|<<=?|->|--|\+\+|&&|\|\||[?:~]|<=>|[-+*/%&|^!=<>]=?|\b(?:and|and_eq|bitand|bitor|not|not_eq|or|or_eq|xor|xor_eq)\b/,boolean:/\b(?:false|true)\b/}),e.languages.insertBefore("cpp","string",{module:{pattern:RegExp(/(\b(?:import|module)\s+)/.source+"(?:"+/"(?:\\(?:\r\n|[\s\S])|[^"\\\r\n])*"|<[^<>\r\n]*>/.source+"|"+/(?:\s*:\s*)?|:\s*/.source.replace(//g,(function(){return n}))+")"),lookbehind:!0,greedy:!0,inside:{string:/^[<"][\s\S]+/,operator:/:/,punctuation:/\./}},"raw-string":{pattern:/R"([^()\\ ]{0,16})\([\s\S]*?\)\1"/,alias:"string",greedy:!0}}),e.languages.insertBefore("cpp","keyword",{"generic-function":{pattern:/\b(?!operator\b)[a-z_]\w*\s*<(?:[^<>]|<[^<>]*>)*>(?=\s*\()/i,inside:{function:/^\w+/,generic:{pattern:/<[\s\S]+/,alias:"class-name",inside:e.languages.cpp}}}}),e.languages.insertBefore("cpp","operator",{"double-colon":{pattern:/::/,alias:"punctuation"}}),e.languages.insertBefore("cpp","class-name",{"base-clause":{pattern:/(\b(?:class|struct)\s+\w+\s*:\s*)[^;{}"'\s]+(?:\s+[^;{}"'\s]+)*(?=\s*[;{])/,lookbehind:!0,greedy:!0,inside:e.languages.extend("cpp",{})}}),e.languages.insertBefore("inside","double-colon",{"class-name":/\b[a-z_]\w*\b(?!\s*::)/i},e.languages.cpp["base-clause"])}(o),function(e){var t=/(?:"(?:\\(?:\r\n|[\s\S])|[^"\\\r\n])*"|'(?:\\(?:\r\n|[\s\S])|[^'\\\r\n])*')/;e.languages.css={comment:/\/\*[\s\S]*?\*\//,atrule:{pattern:/@[\w-](?:[^;{\s]|\s+(?![\s{]))*(?:;|(?=\s*\{))/,inside:{rule:/^@[\w-]+/,"selector-function-argument":{pattern:/(\bselector\s*\(\s*(?![\s)]))(?:[^()\s]|\s+(?![\s)])|\((?:[^()]|\([^()]*\))*\))+(?=\s*\))/,lookbehind:!0,alias:"selector"},keyword:{pattern:/(^|[^\w-])(?:and|not|only|or)(?![\w-])/,lookbehind:!0}}},url:{pattern:RegExp("\\burl\\((?:"+t.source+"|"+/(?:[^\\\r\n()"']|\\[\s\S])*/.source+")\\)","i"),greedy:!0,inside:{function:/^url/i,punctuation:/^\(|\)$/,string:{pattern:RegExp("^"+t.source+"$"),alias:"url"}}},selector:{pattern:RegExp("(^|[{}\\s])[^{}\\s](?:[^{};\"'\\s]|\\s+(?![\\s{])|"+t.source+")*(?=\\s*\\{)"),lookbehind:!0},string:{pattern:t,greedy:!0},property:{pattern:/(^|[^-\w\xA0-\uFFFF])(?!\s)[-_a-z\xA0-\uFFFF](?:(?!\s)[-\w\xA0-\uFFFF])*(?=\s*:)/i,lookbehind:!0},important:/!important\b/i,function:{pattern:/(^|[^-a-z0-9])[-a-z0-9]+(?=\()/i,lookbehind:!0},punctuation:/[(){};:,]/},e.languages.css.atrule.inside.rest=e.languages.css;var n=e.languages.markup;n&&(n.tag.addInlined("style","css"),n.tag.addAttribute("style","css"))}(o),function(e){var t,n=/("|')(?:\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1/;e.languages.css.selector={pattern:e.languages.css.selector.pattern,lookbehind:!0,inside:t={"pseudo-element":/:(?:after|before|first-letter|first-line|selection)|::[-\w]+/,"pseudo-class":/:[-\w]+/,class:/\.[-\w]+/,id:/#[-\w]+/,attribute:{pattern:RegExp("\\[(?:[^[\\]\"']|"+n.source+")*\\]"),greedy:!0,inside:{punctuation:/^\[|\]$/,"case-sensitivity":{pattern:/(\s)[si]$/i,lookbehind:!0,alias:"keyword"},namespace:{pattern:/^(\s*)(?:(?!\s)[-*\w\xA0-\uFFFF])*\|(?!=)/,lookbehind:!0,inside:{punctuation:/\|$/}},"attr-name":{pattern:/^(\s*)(?:(?!\s)[-\w\xA0-\uFFFF])+/,lookbehind:!0},"attr-value":[n,{pattern:/(=\s*)(?:(?!\s)[-\w\xA0-\uFFFF])+(?=\s*$)/,lookbehind:!0}],operator:/[|~*^$]?=/}},"n-th":[{pattern:/(\(\s*)[+-]?\d*[\dn](?:\s*[+-]\s*\d+)?(?=\s*\))/,lookbehind:!0,inside:{number:/[\dn]+/,operator:/[+-]/}},{pattern:/(\(\s*)(?:even|odd)(?=\s*\))/i,lookbehind:!0}],combinator:/>|\+|~|\|\|/,punctuation:/[(),]/}},e.languages.css.atrule.inside["selector-function-argument"].inside=t,e.languages.insertBefore("css","property",{variable:{pattern:/(^|[^-\w\xA0-\uFFFF])--(?!\s)[-_a-z\xA0-\uFFFF](?:(?!\s)[-\w\xA0-\uFFFF])*/i,lookbehind:!0}});var r={pattern:/(\b\d+)(?:%|[a-z]+(?![\w-]))/,lookbehind:!0},o={pattern:/(^|[^\w.-])-?(?:\d+(?:\.\d+)?|\.\d+)/,lookbehind:!0};e.languages.insertBefore("css","function",{operator:{pattern:/(\s)[+\-*\/](?=\s)/,lookbehind:!0},hexcode:{pattern:/\B#[\da-f]{3,8}\b/i,alias:"color"},color:[{pattern:/(^|[^\w-])(?:AliceBlue|AntiqueWhite|Aqua|Aquamarine|Azure|Beige|Bisque|Black|BlanchedAlmond|Blue|BlueViolet|Brown|BurlyWood|CadetBlue|Chartreuse|Chocolate|Coral|CornflowerBlue|Cornsilk|Crimson|Cyan|DarkBlue|DarkCyan|DarkGoldenRod|DarkGr[ae]y|DarkGreen|DarkKhaki|DarkMagenta|DarkOliveGreen|DarkOrange|DarkOrchid|DarkRed|DarkSalmon|DarkSeaGreen|DarkSlateBlue|DarkSlateGr[ae]y|DarkTurquoise|DarkViolet|DeepPink|DeepSkyBlue|DimGr[ae]y|DodgerBlue|FireBrick|FloralWhite|ForestGreen|Fuchsia|Gainsboro|GhostWhite|Gold|GoldenRod|Gr[ae]y|Green|GreenYellow|HoneyDew|HotPink|IndianRed|Indigo|Ivory|Khaki|Lavender|LavenderBlush|LawnGreen|LemonChiffon|LightBlue|LightCoral|LightCyan|LightGoldenRodYellow|LightGr[ae]y|LightGreen|LightPink|LightSalmon|LightSeaGreen|LightSkyBlue|LightSlateGr[ae]y|LightSteelBlue|LightYellow|Lime|LimeGreen|Linen|Magenta|Maroon|MediumAquaMarine|MediumBlue|MediumOrchid|MediumPurple|MediumSeaGreen|MediumSlateBlue|MediumSpringGreen|MediumTurquoise|MediumVioletRed|MidnightBlue|MintCream|MistyRose|Moccasin|NavajoWhite|Navy|OldLace|Olive|OliveDrab|Orange|OrangeRed|Orchid|PaleGoldenRod|PaleGreen|PaleTurquoise|PaleVioletRed|PapayaWhip|PeachPuff|Peru|Pink|Plum|PowderBlue|Purple|Red|RosyBrown|RoyalBlue|SaddleBrown|Salmon|SandyBrown|SeaGreen|SeaShell|Sienna|Silver|SkyBlue|SlateBlue|SlateGr[ae]y|Snow|SpringGreen|SteelBlue|Tan|Teal|Thistle|Tomato|Transparent|Turquoise|Violet|Wheat|White|WhiteSmoke|Yellow|YellowGreen)(?![\w-])/i,lookbehind:!0},{pattern:/\b(?:hsl|rgb)\(\s*\d{1,3}\s*,\s*\d{1,3}%?\s*,\s*\d{1,3}%?\s*\)\B|\b(?:hsl|rgb)a\(\s*\d{1,3}\s*,\s*\d{1,3}%?\s*,\s*\d{1,3}%?\s*,\s*(?:0|0?\.\d+|1)\s*\)\B/i,inside:{unit:r,number:o,function:/[\w-]+(?=\()/,punctuation:/[(),]/}}],entity:/\\[\da-f]{1,8}/i,unit:r,number:o})}(o),o.languages.javascript=o.languages.extend("clike",{"class-name":[o.languages.clike["class-name"],{pattern:/(^|[^$\w\xA0-\uFFFF])(?!\s)[_$A-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\.(?:constructor|prototype))/,lookbehind:!0}],keyword:[{pattern:/((?:^|\})\s*)catch\b/,lookbehind:!0},{pattern:/(^|[^.]|\.\.\.\s*)\b(?:as|assert(?=\s*\{)|async(?=\s*(?:function\b|\(|[$\w\xA0-\uFFFF]|$))|await|break|case|class|const|continue|debugger|default|delete|do|else|enum|export|extends|finally(?=\s*(?:\{|$))|for|from(?=\s*(?:['"]|$))|function|(?:get|set)(?=\s*(?:[#\[$\w\xA0-\uFFFF]|$))|if|implements|import|in|instanceof|interface|let|new|null|of|package|private|protected|public|return|static|super|switch|this|throw|try|typeof|undefined|var|void|while|with|yield)\b/,lookbehind:!0}],function:/#?(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*(?:\.\s*(?:apply|bind|call)\s*)?\()/,number:{pattern:RegExp(/(^|[^\w$])/.source+"(?:"+/NaN|Infinity/.source+"|"+/0[bB][01]+(?:_[01]+)*n?/.source+"|"+/0[oO][0-7]+(?:_[0-7]+)*n?/.source+"|"+/0[xX][\dA-Fa-f]+(?:_[\dA-Fa-f]+)*n?/.source+"|"+/\d+(?:_\d+)*n/.source+"|"+/(?:\d+(?:_\d+)*(?:\.(?:\d+(?:_\d+)*)?)?|\.\d+(?:_\d+)*)(?:[Ee][+-]?\d+(?:_\d+)*)?/.source+")"+/(?![\w$])/.source),lookbehind:!0},operator:/--|\+\+|\*\*=?|=>|&&=?|\|\|=?|[!=]==|<<=?|>>>?=?|[-+*/%&|^!=<>]=?|\.{3}|\?\?=?|\?\.?|[~:]/}),o.languages.javascript["class-name"][0].pattern=/(\b(?:class|extends|implements|instanceof|interface|new)\s+)[\w.\\]+/,o.languages.insertBefore("javascript","keyword",{regex:{pattern:/((?:^|[^$\w\xA0-\uFFFF."'\])\s]|\b(?:return|yield))\s*)\/(?:\[(?:[^\]\\\r\n]|\\.)*\]|\\.|[^/\\\[\r\n])+\/[dgimyus]{0,7}(?=(?:\s|\/\*(?:[^*]|\*(?!\/))*\*\/)*(?:$|[\r\n,.;:})\]]|\/\/))/,lookbehind:!0,greedy:!0,inside:{"regex-source":{pattern:/^(\/)[\s\S]+(?=\/[a-z]*$)/,lookbehind:!0,alias:"language-regex",inside:o.languages.regex},"regex-delimiter":/^\/|\/$/,"regex-flags":/^[a-z]+$/}},"function-variable":{pattern:/#?(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*[=:]\s*(?:async\s*)?(?:\bfunction\b|(?:\((?:[^()]|\([^()]*\))*\)|(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*)\s*=>))/,alias:"function"},parameter:[{pattern:/(function(?:\s+(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*)?\s*\(\s*)(?!\s)(?:[^()\s]|\s+(?![\s)])|\([^()]*\))+(?=\s*\))/,lookbehind:!0,inside:o.languages.javascript},{pattern:/(^|[^$\w\xA0-\uFFFF])(?!\s)[_$a-z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*=>)/i,lookbehind:!0,inside:o.languages.javascript},{pattern:/(\(\s*)(?!\s)(?:[^()\s]|\s+(?![\s)])|\([^()]*\))+(?=\s*\)\s*=>)/,lookbehind:!0,inside:o.languages.javascript},{pattern:/((?:\b|\s|^)(?!(?:as|async|await|break|case|catch|class|const|continue|debugger|default|delete|do|else|enum|export|extends|finally|for|from|function|get|if|implements|import|in|instanceof|interface|let|new|null|of|package|private|protected|public|return|set|static|super|switch|this|throw|try|typeof|undefined|var|void|while|with|yield)(?![$\w\xA0-\uFFFF]))(?:(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*\s*)\(\s*|\]\s*\(\s*)(?!\s)(?:[^()\s]|\s+(?![\s)])|\([^()]*\))+(?=\s*\)\s*\{)/,lookbehind:!0,inside:o.languages.javascript}],constant:/\b[A-Z](?:[A-Z_]|\dx?)*\b/}),o.languages.insertBefore("javascript","string",{hashbang:{pattern:/^#!.*/,greedy:!0,alias:"comment"},"template-string":{pattern:/`(?:\\[\s\S]|\$\{(?:[^{}]|\{(?:[^{}]|\{[^}]*\})*\})+\}|(?!\$\{)[^\\`])*`/,greedy:!0,inside:{"template-punctuation":{pattern:/^`|`$/,alias:"string"},interpolation:{pattern:/((?:^|[^\\])(?:\\{2})*)\$\{(?:[^{}]|\{(?:[^{}]|\{[^}]*\})*\})+\}/,lookbehind:!0,inside:{"interpolation-punctuation":{pattern:/^\$\{|\}$/,alias:"punctuation"},rest:o.languages.javascript}},string:/[\s\S]+/}},"string-property":{pattern:/((?:^|[,{])[ \t]*)(["'])(?:\\(?:\r\n|[\s\S])|(?!\2)[^\\\r\n])*\2(?=\s*:)/m,lookbehind:!0,greedy:!0,alias:"property"}}),o.languages.insertBefore("javascript","operator",{"literal-property":{pattern:/((?:^|[,{])[ \t]*)(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*:)/m,lookbehind:!0,alias:"property"}}),o.languages.markup&&(o.languages.markup.tag.addInlined("script","javascript"),o.languages.markup.tag.addAttribute(/on(?:abort|blur|change|click|composition(?:end|start|update)|dblclick|error|focus(?:in|out)?|key(?:down|up)|load|mouse(?:down|enter|leave|move|out|over|up)|reset|resize|scroll|select|slotchange|submit|unload|wheel)/.source,"javascript")),o.languages.js=o.languages.javascript,function(e){var t=/#(?!\{).+/,n={pattern:/#\{[^}]+\}/,alias:"variable"};e.languages.coffeescript=e.languages.extend("javascript",{comment:t,string:[{pattern:/'(?:\\[\s\S]|[^\\'])*'/,greedy:!0},{pattern:/"(?:\\[\s\S]|[^\\"])*"/,greedy:!0,inside:{interpolation:n}}],keyword:/\b(?:and|break|by|catch|class|continue|debugger|delete|do|each|else|extend|extends|false|finally|for|if|in|instanceof|is|isnt|let|loop|namespace|new|no|not|null|of|off|on|or|own|return|super|switch|then|this|throw|true|try|typeof|undefined|unless|until|when|while|window|with|yes|yield)\b/,"class-member":{pattern:/@(?!\d)\w+/,alias:"variable"}}),e.languages.insertBefore("coffeescript","comment",{"multiline-comment":{pattern:/###[\s\S]+?###/,alias:"comment"},"block-regex":{pattern:/\/{3}[\s\S]*?\/{3}/,alias:"regex",inside:{comment:t,interpolation:n}}}),e.languages.insertBefore("coffeescript","string",{"inline-javascript":{pattern:/`(?:\\[\s\S]|[^\\`])*`/,inside:{delimiter:{pattern:/^`|`$/,alias:"punctuation"},script:{pattern:/[\s\S]+/,alias:"language-javascript",inside:e.languages.javascript}}},"multiline-string":[{pattern:/'''[\s\S]*?'''/,greedy:!0,alias:"string"},{pattern:/"""[\s\S]*?"""/,greedy:!0,alias:"string",inside:{interpolation:n}}]}),e.languages.insertBefore("coffeescript","keyword",{property:/(?!\d)\w+(?=\s*:(?!:))/}),delete e.languages.coffeescript["template-string"],e.languages.coffee=e.languages.coffeescript}(o),function(e){var t=/[*&][^\s[\]{},]+/,n=/!(?:<[\w\-%#;/?:@&=+$,.!~*'()[\]]+>|(?:[a-zA-Z\d-]*!)?[\w\-%#;/?:@&=+$.~*'()]+)?/,r="(?:"+n.source+"(?:[ \t]+"+t.source+")?|"+t.source+"(?:[ \t]+"+n.source+")?)",o=/(?:[^\s\x00-\x08\x0e-\x1f!"#%&'*,\-:>?@[\]`{|}\x7f-\x84\x86-\x9f\ud800-\udfff\ufffe\uffff]|[?:-])(?:[ \t]*(?:(?![#:])|:))*/.source.replace(//g,(function(){return/[^\s\x00-\x08\x0e-\x1f,[\]{}\x7f-\x84\x86-\x9f\ud800-\udfff\ufffe\uffff]/.source})),a=/"(?:[^"\\\r\n]|\\.)*"|'(?:[^'\\\r\n]|\\.)*'/.source;function i(e,t){t=(t||"").replace(/m/g,"")+"m";var n=/([:\-,[{]\s*(?:\s<>[ \t]+)?)(?:<>)(?=[ \t]*(?:$|,|\]|\}|(?:[\r\n]\s*)?#))/.source.replace(/<>/g,(function(){return r})).replace(/<>/g,(function(){return e}));return RegExp(n,t)}e.languages.yaml={scalar:{pattern:RegExp(/([\-:]\s*(?:\s<>[ \t]+)?[|>])[ \t]*(?:((?:\r?\n|\r)[ \t]+)\S[^\r\n]*(?:\2[^\r\n]+)*)/.source.replace(/<>/g,(function(){return r}))),lookbehind:!0,alias:"string"},comment:/#.*/,key:{pattern:RegExp(/((?:^|[:\-,[{\r\n?])[ \t]*(?:<>[ \t]+)?)<>(?=\s*:\s)/.source.replace(/<>/g,(function(){return r})).replace(/<>/g,(function(){return"(?:"+o+"|"+a+")"}))),lookbehind:!0,greedy:!0,alias:"atrule"},directive:{pattern:/(^[ \t]*)%.+/m,lookbehind:!0,alias:"important"},datetime:{pattern:i(/\d{4}-\d\d?-\d\d?(?:[tT]|[ \t]+)\d\d?:\d{2}:\d{2}(?:\.\d*)?(?:[ \t]*(?:Z|[-+]\d\d?(?::\d{2})?))?|\d{4}-\d{2}-\d{2}|\d\d?:\d{2}(?::\d{2}(?:\.\d*)?)?/.source),lookbehind:!0,alias:"number"},boolean:{pattern:i(/false|true/.source,"i"),lookbehind:!0,alias:"important"},null:{pattern:i(/null|~/.source,"i"),lookbehind:!0,alias:"important"},string:{pattern:i(a),lookbehind:!0,greedy:!0},number:{pattern:i(/[+-]?(?:0x[\da-f]+|0o[0-7]+|(?:\d+(?:\.\d*)?|\.\d+)(?:e[+-]?\d+)?|\.inf|\.nan)/.source,"i"),lookbehind:!0},tag:n,important:t,punctuation:/---|[:[\]{}\-,|>?]|\.\.\./},e.languages.yml=e.languages.yaml}(o),function(e){var t=/(?:\\.|[^\\\n\r]|(?:\n|\r\n?)(?![\r\n]))/.source;function n(e){return e=e.replace(//g,(function(){return t})),RegExp(/((?:^|[^\\])(?:\\{2})*)/.source+"(?:"+e+")")}var r=/(?:\\.|``(?:[^`\r\n]|`(?!`))+``|`[^`\r\n]+`|[^\\|\r\n`])+/.source,o=/\|?__(?:\|__)+\|?(?:(?:\n|\r\n?)|(?![\s\S]))/.source.replace(/__/g,(function(){return r})),a=/\|?[ \t]*:?-{3,}:?[ \t]*(?:\|[ \t]*:?-{3,}:?[ \t]*)+\|?(?:\n|\r\n?)/.source;e.languages.markdown=e.languages.extend("markup",{}),e.languages.insertBefore("markdown","prolog",{"front-matter-block":{pattern:/(^(?:\s*[\r\n])?)---(?!.)[\s\S]*?[\r\n]---(?!.)/,lookbehind:!0,greedy:!0,inside:{punctuation:/^---|---$/,"front-matter":{pattern:/\S+(?:\s+\S+)*/,alias:["yaml","language-yaml"],inside:e.languages.yaml}}},blockquote:{pattern:/^>(?:[\t ]*>)*/m,alias:"punctuation"},table:{pattern:RegExp("^"+o+a+"(?:"+o+")*","m"),inside:{"table-data-rows":{pattern:RegExp("^("+o+a+")(?:"+o+")*$"),lookbehind:!0,inside:{"table-data":{pattern:RegExp(r),inside:e.languages.markdown},punctuation:/\|/}},"table-line":{pattern:RegExp("^("+o+")"+a+"$"),lookbehind:!0,inside:{punctuation:/\||:?-{3,}:?/}},"table-header-row":{pattern:RegExp("^"+o+"$"),inside:{"table-header":{pattern:RegExp(r),alias:"important",inside:e.languages.markdown},punctuation:/\|/}}}},code:[{pattern:/((?:^|\n)[ \t]*\n|(?:^|\r\n?)[ \t]*\r\n?)(?: {4}|\t).+(?:(?:\n|\r\n?)(?: {4}|\t).+)*/,lookbehind:!0,alias:"keyword"},{pattern:/^```[\s\S]*?^```$/m,greedy:!0,inside:{"code-block":{pattern:/^(```.*(?:\n|\r\n?))[\s\S]+?(?=(?:\n|\r\n?)^```$)/m,lookbehind:!0},"code-language":{pattern:/^(```).+/,lookbehind:!0},punctuation:/```/}}],title:[{pattern:/\S.*(?:\n|\r\n?)(?:==+|--+)(?=[ \t]*$)/m,alias:"important",inside:{punctuation:/==+$|--+$/}},{pattern:/(^\s*)#.+/m,lookbehind:!0,alias:"important",inside:{punctuation:/^#+|#+$/}}],hr:{pattern:/(^\s*)([*-])(?:[\t ]*\2){2,}(?=\s*$)/m,lookbehind:!0,alias:"punctuation"},list:{pattern:/(^\s*)(?:[*+-]|\d+\.)(?=[\t ].)/m,lookbehind:!0,alias:"punctuation"},"url-reference":{pattern:/!?\[[^\]]+\]:[\t ]+(?:\S+|<(?:\\.|[^>\\])+>)(?:[\t ]+(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\((?:\\.|[^)\\])*\)))?/,inside:{variable:{pattern:/^(!?\[)[^\]]+/,lookbehind:!0},string:/(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\((?:\\.|[^)\\])*\))$/,punctuation:/^[\[\]!:]|[<>]/},alias:"url"},bold:{pattern:n(/\b__(?:(?!_)|_(?:(?!_))+_)+__\b|\*\*(?:(?!\*)|\*(?:(?!\*))+\*)+\*\*/.source),lookbehind:!0,greedy:!0,inside:{content:{pattern:/(^..)[\s\S]+(?=..$)/,lookbehind:!0,inside:{}},punctuation:/\*\*|__/}},italic:{pattern:n(/\b_(?:(?!_)|__(?:(?!_))+__)+_\b|\*(?:(?!\*)|\*\*(?:(?!\*))+\*\*)+\*/.source),lookbehind:!0,greedy:!0,inside:{content:{pattern:/(^.)[\s\S]+(?=.$)/,lookbehind:!0,inside:{}},punctuation:/[*_]/}},strike:{pattern:n(/(~~?)(?:(?!~))+\2/.source),lookbehind:!0,greedy:!0,inside:{content:{pattern:/(^~~?)[\s\S]+(?=\1$)/,lookbehind:!0,inside:{}},punctuation:/~~?/}},"code-snippet":{pattern:/(^|[^\\`])(?:``[^`\r\n]+(?:`[^`\r\n]+)*``(?!`)|`[^`\r\n]+`(?!`))/,lookbehind:!0,greedy:!0,alias:["code","keyword"]},url:{pattern:n(/!?\[(?:(?!\]))+\](?:\([^\s)]+(?:[\t ]+"(?:\\.|[^"\\])*")?\)|[ \t]?\[(?:(?!\]))+\])/.source),lookbehind:!0,greedy:!0,inside:{operator:/^!/,content:{pattern:/(^\[)[^\]]+(?=\])/,lookbehind:!0,inside:{}},variable:{pattern:/(^\][ \t]?\[)[^\]]+(?=\]$)/,lookbehind:!0},url:{pattern:/(^\]\()[^\s)]+/,lookbehind:!0},string:{pattern:/(^[ \t]+)"(?:\\.|[^"\\])*"(?=\)$)/,lookbehind:!0}}}}),["url","bold","italic","strike"].forEach((function(t){["url","bold","italic","strike","code-snippet"].forEach((function(n){t!==n&&(e.languages.markdown[t].inside.content.inside[n]=e.languages.markdown[n])}))})),e.hooks.add("after-tokenize",(function(e){"markdown"!==e.language&&"md"!==e.language||function e(t){if(t&&"string"!=typeof t)for(var n=0,r=t.length;n",quot:'"'},s=String.fromCodePoint||String.fromCharCode;e.languages.md=e.languages.markdown}(o),o.languages.graphql={comment:/#.*/,description:{pattern:/(?:"""(?:[^"]|(?!""")")*"""|"(?:\\.|[^\\"\r\n])*")(?=\s*[a-z_])/i,greedy:!0,alias:"string",inside:{"language-markdown":{pattern:/(^"(?:"")?)(?!\1)[\s\S]+(?=\1$)/,lookbehind:!0,inside:o.languages.markdown}}},string:{pattern:/"""(?:[^"]|(?!""")")*"""|"(?:\\.|[^\\"\r\n])*"/,greedy:!0},number:/(?:\B-|\b)\d+(?:\.\d+)?(?:e[+-]?\d+)?\b/i,boolean:/\b(?:false|true)\b/,variable:/\$[a-z_]\w*/i,directive:{pattern:/@[a-z_]\w*/i,alias:"function"},"attr-name":{pattern:/\b[a-z_]\w*(?=\s*(?:\((?:[^()"]|"(?:\\.|[^\\"\r\n])*")*\))?:)/i,greedy:!0},"atom-input":{pattern:/\b[A-Z]\w*Input\b/,alias:"class-name"},scalar:/\b(?:Boolean|Float|ID|Int|String)\b/,constant:/\b[A-Z][A-Z_\d]*\b/,"class-name":{pattern:/(\b(?:enum|implements|interface|on|scalar|type|union)\s+|&\s*|:\s*|\[)[A-Z_]\w*/,lookbehind:!0},fragment:{pattern:/(\bfragment\s+|\.{3}\s*(?!on\b))[a-zA-Z_]\w*/,lookbehind:!0,alias:"function"},"definition-mutation":{pattern:/(\bmutation\s+)[a-zA-Z_]\w*/,lookbehind:!0,alias:"function"},"definition-query":{pattern:/(\bquery\s+)[a-zA-Z_]\w*/,lookbehind:!0,alias:"function"},keyword:/\b(?:directive|enum|extend|fragment|implements|input|interface|mutation|on|query|repeatable|scalar|schema|subscription|type|union)\b/,operator:/[!=|&]|\.{3}/,"property-query":/\w+(?=\s*\()/,object:/\w+(?=\s*\{)/,punctuation:/[!(){}\[\]:=,]/,property:/\w+/},o.hooks.add("after-tokenize",(function(e){if("graphql"===e.language)for(var t=e.tokens.filter((function(e){return"string"!=typeof e&&"comment"!==e.type&&"scalar"!==e.type})),n=0;n0)){var l=f(/^\{$/,/^\}$/);if(-1===l)continue;for(var s=n;s=0&&p(u,"variable-input")}}}}function c(e){return t[n+e]}function d(e,t){t=t||0;for(var n=0;n?|<|>)?|>[>=]?|\b(?:AND|BETWEEN|DIV|ILIKE|IN|IS|LIKE|NOT|OR|REGEXP|RLIKE|SOUNDS LIKE|XOR)\b/i,punctuation:/[;[\]()`,.]/},function(e){var t=e.languages.javascript["template-string"],n=t.pattern.source,r=t.inside.interpolation,o=r.inside["interpolation-punctuation"],a=r.pattern.source;function i(t,r){if(e.languages[t])return{pattern:RegExp("((?:"+r+")\\s*)"+n),lookbehind:!0,greedy:!0,inside:{"template-punctuation":{pattern:/^`|`$/,alias:"string"},"embedded-code":{pattern:/[\s\S]+/,alias:t}}}}function l(e,t){return"___"+t.toUpperCase()+"_"+e+"___"}function s(t,n,r){var o={code:t,grammar:n,language:r};return e.hooks.run("before-tokenize",o),o.tokens=e.tokenize(o.code,o.grammar),e.hooks.run("after-tokenize",o),o.tokens}function u(t){var n={};n["interpolation-punctuation"]=o;var a=e.tokenize(t,n);if(3===a.length){var i=[1,1];i.push.apply(i,s(a[1],e.languages.javascript,"javascript")),a.splice.apply(a,i)}return new e.Token("interpolation",a,r.alias,t)}function c(t,n,r){var o=e.tokenize(t,{interpolation:{pattern:RegExp(a),lookbehind:!0}}),i=0,c={},d=s(o.map((function(e){if("string"==typeof e)return e;for(var n,o=e.content;-1!==t.indexOf(n=l(i++,r)););return c[n]=o,n})).join(""),n,r),f=Object.keys(c);return i=0,function e(t){for(var n=0;n=f.length)return;var r=t[n];if("string"==typeof r||"string"==typeof r.content){var o=f[i],a="string"==typeof r?r:r.content,l=a.indexOf(o);if(-1!==l){++i;var s=a.substring(0,l),d=u(c[o]),p=a.substring(l+o.length),m=[];if(s&&m.push(s),m.push(d),p){var h=[p];e(h),m.push.apply(m,h)}"string"==typeof r?(t.splice.apply(t,[n,1].concat(m)),n+=m.length-1):r.content=m}}else{var g=r.content;Array.isArray(g)?e(g):e([g])}}}(d),new e.Token(r,d,"language-"+r,t)}e.languages.javascript["template-string"]=[i("css",/\b(?:styled(?:\([^)]*\))?(?:\s*\.\s*\w+(?:\([^)]*\))*)*|css(?:\s*\.\s*(?:global|resolve))?|createGlobalStyle|keyframes)/.source),i("html",/\bhtml|\.\s*(?:inner|outer)HTML\s*\+?=/.source),i("svg",/\bsvg/.source),i("markdown",/\b(?:markdown|md)/.source),i("graphql",/\b(?:gql|graphql(?:\s*\.\s*experimental)?)/.source),i("sql",/\bsql/.source),t].filter(Boolean);var d={javascript:!0,js:!0,typescript:!0,ts:!0,jsx:!0,tsx:!0};function f(e){return"string"==typeof e?e:Array.isArray(e)?e.map(f).join(""):f(e.content)}e.hooks.add("after-tokenize",(function(t){t.language in d&&function t(n){for(var r=0,o=n.length;r]|<(?:[^<>]|<[^<>]*>)*>)*>)?/,lookbehind:!0,greedy:!0,inside:null},builtin:/\b(?:Array|Function|Promise|any|boolean|console|never|number|string|symbol|unknown)\b/}),e.languages.typescript.keyword.push(/\b(?:abstract|declare|is|keyof|readonly|require)\b/,/\b(?:asserts|infer|interface|module|namespace|type)\b(?=\s*(?:[{_$a-zA-Z\xA0-\uFFFF]|$))/,/\btype\b(?=\s*(?:[\{*]|$))/),delete e.languages.typescript.parameter,delete e.languages.typescript["literal-property"];var t=e.languages.extend("typescript",{});delete t["class-name"],e.languages.typescript["class-name"].inside=t,e.languages.insertBefore("typescript","function",{decorator:{pattern:/@[$\w\xA0-\uFFFF]+/,inside:{at:{pattern:/^@/,alias:"operator"},function:/^[\s\S]+/}},"generic-function":{pattern:/#?(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*\s*<(?:[^<>]|<(?:[^<>]|<[^<>]*>)*>)*>(?=\s*\()/,greedy:!0,inside:{function:/^#?(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*/,generic:{pattern:/<[\s\S]+/,alias:"class-name",inside:t}}}}),e.languages.ts=e.languages.typescript}(o),function(e){function t(e,t){return RegExp(e.replace(//g,(function(){return/(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*/.source})),t)}e.languages.insertBefore("javascript","function-variable",{"method-variable":{pattern:RegExp("(\\.\\s*)"+e.languages.javascript["function-variable"].pattern.source),lookbehind:!0,alias:["function-variable","method","function","property-access"]}}),e.languages.insertBefore("javascript","function",{method:{pattern:RegExp("(\\.\\s*)"+e.languages.javascript.function.source),lookbehind:!0,alias:["function","property-access"]}}),e.languages.insertBefore("javascript","constant",{"known-class-name":[{pattern:/\b(?:(?:Float(?:32|64)|(?:Int|Uint)(?:8|16|32)|Uint8Clamped)?Array|ArrayBuffer|BigInt|Boolean|DataView|Date|Error|Function|Intl|JSON|(?:Weak)?(?:Map|Set)|Math|Number|Object|Promise|Proxy|Reflect|RegExp|String|Symbol|WebAssembly)\b/,alias:"class-name"},{pattern:/\b(?:[A-Z]\w*)Error\b/,alias:"class-name"}]}),e.languages.insertBefore("javascript","keyword",{imports:{pattern:t(/(\bimport\b\s*)(?:(?:\s*,\s*(?:\*\s*as\s+|\{[^{}]*\}))?|\*\s*as\s+|\{[^{}]*\})(?=\s*\bfrom\b)/.source),lookbehind:!0,inside:e.languages.javascript},exports:{pattern:t(/(\bexport\b\s*)(?:\*(?:\s*as\s+)?(?=\s*\bfrom\b)|\{[^{}]*\})/.source),lookbehind:!0,inside:e.languages.javascript}}),e.languages.javascript.keyword.unshift({pattern:/\b(?:as|default|export|from|import)\b/,alias:"module"},{pattern:/\b(?:await|break|catch|continue|do|else|finally|for|if|return|switch|throw|try|while|yield)\b/,alias:"control-flow"},{pattern:/\bnull\b/,alias:["null","nil"]},{pattern:/\bundefined\b/,alias:"nil"}),e.languages.insertBefore("javascript","operator",{spread:{pattern:/\.{3}/,alias:"operator"},arrow:{pattern:/=>/,alias:"operator"}}),e.languages.insertBefore("javascript","punctuation",{"property-access":{pattern:t(/(\.\s*)#?/.source),lookbehind:!0},"maybe-class-name":{pattern:/(^|[^$\w\xA0-\uFFFF])[A-Z][$\w\xA0-\uFFFF]+/,lookbehind:!0},dom:{pattern:/\b(?:document|(?:local|session)Storage|location|navigator|performance|window)\b/,alias:"variable"},console:{pattern:/\bconsole(?=\s*\.)/,alias:"class-name"}});for(var n=["function","function-variable","method","method-variable","property-access"],r=0;r*\.{3}(?:[^{}]|)*\})/.source;function a(e,t){return e=e.replace(//g,(function(){return n})).replace(//g,(function(){return r})).replace(//g,(function(){return o})),RegExp(e,t)}o=a(o).source,e.languages.jsx=e.languages.extend("markup",t),e.languages.jsx.tag.pattern=a(/<\/?(?:[\w.:-]+(?:+(?:[\w.:$-]+(?:=(?:"(?:\\[\s\S]|[^\\"])*"|'(?:\\[\s\S]|[^\\'])*'|[^\s{'"/>=]+|))?|))**\/?)?>/.source),e.languages.jsx.tag.inside.tag.pattern=/^<\/?[^\s>\/]*/,e.languages.jsx.tag.inside["attr-value"].pattern=/=(?!\{)(?:"(?:\\[\s\S]|[^\\"])*"|'(?:\\[\s\S]|[^\\'])*'|[^\s'">]+)/,e.languages.jsx.tag.inside.tag.inside["class-name"]=/^[A-Z]\w*(?:\.[A-Z]\w*)*$/,e.languages.jsx.tag.inside.comment=t.comment,e.languages.insertBefore("inside","attr-name",{spread:{pattern:a(//.source),inside:e.languages.jsx}},e.languages.jsx.tag),e.languages.insertBefore("inside","special-attr",{script:{pattern:a(/=/.source),alias:"language-javascript",inside:{"script-punctuation":{pattern:/^=(?=\{)/,alias:"punctuation"},rest:e.languages.jsx}}},e.languages.jsx.tag);var i=function(e){return e?"string"==typeof e?e:"string"==typeof e.content?e.content:e.content.map(i).join(""):""},l=function(t){for(var n=[],r=0;r0&&n[n.length-1].tagName===i(o.content[0].content[1])&&n.pop():"/>"===o.content[o.content.length-1].content||n.push({tagName:i(o.content[0].content[1]),openedBraces:0}):n.length>0&&"punctuation"===o.type&&"{"===o.content?n[n.length-1].openedBraces++:n.length>0&&n[n.length-1].openedBraces>0&&"punctuation"===o.type&&"}"===o.content?n[n.length-1].openedBraces--:a=!0),(a||"string"==typeof o)&&n.length>0&&0===n[n.length-1].openedBraces){var s=i(o);r0&&("string"==typeof t[r-1]||"plain-text"===t[r-1].type)&&(s=i(t[r-1])+s,t.splice(r-1,1),r--),t[r]=new e.Token("plain-text",s,null,s)}o.content&&"string"!=typeof o.content&&l(o.content)}};e.hooks.add("after-tokenize",(function(e){"jsx"!==e.language&&"tsx"!==e.language||l(e.tokens)}))}(o),function(e){e.languages.diff={coord:[/^(?:\*{3}|-{3}|\+{3}).*$/m,/^@@.*@@$/m,/^\d.*$/m]};var t={"deleted-sign":"-","deleted-arrow":"<","inserted-sign":"+","inserted-arrow":">",unchanged:" ",diff:"!"};Object.keys(t).forEach((function(n){var r=t[n],o=[];/^\w+$/.test(n)||o.push(/\w+/.exec(n)[0]),"diff"===n&&o.push("bold"),e.languages.diff[n]={pattern:RegExp("^(?:["+r+"].*(?:\r\n?|\n|(?![\\s\\S])))+","m"),alias:o,inside:{line:{pattern:/(.)(?=[\s\S]).*(?:\r\n?|\n)?/,lookbehind:!0},prefix:{pattern:/[\s\S]/,alias:/\w+/.exec(n)[0]}}}})),Object.defineProperty(e.languages.diff,"PREFIXES",{value:t})}(o),o.languages.git={comment:/^#.*/m,deleted:/^[-\u2013].*/m,inserted:/^\+.*/m,string:/("|')(?:\\.|(?!\1)[^\\\r\n])*\1/,command:{pattern:/^.*\$ git .*$/m,inside:{parameter:/\s--?\w+/}},coord:/^@@.*@@$/m,"commit-sha1":/^commit \w{40}$/m},o.languages.go=o.languages.extend("clike",{string:{pattern:/(^|[^\\])"(?:\\.|[^"\\\r\n])*"|`[^`]*`/,lookbehind:!0,greedy:!0},keyword:/\b(?:break|case|chan|const|continue|default|defer|else|fallthrough|for|func|go(?:to)?|if|import|interface|map|package|range|return|select|struct|switch|type|var)\b/,boolean:/\b(?:_|false|iota|nil|true)\b/,number:[/\b0(?:b[01_]+|o[0-7_]+)i?\b/i,/\b0x(?:[a-f\d_]+(?:\.[a-f\d_]*)?|\.[a-f\d_]+)(?:p[+-]?\d+(?:_\d+)*)?i?(?!\w)/i,/(?:\b\d[\d_]*(?:\.[\d_]*)?|\B\.\d[\d_]*)(?:e[+-]?[\d_]+)?i?(?!\w)/i],operator:/[*\/%^!=]=?|\+[=+]?|-[=-]?|\|[=|]?|&(?:=|&|\^=?)?|>(?:>=?|=)?|<(?:<=?|=|-)?|:=|\.\.\./,builtin:/\b(?:append|bool|byte|cap|close|complex|complex(?:64|128)|copy|delete|error|float(?:32|64)|u?int(?:8|16|32|64)?|imag|len|make|new|panic|print(?:ln)?|real|recover|rune|string|uintptr)\b/}),o.languages.insertBefore("go","string",{char:{pattern:/'(?:\\.|[^'\\\r\n]){0,10}'/,greedy:!0}}),delete o.languages.go["class-name"],function(e){function t(e,t){return"___"+e.toUpperCase()+t+"___"}Object.defineProperties(e.languages["markup-templating"]={},{buildPlaceholders:{value:function(n,r,o,a){if(n.language===r){var i=n.tokenStack=[];n.code=n.code.replace(o,(function(e){if("function"==typeof a&&!a(e))return e;for(var o,l=i.length;-1!==n.code.indexOf(o=t(r,l));)++l;return i[l]=e,o})),n.grammar=e.languages.markup}}},tokenizePlaceholders:{value:function(n,r){if(n.language===r&&n.tokenStack){n.grammar=e.languages[r];var o=0,a=Object.keys(n.tokenStack);!function i(l){for(var s=0;s=a.length);s++){var u=l[s];if("string"==typeof u||u.content&&"string"==typeof u.content){var c=a[o],d=n.tokenStack[c],f="string"==typeof u?u:u.content,p=t(r,c),m=f.indexOf(p);if(m>-1){++o;var h=f.substring(0,m),g=new e.Token(r,e.tokenize(d,n.grammar),"language-"+r,d),v=f.substring(m+p.length),b=[];h&&b.push.apply(b,i([h])),b.push(g),v&&b.push.apply(b,i([v])),"string"==typeof u?l.splice.apply(l,[s,1].concat(b)):u.content=b}}else u.content&&i(u.content)}return l}(n.tokens)}}}})}(o),function(e){e.languages.handlebars={comment:/\{\{![\s\S]*?\}\}/,delimiter:{pattern:/^\{\{\{?|\}\}\}?$/,alias:"punctuation"},string:/(["'])(?:\\.|(?!\1)[^\\\r\n])*\1/,number:/\b0x[\dA-Fa-f]+\b|(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:[Ee][+-]?\d+)?/,boolean:/\b(?:false|true)\b/,block:{pattern:/^(\s*(?:~\s*)?)[#\/]\S+?(?=\s*(?:~\s*)?$|\s)/,lookbehind:!0,alias:"keyword"},brackets:{pattern:/\[[^\]]+\]/,inside:{punctuation:/\[|\]/,variable:/[\s\S]+/}},punctuation:/[!"#%&':()*+,.\/;<=>@\[\\\]^`{|}~]/,variable:/[^!"#%&'()*+,\/;<=>@\[\\\]^`{|}~\s]+/},e.hooks.add("before-tokenize",(function(t){e.languages["markup-templating"].buildPlaceholders(t,"handlebars",/\{\{\{[\s\S]+?\}\}\}|\{\{[\s\S]+?\}\}/g)})),e.hooks.add("after-tokenize",(function(t){e.languages["markup-templating"].tokenizePlaceholders(t,"handlebars")})),e.languages.hbs=e.languages.handlebars}(o),o.languages.json={property:{pattern:/(^|[^\\])"(?:\\.|[^\\"\r\n])*"(?=\s*:)/,lookbehind:!0,greedy:!0},string:{pattern:/(^|[^\\])"(?:\\.|[^\\"\r\n])*"(?!\s*:)/,lookbehind:!0,greedy:!0},comment:{pattern:/\/\/.*|\/\*[\s\S]*?(?:\*\/|$)/,greedy:!0},number:/-?\b\d+(?:\.\d+)?(?:e[+-]?\d+)?\b/i,punctuation:/[{}[\],]/,operator:/:/,boolean:/\b(?:false|true)\b/,null:{pattern:/\bnull\b/,alias:"keyword"}},o.languages.webmanifest=o.languages.json,o.languages.less=o.languages.extend("css",{comment:[/\/\*[\s\S]*?\*\//,{pattern:/(^|[^\\])\/\/.*/,lookbehind:!0}],atrule:{pattern:/@[\w-](?:\((?:[^(){}]|\([^(){}]*\))*\)|[^(){};\s]|\s+(?!\s))*?(?=\s*\{)/,inside:{punctuation:/[:()]/}},selector:{pattern:/(?:@\{[\w-]+\}|[^{};\s@])(?:@\{[\w-]+\}|\((?:[^(){}]|\([^(){}]*\))*\)|[^(){};@\s]|\s+(?!\s))*?(?=\s*\{)/,inside:{variable:/@+[\w-]+/}},property:/(?:@\{[\w-]+\}|[\w-])+(?:\+_?)?(?=\s*:)/,operator:/[+\-*\/]/}),o.languages.insertBefore("less","property",{variable:[{pattern:/@[\w-]+\s*:/,inside:{punctuation:/:/}},/@@?[\w-]+/],"mixin-usage":{pattern:/([{;]\s*)[.#](?!\d)[\w-].*?(?=[(;])/,lookbehind:!0,alias:"function"}}),o.languages.makefile={comment:{pattern:/(^|[^\\])#(?:\\(?:\r\n|[\s\S])|[^\\\r\n])*/,lookbehind:!0},string:{pattern:/(["'])(?:\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1/,greedy:!0},"builtin-target":{pattern:/\.[A-Z][^:#=\s]+(?=\s*:(?!=))/,alias:"builtin"},target:{pattern:/^(?:[^:=\s]|[ \t]+(?![\s:]))+(?=\s*:(?!=))/m,alias:"symbol",inside:{variable:/\$+(?:(?!\$)[^(){}:#=\s]+|(?=[({]))/}},variable:/\$+(?:(?!\$)[^(){}:#=\s]+|\([@*%<^+?][DF]\)|(?=[({]))/,keyword:/-include\b|\b(?:define|else|endef|endif|export|ifn?def|ifn?eq|include|override|private|sinclude|undefine|unexport|vpath)\b/,function:{pattern:/(\()(?:abspath|addsuffix|and|basename|call|dir|error|eval|file|filter(?:-out)?|findstring|firstword|flavor|foreach|guile|if|info|join|lastword|load|notdir|or|origin|patsubst|realpath|shell|sort|strip|subst|suffix|value|warning|wildcard|word(?:list|s)?)(?=[ \t])/,lookbehind:!0},operator:/(?:::|[?:+!])?=|[|@]/,punctuation:/[:;(){}]/},o.languages.objectivec=o.languages.extend("c",{string:{pattern:/@?"(?:\\(?:\r\n|[\s\S])|[^"\\\r\n])*"/,greedy:!0},keyword:/\b(?:asm|auto|break|case|char|const|continue|default|do|double|else|enum|extern|float|for|goto|if|in|inline|int|long|register|return|self|short|signed|sizeof|static|struct|super|switch|typedef|typeof|union|unsigned|void|volatile|while)\b|(?:@interface|@end|@implementation|@protocol|@class|@public|@protected|@private|@property|@try|@catch|@finally|@throw|@synthesize|@dynamic|@selector)\b/,operator:/-[->]?|\+\+?|!=?|<>?=?|==?|&&?|\|\|?|[~^%?*\/@]/}),delete o.languages.objectivec["class-name"],o.languages.objc=o.languages.objectivec,o.languages.ocaml={comment:{pattern:/\(\*[\s\S]*?\*\)/,greedy:!0},char:{pattern:/'(?:[^\\\r\n']|\\(?:.|[ox]?[0-9a-f]{1,3}))'/i,greedy:!0},string:[{pattern:/"(?:\\(?:[\s\S]|\r\n)|[^\\\r\n"])*"/,greedy:!0},{pattern:/\{([a-z_]*)\|[\s\S]*?\|\1\}/,greedy:!0}],number:[/\b(?:0b[01][01_]*|0o[0-7][0-7_]*)\b/i,/\b0x[a-f0-9][a-f0-9_]*(?:\.[a-f0-9_]*)?(?:p[+-]?\d[\d_]*)?(?!\w)/i,/\b\d[\d_]*(?:\.[\d_]*)?(?:e[+-]?\d[\d_]*)?(?!\w)/i],directive:{pattern:/\B#\w+/,alias:"property"},label:{pattern:/\B~\w+/,alias:"property"},"type-variable":{pattern:/\B'\w+/,alias:"function"},variant:{pattern:/`\w+/,alias:"symbol"},keyword:/\b(?:as|assert|begin|class|constraint|do|done|downto|else|end|exception|external|for|fun|function|functor|if|in|include|inherit|initializer|lazy|let|match|method|module|mutable|new|nonrec|object|of|open|private|rec|sig|struct|then|to|try|type|val|value|virtual|when|where|while|with)\b/,boolean:/\b(?:false|true)\b/,"operator-like-punctuation":{pattern:/\[[<>|]|[>|]\]|\{<|>\}/,alias:"punctuation"},operator:/\.[.~]|:[=>]|[=<>@^|&+\-*\/$%!?~][!$%&*+\-.\/:<=>?@^|~]*|\b(?:and|asr|land|lor|lsl|lsr|lxor|mod|or)\b/,punctuation:/;;|::|[(){}\[\].,:;#]|\b_\b/},o.languages.python={comment:{pattern:/(^|[^\\])#.*/,lookbehind:!0,greedy:!0},"string-interpolation":{pattern:/(?:f|fr|rf)(?:("""|''')[\s\S]*?\1|("|')(?:\\.|(?!\2)[^\\\r\n])*\2)/i,greedy:!0,inside:{interpolation:{pattern:/((?:^|[^{])(?:\{\{)*)\{(?!\{)(?:[^{}]|\{(?!\{)(?:[^{}]|\{(?!\{)(?:[^{}])+\})+\})+\}/,lookbehind:!0,inside:{"format-spec":{pattern:/(:)[^:(){}]+(?=\}$)/,lookbehind:!0},"conversion-option":{pattern:/![sra](?=[:}]$)/,alias:"punctuation"},rest:null}},string:/[\s\S]+/}},"triple-quoted-string":{pattern:/(?:[rub]|br|rb)?("""|''')[\s\S]*?\1/i,greedy:!0,alias:"string"},string:{pattern:/(?:[rub]|br|rb)?("|')(?:\\.|(?!\1)[^\\\r\n])*\1/i,greedy:!0},function:{pattern:/((?:^|\s)def[ \t]+)[a-zA-Z_]\w*(?=\s*\()/g,lookbehind:!0},"class-name":{pattern:/(\bclass\s+)\w+/i,lookbehind:!0},decorator:{pattern:/(^[\t ]*)@\w+(?:\.\w+)*/m,lookbehind:!0,alias:["annotation","punctuation"],inside:{punctuation:/\./}},keyword:/\b(?:_(?=\s*:)|and|as|assert|async|await|break|case|class|continue|def|del|elif|else|except|exec|finally|for|from|global|if|import|in|is|lambda|match|nonlocal|not|or|pass|print|raise|return|try|while|with|yield)\b/,builtin:/\b(?:__import__|abs|all|any|apply|ascii|basestring|bin|bool|buffer|bytearray|bytes|callable|chr|classmethod|cmp|coerce|compile|complex|delattr|dict|dir|divmod|enumerate|eval|execfile|file|filter|float|format|frozenset|getattr|globals|hasattr|hash|help|hex|id|input|int|intern|isinstance|issubclass|iter|len|list|locals|long|map|max|memoryview|min|next|object|oct|open|ord|pow|property|range|raw_input|reduce|reload|repr|reversed|round|set|setattr|slice|sorted|staticmethod|str|sum|super|tuple|type|unichr|unicode|vars|xrange|zip)\b/,boolean:/\b(?:False|None|True)\b/,number:/\b0(?:b(?:_?[01])+|o(?:_?[0-7])+|x(?:_?[a-f0-9])+)\b|(?:\b\d+(?:_\d+)*(?:\.(?:\d+(?:_\d+)*)?)?|\B\.\d+(?:_\d+)*)(?:e[+-]?\d+(?:_\d+)*)?j?(?!\w)/i,operator:/[-+%=]=?|!=|:=|\*\*?=?|\/\/?=?|<[<=>]?|>[=>]?|[&|^~]/,punctuation:/[{}[\];(),.:]/},o.languages.python["string-interpolation"].inside.interpolation.inside.rest=o.languages.python,o.languages.py=o.languages.python,o.languages.reason=o.languages.extend("clike",{string:{pattern:/"(?:\\(?:\r\n|[\s\S])|[^\\\r\n"])*"/,greedy:!0},"class-name":/\b[A-Z]\w*/,keyword:/\b(?:and|as|assert|begin|class|constraint|do|done|downto|else|end|exception|external|for|fun|function|functor|if|in|include|inherit|initializer|lazy|let|method|module|mutable|new|nonrec|object|of|open|or|private|rec|sig|struct|switch|then|to|try|type|val|virtual|when|while|with)\b/,operator:/\.{3}|:[:=]|\|>|->|=(?:==?|>)?|<=?|>=?|[|^?'#!~`]|[+\-*\/]\.?|\b(?:asr|land|lor|lsl|lsr|lxor|mod)\b/}),o.languages.insertBefore("reason","class-name",{char:{pattern:/'(?:\\x[\da-f]{2}|\\o[0-3][0-7][0-7]|\\\d{3}|\\.|[^'\\\r\n])'/,greedy:!0},constructor:/\b[A-Z]\w*\b(?!\s*\.)/,label:{pattern:/\b[a-z]\w*(?=::)/,alias:"symbol"}}),delete o.languages.reason.function,function(e){e.languages.sass=e.languages.extend("css",{comment:{pattern:/^([ \t]*)\/[\/*].*(?:(?:\r?\n|\r)\1[ \t].+)*/m,lookbehind:!0,greedy:!0}}),e.languages.insertBefore("sass","atrule",{"atrule-line":{pattern:/^(?:[ \t]*)[@+=].+/m,greedy:!0,inside:{atrule:/(?:@[\w-]+|[+=])/}}}),delete e.languages.sass.atrule;var t=/\$[-\w]+|#\{\$[-\w]+\}/,n=[/[+*\/%]|[=!]=|<=?|>=?|\b(?:and|not|or)\b/,{pattern:/(\s)-(?=\s)/,lookbehind:!0}];e.languages.insertBefore("sass","property",{"variable-line":{pattern:/^[ \t]*\$.+/m,greedy:!0,inside:{punctuation:/:/,variable:t,operator:n}},"property-line":{pattern:/^[ \t]*(?:[^:\s]+ *:.*|:[^:\s].*)/m,greedy:!0,inside:{property:[/[^:\s]+(?=\s*:)/,{pattern:/(:)[^:\s]+/,lookbehind:!0}],punctuation:/:/,variable:t,operator:n,important:e.languages.sass.important}}}),delete e.languages.sass.property,delete e.languages.sass.important,e.languages.insertBefore("sass","punctuation",{selector:{pattern:/^([ \t]*)\S(?:,[^,\r\n]+|[^,\r\n]*)(?:,[^,\r\n]+)*(?:,(?:\r?\n|\r)\1[ \t]+\S(?:,[^,\r\n]+|[^,\r\n]*)(?:,[^,\r\n]+)*)*/m,lookbehind:!0,greedy:!0}})}(o),o.languages.scss=o.languages.extend("css",{comment:{pattern:/(^|[^\\])(?:\/\*[\s\S]*?\*\/|\/\/.*)/,lookbehind:!0},atrule:{pattern:/@[\w-](?:\([^()]+\)|[^()\s]|\s+(?!\s))*?(?=\s+[{;])/,inside:{rule:/@[\w-]+/}},url:/(?:[-a-z]+-)?url(?=\()/i,selector:{pattern:/(?=\S)[^@;{}()]?(?:[^@;{}()\s]|\s+(?!\s)|#\{\$[-\w]+\})+(?=\s*\{(?:\}|\s|[^}][^:{}]*[:{][^}]))/,inside:{parent:{pattern:/&/,alias:"important"},placeholder:/%[-\w]+/,variable:/\$[-\w]+|#\{\$[-\w]+\}/}},property:{pattern:/(?:[-\w]|\$[-\w]|#\{\$[-\w]+\})+(?=\s*:)/,inside:{variable:/\$[-\w]+|#\{\$[-\w]+\}/}}}),o.languages.insertBefore("scss","atrule",{keyword:[/@(?:content|debug|each|else(?: if)?|extend|for|forward|function|if|import|include|mixin|return|use|warn|while)\b/i,{pattern:/( )(?:from|through)(?= )/,lookbehind:!0}]}),o.languages.insertBefore("scss","important",{variable:/\$[-\w]+|#\{\$[-\w]+\}/}),o.languages.insertBefore("scss","function",{"module-modifier":{pattern:/\b(?:as|hide|show|with)\b/i,alias:"keyword"},placeholder:{pattern:/%[-\w]+/,alias:"selector"},statement:{pattern:/\B!(?:default|optional)\b/i,alias:"keyword"},boolean:/\b(?:false|true)\b/,null:{pattern:/\bnull\b/,alias:"keyword"},operator:{pattern:/(\s)(?:[-+*\/%]|[=!]=|<=?|>=?|and|not|or)(?=\s)/,lookbehind:!0}}),o.languages.scss.atrule.inside.rest=o.languages.scss,function(e){var t={pattern:/(\b\d+)(?:%|[a-z]+)/,lookbehind:!0},n={pattern:/(^|[^\w.-])-?(?:\d+(?:\.\d+)?|\.\d+)/,lookbehind:!0},r={comment:{pattern:/(^|[^\\])(?:\/\*[\s\S]*?\*\/|\/\/.*)/,lookbehind:!0},url:{pattern:/\burl\((["']?).*?\1\)/i,greedy:!0},string:{pattern:/("|')(?:(?!\1)[^\\\r\n]|\\(?:\r\n|[\s\S]))*\1/,greedy:!0},interpolation:null,func:null,important:/\B!(?:important|optional)\b/i,keyword:{pattern:/(^|\s+)(?:(?:else|for|if|return|unless)(?=\s|$)|@[\w-]+)/,lookbehind:!0},hexcode:/#[\da-f]{3,6}/i,color:[/\b(?:AliceBlue|AntiqueWhite|Aqua|Aquamarine|Azure|Beige|Bisque|Black|BlanchedAlmond|Blue|BlueViolet|Brown|BurlyWood|CadetBlue|Chartreuse|Chocolate|Coral|CornflowerBlue|Cornsilk|Crimson|Cyan|DarkBlue|DarkCyan|DarkGoldenRod|DarkGr[ae]y|DarkGreen|DarkKhaki|DarkMagenta|DarkOliveGreen|DarkOrange|DarkOrchid|DarkRed|DarkSalmon|DarkSeaGreen|DarkSlateBlue|DarkSlateGr[ae]y|DarkTurquoise|DarkViolet|DeepPink|DeepSkyBlue|DimGr[ae]y|DodgerBlue|FireBrick|FloralWhite|ForestGreen|Fuchsia|Gainsboro|GhostWhite|Gold|GoldenRod|Gr[ae]y|Green|GreenYellow|HoneyDew|HotPink|IndianRed|Indigo|Ivory|Khaki|Lavender|LavenderBlush|LawnGreen|LemonChiffon|LightBlue|LightCoral|LightCyan|LightGoldenRodYellow|LightGr[ae]y|LightGreen|LightPink|LightSalmon|LightSeaGreen|LightSkyBlue|LightSlateGr[ae]y|LightSteelBlue|LightYellow|Lime|LimeGreen|Linen|Magenta|Maroon|MediumAquaMarine|MediumBlue|MediumOrchid|MediumPurple|MediumSeaGreen|MediumSlateBlue|MediumSpringGreen|MediumTurquoise|MediumVioletRed|MidnightBlue|MintCream|MistyRose|Moccasin|NavajoWhite|Navy|OldLace|Olive|OliveDrab|Orange|OrangeRed|Orchid|PaleGoldenRod|PaleGreen|PaleTurquoise|PaleVioletRed|PapayaWhip|PeachPuff|Peru|Pink|Plum|PowderBlue|Purple|Red|RosyBrown|RoyalBlue|SaddleBrown|Salmon|SandyBrown|SeaGreen|SeaShell|Sienna|Silver|SkyBlue|SlateBlue|SlateGr[ae]y|Snow|SpringGreen|SteelBlue|Tan|Teal|Thistle|Tomato|Transparent|Turquoise|Violet|Wheat|White|WhiteSmoke|Yellow|YellowGreen)\b/i,{pattern:/\b(?:hsl|rgb)\(\s*\d{1,3}\s*,\s*\d{1,3}%?\s*,\s*\d{1,3}%?\s*\)\B|\b(?:hsl|rgb)a\(\s*\d{1,3}\s*,\s*\d{1,3}%?\s*,\s*\d{1,3}%?\s*,\s*(?:0|0?\.\d+|1)\s*\)\B/i,inside:{unit:t,number:n,function:/[\w-]+(?=\()/,punctuation:/[(),]/}}],entity:/\\[\da-f]{1,8}/i,unit:t,boolean:/\b(?:false|true)\b/,operator:[/~|[+!\/%<>?=]=?|[-:]=|\*[*=]?|\.{2,3}|&&|\|\||\B-\B|\b(?:and|in|is(?: a| defined| not|nt)?|not|or)\b/],number:n,punctuation:/[{}()\[\];:,]/};r.interpolation={pattern:/\{[^\r\n}:]+\}/,alias:"variable",inside:{delimiter:{pattern:/^\{|\}$/,alias:"punctuation"},rest:r}},r.func={pattern:/[\w-]+\([^)]*\).*/,inside:{function:/^[^(]+/,rest:r}},e.languages.stylus={"atrule-declaration":{pattern:/(^[ \t]*)@.+/m,lookbehind:!0,inside:{atrule:/^@[\w-]+/,rest:r}},"variable-declaration":{pattern:/(^[ \t]*)[\w$-]+\s*.?=[ \t]*(?:\{[^{}]*\}|\S.*|$)/m,lookbehind:!0,inside:{variable:/^\S+/,rest:r}},statement:{pattern:/(^[ \t]*)(?:else|for|if|return|unless)[ \t].+/m,lookbehind:!0,inside:{keyword:/^\S+/,rest:r}},"property-declaration":{pattern:/((?:^|\{)([ \t]*))(?:[\w-]|\{[^}\r\n]+\})+(?:\s*:\s*|[ \t]+)(?!\s)[^{\r\n]*(?:;|[^{\r\n,]$(?!(?:\r?\n|\r)(?:\{|\2[ \t])))/m,lookbehind:!0,inside:{property:{pattern:/^[^\s:]+/,inside:{interpolation:r.interpolation}},rest:r}},selector:{pattern:/(^[ \t]*)(?:(?=\S)(?:[^{}\r\n:()]|::?[\w-]+(?:\([^)\r\n]*\)|(?![\w-]))|\{[^}\r\n]+\})+)(?:(?:\r?\n|\r)(?:\1(?:(?=\S)(?:[^{}\r\n:()]|::?[\w-]+(?:\([^)\r\n]*\)|(?![\w-]))|\{[^}\r\n]+\})+)))*(?:,$|\{|(?=(?:\r?\n|\r)(?:\{|\1[ \t])))/m,lookbehind:!0,inside:{interpolation:r.interpolation,comment:r.comment,punctuation:/[{},]/}},func:r.func,string:r.string,comment:{pattern:/(^|[^\\])(?:\/\*[\s\S]*?\*\/|\/\/.*)/,lookbehind:!0,greedy:!0},interpolation:r.interpolation,punctuation:/[{}()\[\];:.]/}}(o),function(e){var t=e.util.clone(e.languages.typescript);e.languages.tsx=e.languages.extend("jsx",t),delete e.languages.tsx.parameter,delete e.languages.tsx["literal-property"];var n=e.languages.tsx.tag;n.pattern=RegExp(/(^|[^\w$]|(?=<\/))/.source+"(?:"+n.pattern.source+")",n.pattern.flags),n.lookbehind=!0}(o),o.languages.wasm={comment:[/\(;[\s\S]*?;\)/,{pattern:/;;.*/,greedy:!0}],string:{pattern:/"(?:\\[\s\S]|[^"\\])*"/,greedy:!0},keyword:[{pattern:/\b(?:align|offset)=/,inside:{operator:/=/}},{pattern:/\b(?:(?:f32|f64|i32|i64)(?:\.(?:abs|add|and|ceil|clz|const|convert_[su]\/i(?:32|64)|copysign|ctz|demote\/f64|div(?:_[su])?|eqz?|extend_[su]\/i32|floor|ge(?:_[su])?|gt(?:_[su])?|le(?:_[su])?|load(?:(?:8|16|32)_[su])?|lt(?:_[su])?|max|min|mul|neg?|nearest|or|popcnt|promote\/f32|reinterpret\/[fi](?:32|64)|rem_[su]|rot[lr]|shl|shr_[su]|sqrt|store(?:8|16|32)?|sub|trunc(?:_[su]\/f(?:32|64))?|wrap\/i64|xor))?|memory\.(?:grow|size))\b/,inside:{punctuation:/\./}},/\b(?:anyfunc|block|br(?:_if|_table)?|call(?:_indirect)?|data|drop|elem|else|end|export|func|get_(?:global|local)|global|if|import|local|loop|memory|module|mut|nop|offset|param|result|return|select|set_(?:global|local)|start|table|tee_local|then|type|unreachable)\b/],variable:/\$[\w!#$%&'*+\-./:<=>?@\\^`|~]+/,number:/[+-]?\b(?:\d(?:_?\d)*(?:\.\d(?:_?\d)*)?(?:[eE][+-]?\d(?:_?\d)*)?|0x[\da-fA-F](?:_?[\da-fA-F])*(?:\.[\da-fA-F](?:_?[\da-fA-D])*)?(?:[pP][+-]?\d(?:_?\d)*)?)\b|\binf\b|\bnan(?::0x[\da-fA-F](?:_?[\da-fA-D])*)?\b/,punctuation:/[()]/};const a=o},905:()=>{!function(e){var t=e.languages.powershell={comment:[{pattern:/(^|[^`])<#[\s\S]*?#>/,lookbehind:!0},{pattern:/(^|[^`])#.*/,lookbehind:!0}],string:[{pattern:/"(?:`[\s\S]|[^`"])*"/,greedy:!0,inside:null},{pattern:/'(?:[^']|'')*'/,greedy:!0}],namespace:/\[[a-z](?:\[(?:\[[^\]]*\]|[^\[\]])*\]|[^\[\]])*\]/i,boolean:/\$(?:false|true)\b/i,variable:/\$\w+\b/,function:[/\b(?:Add|Approve|Assert|Backup|Block|Checkpoint|Clear|Close|Compare|Complete|Compress|Confirm|Connect|Convert|ConvertFrom|ConvertTo|Copy|Debug|Deny|Disable|Disconnect|Dismount|Edit|Enable|Enter|Exit|Expand|Export|Find|ForEach|Format|Get|Grant|Group|Hide|Import|Initialize|Install|Invoke|Join|Limit|Lock|Measure|Merge|Move|New|Open|Optimize|Out|Ping|Pop|Protect|Publish|Push|Read|Receive|Redo|Register|Remove|Rename|Repair|Request|Reset|Resize|Resolve|Restart|Restore|Resume|Revoke|Save|Search|Select|Send|Set|Show|Skip|Sort|Split|Start|Step|Stop|Submit|Suspend|Switch|Sync|Tee|Test|Trace|Unblock|Undo|Uninstall|Unlock|Unprotect|Unpublish|Unregister|Update|Use|Wait|Watch|Where|Write)-[a-z]+\b/i,/\b(?:ac|cat|chdir|clc|cli|clp|clv|compare|copy|cp|cpi|cpp|cvpa|dbp|del|diff|dir|ebp|echo|epal|epcsv|epsn|erase|fc|fl|ft|fw|gal|gbp|gc|gci|gcs|gdr|gi|gl|gm|gp|gps|group|gsv|gu|gv|gwmi|iex|ii|ipal|ipcsv|ipsn|irm|iwmi|iwr|kill|lp|ls|measure|mi|mount|move|mp|mv|nal|ndr|ni|nv|ogv|popd|ps|pushd|pwd|rbp|rd|rdr|ren|ri|rm|rmdir|rni|rnp|rp|rv|rvpa|rwmi|sal|saps|sasv|sbp|sc|select|set|shcm|si|sl|sleep|sls|sort|sp|spps|spsv|start|sv|swmi|tee|trcm|type|write)\b/i],keyword:/\b(?:Begin|Break|Catch|Class|Continue|Data|Define|Do|DynamicParam|Else|ElseIf|End|Exit|Filter|Finally|For|ForEach|From|Function|If|InlineScript|Parallel|Param|Process|Return|Sequence|Switch|Throw|Trap|Try|Until|Using|Var|While|Workflow)\b/i,operator:{pattern:/(^|\W)(?:!|-(?:b?(?:and|x?or)|as|(?:Not)?(?:Contains|In|Like|Match)|eq|ge|gt|is(?:Not)?|Join|le|lt|ne|not|Replace|sh[lr])\b|-[-=]?|\+[+=]?|[*\/%]=?)/i,lookbehind:!0},punctuation:/[|{}[\];(),.]/};t.string[0].inside={function:{pattern:/(^|[^`])\$\((?:\$\([^\r\n()]*\)|(?!\$\()[^\r\n)])*\)/,lookbehind:!0,inside:t},boolean:t.boolean,variable:t.variable}}(Prism)},2433:(e,t,n)=>{var r={"./prism-powershell":905};function o(e){var t=a(e);return n(t)}function a(e){if(!n.o(r,e)){var t=new Error("Cannot find module '"+e+"'");throw t.code="MODULE_NOT_FOUND",t}return r[e]}o.keys=function(){return Object.keys(r)},o.resolve=a,e.exports=o,o.id=2433},2694:(e,t,n)=>{"use strict";var r=n(6925);function o(){}function a(){}a.resetWarningCache=o,e.exports=function(){function e(e,t,n,o,a,i){if(i!==r){var l=new Error("Calling PropTypes validators directly is not supported by the `prop-types` package. Use PropTypes.checkPropTypes() to call them. Read more at http://fb.me/use-check-prop-types");throw l.name="Invariant Violation",l}}function t(){return e}e.isRequired=e;var n={array:e,bigint:e,bool:e,func:e,number:e,object:e,string:e,symbol:e,any:e,arrayOf:t,element:e,elementType:e,instanceOf:t,node:e,objectOf:t,oneOf:t,oneOfType:t,shape:t,exact:t,checkPropTypes:a,resetWarningCache:o};return n.PropTypes=n,n}},5556:(e,t,n)=>{e.exports=n(2694)()},6925:e=>{"use strict";e.exports="SECRET_DO_NOT_PASS_THIS_OR_YOU_WILL_BE_FIRED"},2551:(e,t,n)=>{"use strict";var r=n(6540),o=n(5228),a=n(9982);function i(e){for(var t="https://reactjs.org/docs/error-decoder.html?invariant="+e,n=1;n