diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index 5f9a95f..8eaa2fa 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -109,14 +109,6 @@ export default withMermaid( text: 'Overview', link: '/engineering/demo/', }, - { - text: 'Vision for Demo System', - link: '/engineering/demo/demo-system-mindmap', - }, - { - text: '1st Impl Attempt', - link: '/engineering/demo/impl-notes-take-1', - }, { text: 'Installing KDCC', link: '/engineering/demo/installing-kdcc.md', diff --git a/docs/engineering/demo/demo-system-mindmap.md b/docs/engineering/demo/demo-system-mindmap.md deleted file mode 100644 index e70c062..0000000 --- a/docs/engineering/demo/demo-system-mindmap.md +++ /dev/null @@ -1,75 +0,0 @@ ---- -title: Demo System Vision Mindmap -# mermaidTheme: default -mermaidTheme: neutral -# mermaidTheme: dark -# mermaidTheme: forest -# mermaidTheme: base # customizable, see https://mermaid.js.org/config/theming.html ---- - -# Demo System Vision Mindmap - -This is visualization of the ways we can use our demo system. - -## Mindmap - -```mermaid -mindmap - root(Koor Demo System) - ksd ))KSD on KDS(( - learn ))learn(( - safe {{safe environment to see how things work}} - practice K8s - deploy KSD - exp {{experiments}} - observe what happens when... - discover lower boundaries for "production" - upgrades from A to B - adding storage nodes - recovery when node goes offline - show ))showcase(( - op {{Koor Operator}} - insufficient resource warnings - dash {{Koor Dashboard}} - Grafana dashboards - license verification - troubleshooting scripts - ver {{Version Manager}} - available updates - assisted upgrades - sso {{SSO}} - share ))share(( - set {{try different Rook Ceph settings}} - opt {{optimizations}} - discover optimal settings - use {{use cases}} - for block, object and file - share on Knowledge Center -``` - -## Learn - -First and foremost, this demo is a great way to learn about the Kubernetes environment and the components that have to work together. Building and running the system is the best way to gain practical experience with the challenges that come with working with so much complex technology. - -* Kubernetes is complex -* Ceph is complex -* Networking is complex -* Security is complex - -Putting them all together is even more challenging. Keeping things running under various pressures and stresses of usage is another challenge. Diagnosing and fixing problems is a third. Our demo system needs to reflect a realistic production system, even if it is at the small end of things in terms of size. - -The demo system is a safe place to try things. Anyone who really wants to learn can follow the procedures to set up a new system, and help care for the system we have running. - -Also, this is a great basis for experimentation. Start from the known stable system and see what happens with various changes. Try an upgrade either by hand or using the Koor Version Manager (someday). Try adding storage capacity. See what happens when a node is shut down, then restored. There is no limit to the number of real-world situations we can try using our demo system. - - -## Showcase - -One of the most important uses of our demo system will be...wait for it...to *demonstrate* how KSD helps manage Rook Ceph data storage. We will use the system to show what KSD can do. We will start with a simple demo since KSD is just getting started. Over time, the demo will become more elaborate and impressive. - -This may be the single most important factor in convincing people to adopt KSD. - - -## Share - -Closely related to learning and experiments is our ability to set up various use cases for Ceph-managed data. The configuration for each use case can be copied from the demo system into our Knowledge Base. We can also include configuration pattern in KSD that customers can apply to their environments with a click or two. Easy setup, quick tuning. diff --git a/docs/engineering/demo/impl-notes-take-1.md b/docs/engineering/demo/impl-notes-take-1.md deleted file mode 100644 index be6ccd7..0000000 --- a/docs/engineering/demo/impl-notes-take-1.md +++ /dev/null @@ -1,297 +0,0 @@ ---- -title: Demo system - Dave's manual implementation notes ---- - -# Demo system - Dave's manual implementation notes - -These notes capture Dave's attempt to set up a demo system for shoring Koor Rook Ceph in Kubernetes. The [overview](index) defines the target system. As with all plans, they go bad at the first sign of trouble. The discovery of what worked is documented here. - -A word of caution, the fact that something "worked" in that I was able to continue with the next step does not mean that I made the best decisions. A big part of the this exercise is to learn how all of this works, and there are a lot of moving parts. - -Basically, apply your own judgement and critical thinking, and experiment with alternate approaches to everything. - - -## Target requirements - -For this exercise, I am attempting to set up a minimum environment that would be suitable for production. That means I will not use minikube, favoring kubeadm and other production-grade tooling. The system will use multiple hosts. However, on Hetzner where this is being provisioned, there are limits of 10 vCPU and 8 hosts. The minimum number of CPUs per K8s node is 2, so we are limited to 5 hosts. Given that constraint we will start by setting up: - -* 1 control plane node -* 1 worker node -* 3 storage nodes - -This system would not be suitable for mission-critical use. For example, we recommend 4 storage nodes as a minimum to allow for recovery from a single node failure. However, this will be a good starting point to expand on by by adding more nodes and capacity: CPUs, memory, drives, etc. - - -## General approach - -After 3 or 4 attempts, a basic pattern for setting things up by hand is emerging. - -1. Provision machines, or define what to provision - 1. We are using Hetzner for VM hosting by the minute. That's a good approach for temporary environments. The smallest host with 2 CPU is about 3.5 Euro per month. I don't think it gets much cheaper. - 2. We should use a provisioning tool, like [Terraform](https://www.terraform.io/). In fact, if we use KubeOne to set up the Kubernetes cluster, it uses Terraform under the hood for host provisioning. KubeOne integrates with AWS, CGP, Hetzner and others. - 3. Provisioning by hand works, too. -2. Set up pre-requisites for K8s - 1. Each host needs a container runtime: containerd or CRI-O - 1. Ubuntu and Debian have packages for containrd. Install that and configure according to instructions. - 2. Set up networking, CNI, plugins, etc. -3. Install K8s and set up a cluster - 1. TBD -4. Use Koor Operator to set up data storage -- it should be that easy if the rest was done right - 1. TBD - - -## Manual installation on Ubuntu - -### Provisioning servers (VMs) on Hetzner - -Hardware and network - -* The minimum spec is CPX11: 2 vCPU, 2GB RAM, 40GB local SSD, 3.85 Euro / mo - * Select Ubuntu 22.04 (or choose a different OS, but you will need to make adjustments as you go through these instructions) - * Small additional charge for public IPv4 - * Worth thinking about what needs access from outside, limitation on number of public IPs -* Drives are provisioned separately and attached to a host - * Available in 10 GB increments up to 10 TB - * Need 1 per storage node to start. Expand to 2 per node once things are running. - * They come with a file system: XDF or EXT4 -- will need to remove that before setting up Ceph -* Also available: load balancers, floating IPs, networks (e.g., private between nodes might be handy), firewalls and security keys. - - -### Logging in - -These tips might save you some time and frustration when logging in to your hosts. - -1. As you are provision a server, you can created an SSH key or use an existing key. Do this: create one, then use the same one for all hosts. The private side of that key goes on your local machine. -2. You can log in as root over SSH. By default, root does not allow login/password over SSH. [Directions are here](https://docs.hetzner.com/cloud/servers/getting-started/connecting-to-the-server). - 1. If you ever need to reset a password, do it from the Cloud Console. [Resetting the password](https://docs.hetzner.com/cloud/servers/getting-started/rescue-system) -3. Then you have to set up local SSH stuff - 1. ~/.ssh/config needs an entry like this: - -```sh -Host hetzner.com - HostName 5.78.69.87 - User root - PreferredAuthentications publickey - IdentityFile ~/.ssh/hetzner-id_ed25519 -``` - -You probably have at least github.com in there as well, since you are signing your commits, right? Just append to what is there. - -Then you can connect with: `ssh hetzner.com` - -The first time you connect to a server, you'll want to accept the host fingerprint. - - -### Set up containerd - -* provisioned 4 VMs: 1 control plane and 3 data nodes -* setting up all at the same time -* Ubuntu 22.04 - -Setting up each machine with: - -* `apt update` -* `apt upgrade` -* reboot: `shutdown -r now` -* `apt install containerd` -* `curl -L https://raw.githubusercontent.com/containerd/containerd/main/containerd.service -o /usr/local/lib/systemd/system/containerd.service` - * edit and change - * `ExecStartPre=-/usr/sbin/modprobe overlay` - * `ExecStart=/usr/bin/containerd` -* `systemctl daemon-reload` -* `systemctl enable --now containerd` -* `mkdir -p /etc/containerd` -* `containerd config default > /etc/containerd/config.toml` -* `vi /etc/containerd/config.toml` - * change `SystemdCgroup = true` -* restart: `systemctl restart containerd` - -Regarding cgroup drivers, Ubuntu uses systemd. Looks like we don't need to worry about them since k8s v1.22 if using kubeadm, which we are. - -### Install CNI plugins - -```sh -wget https://github.com/containernetworking/plugins/releases/download/v1.3.0/cni-plugins-linux-amd64-v1.3.0.tgz -wget https://github.com/containernetworking/plugins/releases/download/v1.3.0/cni-plugins-linux-amd64-v1.3.0.tgz.sha256 -cat cni-plugins-linux-amd64-v1.3.0.tgz.sha256 -sha256sum cni-plugins-linux-amd64-v1.3.0.tgz -mkdir -p /opt/cni/bin -tar Cxzvf /opt/cni/bin cni-plugins-linux-amd64-v1.3.0.tgz -``` - -### How to use containerd - -For reference: [containerd basic usage](https://github.com/containerd/nerdctl#readme) - - -### Install kubeadm, kubectl, kubelet - -Use the following commands to: -* Make sure dependencies are in place. -* Get Google Cloud signing key. -* Add K8s apt repo. -* Update apt package index, install kubelet, kubeadm and kubectl, and pin their versions. - -```sh -apt-get update -apt-get install -y apt-transport-https ca-certificates curl gpg -curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-archive-keyring.gpg -echo "deb [signed-by=/etc/apt/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list -sudo apt-get update -sudo apt-get install -y kubelet kubeadm kubectl -sudo apt-mark hold kubelet kubeadm kubectl -``` - -### Forwarding IPv4 and letting iptables see bridged traffic - -```sh -cat <=8.16.0" - } + "peer": true }, "node_modules/shiki": { "version": "0.14.4", @@ -2499,10 +2496,14 @@ } }, "node_modules/uuid": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", - "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==", + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", "dev": true, + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], "bin": { "uuid": "dist/bin/uuid" } @@ -2526,9 +2527,9 @@ } }, "node_modules/vite": { - "version": "4.4.9", - "resolved": "https://registry.npmjs.org/vite/-/vite-4.4.9.tgz", - "integrity": "sha512-2mbUn2LlUmNASWwSCNSJ/EG2HuSRTnVNaydp6vMCm5VIqJsjMfbIWtbH2kDuwUVW5mMUKKZvGPX/rqeqVvv1XA==", + "version": "4.4.11", + "resolved": "https://registry.npmjs.org/vite/-/vite-4.4.11.tgz", + "integrity": "sha512-ksNZJlkcU9b0lBwAGZGGaZHCMqHsc8OpgtoYhsQ4/I2v5cnpmmmqe5pM4nv/4Hn6G/2GhTdj0DhZh2e+Er1q5A==", "dev": true, "dependencies": { "esbuild": "^0.18.10", @@ -2616,16 +2617,16 @@ } }, "node_modules/vitepress-plugin-mermaid": { - "version": "2.0.14", - "resolved": "https://registry.npmjs.org/vitepress-plugin-mermaid/-/vitepress-plugin-mermaid-2.0.14.tgz", - "integrity": "sha512-W3MEzrOB0xH8cSsc3ztliVPldTTr8TPZ4KGqC0uNmRKNbwsqT2Tl96R4ZXgqARLLawcb+0vyjU9wnOALRYTdxQ==", + "version": "2.0.15", + "resolved": "https://registry.npmjs.org/vitepress-plugin-mermaid/-/vitepress-plugin-mermaid-2.0.15.tgz", + "integrity": "sha512-63tTqK4WvYXSnT1jY+Ex0htUCb4XGQmHeIexvqZHkVOBruFiDFG0f+ishLnS1F+1IYkVkyYHacv7PDX3wUP9hQ==", "dev": true, "optionalDependencies": { "@mermaid-js/mermaid-mindmap": "^9.3.0" }, "peerDependencies": { "mermaid": "10", - "vitepress": "^0.21.6 || ^1.0.0 || ^1.0.0-alpha" + "vitepress": "^1.0.0 || ^1.0.0-alpha" } }, "node_modules/vscode-oniguruma": { diff --git a/package.json b/package.json index 869d0de..8d447dc 100644 --- a/package.json +++ b/package.json @@ -1,8 +1,8 @@ { "devDependencies": { - "mermaid": "^10.3.0", - "vitepress": "^1.0.0-rc.15", - "vitepress-plugin-mermaid": "^2.0.13" + "mermaid": "^10.5.0", + "vitepress": "^1.0.0-rc.20", + "vitepress-plugin-mermaid": "^2.0.15" }, "scripts": { "docs:dev": "vitepress dev docs",