operations: add k8s setup examples that work (#640)

This adds an initial example of running everything
locally and ephemerally so that you can complete
the getting started tutorial on your own k8s
environment.

.prettierrc.yaml

@@ -0,0 +1,5 @@
+# @format
+
+semi: false
+singleQuote: true
+proseWrap: always

operations/k8s/README.md

@@ -0,0 +1,42 @@
+# Kubernetes
+
+## Components
+
+The resources have been split into separate components:
+
+- [base](base) - this includes the API server and the coordinator.
+- [components/postgres](components/postgres) - a simple, ephemeral example of
+  using postgres for all database operations including the vector store.
+- [components/minio](components/minio) - an ephemeral example of using S3 for
+  blog storage.
+- [components/extractors](components/extractors) - extractors are published as
+  common containers, this component is used by all the extractors, such as
+  [minilm-l6](components/minilm-l6) to provide extraction.
+
+> [!NOTE] The API server comes with an ingress resource by default that exposes
+> the api at `/`. Make sure to change this if you'd like it at a different
+> location.
+
+To run locally, you can apply the [local](local) setup and then go through the
+getting started guide.
+
+```bash
+kubectl apply -k local
+```
+
+## Cluster Standup
+
+### Local
+
+One way to create a cluster is using k3d. This will run a lightweight version of
+Kubernetes ([k3s][k3s]) entirely within docker on your local system.
+
+[k3s]: https://k3s.io
+
+```bash
+k3d cluster create -p "8081:80@loadbalancer" indexify
+```
+
+When using this setup, Indexify will be exposed via k3d's ingress which will be
+[http://localhost:8081](http://localhost:8081). You'll want to configure
+`IndexifyClient(service_url="http://localhost:8081")`.

operations/k8s/kustomize/base/api.yaml

@@ -0,0 +1,100 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: api
+  labels:
+    app.kubernetes.io/component: api
+spec:
+  ports:
+    - port: 8900
+  selector:
+    app.kubernetes.io/component: api
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: ingress
+  labels:
+    app.kubernetes.io/component: api
+spec:
+  rules:
+    - http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: api
+                port:
+                  number: 8900
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: api
+  labels:
+    app.kubernetes.io/component: api
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: api
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: api
+
+    spec:
+      initContainers:
+        - name: concat
+          image: alpine:3
+          command: ['/bin/sh', '-c']
+          args:
+            - find /fragments -type f | xargs -I{} sh -c "cat {}; echo ''" > /config/config.yaml
+
+          volumeMounts:
+            - mountPath: /config
+              name: config
+            - mountPath: /fragments
+              name: fragments
+
+      containers:
+        - name: indexify
+          image: tensorlake/indexify:stable
+
+          command: ['indexify']
+          args:
+            - server
+            - --config-path
+            - ./config/config.yaml
+
+          env:
+            - name: AWS_ENDPOINT_URL
+              value: http://blob-store:9000
+            - name: AWS_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  name: blob-store
+                  key: AWS_ACCESS_KEY_ID
+            - name: AWS_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: blob-store
+                  key: AWS_SECRET_ACCESS_KEY
+
+          volumeMounts:
+            - mountPath: /indexify/config
+              name: config
+              readOnly: true
+            - mountPath: /data
+              name: data
+
+      volumes:
+        - name: fragments
+          configMap:
+            name: indexify
+        - name: config
+          emptyDir: {}
+        - name: data
+          emptyDir: {}

operations/k8s/kustomize/base/config.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: indexify
+data:
+  config.yaml: |-
+    coordinator_addr: coordinator:8950
+    raft_port: 8970
+    seed_node: localhost:8970
+    node_id: 0
+
+    state_store:
+      path: /data/state

operations/k8s/kustomize/base/coordinator.yaml

@@ -0,0 +1,66 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: coordinator
+spec:
+  ports:
+    - port: 8950
+  selector:
+    app.kubernetes.io/component: coordinator
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: coordinator
+  labels:
+    app.kubernetes.io/component: coordinator
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: coordinator
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: coordinator
+
+    spec:
+      initContainers:
+        - name: concat
+          image: alpine:3
+          command: ['/bin/sh', '-c']
+          args:
+            - find /fragments -type f | xargs -I{} sh -c "cat {}; echo ''" > /config/config.yaml
+
+          volumeMounts:
+            - mountPath: /config
+              name: config
+            - mountPath: /fragments
+              name: fragments
+
+      containers:
+        - name: indexify
+          image: tensorlake/indexify:stable
+
+          command: ['indexify']
+          args:
+            - coordinator
+            - --config-path
+            - ./config/config.yaml
+
+          volumeMounts:
+            - mountPath: /indexify/config
+              name: config
+              readOnly: true
+            - mountPath: /data
+              name: data
+
+      volumes:
+        - name: fragments
+          configMap:
+            name: indexify
+        - name: config
+          emptyDir: {}
+        - name: data
+          emptyDir: {}

operations/k8s/kustomize/base/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - api.yaml
+  - coordinator.yaml
+  - config.yaml

operations/k8s/kustomize/components/chunker/kustomization.yaml

@@ -0,0 +1,44 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+components:
+  - ../extractor
+
+images:
+  - name: tensorlake/extractor:latest
+    newName: tensorlake/chunk-extractor
+    newTag: latest
+
+patches:
+  - target:
+      group: apps
+      version: v1
+      kind: Deployment
+      name: extractor
+    patch: |-
+      - op: replace
+        path: /metadata/name
+        value: chunker
+      - op: add
+        path: /spec/selector/matchLabels/app.kubernetes.io~1name
+        value: chunker
+      - op: add
+        path: /metadata/labels/app.kubernetes.io~1name
+        value: chunker
+      - op: add
+        path: /spec/template/metadata/labels/app.kubernetes.io~1name
+        value: chunker
+  - target:
+      version: v1
+      kind: Service
+      name: extractor
+    patch: |-
+      - op: replace
+        path: /metadata/name
+        value: chunker
+      - op: add
+        path: /metadata/labels/app.kubernetes.io~1name
+        value: chunker
+      - op: add
+        path: /spec/selector/app.kubernetes.io~1name
+        value: chunker

operations/k8s/kustomize/components/extractor/extractor.yaml

@@ -0,0 +1,41 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: extractor
+  labels:
+    app.kubernetes.io/component: extractor
+spec:
+  ports:
+    - port: 9501
+  selector:
+    app.kubernetes.io/component: extractor
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: extractor
+  labels:
+    app.kubernetes.io/component: extractor
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/component: extractor
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/component: extractor
+    spec:
+      containers:
+        - name: extractor
+          image: tensorlake/extractor:latest
+
+          command: ['/bin/bash', '-c']
+          args:
+            - |-
+              indexify-extractor join-server \
+              --coordinator-addr coordinator:8950 \
+              --ingestion-addr api:8900 \
+              --advertise-addr $(hostname | cut -d"-" -f1):9501 \
+              --listen-port 9501

operations/k8s/kustomize/components/extractor/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+
+resources:
+  - extractor.yaml

operations/k8s/kustomize/components/minilm-l6/kustomization.yaml

@@ -0,0 +1,44 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+components:
+  - ../extractor
+
+images:
+  - name: tensorlake/extractor:latest
+    newName: tensorlake/minilm-l6
+    newTag: latest
+
+patches:
+  - target:
+      group: apps
+      version: v1
+      kind: Deployment
+      name: extractor
+    patch: |-
+      - op: replace
+        path: /metadata/name
+        value: minilm-l6
+      - op: add
+        path: /spec/selector/matchLabels/app.kubernetes.io~1name
+        value: minilm-l6
+      - op: add
+        path: /metadata/labels/app.kubernetes.io~1name
+        value: minilm-l6
+      - op: add
+        path: /spec/template/metadata/labels/app.kubernetes.io~1name
+        value: minilm-l6
+  - target:
+      version: v1
+      kind: Service
+      name: extractor
+    patch: |-
+      - op: replace
+        path: /metadata/name
+        value: minilm-l6
+      - op: add
+        path: /metadata/labels/app.kubernetes.io~1name
+        value: minilm-l6
+      - op: add
+        path: /spec/selector/app.kubernetes.io~1name
+        value: minilm-l6