Merge pull request rh-aiservices-bu#42 from rh-aiservices-bu/feature/…

…robert_documentation

Sanity doc and empty claim processing doc

content/modules/ROOT/pages/03-07-sanity-check.adoc

@@ -1,10 +1,45 @@
+:imagesdir: ../assets/images
 = Sanity-Check Pipeline
 
+== What will the pipeline do?
+To make sure that everything works as we would expect it, and that the model has not been tampered with, we create a sanity pipeline that tests the model through its endoint. +
+We will test the response time, the response quality, and that the model hash has not changed.
+
 == Deploy a pipeline Server in your project
+Start by deploying a pipeline server that can we can use to run pipelines with:
+
+- Go to your Data Science Project
+- Scroll down to Pipelines and press "Create a pipeline server"
+- Select the existing data connection "Pipelines"
+- Press Create
+
+We need to connect the pipeline server with a data connection as the pipeline server will use the S3 storage to store logs and other artifacts.
 
 == Deploy a sanity-check pipeline
+There are two pipeline files, one *sanity_check.pipeline* and one *sanity_check.yaml* file. 
+
+The .pipeline file can be opened in Elyra to be visualliy modified and ran, while the .yaml file can be imported into the pipeline server through the dashboard. +
+Here we will be running the pipeline through Elyra.
 
 == Ad-Hoc execution
+Running it through Elyra is the same as doing an ad-hoc execution of the pipeline, as opposed to importing the pipeline which won't automatically execute it.
+
+Start by going to your workbench and open up the .pipeline file. +
+Here we can see that the pipeline consists of 3 tests: a response quality test, a response time test, and a security test. Feel free to peek into each of the python files by double clicking on the nodes to see what they do. +
+After the tests have been ran, we have a final function that will summarize the results and log them.
+
+To run the pipeline, press the Play button in the menu bar.
+
+image::03/07-elyra-pipeline.png[elyra sanity pipeline]
+
+When you get a popup that says "Job submission to Data Science Pipelines succeeded", press the link "Run details" to see how the pipeline is progessing.
 
 == Schedule execution
+We can also schedule an execution so that the sanity check is ran at appropriate intervals. + 
+To do that, go back to the dashboard, find the pipeline you just ran, press the 3 dots at the very right, and press "Create run". +
+
+image::03/07-create-run.png[create run]
 
+From this page, you are able to create both ad-hoc and scheduled pipeline runs. +
+Name your pipeline something like "Scheduled Sanity Pipeline", select the option "Schedule recurring run", set it to run periodically every hour, and press Create. + 
+We can now leave the sanity pipeline to do its thing while we are comfortable that it will start failing if something is wrong with our LLM setup.

content/modules/ROOT/pages/05-05-process-claims.adoc

@@ -0,0 +1,6 @@
+:imagesdir: ../assets/images
+= Pipeline for processing claims
+
+== Run the pipeline
+
+== Check the results

lab-materials/03/07/sanity_check.yaml

@@ -0,0 +1,327 @@
+apiVersion: tekton.dev/v1beta1
+kind: PipelineRun
+metadata:
+  name: sanity-check
+  annotations:
+    tekton.dev/output_artifacts: '{"run-a-file": [{"key": "artifacts/$PIPELINERUN/run-a-file/mlpipeline-metrics.tgz",
+      "name": "mlpipeline-metrics", "path": "/tmp/mlpipeline-metrics.json"}, {"key":
+      "artifacts/$PIPELINERUN/run-a-file/mlpipeline-ui-metadata.tgz", "name": "mlpipeline-ui-metadata",
+      "path": "/tmp/mlpipeline-ui-metadata.json"}], "run-a-file-2": [{"key": "artifacts/$PIPELINERUN/run-a-file-2/mlpipeline-metrics.tgz",
+      "name": "mlpipeline-metrics", "path": "/tmp/mlpipeline-metrics.json"}, {"key":
+      "artifacts/$PIPELINERUN/run-a-file-2/mlpipeline-ui-metadata.tgz", "name": "mlpipeline-ui-metadata",
+      "path": "/tmp/mlpipeline-ui-metadata.json"}], "run-a-file-3": [{"key": "artifacts/$PIPELINERUN/run-a-file-3/mlpipeline-metrics.tgz",
+      "name": "mlpipeline-metrics", "path": "/tmp/mlpipeline-metrics.json"}, {"key":
+      "artifacts/$PIPELINERUN/run-a-file-3/mlpipeline-ui-metadata.tgz", "name": "mlpipeline-ui-metadata",
+      "path": "/tmp/mlpipeline-ui-metadata.json"}], "run-a-file-4": [{"key": "artifacts/$PIPELINERUN/run-a-file-4/mlpipeline-metrics.tgz",
+      "name": "mlpipeline-metrics", "path": "/tmp/mlpipeline-metrics.json"}, {"key":
+      "artifacts/$PIPELINERUN/run-a-file-4/mlpipeline-ui-metadata.tgz", "name": "mlpipeline-ui-metadata",
+      "path": "/tmp/mlpipeline-ui-metadata.json"}]}'
+    tekton.dev/input_artifacts: '{}'
+    tekton.dev/artifact_bucket: mlpipeline
+    tekton.dev/artifact_endpoint: minio-service.kubeflow:9000
+    tekton.dev/artifact_endpoint_scheme: http://
+    tekton.dev/artifact_items: '{"run-a-file": [["mlpipeline-metrics", "/tmp/mlpipeline-metrics.json"],
+      ["mlpipeline-ui-metadata", "/tmp/mlpipeline-ui-metadata.json"]], "run-a-file-2":
+      [["mlpipeline-metrics", "/tmp/mlpipeline-metrics.json"], ["mlpipeline-ui-metadata",
+      "/tmp/mlpipeline-ui-metadata.json"]], "run-a-file-3": [["mlpipeline-metrics",
+      "/tmp/mlpipeline-metrics.json"], ["mlpipeline-ui-metadata", "/tmp/mlpipeline-ui-metadata.json"]],
+      "run-a-file-4": [["mlpipeline-metrics", "/tmp/mlpipeline-metrics.json"], ["mlpipeline-ui-metadata",
+      "/tmp/mlpipeline-ui-metadata.json"]]}'
+    sidecar.istio.io/inject: "false"
+    tekton.dev/template: ''
+    pipelines.kubeflow.org/big_data_passing_format: $(workspaces.$TASK_NAME.path)/artifacts/$ORIG_PR_NAME/$TASKRUN_NAME/$TASK_PARAM_NAME
+    pipelines.kubeflow.org/pipeline_spec: '{"name": "sanity_check"}'
+  labels:
+    pipelines.kubeflow.org/pipelinename: ''
+    pipelines.kubeflow.org/generation: ''
+spec:
+  pipelineSpec:
+    tasks:
+    - name: run-a-file
+      taskSpec:
+        steps:
+        - name: main
+          args:
+          - |
+            sh -c "mkdir -p ./jupyter-work-dir && cd ./jupyter-work-dir"
+            sh -c "echo 'Downloading file:///opt/app-root/bin/utils/bootstrapper.py' && curl --fail -H 'Cache-Control: no-cache' -L file:///opt/app-root/bin/utils/bootstrapper.py --output bootstrapper.py"
+            sh -c "echo 'Downloading file:///opt/app-root/bin/utils/requirements-elyra.txt' && curl --fail -H 'Cache-Control: no-cache' -L file:///opt/app-root/bin/utils/requirements-elyra.txt --output requirements-elyra.txt"
+            sh -c "python3 -m pip install  packaging && python3 -m pip freeze > requirements-current.txt && python3 bootstrapper.py --pipeline-name 'sanity_check' --cos-endpoint 'https://minio-api-robert-serving-test.apps.rhods-internal.61tk.p1.openshiftapps.com' --cos-bucket 'pipeline-bucket' --cos-directory 'sanity_check-1215145442' --cos-dependencies-archive 'test_response_quality-d0510f0c-fc50-42fc-a598-97e26b34ed88.tar.gz' --file 'insurance-claim-processing/lab-materials/03/test_response_quality.py' --outputs 'quality_result.json' "
+          command:
+          - sh
+          - -c
+          env:
+          - name: AWS_ACCESS_KEY_ID
+            valueFrom:
+              secretKeyRef:
+                key: AWS_ACCESS_KEY_ID
+                name: aws-connection-minio-pipeline
+          - name: AWS_SECRET_ACCESS_KEY
+            valueFrom:
+              secretKeyRef:
+                key: AWS_SECRET_ACCESS_KEY
+                name: aws-connection-minio-pipeline
+          - name: ELYRA_RUNTIME_ENV
+            value: kfp
+          - name: ELYRA_ENABLE_PIPELINE_INFO
+            value: "True"
+          - name: ELYRA_WRITABLE_CONTAINER_DIR
+            value: /tmp
+          - name: ELYRA_RUN_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.annotations['pipelines.kubeflow.org/run_name']
+          - name: LLM_ENDPOINT
+            valueFrom:
+              secretKeyRef:
+                key: LLM_ENDPOINT
+                name: llm-info
+          - name: MODEL_SHA
+            valueFrom:
+              secretKeyRef:
+                key: MODEL_SHA
+                name: llm-info
+          - name: SERVING_SHA
+            valueFrom:
+              secretKeyRef:
+                key: SERVING_SHA
+                name: llm-info
+          image: quay.io/rlundber/rh1/sanity_pipeline:1.7
+        stepTemplate:
+          volumeMounts:
+          - name: mlpipeline-metrics
+            mountPath: /tmp
+        volumes:
+        - name: mlpipeline-metrics
+          emptyDir: {}
+        metadata:
+          labels:
+            elyra/node-type: notebook-script
+            elyra/pipeline-name: sanity_check
+            elyra/pipeline-version: ''
+            elyra/experiment-name: ''
+            elyra/node-name: test_response_quality
+            pipelines.kubeflow.org/cache_enabled: "true"
+          annotations:
+            elyra/node-file-name: insurance-claim-processing/lab-materials/03/test_response_quality.py
+            elyra/pipeline-source: sanity_check.pipeline
+            pipelines.kubeflow.org/task_display_name: test_response_quality
+            pipelines.kubeflow.org/component_spec_digest: '{"name": "Run a file",
+              "outputs": [], "version": "Run a file@sha256=7e955de34869fba374b1d87fd591922910d2f2490d11e7fdd1312ae0749a0742"}'
+    - name: run-a-file-2
+      taskSpec:
+        steps:
+        - name: main
+          args:
+          - |
+            sh -c "mkdir -p ./jupyter-work-dir && cd ./jupyter-work-dir"
+            sh -c "echo 'Downloading file:///opt/app-root/bin/utils/bootstrapper.py' && curl --fail -H 'Cache-Control: no-cache' -L file:///opt/app-root/bin/utils/bootstrapper.py --output bootstrapper.py"
+            sh -c "echo 'Downloading file:///opt/app-root/bin/utils/requirements-elyra.txt' && curl --fail -H 'Cache-Control: no-cache' -L file:///opt/app-root/bin/utils/requirements-elyra.txt --output requirements-elyra.txt"
+            sh -c "python3 -m pip install  packaging && python3 -m pip freeze > requirements-current.txt && python3 bootstrapper.py --pipeline-name 'sanity_check' --cos-endpoint 'https://minio-api-robert-serving-test.apps.rhods-internal.61tk.p1.openshiftapps.com' --cos-bucket 'pipeline-bucket' --cos-directory 'sanity_check-1215145442' --cos-dependencies-archive 'test_responsetime-c50bb14f-a036-4af1-b5dc-21e48eb80f7f.tar.gz' --file 'insurance-claim-processing/lab-materials/03/test_responsetime.py' --outputs 'responsetime_result.json' "
+          command:
+          - sh
+          - -c
+          env:
+          - name: AWS_ACCESS_KEY_ID
+            valueFrom:
+              secretKeyRef:
+                key: AWS_ACCESS_KEY_ID
+                name: aws-connection-minio-pipeline
+          - name: AWS_SECRET_ACCESS_KEY
+            valueFrom:
+              secretKeyRef:
+                key: AWS_SECRET_ACCESS_KEY
+                name: aws-connection-minio-pipeline
+          - name: ELYRA_RUNTIME_ENV
+            value: kfp
+          - name: ELYRA_ENABLE_PIPELINE_INFO
+            value: "True"
+          - name: ELYRA_WRITABLE_CONTAINER_DIR
+            value: /tmp
+          - name: ELYRA_RUN_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.annotations['pipelines.kubeflow.org/run_name']
+          - name: LLM_ENDPOINT
+            valueFrom:
+              secretKeyRef:
+                key: LLM_ENDPOINT
+                name: llm-info
+          - name: MODEL_SHA
+            valueFrom:
+              secretKeyRef:
+                key: MODEL_SHA
+                name: llm-info
+          - name: SERVING_SHA
+            valueFrom:
+              secretKeyRef:
+                key: SERVING_SHA
+                name: llm-info
+          image: quay.io/modh/runtime-images@sha256:7dd23e58291cad7a0ab4a8e04bda06492f2c027eb33b226358380db58dcdd60b
+        stepTemplate:
+          volumeMounts:
+          - name: mlpipeline-metrics
+            mountPath: /tmp
+        volumes:
+        - name: mlpipeline-metrics
+          emptyDir: {}
+        metadata:
+          labels:
+            elyra/node-type: notebook-script
+            elyra/pipeline-name: sanity_check
+            elyra/pipeline-version: ''
+            elyra/experiment-name: ''
+            elyra/node-name: test_responsetime
+            pipelines.kubeflow.org/cache_enabled: "true"
+          annotations:
+            elyra/node-file-name: insurance-claim-processing/lab-materials/03/test_responsetime.py
+            elyra/pipeline-source: sanity_check.pipeline
+            pipelines.kubeflow.org/task_display_name: test_responsetime
+            pipelines.kubeflow.org/component_spec_digest: '{"name": "Run a file",
+              "outputs": [], "version": "Run a file@sha256=afa090f686bb464016ab6640a960b8e31b88c415166d799c2c734f6e9ba58f6a"}'
+    - name: run-a-file-3
+      taskSpec:
+        steps:
+        - name: main
+          args:
+          - |
+            sh -c "mkdir -p ./jupyter-work-dir && cd ./jupyter-work-dir"
+            sh -c "echo 'Downloading file:///opt/app-root/bin/utils/bootstrapper.py' && curl --fail -H 'Cache-Control: no-cache' -L file:///opt/app-root/bin/utils/bootstrapper.py --output bootstrapper.py"
+            sh -c "echo 'Downloading file:///opt/app-root/bin/utils/requirements-elyra.txt' && curl --fail -H 'Cache-Control: no-cache' -L file:///opt/app-root/bin/utils/requirements-elyra.txt --output requirements-elyra.txt"
+            sh -c "python3 -m pip install  packaging && python3 -m pip freeze > requirements-current.txt && python3 bootstrapper.py --pipeline-name 'sanity_check' --cos-endpoint 'https://minio-api-robert-serving-test.apps.rhods-internal.61tk.p1.openshiftapps.com' --cos-bucket 'pipeline-bucket' --cos-directory 'sanity_check-1215145442' --cos-dependencies-archive 'test_security-6b595dc7-afb8-46bb-bf52-7cd695ddafb8.tar.gz' --file 'insurance-claim-processing/lab-materials/03/test_security.py' --outputs 'security_result.json' "
+          command:
+          - sh
+          - -c
+          env:
+          - name: AWS_ACCESS_KEY_ID
+            valueFrom:
+              secretKeyRef:
+                key: AWS_ACCESS_KEY_ID
+                name: aws-connection-minio-pipeline
+          - name: AWS_SECRET_ACCESS_KEY
+            valueFrom:
+              secretKeyRef:
+                key: AWS_SECRET_ACCESS_KEY
+                name: aws-connection-minio-pipeline
+          - name: ELYRA_RUNTIME_ENV
+            value: kfp
+          - name: ELYRA_ENABLE_PIPELINE_INFO
+            value: "True"
+          - name: ELYRA_WRITABLE_CONTAINER_DIR
+            value: /tmp
+          - name: ELYRA_RUN_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.annotations['pipelines.kubeflow.org/run_name']
+          - name: LLM_ENDPOINT
+            valueFrom:
+              secretKeyRef:
+                key: LLM_ENDPOINT
+                name: llm-info
+          - name: MODEL_SHA
+            valueFrom:
+              secretKeyRef:
+                key: MODEL_SHA
+                name: llm-info
+          - name: SERVING_SHA
+            valueFrom:
+              secretKeyRef:
+                key: SERVING_SHA
+                name: llm-info
+          image: quay.io/modh/runtime-images@sha256:7dd23e58291cad7a0ab4a8e04bda06492f2c027eb33b226358380db58dcdd60b
+        stepTemplate:
+          volumeMounts:
+          - name: mlpipeline-metrics
+            mountPath: /tmp
+        volumes:
+        - name: mlpipeline-metrics
+          emptyDir: {}
+        metadata:
+          labels:
+            elyra/node-type: notebook-script
+            elyra/pipeline-name: sanity_check
+            elyra/pipeline-version: ''
+            elyra/experiment-name: ''
+            elyra/node-name: test_security
+            pipelines.kubeflow.org/cache_enabled: "true"
+          annotations:
+            elyra/node-file-name: insurance-claim-processing/lab-materials/03/test_security.py
+            elyra/pipeline-source: sanity_check.pipeline
+            pipelines.kubeflow.org/task_display_name: test_security
+            pipelines.kubeflow.org/component_spec_digest: '{"name": "Run a file",
+              "outputs": [], "version": "Run a file@sha256=a523a1dc72737c6cfaf07342df759a61f68c9b45bf34ab33d3f20cb496ae8083"}'
+    - name: run-a-file-4
+      taskSpec:
+        steps:
+        - name: main
+          args:
+          - |
+            sh -c "mkdir -p ./jupyter-work-dir && cd ./jupyter-work-dir"
+            sh -c "echo 'Downloading file:///opt/app-root/bin/utils/bootstrapper.py' && curl --fail -H 'Cache-Control: no-cache' -L file:///opt/app-root/bin/utils/bootstrapper.py --output bootstrapper.py"
+            sh -c "echo 'Downloading file:///opt/app-root/bin/utils/requirements-elyra.txt' && curl --fail -H 'Cache-Control: no-cache' -L file:///opt/app-root/bin/utils/requirements-elyra.txt --output requirements-elyra.txt"
+            sh -c "python3 -m pip install  packaging && python3 -m pip freeze > requirements-current.txt && python3 bootstrapper.py --pipeline-name 'sanity_check' --cos-endpoint 'https://minio-api-robert-serving-test.apps.rhods-internal.61tk.p1.openshiftapps.com' --cos-bucket 'pipeline-bucket' --cos-directory 'sanity_check-1215145442' --cos-dependencies-archive 'summarize_results-6b99ceae-d124-4758-904a-03e1a49fe56d.tar.gz' --file 'insurance-claim-processing/lab-materials/03/summarize_results.py' --inputs 'security_result.json;responsetime_result.json;quality_result.json' --outputs 'results.json' "
+          command:
+          - sh
+          - -c
+          env:
+          - name: AWS_ACCESS_KEY_ID
+            valueFrom:
+              secretKeyRef:
+                key: AWS_ACCESS_KEY_ID
+                name: aws-connection-minio-pipeline
+          - name: AWS_SECRET_ACCESS_KEY
+            valueFrom:
+              secretKeyRef:
+                key: AWS_SECRET_ACCESS_KEY
+                name: aws-connection-minio-pipeline
+          - name: ELYRA_RUNTIME_ENV
+            value: kfp
+          - name: ELYRA_ENABLE_PIPELINE_INFO
+            value: "True"
+          - name: ELYRA_WRITABLE_CONTAINER_DIR
+            value: /tmp
+          - name: ELYRA_RUN_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.annotations['pipelines.kubeflow.org/run_name']
+          - name: LLM_ENDPOINT
+            valueFrom:
+              secretKeyRef:
+                key: LLM_ENDPOINT
+                name: llm-info
+          - name: MODEL_SHA
+            valueFrom:
+              secretKeyRef:
+                key: MODEL_SHA
+                name: llm-info
+          - name: SERVING_SHA
+            valueFrom:
+              secretKeyRef:
+                key: SERVING_SHA
+                name: llm-info
+          image: quay.io/modh/runtime-images@sha256:7dd23e58291cad7a0ab4a8e04bda06492f2c027eb33b226358380db58dcdd60b
+        stepTemplate:
+          volumeMounts:
+          - name: mlpipeline-metrics
+            mountPath: /tmp
+        volumes:
+        - name: mlpipeline-metrics
+          emptyDir: {}
+        metadata:
+          labels:
+            elyra/node-type: notebook-script
+            elyra/pipeline-name: sanity_check
+            elyra/pipeline-version: ''
+            elyra/experiment-name: ''
+            elyra/node-name: summarize_results
+            pipelines.kubeflow.org/cache_enabled: "true"
+          annotations:
+            elyra/node-file-name: insurance-claim-processing/lab-materials/03/summarize_results.py
+            elyra/pipeline-source: sanity_check.pipeline
+            pipelines.kubeflow.org/task_display_name: summarize_results
+            pipelines.kubeflow.org/component_spec_digest: '{"name": "Run a file",
+              "outputs": [], "version": "Run a file@sha256=9e2cfcee5f1d1f9eb84f47713866930314f31eb4ea3e7f564394dce005843176"}'
+      runAfter:
+      - run-a-file
+      - run-a-file-2
+      - run-a-file-3