diff --git a/content/modules/ROOT/assets/images/03/07-create-run.png b/content/modules/ROOT/assets/images/03/07-create-run.png new file mode 100644 index 00000000..c667dfff Binary files /dev/null and b/content/modules/ROOT/assets/images/03/07-create-run.png differ diff --git a/content/modules/ROOT/assets/images/03/07-elyra-pipeline.png b/content/modules/ROOT/assets/images/03/07-elyra-pipeline.png new file mode 100644 index 00000000..4ef52b41 Binary files /dev/null and b/content/modules/ROOT/assets/images/03/07-elyra-pipeline.png differ diff --git a/content/modules/ROOT/pages/03-07-sanity-check.adoc b/content/modules/ROOT/pages/03-07-sanity-check.adoc index eda33fe2..87585bc4 100644 --- a/content/modules/ROOT/pages/03-07-sanity-check.adoc +++ b/content/modules/ROOT/pages/03-07-sanity-check.adoc @@ -1,10 +1,45 @@ +:imagesdir: ../assets/images = Sanity-Check Pipeline +== What will the pipeline do? +To make sure that everything works as we would expect it, and that the model has not been tampered with, we create a sanity pipeline that tests the model through its endoint. + +We will test the response time, the response quality, and that the model hash has not changed. + == Deploy a pipeline Server in your project +Start by deploying a pipeline server that can we can use to run pipelines with: + +- Go to your Data Science Project +- Scroll down to Pipelines and press "Create a pipeline server" +- Select the existing data connection "Pipelines" +- Press Create + +We need to connect the pipeline server with a data connection as the pipeline server will use the S3 storage to store logs and other artifacts. == Deploy a sanity-check pipeline +There are two pipeline files, one *sanity_check.pipeline* and one *sanity_check.yaml* file. + +The .pipeline file can be opened in Elyra to be visualliy modified and ran, while the .yaml file can be imported into the pipeline server through the dashboard. + +Here we will be running the pipeline through Elyra. == Ad-Hoc execution +Running it through Elyra is the same as doing an ad-hoc execution of the pipeline, as opposed to importing the pipeline which won't automatically execute it. + +Start by going to your workbench and open up the .pipeline file. + +Here we can see that the pipeline consists of 3 tests: a response quality test, a response time test, and a security test. Feel free to peek into each of the python files by double clicking on the nodes to see what they do. + +After the tests have been ran, we have a final function that will summarize the results and log them. + +To run the pipeline, press the Play button in the menu bar. + +image::03/07-elyra-pipeline.png[elyra sanity pipeline] + +When you get a popup that says "Job submission to Data Science Pipelines succeeded", press the link "Run details" to see how the pipeline is progessing. == Schedule execution +We can also schedule an execution so that the sanity check is ran at appropriate intervals. + +To do that, go back to the dashboard, find the pipeline you just ran, press the 3 dots at the very right, and press "Create run". + + +image::03/07-create-run.png[create run] +From this page, you are able to create both ad-hoc and scheduled pipeline runs. + +Name your pipeline something like "Scheduled Sanity Pipeline", select the option "Schedule recurring run", set it to run periodically every hour, and press Create. + +We can now leave the sanity pipeline to do its thing while we are comfortable that it will start failing if something is wrong with our LLM setup. \ No newline at end of file diff --git a/content/modules/ROOT/pages/05-05-process-claims.adoc b/content/modules/ROOT/pages/05-05-process-claims.adoc new file mode 100644 index 00000000..3fde532d --- /dev/null +++ b/content/modules/ROOT/pages/05-05-process-claims.adoc @@ -0,0 +1,6 @@ +:imagesdir: ../assets/images += Pipeline for processing claims + +== Run the pipeline + +== Check the results \ No newline at end of file diff --git a/lab-materials/03/07/sanity_check.yaml b/lab-materials/03/07/sanity_check.yaml new file mode 100644 index 00000000..ce777e8a --- /dev/null +++ b/lab-materials/03/07/sanity_check.yaml @@ -0,0 +1,327 @@ +apiVersion: tekton.dev/v1beta1 +kind: PipelineRun +metadata: + name: sanity-check + annotations: + tekton.dev/output_artifacts: '{"run-a-file": [{"key": "artifacts/$PIPELINERUN/run-a-file/mlpipeline-metrics.tgz", + "name": "mlpipeline-metrics", "path": "/tmp/mlpipeline-metrics.json"}, {"key": + "artifacts/$PIPELINERUN/run-a-file/mlpipeline-ui-metadata.tgz", "name": "mlpipeline-ui-metadata", + "path": "/tmp/mlpipeline-ui-metadata.json"}], "run-a-file-2": [{"key": "artifacts/$PIPELINERUN/run-a-file-2/mlpipeline-metrics.tgz", + "name": "mlpipeline-metrics", "path": "/tmp/mlpipeline-metrics.json"}, {"key": + "artifacts/$PIPELINERUN/run-a-file-2/mlpipeline-ui-metadata.tgz", "name": "mlpipeline-ui-metadata", + "path": "/tmp/mlpipeline-ui-metadata.json"}], "run-a-file-3": [{"key": "artifacts/$PIPELINERUN/run-a-file-3/mlpipeline-metrics.tgz", + "name": "mlpipeline-metrics", "path": "/tmp/mlpipeline-metrics.json"}, {"key": + "artifacts/$PIPELINERUN/run-a-file-3/mlpipeline-ui-metadata.tgz", "name": "mlpipeline-ui-metadata", + "path": "/tmp/mlpipeline-ui-metadata.json"}], "run-a-file-4": [{"key": "artifacts/$PIPELINERUN/run-a-file-4/mlpipeline-metrics.tgz", + "name": "mlpipeline-metrics", "path": "/tmp/mlpipeline-metrics.json"}, {"key": + "artifacts/$PIPELINERUN/run-a-file-4/mlpipeline-ui-metadata.tgz", "name": "mlpipeline-ui-metadata", + "path": "/tmp/mlpipeline-ui-metadata.json"}]}' + tekton.dev/input_artifacts: '{}' + tekton.dev/artifact_bucket: mlpipeline + tekton.dev/artifact_endpoint: minio-service.kubeflow:9000 + tekton.dev/artifact_endpoint_scheme: http:// + tekton.dev/artifact_items: '{"run-a-file": [["mlpipeline-metrics", "/tmp/mlpipeline-metrics.json"], + ["mlpipeline-ui-metadata", "/tmp/mlpipeline-ui-metadata.json"]], "run-a-file-2": + [["mlpipeline-metrics", "/tmp/mlpipeline-metrics.json"], ["mlpipeline-ui-metadata", + "/tmp/mlpipeline-ui-metadata.json"]], "run-a-file-3": [["mlpipeline-metrics", + "/tmp/mlpipeline-metrics.json"], ["mlpipeline-ui-metadata", "/tmp/mlpipeline-ui-metadata.json"]], + "run-a-file-4": [["mlpipeline-metrics", "/tmp/mlpipeline-metrics.json"], ["mlpipeline-ui-metadata", + "/tmp/mlpipeline-ui-metadata.json"]]}' + sidecar.istio.io/inject: "false" + tekton.dev/template: '' + pipelines.kubeflow.org/big_data_passing_format: $(workspaces.$TASK_NAME.path)/artifacts/$ORIG_PR_NAME/$TASKRUN_NAME/$TASK_PARAM_NAME + pipelines.kubeflow.org/pipeline_spec: '{"name": "sanity_check"}' + labels: + pipelines.kubeflow.org/pipelinename: '' + pipelines.kubeflow.org/generation: '' +spec: + pipelineSpec: + tasks: + - name: run-a-file + taskSpec: + steps: + - name: main + args: + - | + sh -c "mkdir -p ./jupyter-work-dir && cd ./jupyter-work-dir" + sh -c "echo 'Downloading file:///opt/app-root/bin/utils/bootstrapper.py' && curl --fail -H 'Cache-Control: no-cache' -L file:///opt/app-root/bin/utils/bootstrapper.py --output bootstrapper.py" + sh -c "echo 'Downloading file:///opt/app-root/bin/utils/requirements-elyra.txt' && curl --fail -H 'Cache-Control: no-cache' -L file:///opt/app-root/bin/utils/requirements-elyra.txt --output requirements-elyra.txt" + sh -c "python3 -m pip install packaging && python3 -m pip freeze > requirements-current.txt && python3 bootstrapper.py --pipeline-name 'sanity_check' --cos-endpoint 'https://minio-api-robert-serving-test.apps.rhods-internal.61tk.p1.openshiftapps.com' --cos-bucket 'pipeline-bucket' --cos-directory 'sanity_check-1215145442' --cos-dependencies-archive 'test_response_quality-d0510f0c-fc50-42fc-a598-97e26b34ed88.tar.gz' --file 'insurance-claim-processing/lab-materials/03/test_response_quality.py' --outputs 'quality_result.json' " + command: + - sh + - -c + env: + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: AWS_ACCESS_KEY_ID + name: aws-connection-minio-pipeline + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: AWS_SECRET_ACCESS_KEY + name: aws-connection-minio-pipeline + - name: ELYRA_RUNTIME_ENV + value: kfp + - name: ELYRA_ENABLE_PIPELINE_INFO + value: "True" + - name: ELYRA_WRITABLE_CONTAINER_DIR + value: /tmp + - name: ELYRA_RUN_NAME + valueFrom: + fieldRef: + fieldPath: metadata.annotations['pipelines.kubeflow.org/run_name'] + - name: LLM_ENDPOINT + valueFrom: + secretKeyRef: + key: LLM_ENDPOINT + name: llm-info + - name: MODEL_SHA + valueFrom: + secretKeyRef: + key: MODEL_SHA + name: llm-info + - name: SERVING_SHA + valueFrom: + secretKeyRef: + key: SERVING_SHA + name: llm-info + image: quay.io/rlundber/rh1/sanity_pipeline:1.7 + stepTemplate: + volumeMounts: + - name: mlpipeline-metrics + mountPath: /tmp + volumes: + - name: mlpipeline-metrics + emptyDir: {} + metadata: + labels: + elyra/node-type: notebook-script + elyra/pipeline-name: sanity_check + elyra/pipeline-version: '' + elyra/experiment-name: '' + elyra/node-name: test_response_quality + pipelines.kubeflow.org/cache_enabled: "true" + annotations: + elyra/node-file-name: insurance-claim-processing/lab-materials/03/test_response_quality.py + elyra/pipeline-source: sanity_check.pipeline + pipelines.kubeflow.org/task_display_name: test_response_quality + pipelines.kubeflow.org/component_spec_digest: '{"name": "Run a file", + "outputs": [], "version": "Run a file@sha256=7e955de34869fba374b1d87fd591922910d2f2490d11e7fdd1312ae0749a0742"}' + - name: run-a-file-2 + taskSpec: + steps: + - name: main + args: + - | + sh -c "mkdir -p ./jupyter-work-dir && cd ./jupyter-work-dir" + sh -c "echo 'Downloading file:///opt/app-root/bin/utils/bootstrapper.py' && curl --fail -H 'Cache-Control: no-cache' -L file:///opt/app-root/bin/utils/bootstrapper.py --output bootstrapper.py" + sh -c "echo 'Downloading file:///opt/app-root/bin/utils/requirements-elyra.txt' && curl --fail -H 'Cache-Control: no-cache' -L file:///opt/app-root/bin/utils/requirements-elyra.txt --output requirements-elyra.txt" + sh -c "python3 -m pip install packaging && python3 -m pip freeze > requirements-current.txt && python3 bootstrapper.py --pipeline-name 'sanity_check' --cos-endpoint 'https://minio-api-robert-serving-test.apps.rhods-internal.61tk.p1.openshiftapps.com' --cos-bucket 'pipeline-bucket' --cos-directory 'sanity_check-1215145442' --cos-dependencies-archive 'test_responsetime-c50bb14f-a036-4af1-b5dc-21e48eb80f7f.tar.gz' --file 'insurance-claim-processing/lab-materials/03/test_responsetime.py' --outputs 'responsetime_result.json' " + command: + - sh + - -c + env: + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: AWS_ACCESS_KEY_ID + name: aws-connection-minio-pipeline + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: AWS_SECRET_ACCESS_KEY + name: aws-connection-minio-pipeline + - name: ELYRA_RUNTIME_ENV + value: kfp + - name: ELYRA_ENABLE_PIPELINE_INFO + value: "True" + - name: ELYRA_WRITABLE_CONTAINER_DIR + value: /tmp + - name: ELYRA_RUN_NAME + valueFrom: + fieldRef: + fieldPath: metadata.annotations['pipelines.kubeflow.org/run_name'] + - name: LLM_ENDPOINT + valueFrom: + secretKeyRef: + key: LLM_ENDPOINT + name: llm-info + - name: MODEL_SHA + valueFrom: + secretKeyRef: + key: MODEL_SHA + name: llm-info + - name: SERVING_SHA + valueFrom: + secretKeyRef: + key: SERVING_SHA + name: llm-info + image: quay.io/modh/runtime-images@sha256:7dd23e58291cad7a0ab4a8e04bda06492f2c027eb33b226358380db58dcdd60b + stepTemplate: + volumeMounts: + - name: mlpipeline-metrics + mountPath: /tmp + volumes: + - name: mlpipeline-metrics + emptyDir: {} + metadata: + labels: + elyra/node-type: notebook-script + elyra/pipeline-name: sanity_check + elyra/pipeline-version: '' + elyra/experiment-name: '' + elyra/node-name: test_responsetime + pipelines.kubeflow.org/cache_enabled: "true" + annotations: + elyra/node-file-name: insurance-claim-processing/lab-materials/03/test_responsetime.py + elyra/pipeline-source: sanity_check.pipeline + pipelines.kubeflow.org/task_display_name: test_responsetime + pipelines.kubeflow.org/component_spec_digest: '{"name": "Run a file", + "outputs": [], "version": "Run a file@sha256=afa090f686bb464016ab6640a960b8e31b88c415166d799c2c734f6e9ba58f6a"}' + - name: run-a-file-3 + taskSpec: + steps: + - name: main + args: + - | + sh -c "mkdir -p ./jupyter-work-dir && cd ./jupyter-work-dir" + sh -c "echo 'Downloading file:///opt/app-root/bin/utils/bootstrapper.py' && curl --fail -H 'Cache-Control: no-cache' -L file:///opt/app-root/bin/utils/bootstrapper.py --output bootstrapper.py" + sh -c "echo 'Downloading file:///opt/app-root/bin/utils/requirements-elyra.txt' && curl --fail -H 'Cache-Control: no-cache' -L file:///opt/app-root/bin/utils/requirements-elyra.txt --output requirements-elyra.txt" + sh -c "python3 -m pip install packaging && python3 -m pip freeze > requirements-current.txt && python3 bootstrapper.py --pipeline-name 'sanity_check' --cos-endpoint 'https://minio-api-robert-serving-test.apps.rhods-internal.61tk.p1.openshiftapps.com' --cos-bucket 'pipeline-bucket' --cos-directory 'sanity_check-1215145442' --cos-dependencies-archive 'test_security-6b595dc7-afb8-46bb-bf52-7cd695ddafb8.tar.gz' --file 'insurance-claim-processing/lab-materials/03/test_security.py' --outputs 'security_result.json' " + command: + - sh + - -c + env: + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: AWS_ACCESS_KEY_ID + name: aws-connection-minio-pipeline + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: AWS_SECRET_ACCESS_KEY + name: aws-connection-minio-pipeline + - name: ELYRA_RUNTIME_ENV + value: kfp + - name: ELYRA_ENABLE_PIPELINE_INFO + value: "True" + - name: ELYRA_WRITABLE_CONTAINER_DIR + value: /tmp + - name: ELYRA_RUN_NAME + valueFrom: + fieldRef: + fieldPath: metadata.annotations['pipelines.kubeflow.org/run_name'] + - name: LLM_ENDPOINT + valueFrom: + secretKeyRef: + key: LLM_ENDPOINT + name: llm-info + - name: MODEL_SHA + valueFrom: + secretKeyRef: + key: MODEL_SHA + name: llm-info + - name: SERVING_SHA + valueFrom: + secretKeyRef: + key: SERVING_SHA + name: llm-info + image: quay.io/modh/runtime-images@sha256:7dd23e58291cad7a0ab4a8e04bda06492f2c027eb33b226358380db58dcdd60b + stepTemplate: + volumeMounts: + - name: mlpipeline-metrics + mountPath: /tmp + volumes: + - name: mlpipeline-metrics + emptyDir: {} + metadata: + labels: + elyra/node-type: notebook-script + elyra/pipeline-name: sanity_check + elyra/pipeline-version: '' + elyra/experiment-name: '' + elyra/node-name: test_security + pipelines.kubeflow.org/cache_enabled: "true" + annotations: + elyra/node-file-name: insurance-claim-processing/lab-materials/03/test_security.py + elyra/pipeline-source: sanity_check.pipeline + pipelines.kubeflow.org/task_display_name: test_security + pipelines.kubeflow.org/component_spec_digest: '{"name": "Run a file", + "outputs": [], "version": "Run a file@sha256=a523a1dc72737c6cfaf07342df759a61f68c9b45bf34ab33d3f20cb496ae8083"}' + - name: run-a-file-4 + taskSpec: + steps: + - name: main + args: + - | + sh -c "mkdir -p ./jupyter-work-dir && cd ./jupyter-work-dir" + sh -c "echo 'Downloading file:///opt/app-root/bin/utils/bootstrapper.py' && curl --fail -H 'Cache-Control: no-cache' -L file:///opt/app-root/bin/utils/bootstrapper.py --output bootstrapper.py" + sh -c "echo 'Downloading file:///opt/app-root/bin/utils/requirements-elyra.txt' && curl --fail -H 'Cache-Control: no-cache' -L file:///opt/app-root/bin/utils/requirements-elyra.txt --output requirements-elyra.txt" + sh -c "python3 -m pip install packaging && python3 -m pip freeze > requirements-current.txt && python3 bootstrapper.py --pipeline-name 'sanity_check' --cos-endpoint 'https://minio-api-robert-serving-test.apps.rhods-internal.61tk.p1.openshiftapps.com' --cos-bucket 'pipeline-bucket' --cos-directory 'sanity_check-1215145442' --cos-dependencies-archive 'summarize_results-6b99ceae-d124-4758-904a-03e1a49fe56d.tar.gz' --file 'insurance-claim-processing/lab-materials/03/summarize_results.py' --inputs 'security_result.json;responsetime_result.json;quality_result.json' --outputs 'results.json' " + command: + - sh + - -c + env: + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: AWS_ACCESS_KEY_ID + name: aws-connection-minio-pipeline + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: AWS_SECRET_ACCESS_KEY + name: aws-connection-minio-pipeline + - name: ELYRA_RUNTIME_ENV + value: kfp + - name: ELYRA_ENABLE_PIPELINE_INFO + value: "True" + - name: ELYRA_WRITABLE_CONTAINER_DIR + value: /tmp + - name: ELYRA_RUN_NAME + valueFrom: + fieldRef: + fieldPath: metadata.annotations['pipelines.kubeflow.org/run_name'] + - name: LLM_ENDPOINT + valueFrom: + secretKeyRef: + key: LLM_ENDPOINT + name: llm-info + - name: MODEL_SHA + valueFrom: + secretKeyRef: + key: MODEL_SHA + name: llm-info + - name: SERVING_SHA + valueFrom: + secretKeyRef: + key: SERVING_SHA + name: llm-info + image: quay.io/modh/runtime-images@sha256:7dd23e58291cad7a0ab4a8e04bda06492f2c027eb33b226358380db58dcdd60b + stepTemplate: + volumeMounts: + - name: mlpipeline-metrics + mountPath: /tmp + volumes: + - name: mlpipeline-metrics + emptyDir: {} + metadata: + labels: + elyra/node-type: notebook-script + elyra/pipeline-name: sanity_check + elyra/pipeline-version: '' + elyra/experiment-name: '' + elyra/node-name: summarize_results + pipelines.kubeflow.org/cache_enabled: "true" + annotations: + elyra/node-file-name: insurance-claim-processing/lab-materials/03/summarize_results.py + elyra/pipeline-source: sanity_check.pipeline + pipelines.kubeflow.org/task_display_name: summarize_results + pipelines.kubeflow.org/component_spec_digest: '{"name": "Run a file", + "outputs": [], "version": "Run a file@sha256=9e2cfcee5f1d1f9eb84f47713866930314f31eb4ea3e7f564394dce005843176"}' + runAfter: + - run-a-file + - run-a-file-2 + - run-a-file-3