ci_tests/model_convergence.ipynb

{
    "cells": [
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "# Model Convergence and Test/Validation Analysis"
            ]
        },
        {
            "attachments": {},
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "### The Mission\n",
                "\n",
                "This is a **very basic** continuous validation test - which acts more as a playground for you to do analysis of train and validation loss. You may test for general acceptability of the validation loss/accuracy or even set thresholds for e.g. underfitting and overfitting.\n",
                "In this test, we download both binaries for the pickled **train_loss.pkl** and **val_los.pkl** from the Bucket in our Cloud Object Storage instance.\n",
                "\n",
                "Happy testing! 🥳"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 1,
            "metadata": {},
            "outputs": [],
            "source": [
                "import os\n",
                "import pickle\n",
                "from ibm_watson_studio_pipelines import WSPipelines"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "# Get Cloud API Key from Pipelines Params (Global Parameter)\n",
                "CLOUD_API_KEY = os.getenv('cloud_api_key')"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 3,
            "metadata": {},
            "outputs": [],
            "source": [
                "## Retrieve cos credentials from pipeline parameters\n",
                "import json\n",
                "# Get json from environment and convert to string\n",
                "project_cos_credentials = json.loads(os.environ['project_cos_credentials'])\n",
                "mlops_cos_credentials = json.loads(os.getenv('mlops_cos_credentials'))\n",
                "\n",
                "## PROJECT COS \n",
                "AUTH_ENDPOINT = project_cos_credentials['AUTH_ENDPOINT']\n",
                "#ENDPOINT_URL = project_cos_credentials['ENDPOINT_URL']\n",
                "#API_KEY_COS = project_cos_credentials['API_KEY']\n",
                "#BUCKET_PROJECT_COS = project_cos_credentials['BUCKET']\n",
                "\n",
                "## MLOPS COS\n",
                "ENDPOINT_URL_MLOPS = mlops_cos_credentials['ENDPOINT_URL']\n",
                "API_KEY_MLOPS = mlops_cos_credentials['API_KEY']\n",
                "CRN_MLOPS = mlops_cos_credentials['CRN']\n",
                "BUCKET_MLOPS  = mlops_cos_credentials['BUCKET']"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 4,
            "metadata": {},
            "outputs": [],
            "source": [
                "from ibm_botocore.client import Config\n",
                "import ibm_boto3\n",
                "\n",
                "def download_file_cos(local_file_name,key):\n",
                "    cos = ibm_boto3.client(service_name='s3',\n",
                "                           ibm_api_key_id=API_KEY_MLOPS,\n",
                "                           ibm_service_instance_id=CRN_MLOPS,\n",
                "                           ibm_auth_endpoint=AUTH_ENDPOINT,\n",
                "                           config=Config(signature_version='oauth'),\n",
                "                           endpoint_url=ENDPOINT_URL_MLOPS)\n",
                "    try:\n",
                "        res=cos.download_file(Bucket=BUCKET_MLOPS,Key=key,Filename=local_file_name)\n",
                "    except Exception as e:\n",
                "        print(Exception, e)\n",
                "    else:\n",
                "        print('File Downloaded')\n",
                "        \n",
                "def load_loss(key, filename):\n",
                "    download_file_cos(key,filename)\n",
                "    with open (filename,\"rb\") as f:\n",
                "        loss = pickle.load(f)\n",
                "    return loss"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 5,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "File Downloaded\n"
                    ]
                }
            ],
            "source": [
                "# Download and deserialize pickled dictionary\n",
                "train_loss = load_loss('train_loss.pkl','train_loss.pkl')\n",
                "val_loss   = load_loss('val_loss.pkl', 'val_loss.pkl')\n",
                "\n",
                "train_loss = train_loss['binary_logloss']\n",
                "val_loss = val_loss['binary_logloss']"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Test if the model training converges"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 7,
            "metadata": {},
            "outputs": [],
            "source": [
                "def check_loss_converge(loss):\n",
                "    # check every 5 iterations if the loss has decreased\n",
                "    coverges = [loss[i]>loss[i+4] for i in range(0, len(loss),5)]\n",
                "    return all(coverges)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 12,
            "metadata": {},
            "outputs": [],
            "source": [
                "training_converges = check_loss_converge(train_loss)\n",
                "validation_converges = check_loss_converge(val_loss)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Save the test results to pipeline"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 14,
            "metadata": {},
            "outputs": [],
            "source": [
                "params = {}\n",
                "params['train-loss_converges'] = training_converges\n",
                "params['val-loss_converges'] = validation_converges"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 15,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Running outside of Watson Studio Pipeline - storing results in the local filesystem for testing purposes...\n",
                        "\n",
                        "  output paths:\n",
                        "    - \"train_converge\": .ibm_watson_studio_pipelines/results/train_converge\n"
                    ]
                },
                {
                    "data": {
                        "text/plain": [
                            "<ibm_cloud_sdk_core.detailed_response.DetailedResponse at 0x7f171f61df40>"
                        ]
                    },
                    "execution_count": 15,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "pipelines_client = WSPipelines.from_apikey(apikey=CLOUD_API_KEY)\n",
                "pipelines_client.store_results(params)"
            ]
        }
    ],
    "metadata": {
        "kernelspec": {
            "display_name": "Python 3",
            "language": "python",
            "name": "python3"
        },
        "language_info": {
            "codemirror_mode": {
                "name": "ipython",
                "version": 3
            },
            "file_extension": ".py",
            "mimetype": "text/x-python",
            "name": "python",
            "nbconvert_exporter": "python",
            "pygments_lexer": "ipython3",
            "version": "3.10.10"
        },
        "vscode": {
            "interpreter": {
                "hash": "bd385fe162c5ca0c84973b7dd5c518456272446b2b64e67c2a69f949ca7a1754"
            }
        }
    },
    "nbformat": 4,
    "nbformat_minor": 1
}