From 0deb02dd46060a86b8a9ebc117479645c9e21e0a Mon Sep 17 00:00:00 2001 From: jtsextonMITRE <45762017+jtsextonMITRE@users.noreply.github.com> Date: Fri, 6 Sep 2024 16:12:39 -0400 Subject: [PATCH] examples: update notebook for documentation and slight cleanliness --- .../demo.ipynb | 292 +++++++++--------- 1 file changed, 139 insertions(+), 153 deletions(-) diff --git a/examples/v1-client-tensorflow-mnist-classifier/demo.ipynb b/examples/v1-client-tensorflow-mnist-classifier/demo.ipynb index a00977d48..9566c7a07 100644 --- a/examples/v1-client-tensorflow-mnist-classifier/demo.ipynb +++ b/examples/v1-client-tensorflow-mnist-classifier/demo.ipynb @@ -31,11 +31,39 @@ }, { "cell_type": "code", - "execution_count": 94, + "execution_count": null, "metadata": { "tags": [] }, "outputs": [], + "source": [ + "EXPERIMENT_NAME = \"mnist_fgm\"\n", + "EXPERIMENT_DESC = \"applying the fast gradient sign (FGM) attack to a classifier trained on MNIST\"\n", + "QUEUE_NAME = 'tensorflow_cpu'\n", + "QUEUE_DESC = 'Tensorflow CPU Queue'\n", + "PLUGIN_FILES = '../task-plugins/dioptra_custom/vc/'\n", + "MODEL_NAME = \"mnist_classifier\"\n", + "\n", + "# Default address for accessing the RESTful API service\n", + "RESTAPI_ADDRESS = \"http://localhost:20080\"\n", + "\n", + "# Set DIOPTRA_RESTAPI_URI variable if not defined, used to connect to RESTful API service\n", + "if os.getenv(\"DIOPTRA_RESTAPI_URI\") is None:\n", + " os.environ[\"DIOPTRA_RESTAPI_URI\"] = RESTAPI_ADDRESS\n", + "\n", + "# Default address for accessing the MLFlow Tracking server\n", + "MLFLOW_TRACKING_URI = \"http://localhost:35000\"\n", + "\n", + "# Set MLFLOW_TRACKING_URI variable, used to connect to MLFlow Tracking service\n", + "if os.getenv(\"MLFLOW_TRACKING_URI\") is None:\n", + " os.environ[\"MLFLOW_TRACKING_URI\"] = MLFLOW_TRACKING_URI" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "# Import packages from the Python standard library\n", "import importlib.util\n", @@ -46,7 +74,15 @@ "import warnings\n", "from pathlib import Path\n", "from IPython.display import display, clear_output\n", + "import logging\n", + "import structlog\n", + "import yaml\n", "\n", + "# Filter out warning messages\n", + "warnings.filterwarnings(\"ignore\")\n", + "structlog.configure(\n", + " wrapper_class=structlog.make_filtering_bound_logger(logging.CRITICAL),\n", + ")\n", "\n", "def register_python_source_file(module_name: str, filepath: Path) -> None:\n", " \"\"\"Import a source file directly.\n", @@ -63,42 +99,11 @@ " module = importlib.util.module_from_spec(spec)\n", " sys.modules[module_name] = module\n", " spec.loader.exec_module(module)\n", - "\n", - "\n", - "# Filter out warning messages\n", - "warnings.filterwarnings(\"ignore\")\n", - "\n", - "# Experiment name\n", - "EXPERIMENT_NAME = \"mnist_fgm\"\n", - "EXPERIMENT_DESC = \"applying the fast gradient sign (FGM) attack to a classifier trained on MNIST\"\n", - "\n", - "# Default address for accessing the RESTful API service\n", - "RESTAPI_ADDRESS = \"http://localhost:20080\"\n", - "\n", - "# Set DIOPTRA_RESTAPI_URI variable if not defined, used to connect to RESTful API service\n", - "if os.getenv(\"DIOPTRA_RESTAPI_URI\") is None:\n", - " os.environ[\"DIOPTRA_RESTAPI_URI\"] = RESTAPI_ADDRESS\n", - "\n", - "# Default address for accessing the MLFlow Tracking server\n", - "MLFLOW_TRACKING_URI = \"http://localhost:35000\"\n", - "\n", - "# Set MLFLOW_TRACKING_URI variable, used to connect to MLFlow Tracking service\n", - "if os.getenv(\"MLFLOW_TRACKING_URI\") is None:\n", - " os.environ[\"MLFLOW_TRACKING_URI\"] = MLFLOW_TRACKING_URI\n", - "\n", - "\n", - "# Register the examples/scripts directory as a Python module\n", "register_python_source_file(\"scripts\", Path(\"..\", \"scripts\", \"__init__.py\"))\n", "\n", + "# Register the examples/scripts directory as a Python module\n", "from scripts.client import DioptraClient\n", - "from scripts.utils import make_tar\n", - "\n", - "# Import third-party Python packages\n", - "import numpy as np\n", - "from mlflow.tracking import MlflowClient\n", - "\n", - "# Create random number generator\n", - "rng = np.random.default_rng(54399264723942495723666216079516778448)" + "from scripts.utils import make_tar" ] }, { @@ -171,7 +176,7 @@ }, { "cell_type": "code", - "execution_count": 79, + "execution_count": null, "metadata": { "tags": [] }, @@ -180,30 +185,18 @@ "client = DioptraClient()" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It is necessary to login to the RESTAPI to be able to perform any functions. Here we create a user if it is not created already, and login with it." + ] + }, { "cell_type": "code", - "execution_count": 80, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[2m2024-09-06 15:08:50\u001b[0m [\u001b[31m\u001b[1merror \u001b[0m] \u001b[1mError code 400 returned. \u001b[0m \u001b[36mdata\u001b[0m=\u001b[35m{'username': 'pluginuser', 'email': 'pluginuser@dioptra.nccoe.nist.gov', 'password': 'pleasemakesuretoPLUGINthecomputer', 'confirmPassword': 'pleasemakesuretoPLUGINthecomputer'}\u001b[0m \u001b[36mmethod\u001b[0m=\u001b[35mPOST\u001b[0m \u001b[36mresponse\u001b[0m=\u001b[35m{\"message\": \"Bad Request - The username on the registration form is not available. Please select another and resubmit.\"}\n", - "\u001b[0m \u001b[36murl\u001b[0m=\u001b[35mhttp://localhost:20080/api/v1/users/\u001b[0m\n" - ] - }, - { - "data": { - "text/plain": [ - "{'username': 'pluginuser', 'status': 'Login successful'}" - ] - }, - "execution_count": 80, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "try:\n", " client.users.create('pluginuser','pluginuser@dioptra.nccoe.nist.gov','pleasemakesuretoPLUGINthecomputer','pleasemakesuretoPLUGINthecomputer')\n", @@ -216,14 +209,12 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We need to register an experiment under which to collect our job runs.\n", - "The code below checks if the relevant experiment named `\"mnist\"` exists.\n", - "If it does, then it just returns info about the experiment, if it doesn't, it then registers the new experiment." + "The following function can be used to clear all experiments, entrypoints, jobs, models, plugins, tags, and queues in the database, if a fresh start is desired. It is not currently used anywhere in this notebook, but is included for utility." ] }, { "cell_type": "code", - "execution_count": 87, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -231,18 +222,18 @@ " for d in client.experiments.get_all(pageLength=100000)['data']:\n", " client.experiments.delete_by_id(d['id'])\n", " for d in client.entrypoints.get_all(pageLength=100000)['data']:\n", - " client.entrypoints.delete_by_id(d['id'])\n", + " client.entrypoints.delete_by_id(d['id'])\n", " for d in client.jobs.get_all(pageLength=100000)['data']:\n", - " client.jobs.delete_by_id(d['id'])\n", + " client.jobs.delete_by_id(d['id'])\n", " for d in client.models.get_all(pageLength=100000)['data']:\n", - " client.models.delete_by_id(d['id'])\n", + " client.models.delete_by_id(d['id'])\n", " for d in client.plugins.get_all(pageLength=100000)['data']:\n", " try:\n", " client.plugins.delete_by_id(d['id'])\n", " except:\n", " pass\n", " for d in client.tags.get_all(pageLength=100000)['data']:\n", - " client.tags.delete_by_id(d['id'])\n", + " client.tags.delete_by_id(d['id'])\n", " for d in client.pluginParameterTypes.get_all(pageLength=100000)['data']:\n", " try:\n", " client.pluginParameterTypes.delete_by_id(d['id'])\n", @@ -252,24 +243,24 @@ " client.queues.delete_by_id(d['id'])" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The following functions are used for registering plugins located in the `../examples/task-plugins/` folder, associating them with endpoints in the ./src/ folder, and then associating those endpoints with an experiment. When `run_experiment` is called, it will create plugins based on the YML files provided, and upload any additional files in the directory specified by `PLUGIN_FILES` at the top of the notebook." + ] + }, { "cell_type": "code", - "execution_count": 88, + "execution_count": null, "metadata": { "scrolled": true, "tags": [] }, "outputs": [], "source": [ - "import logging\n", - "import structlog\n", - "\n", - "structlog.configure(\n", - " wrapper_class=structlog.make_filtering_bound_logger(logging.CRITICAL),\n", - ")\n", - "\n", - "import yaml\n", "basic_types = ['integer', 'string', 'number', 'any', 'boolean', 'null']\n", + "\n", "def create_or_get_experiment(group, name, description, entrypoints):\n", " found = None\n", " for exp in client.experiments.get_all(search=name,pageLength=100000)['data']:\n", @@ -486,37 +477,29 @@ ] }, { - "cell_type": "code", - "execution_count": 89, - "metadata": {}, - "outputs": [], - "source": [ - "queue_name = 'tensorflow_cpu'\n", - "queue_desc = 'Tensorflow CPU Queue'\n", - "plugin_files = '../task-plugins/dioptra_custom/vc/'\n", - "job_time_limit = '1h'\n", - "model_name = \"mnist_classifier\"" - ] - }, - { - "cell_type": "code", - "execution_count": 90, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "delete_all()" + "`run_experiment` uses the helper functions above to do the following tasks:\n", + " - create a queue specified by `QUEUE_NAME` if needed\n", + " - upload the plugins used by the specified `entrypoint` \n", + " - upload any other plugin files in the directory `PLUGIN_FILES`\n", + " - register the entrypoint in Dioptra\n", + " - create the experiment (if needed) and associate the entrypoint with the experiment\n", + " - start a job for the specified `entrypoint` on the queue `QUEUE_NAME`\n", + "Note that any parameters passed in to `parameters` will overwrite the defaults in the specified YML file." ] }, { "cell_type": "code", - "execution_count": 91, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "def run_experiment(entrypoint, entrypoint_name, entrypoint_desc, parameters={}):\n", + "def run_experiment(entrypoint, entrypoint_name, entrypoint_desc, job_time_limit, parameters={}):\n", " upload = get_plugins_to_register(entrypoint, {})\n", - " upload = add_missing_plugin_files(plugin_files, upload)\n", - " queue = create_or_get_queue(1, queue_name, queue_desc)\n", + " upload = add_missing_plugin_files(PLUGIN_FILES, upload)\n", + " queue = create_or_get_queue(1, QUEUE_NAME, QUEUE_DESC)\n", " queues = [queue['id']]\n", " plugins = register_plugins(1,upload)\n", " entrypoint = register_entrypoint(1, entrypoint_name, entrypoint_desc, queues, plugins, entrypoint, parameters)\n", @@ -525,23 +508,15 @@ ] }, { - "cell_type": "code", - "execution_count": 92, - "metadata": { - "scrolled": true - }, - "outputs": [], + "cell_type": "markdown", + "metadata": {}, "source": [ - "entrypoint = 'src/train.yml'\n", - "entrypoint_name = 'train'\n", - "entrypoint_desc = 'training a classifier on MNIST'\n", - "\n", - "training_job = run_experiment(entrypoint, entrypoint_name, entrypoint_desc, {\"epochs_p\":1})\n" + "`wait_for_job` stalls til the previous job was finished, which is useful for jobs which depend on the output of other jobs." ] }, { "cell_type": "code", - "execution_count": 104, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -559,80 +534,91 @@ ] }, { - "cell_type": "code", - "execution_count": 105, + "cell_type": "markdown", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'Job finished. Starting next job'" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], "source": [ - "entrypoint = 'src/fgm.yml'\n", - "entrypoint_name = 'fgm'\n", - "entrypoint_desc = 'generating examples on mnist_classifier using the fgm attack'\n", - "\n", - "wait_for_job(training_job, entrypoint_name)\n", - "fgm_job = run_experiment(entrypoint, entrypoint_name, entrypoint_desc, {\"training_job_id\": training_job['id']})\n" + "Next, we need to train our model. This particular entrypoint uses a LeNet-5 model.\n", + "Depending on the specs of your computer, it can take 5-20 minutes or longer to complete.\n", + "If you are fortunate enough to have access to a dedicated GPU, then the training time will be much shorter." ] }, { "cell_type": "code", - "execution_count": 106, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'Job finished. Starting next job'" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], "source": [ - "entrypoint = 'src/infer.yml'\n", - "entrypoint_name = 'infer'\n", - "entrypoint_desc = 'evaluating performance of mnist_classifier on generated fgm examples'\n", + "entrypoint = 'src/train.yml'\n", + "entrypoint_name = 'train'\n", + "entrypoint_desc = 'training a classifier on MNIST'\n", + "job_time_limit = '1h'\n", "\n", - "wait_for_job(fgm_job, entrypoint_name)\n", - "infer_job = run_experiment(entrypoint, entrypoint_name, entrypoint_desc, {\"fgm_job_id\": fgm_job['id'], \"training_job_id\": training_job['id']})\n" + "training_job = run_experiment(entrypoint, \n", + " entrypoint_name, \n", + " entrypoint_desc,\n", + " job_time_limit,\n", + " {\"epochs_p\":1})\n" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], - "source": [] + "source": [ + "Now that we have trained a model, next we will apply the fast-gradient method (FGM) evasion attack on it to generate adversarial images.\n", + "\n", + "This specific workflow is an example of jobs that contain dependencies, as the metric evaluation jobs cannot start until the adversarial image generation jobs have completed, and the adversarial image generation job cannot start until the training job has completed.\n", + "\n", + "Note that the training_job id is needed to tell the FGM attack which model to generate examples against." + ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "entrypoint = 'src/fgm.yml'\n", + "entrypoint_name = 'fgm'\n", + "entrypoint_desc = 'generating examples on mnist_classifier using the fgm attack'\n", + "job_time_limit = '1h'\n", + "\n", + "wait_for_job(training_job, entrypoint_name)\n", + "fgm_job = run_experiment(entrypoint,\n", + " entrypoint_name,\n", + " entrypoint_desc,\n", + " job_time_limit,\n", + " {\"training_job_id\": training_job['id']})\n" + ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], - "source": [] + "source": [ + "Finally, we can test out the results of our adversarial attack on the model we trained earlier. This will wait for the FGM job to finish, and then evaluate the model's performance on the adversarial examples. Note that we need to know both the `fgm_job` id as well as the `training_job` id, so that this entrypoint knows which run's adversarial examples to test against which model. \n", + "\n", + "The previous runs are all stored in Dioptra as well, so you can always go back later and retrieve examples, models, and even the code used to create them." + ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "entrypoint = 'src/infer.yml'\n", + "entrypoint_name = 'infer'\n", + "entrypoint_desc = 'evaluating performance of mnist_classifier on generated fgm examples'\n", + "job_time_limit = '1h'\n", + "\n", + "wait_for_job(fgm_job, entrypoint_name)\n", + "infer_job = run_experiment(entrypoint, \n", + " entrypoint_name,\n", + " entrypoint_desc,\n", + " job_time_limit,\n", + " {\"fgm_job_id\": fgm_job['id'], \"training_job_id\": training_job['id']})\n" + ] } ], "metadata": {