Skip to content

Commit

Permalink
Fixed removing globals
Browse files Browse the repository at this point in the history
  • Loading branch information
@JaumeAmoresDS
JaumeAmoresDS committed Apr 4, 2024
1 parent 868963b commit 97c622d
Show file tree
Hide file tree
Showing 2 changed files with 126 additions and 19 deletions.
2 changes: 2 additions & 0 deletions posts/data_science/.gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,5 @@ test_pipeline
training
preprocessing
inference
*.py
*.json
143 changes: 124 additions & 19 deletions posts/data_science/hello_world.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3674,10 +3674,18 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 1,
"id": "5bcbc199",
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Writing pipeline_input.json\n"
]
}
],
"source": [
"%%writefile pipeline_input.json\n",
"{\n",
Expand Down Expand Up @@ -3706,10 +3714,39 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 11,
"id": "5138a417",
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Overwriting hello_world_pipeline.py\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Bad pipe message: %s [b'1~\\xd1B\\xe9k\\xf8\\x8beF\\x0bi8_~\\n\\xa2F \\xfec\\x82\\xc3\\xb3^P&\\xb5A\\xd2\\xbb\\xa6\\xb1\\xc5\\xef\\x04\\xb7C\\xa9\\xeb\\xb2U\\xed\\x81\\x80M\\x05\\xf7\\x85H\\xc8\\x00\\x08\\x13\\x02\\x13\\x03\\x13\\x01\\x00\\xff\\x01\\x00\\x00\\x8f\\x00\\x00\\x00\\x0e\\x00\\x0c\\x00\\x00\\t127.0.0.1\\x00\\x0b\\x00\\x04\\x03\\x00\\x01\\x02\\x00\\n\\x00\\x0c\\x00\\n\\x00\\x1d\\x00\\x17\\x00\\x1e\\x00\\x19\\x00\\x18\\x00#\\x00\\x00\\x00\\x16\\x00\\x00\\x00\\x17\\x00\\x00\\x00\\r\\x00\\x1e\\x00']\n",
"Bad pipe message: %s [b'\\x03\\x05\\x03\\x06\\x03\\x08\\x07\\x08\\x08\\x08\\t\\x08\\n\\x08\\x0b\\x08\\x04\\x08\\x05\\x08\\x06\\x04\\x01\\x05\\x01\\x06\\x01']\n",
"Bad pipe message: %s [b\"\\xcd\\xe4\\xaew\\x91`\\x94\\xc6fO\\xea\\x92 \\xc7b\\xb15G\\x00\\x00|\\xc0,\\xc00\\x00\\xa3\\x00\\x9f\\xcc\\xa9\\xcc\\xa8\\xcc\\xaa\\xc0\\xaf\\xc0\\xad\\xc0\\xa3\\xc0\\x9f\\xc0]\\xc0a\\xc0W\\xc0S\\xc0+\\xc0/\\x00\\xa2\\x00\\x9e\\xc0\\xae\\xc0\\xac\\xc0\\xa2\\xc0\\x9e\\xc0\\\\\\xc0`\\xc0V\\xc0R\\xc0$\\xc0(\\x00k\\x00j\\xc0#\\xc0'\\x00g\\x00@\\xc0\\n\\xc0\\x14\\x009\\x008\\xc0\\t\\xc0\\x13\\x003\\x002\\x00\\x9d\\xc0\\xa1\\xc0\\x9d\\xc0Q\\x00\\x9c\\xc0\\xa0\\xc0\\x9c\\xc0P\\x00=\\x00<\\x005\\x00/\\x00\\x9a\\x00\\x99\\xc0\\x07\\xc0\\x11\\x00\\x96\\x00\\x05\\x00\\xff\\x01\\x00\\x00j\\x00\\x00\\x00\\x0e\\x00\\x0c\\x00\\x00\\t127.0.0.1\\x00\\x0b\\x00\\x04\\x03\\x00\\x01\\x02\\x00\\n\\x00\\x0c\\x00\\n\\x00\\x1d\\x00\\x17\\x00\\x1e\\x00\\x19\\x00\\x18\\x00#\\x00\\x00\\x00\\x16\\x00\\x00\\x00\\x17\\x00\\x00\\x00\\r\\x000\\x00.\\x04\\x03\\x05\\x03\\x06\\x03\\x08\\x07\\x08\\x08\\x08\\t\\x08\\n\\x08\\x0b\\x08\\x04\\x08\\x05\\x08\\x06\\x04\\x01\\x05\\x01\\x06\\x01\\x03\\x03\\x02\\x03\\x03\\x01\\x02\\x01\\x03\", b'\\x02']\n",
"Bad pipe message: %s [b'\\x05\\x02\\x06']\n",
"Bad pipe message: %s [b'\\x0b\"a7\\xfa\\xa1#\\xf3\\x88\\xcc%\\xae\\xb45\\xbc\\xdbY\\xb8\\x00\\x00\\xa6\\xc0,\\xc00\\x00\\xa3\\x00\\x9f\\xcc\\xa9\\xcc\\xa8\\xcc\\xaa\\xc0\\xaf\\xc0\\xad\\xc0\\xa3\\xc0\\x9f\\xc0]\\xc0a\\xc0W\\xc0S\\xc0+\\xc0/\\x00\\xa2\\x00\\x9e\\xc0\\xae\\xc0\\xac\\xc0\\xa2\\xc0\\x9e\\xc0\\\\\\xc0`\\xc0V\\xc0R\\xc0$\\xc0(\\x00k\\x00j\\xc0s\\xc0w\\x00\\xc4\\x00\\xc3\\xc0#\\xc0\\'\\x00g\\x00@\\xc0r\\xc0v\\x00\\xbe\\x00\\xbd\\xc0\\n\\xc0\\x14\\x009\\x008\\x00\\x88\\x00\\x87\\xc0\\t\\xc0\\x13\\x003\\x002\\x00\\x9a\\x00\\x99\\x00E\\x00D\\xc0\\x07\\xc0\\x11\\xc0\\x08\\xc0\\x12\\x00\\x16\\x00\\x13\\x00\\x9d\\xc0\\xa1\\xc0\\x9d\\xc0Q\\x00\\x9c\\xc0\\xa0\\xc0\\x9c\\xc0P\\x00=\\x00\\xc0\\x00<\\x00\\xba\\x005\\x00\\x84\\x00/\\x00\\x96\\x00A\\x00\\x05\\x00']\n",
"Bad pipe message: %s [b'\\xff\\x01\\x00\\x00j\\x00\\x00\\x00\\x0e']\n",
"Bad pipe message: %s [b'7\\x992kK\\xc1\\xdbF\\x8b\\xb1\\xe1l\\x19\\xbfA<\\x1fc\\x00\\x00>\\xc0\\x14\\xc0\\n\\x009\\x008\\x007\\x006\\xc0\\x0f\\xc0\\x05\\x005\\xc0\\x13\\xc0\\t\\x003\\x002\\x001\\x000\\xc0\\x0e\\xc0\\x04\\x00/\\x00\\x9a\\x00\\x99\\x00\\x98\\x00\\x97\\x00\\x96\\x00\\x07\\xc0\\x11\\xc0\\x07\\xc0\\x0c\\xc0\\x02\\x00\\x05\\x00\\x04\\x00\\xff\\x02\\x01\\x00\\x00C\\x00\\x00\\x00\\x0e\\x00\\x0c\\x00\\x00\\t127.0.0.1\\x00\\x0b\\x00\\x04\\x03\\x00\\x01\\x02']\n",
"Bad pipe message: %s [b'\\xcf\\xf4\\xe1\\xe6\\xec\\xa7$\\x0f\\x95\\xd8\\xe3X\\xfcm\\x93\\xc2\\x06?\\x00\\x00\\xa2\\xc0\\x14\\xc0\\n\\x009\\x008\\x007\\x006\\x00\\x88\\x00\\x87\\x00\\x86\\x00\\x85\\xc0\\x19\\x00:\\x00\\x89\\xc0\\x0f\\xc0\\x05\\x005\\x00\\x84\\xc0\\x13\\xc0\\t\\x003\\x002\\x001\\x000\\x00\\x9a\\x00\\x99\\x00\\x98\\x00\\x97\\x00E\\x00D\\x00C\\x00B\\xc0\\x18\\x004\\x00\\x9b\\x00F\\xc0\\x0e\\xc0\\x04\\x00/\\x00\\x96\\x00A\\x00\\x07\\xc0\\x11\\xc0\\x07\\xc0\\x16\\x00\\x18\\xc0\\x0c\\xc0\\x02\\x00\\x05\\x00\\x04\\xc0\\x12\\xc0\\x08\\x00\\x16\\x00\\x13\\x00', b'\\r\\xc0\\x17\\x00\\x1b\\xc0\\r\\xc0\\x03\\x00\\n\\x00\\x15\\x00\\x12']\n",
"Bad pipe message: %s [b'Nv\\xa8\\xf4\\x13\\xe0t\\x12q\\xe1\\xb8\\xa4\\xe0(\\xaflvv']\n",
"Bad pipe message: %s [b'\\xd5\\xc3({`3.\\xe2\\xad\\xa8.6\\x01\\xe9\\xb4\\x06\\xba\\xb1\\x00\\x00>\\xc0\\x14\\xc0\\n\\x009\\x008\\x007\\x006\\xc0\\x0f\\xc0\\x05\\x005\\xc0\\x13\\xc0\\t\\x003\\x002\\x001\\x000\\xc0\\x0e\\xc0\\x04\\x00/\\x00\\x9a\\x00\\x99\\x00\\x98\\x00\\x97\\x00\\x96\\x00\\x07\\xc0\\x11\\xc0\\x07\\xc0\\x0c\\xc0\\x02\\x00\\x05\\x00\\x04\\x00\\xff\\x02\\x01\\x00\\x15\\x03\\x00']\n",
"Bad pipe message: %s [b\"\\xa7\\xe2\\x12\\x89\\xeaUh\\xb9\\x1a\\xc3\\x92\\x85\\x1f\\xd4\\xf7\\xaf\\xb4\\xcd\\x00\\x00\\x86\\xc00\\xc0,\\xc0(\\xc0$\\xc0\\x14\\xc0\\n\\x00\\xa5\\x00\\xa3\\x00\\xa1\\x00\\x9f\\x00k\\x00j\\x00i\\x00h\\x009\\x008\\x007\\x006\\xc02\\xc0.\\xc0*\\xc0&\\xc0\\x0f\\xc0\\x05\\x00\\x9d\\x00=\\x005\\xc0/\\xc0+\\xc0'\\xc0#\\xc0\\x13\\xc0\\t\\x00\\xa4\\x00\\xa2\\x00\\xa0\\x00\\x9e\\x00g\\x00@\\x00?\\x00>\\x003\\x002\\x001\\x000\\xc01\\xc0-\\xc0)\\xc0%\\xc0\\x0e\\xc0\\x04\\x00\\x9c\\x00<\\x00/\\x00\\x9a\\x00\\x99\\x00\\x98\\x00\\x97\\x00\\x96\\x00\\x07\\xc0\\x11\\xc0\\x07\\xc0\\x0c\\xc0\\x02\\x00\\x05\\x00\\x04\\x00\\xff\\x02\\x01\\x00\\x00g\\x00\\x00\\x00\\x0e\\x00\\x0c\\x00\\x00\\t127.0.0.1\\x00\\x0b\\x00\\x04\\x03\\x00\\x01\\x02\\x00\\n\\x00\\x1c\\x00\\x1a\\x00\\x17\\x00\\x19\\x00\\x1c\\x00\\x1b\\x00\\x18\\x00\\x1a\\x00\\x16\\x00\\x0e\\x00\\r\\x00\\x0b\\x00\\x0c\\x00\\t\\x00\\n\\x00#\\x00\\x00\\x00\\r\\x00 \\x00\\x1e\\x06\\x01\\x06\\x02\\x06\\x03\\x05\\x01\\x05\\x02\\x05\\x03\\x04\\x01\\x04\\x02\", b'\\x03\\x01\\x03', b'\\x03', b'\\x02', b'\\x03']\n",
"Bad pipe message: %s [b'\\xc7\\x95X[|\\x00\\xb9,o\\xa7\\x95\\xdd\\xee\\xd3A\\xd2g\\x06\\x00\\x00\\xf4\\xc00\\xc0,\\xc0(\\xc0$\\xc0\\x14\\xc0\\n\\x00\\xa5\\x00\\xa3\\x00\\xa1\\x00\\x9f\\x00k\\x00j\\x00i\\x00h\\x009\\x008\\x007\\x006\\x00\\x88\\x00\\x87\\x00\\x86\\x00\\x85\\xc0\\x19\\x00\\xa7\\x00', b\":\\x00\\x89\\xc02\\xc0.\\xc0*\\xc0&\\xc0\\x0f\\xc0\\x05\\x00\\x9d\\x00=\\x005\\x00\\x84\\xc0/\\xc0+\\xc0'\\xc0#\\xc0\\x13\\xc0\\t\\x00\\xa4\\x00\\xa2\\x00\\xa0\\x00\\x9e\\x00g\\x00@\\x00?\\x00>\\x003\\x002\\x001\\x000\\x00\\x9a\\x00\\x99\\x00\\x98\\x00\\x97\\x00E\\x00D\\x00C\\x00B\\xc0\\x18\\x00\\xa6\\x00l\\x004\\x00\\x9b\\x00F\\xc01\\xc0-\\xc0)\\xc0%\\xc0\\x0e\\xc0\\x04\\x00\\x9c\\x00<\\x00/\\x00\\x96\\x00\"]\n",
"Bad pipe message: %s [b'\\x07\\xc0\\x11\\xc0\\x07\\xc0\\x16\\x00\\x18\\xc0\\x0c\\xc0\\x02\\x00\\x05\\x00\\x04\\xc0\\x12\\xc0\\x08\\x00\\x16\\x00\\x13\\x00\\x10\\x00\\r\\xc0\\x17\\x00\\x1b\\xc0\\r\\xc0\\x03\\x00\\n\\x00\\x15\\x00\\x12\\x00\\x0f\\x00\\x0c\\x00\\x1a\\x00\\t\\x00\\x14\\x00\\x11\\x00\\x19\\x00\\x08\\x00\\x06\\x00\\x17\\x00']\n",
"Bad pipe message: %s [b'\\x10\\xc0']\n",
"Bad pipe message: %s [b'\\x15\\xc0\\x0b\\xc0\\x01']\n"
]
}
],
"source": [
"%%writefile hello_world_pipeline.py\n",
"# -------------------------------------------------------------------------------------\n",
Expand All @@ -3718,10 +3755,11 @@
"# Standard imports\n",
"import os\n",
"import argparse\n",
"import json\n",
"\n",
"# Third-party imports\n",
"import pandas as pd\n",
"from sklearn import Bunch\n",
"from sklearn.utils import Bunch\n",
"\n",
"# AML imports\n",
"from azure.ai.ml import (\n",
Expand Down Expand Up @@ -3891,19 +3929,21 @@
"# -------------------------------------------------------------------------------------\n",
"def run_pipeline (\n",
" config_path: str=\"./pipeline_input.json\",\n",
" experiment_name=\"hello-world-experiment\",\n",
"):\n",
"\n",
" # Read config json file\n",
" with open (config_path,\"rt\") as config_file\n",
" with open (config_path,\"rt\") as config_file:\n",
" config = json.load (config_file)\n",
"\n",
" # Convert config dictionary into a Bunch object.\n",
" # This allows to get access to fields as object attributes\n",
" # Which I find more convenient.\n",
" # Convert config dictionary into a Bunch object. This is a \n",
" # sub-class of dict which allows to get access to fields \n",
" # as object attributes (a bit more convenient IMO) in addition \n",
" # to also allowing the dict syntax.\n",
" config = Bunch (**config)\n",
"\n",
" # Build pipeline \n",
" three_components_pipeline = three_components_pipeline(\n",
" three_components_pipeline_object = three_components_pipeline(\n",
" # first preprocessing component\n",
" preprocessing_training_input_file=Input(type=\"uri_file\", path=config.preprocessing_training_input_file),\n",
" preprocessing_training_output_filename=config.preprocessing_training_output_filename,\n",
Expand All @@ -3921,9 +3961,9 @@
" )\n",
"\n",
" three_components_pipeline_job = ml_client.jobs.create_or_update(\n",
" three_components_pipeline,\n",
" three_components_pipeline_object,\n",
" # Project's name\n",
" experiment_name=config.experiment_name,\n",
" experiment_name=experiment_name,\n",
" )\n",
"\n",
" # ----------------------------------------------------\n",
Expand Down Expand Up @@ -3973,6 +4013,46 @@
" main ()"
]
},
{
"cell_type": "markdown",
"id": "37071d5f-80f9-4f4e-aa7a-fc6db05b037d",
"metadata": {},
"source": [
"We can now run the script from command line:"
]
},
{
"cell_type": "markdown",
"id": "3062e7af-3133-48b7-8137-b29994d300b4",
"metadata": {},
"source": [
"First we open up a terminal and activate a conda environment that uses the python SDK version 2. Let's see what environments we currently have:\n",
"\n",
"```bash\n",
"conda env list\n",
"```\n",
"\n",
"As of now, this provides the following list:\n",
"```\n",
"# conda environments:\n",
"#\n",
"base /anaconda\n",
"azureml_py310_sdkv2 /anaconda/envs/azureml_py310_sdkv2\n",
"azureml_py38 * /anaconda/envs/azureml_py38\n",
"azureml_py38_PT_TF /anaconda/envs/azureml_py38_PT_TF\n",
"jupyter_env /anaconda/envs/jupyter_env\n",
"```\n",
"\n",
"From that list, the conda environment using SDK v2 is `azureml_py310_sdkv2` (use another if that changes when you read this)\n",
"\n",
"We activate it and run our script:\n",
"\n",
"```bash\n",
"conda activate azureml_py310_sdkv2\n",
"python hello_world_pipeline.py\n",
"```"
]
},
{
"cell_type": "markdown",
"id": "35ca4a4d",
Expand All @@ -3992,10 +4072,18 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 14,
"id": "3b093697",
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Overwriting hello_world_pipeline.py\n"
]
}
],
"source": [
"%%writefile hello_world_pipeline.py\n",
"# -------------------------------------------------------------------------------------\n",
Expand All @@ -4004,10 +4092,11 @@
"# Standard imports\n",
"import os\n",
"import argparse\n",
"import json\n",
"\n",
"# Third-party imports\n",
"import pandas as pd\n",
"from sklearn import Bunch\n",
"from sklearn.utils import Bunch\n",
"\n",
"# AML imports\n",
"from azure.ai.ml import (\n",
Expand Down Expand Up @@ -4075,6 +4164,19 @@
" inference_output_filename: str\n",
" Name of file containing the output data with inference results.\n",
" \"\"\"\n",
" \n",
" print (\n",
" \"Running hello-world pipeline with args\", \n",
" preprocessing_training_input_file,\n",
" preprocessing_training_output_filename,\n",
" x,\n",
" preprocessing_test_input_file,\n",
" preprocessing_test_output_filename,\n",
" training_output_filename,\n",
" inference_output_filename,\n",
" sep=\"\\n\",\n",
" )\n",
" \n",
" # -------------------------------------------------------------------------------------\n",
" # Preprocessing\n",
" # -------------------------------------------------------------------------------------\n",
Expand Down Expand Up @@ -4189,10 +4291,11 @@
"# -------------------------------------------------------------------------------------\n",
"def run_pipeline (\n",
" config_path: str=\"./pipeline_input.json\",\n",
" experiment_name=\"hello-world-experiment\",\n",
"):\n",
"\n",
" # Read config json file\n",
" with open (config_path,\"rt\") as config_file\n",
" with open (config_path,\"rt\") as config_file:\n",
" config = json.load (config_file)\n",
"\n",
" # Convert config dictionary into a Bunch object.\n",
Expand All @@ -4204,7 +4307,7 @@
" ml_client = connect ()\n",
"\n",
" # Build pipeline \n",
" three_components_pipeline = three_components_pipeline(\n",
" three_components_pipeline_object = three_components_pipeline(\n",
" # first preprocessing component\n",
" preprocessing_training_input_file=Input(type=\"uri_file\", path=config.preprocessing_training_input_file),\n",
" preprocessing_training_output_filename=config.preprocessing_training_output_filename,\n",
Expand All @@ -4222,9 +4325,9 @@
" )\n",
"\n",
" three_components_pipeline_job = ml_client.jobs.create_or_update(\n",
" three_components_pipeline,\n",
" three_components_pipeline_object,\n",
" # Project's name\n",
" experiment_name=config.experiment_name,\n",
" experiment_name=experiment_name,\n",
" )\n",
"\n",
" # ----------------------------------------------------\n",
Expand Down Expand Up @@ -4254,6 +4357,8 @@
"\n",
" args = parser.parse_args()\n",
" \n",
" print (\"Running hello-world pipeline with args\", args)\n",
" \n",
" return args\n",
"\n",
"\n",
Expand Down

0 comments on commit 97c622d

Please sign in to comment.