From ee576a49d589c3b6fbec145e380e0e4eb0c0d7c1 Mon Sep 17 00:00:00 2001 From: Jaume Amores Date: Fri, 29 Mar 2024 09:00:46 +0000 Subject: [PATCH] Fixded bug in inference --- posts/data_science/hello_world.ipynb | 43 ++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/posts/data_science/hello_world.ipynb b/posts/data_science/hello_world.ipynb index cf8b093..a8bc8f9 100644 --- a/posts/data_science/hello_world.ipynb +++ b/posts/data_science/hello_world.ipynb @@ -3133,7 +3133,7 @@ }, { "cell_type": "code", - "execution_count": 70, + "execution_count": 74, "id": "50fa331d", "metadata": {}, "outputs": [ @@ -3265,10 +3265,12 @@ " \n", " args = parse_args ()\n", " read_and_inference (\n", - " test_data=args.test_data, \n", - " test_data_file_name=args.test_data_file_name,\n", - " model_path=args.model, \n", - " inference_data=args.inference_output,\n", + " preprocessed_input_folder=args.preprocessed_input_folder,\n", + " preprocessed_input_filename=args.preprocessed_input_filename,\n", + " model_input_folder=args.model_input_folder,\n", + " model_input_filename=args.model_input_filename,\n", + " output_folder=args.output_folder,\n", + " output_filename=args.output_filename,\n", " )\n", "\n", "if __name__ == \"__main__\":\n", @@ -3277,7 +3279,7 @@ }, { "cell_type": "code", - "execution_count": 71, + "execution_count": 75, "id": "a759377b", "metadata": {}, "outputs": [ @@ -3285,7 +3287,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "\u001b[32mUploading inference (0.0 MBs): 100%|██████████| 3881/3881 [00:00<00:00, 323452.96it/s]\n", + "\u001b[32mUploading inference (0.0 MBs): 100%|██████████| 4046/4046 [00:00<00:00, 151355.71it/s]\n", "\u001b[39m\n", "\n" ] @@ -3516,7 +3518,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 73, "id": "f8050551", "metadata": {}, "outputs": [ @@ -3545,7 +3547,30 @@ "[2024-03-29 08:45:11Z] Submitting 2 runs, first five are: f39c6442:de45cd94-3bf9-4405-9b9f-59782424a8fb,feeb5198:0b543053-d140-4699-9c29-1ec2e755898e\n", "[2024-03-29 08:50:08Z] Completing processing run id de45cd94-3bf9-4405-9b9f-59782424a8fb.\n", "[2024-03-29 08:50:33Z] Completing processing run id 0b543053-d140-4699-9c29-1ec2e755898e.\n", - "[2024-03-29 08:50:34Z] Submitting 1 runs, first five are: 5731fc43:46a9ab31-8a68-42ee-a377-a5e84cbad37d\n" + "[2024-03-29 08:50:34Z] Submitting 1 runs, first five are: 5731fc43:46a9ab31-8a68-42ee-a377-a5e84cbad37d\n", + "[2024-03-29 08:55:36Z] Completing processing run id 46a9ab31-8a68-42ee-a377-a5e84cbad37d.\n", + "[2024-03-29 08:55:36Z] Submitting 1 runs, first five are: 5af83f98:fd50a27d-6687-431a-88d3-b8ba3d8799f4\n", + "[2024-03-29 08:56:12Z] Execution of experiment failed, update experiment status and cancel running nodes.\n", + "\n", + "Execution Summary\n", + "=================\n", + "RunId: busy_toe_03bv5yshzh\n", + "Web View: https://ml.azure.com/runs/busy_toe_03bv5yshzh?wsid=/subscriptions/6af6741b-f140-48c2-84ca-027a27365026/resourcegroups/helloworld/workspaces/helloworld\n" + ] + }, + { + "ename": "JobException", + "evalue": "Exception : \n {\n \"error\": {\n \"code\": \"UserError\",\n \"message\": \"Pipeline has failed child jobs. Failed nodes: /inference_job. For more details and logs, please go to the job detail page and check the child jobs.\",\n \"message_format\": \"Pipeline has failed child jobs. {0}\",\n \"message_parameters\": {},\n \"reference_code\": \"PipelineHasStepJobFailed\",\n \"details\": []\n },\n \"environment\": \"eastus2\",\n \"location\": \"eastus2\",\n \"time\": \"2024-03-29T08:56:11.952518Z\",\n \"component_name\": \"\"\n} ", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mJobException\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[73], line 95\u001b[0m\n\u001b[1;32m 86\u001b[0m three_components_pipeline_job \u001b[38;5;241m=\u001b[39m ml_client\u001b[38;5;241m.\u001b[39mjobs\u001b[38;5;241m.\u001b[39mcreate_or_update(\n\u001b[1;32m 87\u001b[0m three_components_pipeline,\n\u001b[1;32m 88\u001b[0m \u001b[38;5;66;03m# Project's name\u001b[39;00m\n\u001b[1;32m 89\u001b[0m experiment_name\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124me2e_three_components_refactored\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 90\u001b[0m )\n\u001b[1;32m 92\u001b[0m \u001b[38;5;66;03m# ----------------------------------------------------\u001b[39;00m\n\u001b[1;32m 93\u001b[0m \u001b[38;5;66;03m# Pipeline running\u001b[39;00m\n\u001b[1;32m 94\u001b[0m \u001b[38;5;66;03m# ----------------------------------------------------\u001b[39;00m\n\u001b[0;32m---> 95\u001b[0m \u001b[43mml_client\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mjobs\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m(\u001b[49m\u001b[43mthree_components_pipeline_job\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mname\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages/azure/core/tracing/decorator.py:76\u001b[0m, in \u001b[0;36mdistributed_trace..decorator..wrapper_use_tracer\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 74\u001b[0m span_impl_type \u001b[38;5;241m=\u001b[39m settings\u001b[38;5;241m.\u001b[39mtracing_implementation()\n\u001b[1;32m 75\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m span_impl_type \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m---> 76\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 78\u001b[0m \u001b[38;5;66;03m# Merge span is parameter is set, but only if no explicit parent are passed\u001b[39;00m\n\u001b[1;32m 79\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m merge_span \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m passed_in_parent:\n", + "File \u001b[0;32m/anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages/azure/ai/ml/_telemetry/activity.py:263\u001b[0m, in \u001b[0;36mmonitor_with_activity..monitor..wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 260\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(f)\n\u001b[1;32m 261\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwrapper\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 262\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m log_activity(logger, activity_name \u001b[38;5;129;01mor\u001b[39;00m f\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m, activity_type, custom_dimensions):\n\u001b[0;32m--> 263\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mf\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages/azure/ai/ml/operations/_job_operations.py:661\u001b[0m, in \u001b[0;36mJobOperations.stream\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m 658\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m _is_pipeline_child_job(job_object):\n\u001b[1;32m 659\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m PipelineChildJobError(job_id\u001b[38;5;241m=\u001b[39mjob_object\u001b[38;5;241m.\u001b[39mid)\n\u001b[0;32m--> 661\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_stream_logs_until_completion\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 662\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_runs_operations\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mjob_object\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_datastore_operations\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrequests_pipeline\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_requests_pipeline\u001b[49m\n\u001b[1;32m 663\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages/azure/ai/ml/operations/_job_ops_helper.py:312\u001b[0m, in \u001b[0;36mstream_logs_until_completion\u001b[0;34m(run_operations, job_resource, datastore_operations, raise_exception_on_failed_job, requests_pipeline)\u001b[0m\n\u001b[1;32m 310\u001b[0m file_handle\u001b[38;5;241m.\u001b[39mwrite(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 311\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 312\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m JobException(\n\u001b[1;32m 313\u001b[0m message\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mException : \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mformat(json\u001b[38;5;241m.\u001b[39mdumps(error, indent\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m4\u001b[39m)),\n\u001b[1;32m 314\u001b[0m target\u001b[38;5;241m=\u001b[39mErrorTarget\u001b[38;5;241m.\u001b[39mJOB,\n\u001b[1;32m 315\u001b[0m no_personal_data_message\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mException raised on failed job.\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 316\u001b[0m error_category\u001b[38;5;241m=\u001b[39mErrorCategory\u001b[38;5;241m.\u001b[39mSYSTEM_ERROR,\n\u001b[1;32m 317\u001b[0m )\n\u001b[1;32m 319\u001b[0m file_handle\u001b[38;5;241m.\u001b[39mwrite(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 320\u001b[0m file_handle\u001b[38;5;241m.\u001b[39mflush()\n", + "\u001b[0;31mJobException\u001b[0m: Exception : \n {\n \"error\": {\n \"code\": \"UserError\",\n \"message\": \"Pipeline has failed child jobs. Failed nodes: /inference_job. For more details and logs, please go to the job detail page and check the child jobs.\",\n \"message_format\": \"Pipeline has failed child jobs. {0}\",\n \"message_parameters\": {},\n \"reference_code\": \"PipelineHasStepJobFailed\",\n \"details\": []\n },\n \"environment\": \"eastus2\",\n \"location\": \"eastus2\",\n \"time\": \"2024-03-29T08:56:11.952518Z\",\n \"component_name\": \"\"\n} " ] } ],