Skip to content

Commit

Permalink
Fixded bug in inference
Browse files Browse the repository at this point in the history
  • Loading branch information
@JaumeAmoresDS
JaumeAmoresDS committed Mar 29, 2024
1 parent 0c88a77 commit ee576a4
Showing 1 changed file with 34 additions and 9 deletions.
43 changes: 34 additions & 9 deletions posts/data_science/hello_world.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3133,7 +3133,7 @@
},
{
"cell_type": "code",
"execution_count": 70,
"execution_count": 74,
"id": "50fa331d",
"metadata": {},
"outputs": [
Expand Down Expand Up @@ -3265,10 +3265,12 @@
" \n",
" args = parse_args ()\n",
" read_and_inference (\n",
" test_data=args.test_data, \n",
" test_data_file_name=args.test_data_file_name,\n",
" model_path=args.model, \n",
" inference_data=args.inference_output,\n",
" preprocessed_input_folder=args.preprocessed_input_folder,\n",
" preprocessed_input_filename=args.preprocessed_input_filename,\n",
" model_input_folder=args.model_input_folder,\n",
" model_input_filename=args.model_input_filename,\n",
" output_folder=args.output_folder,\n",
" output_filename=args.output_filename,\n",
" )\n",
"\n",
"if __name__ == \"__main__\":\n",
Expand All @@ -3277,15 +3279,15 @@
},
{
"cell_type": "code",
"execution_count": 71,
"execution_count": 75,
"id": "a759377b",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[32mUploading inference (0.0 MBs): 100%|██████████| 3881/3881 [00:00<00:00, 323452.96it/s]\n",
"\u001b[32mUploading inference (0.0 MBs): 100%|██████████| 4046/4046 [00:00<00:00, 151355.71it/s]\n",
"\u001b[39m\n",
"\n"
]
Expand Down Expand Up @@ -3516,7 +3518,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 73,
"id": "f8050551",
"metadata": {},
"outputs": [
Expand Down Expand Up @@ -3545,7 +3547,30 @@
"[2024-03-29 08:45:11Z] Submitting 2 runs, first five are: f39c6442:de45cd94-3bf9-4405-9b9f-59782424a8fb,feeb5198:0b543053-d140-4699-9c29-1ec2e755898e\n",
"[2024-03-29 08:50:08Z] Completing processing run id de45cd94-3bf9-4405-9b9f-59782424a8fb.\n",
"[2024-03-29 08:50:33Z] Completing processing run id 0b543053-d140-4699-9c29-1ec2e755898e.\n",
"[2024-03-29 08:50:34Z] Submitting 1 runs, first five are: 5731fc43:46a9ab31-8a68-42ee-a377-a5e84cbad37d\n"
"[2024-03-29 08:50:34Z] Submitting 1 runs, first five are: 5731fc43:46a9ab31-8a68-42ee-a377-a5e84cbad37d\n",
"[2024-03-29 08:55:36Z] Completing processing run id 46a9ab31-8a68-42ee-a377-a5e84cbad37d.\n",
"[2024-03-29 08:55:36Z] Submitting 1 runs, first five are: 5af83f98:fd50a27d-6687-431a-88d3-b8ba3d8799f4\n",
"[2024-03-29 08:56:12Z] Execution of experiment failed, update experiment status and cancel running nodes.\n",
"\n",
"Execution Summary\n",
"=================\n",
"RunId: busy_toe_03bv5yshzh\n",
"Web View: https://ml.azure.com/runs/busy_toe_03bv5yshzh?wsid=/subscriptions/6af6741b-f140-48c2-84ca-027a27365026/resourcegroups/helloworld/workspaces/helloworld\n"
]
},
{
"ename": "JobException",
"evalue": "Exception : \n {\n \"error\": {\n \"code\": \"UserError\",\n \"message\": \"Pipeline has failed child jobs. Failed nodes: /inference_job. For more details and logs, please go to the job detail page and check the child jobs.\",\n \"message_format\": \"Pipeline has failed child jobs. {0}\",\n \"message_parameters\": {},\n \"reference_code\": \"PipelineHasStepJobFailed\",\n \"details\": []\n },\n \"environment\": \"eastus2\",\n \"location\": \"eastus2\",\n \"time\": \"2024-03-29T08:56:11.952518Z\",\n \"component_name\": \"\"\n} ",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mJobException\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[73], line 95\u001b[0m\n\u001b[1;32m 86\u001b[0m three_components_pipeline_job \u001b[38;5;241m=\u001b[39m ml_client\u001b[38;5;241m.\u001b[39mjobs\u001b[38;5;241m.\u001b[39mcreate_or_update(\n\u001b[1;32m 87\u001b[0m three_components_pipeline,\n\u001b[1;32m 88\u001b[0m \u001b[38;5;66;03m# Project's name\u001b[39;00m\n\u001b[1;32m 89\u001b[0m experiment_name\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124me2e_three_components_refactored\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 90\u001b[0m )\n\u001b[1;32m 92\u001b[0m \u001b[38;5;66;03m# ----------------------------------------------------\u001b[39;00m\n\u001b[1;32m 93\u001b[0m \u001b[38;5;66;03m# Pipeline running\u001b[39;00m\n\u001b[1;32m 94\u001b[0m \u001b[38;5;66;03m# ----------------------------------------------------\u001b[39;00m\n\u001b[0;32m---> 95\u001b[0m \u001b[43mml_client\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mjobs\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m(\u001b[49m\u001b[43mthree_components_pipeline_job\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mname\u001b[49m\u001b[43m)\u001b[49m\n",
"File \u001b[0;32m/anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages/azure/core/tracing/decorator.py:76\u001b[0m, in \u001b[0;36mdistributed_trace.<locals>.decorator.<locals>.wrapper_use_tracer\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 74\u001b[0m span_impl_type \u001b[38;5;241m=\u001b[39m settings\u001b[38;5;241m.\u001b[39mtracing_implementation()\n\u001b[1;32m 75\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m span_impl_type \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m---> 76\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 78\u001b[0m \u001b[38;5;66;03m# Merge span is parameter is set, but only if no explicit parent are passed\u001b[39;00m\n\u001b[1;32m 79\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m merge_span \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m passed_in_parent:\n",
"File \u001b[0;32m/anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages/azure/ai/ml/_telemetry/activity.py:263\u001b[0m, in \u001b[0;36mmonitor_with_activity.<locals>.monitor.<locals>.wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 260\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(f)\n\u001b[1;32m 261\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwrapper\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 262\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m log_activity(logger, activity_name \u001b[38;5;129;01mor\u001b[39;00m f\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m, activity_type, custom_dimensions):\n\u001b[0;32m--> 263\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mf\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
"File \u001b[0;32m/anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages/azure/ai/ml/operations/_job_operations.py:661\u001b[0m, in \u001b[0;36mJobOperations.stream\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m 658\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m _is_pipeline_child_job(job_object):\n\u001b[1;32m 659\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m PipelineChildJobError(job_id\u001b[38;5;241m=\u001b[39mjob_object\u001b[38;5;241m.\u001b[39mid)\n\u001b[0;32m--> 661\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_stream_logs_until_completion\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 662\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_runs_operations\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mjob_object\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_datastore_operations\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrequests_pipeline\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_requests_pipeline\u001b[49m\n\u001b[1;32m 663\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
"File \u001b[0;32m/anaconda/envs/azureml_py310_sdkv2/lib/python3.10/site-packages/azure/ai/ml/operations/_job_ops_helper.py:312\u001b[0m, in \u001b[0;36mstream_logs_until_completion\u001b[0;34m(run_operations, job_resource, datastore_operations, raise_exception_on_failed_job, requests_pipeline)\u001b[0m\n\u001b[1;32m 310\u001b[0m file_handle\u001b[38;5;241m.\u001b[39mwrite(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 311\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 312\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m JobException(\n\u001b[1;32m 313\u001b[0m message\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mException : \u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mformat(json\u001b[38;5;241m.\u001b[39mdumps(error, indent\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m4\u001b[39m)),\n\u001b[1;32m 314\u001b[0m target\u001b[38;5;241m=\u001b[39mErrorTarget\u001b[38;5;241m.\u001b[39mJOB,\n\u001b[1;32m 315\u001b[0m no_personal_data_message\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mException raised on failed job.\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 316\u001b[0m error_category\u001b[38;5;241m=\u001b[39mErrorCategory\u001b[38;5;241m.\u001b[39mSYSTEM_ERROR,\n\u001b[1;32m 317\u001b[0m )\n\u001b[1;32m 319\u001b[0m file_handle\u001b[38;5;241m.\u001b[39mwrite(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 320\u001b[0m file_handle\u001b[38;5;241m.\u001b[39mflush()\n",
"\u001b[0;31mJobException\u001b[0m: Exception : \n {\n \"error\": {\n \"code\": \"UserError\",\n \"message\": \"Pipeline has failed child jobs. Failed nodes: /inference_job. For more details and logs, please go to the job detail page and check the child jobs.\",\n \"message_format\": \"Pipeline has failed child jobs. {0}\",\n \"message_parameters\": {},\n \"reference_code\": \"PipelineHasStepJobFailed\",\n \"details\": []\n },\n \"environment\": \"eastus2\",\n \"location\": \"eastus2\",\n \"time\": \"2024-03-29T08:56:11.952518Z\",\n \"component_name\": \"\"\n} "
]
}
],
Expand Down

0 comments on commit ee576a4

Please sign in to comment.