From f100ba16049677fb4feb904fc013158882fe347d Mon Sep 17 00:00:00 2001 From: Kevin Date: Fri, 19 Jul 2024 18:10:33 -0400 Subject: [PATCH] rm default namespace from demo notebooks Signed-off-by: Kevin --- .../additional-demos/hf_interactive.ipynb | 1039 ++++++----- .../additional-demos/local_interactive.ipynb | 7 +- .../additional-demos/ray_job_client.ipynb | 1 - demo-notebooks/guided-demos/0_basic_ray.ipynb | 1 - .../guided-demos/1_cluster_job_client.ipynb | 1 - .../guided-demos/2_basic_interactive.ipynb | 6 +- .../notebook-ex-outputs/0_basic_ray.ipynb | 45 +- .../1_cluster_job_client.ipynb | 1 - .../2_basic_interactive.ipynb | 1580 ++++++++--------- .../preview_nbs/0_basic_ray.ipynb | 1 - .../preview_nbs/1_cluster_job_client.ipynb | 1 - .../preview_nbs/2_basic_interactive.ipynb | 6 +- 12 files changed, 1337 insertions(+), 1352 deletions(-) diff --git a/demo-notebooks/additional-demos/hf_interactive.ipynb b/demo-notebooks/additional-demos/hf_interactive.ipynb index ad5524513..fe4cd429d 100644 --- a/demo-notebooks/additional-demos/hf_interactive.ipynb +++ b/demo-notebooks/additional-demos/hf_interactive.ipynb @@ -90,7 +90,6 @@ "# Create our cluster and submit\n", "# The SDK will try to find the name of your default local queue based on the annotation \"kueue.x-k8s.io/default-queue\": \"true\" unless you specify the local queue manually below\n", "cluster = Cluster(ClusterConfiguration(name='hfgputest', \n", - " namespace=\"default\", # Update to your namespace\n", " head_gpus=1, # For GPU enabled workloads set the head_gpus and num_gpus\n", " num_gpus=1,\n", " num_workers=1,\n", @@ -153,13 +152,13 @@ ], "text/plain": [ "╭─────────────────────────╮\n", - "│ \u001B[3m \u001B[0m\u001B[1;3m 🚀 List of CodeFlare\u001B[0m\u001B[3m \u001B[0m │\n", - "│ \u001B[3m \u001B[0m\u001B[1;3mclusters in queue🚀\u001B[0m\u001B[3m \u001B[0m │\n", + "│ \u001b[3m \u001b[0m\u001b[1;3m 🚀 List of CodeFlare\u001b[0m\u001b[3m \u001b[0m │\n", + "│ \u001b[3m \u001b[0m\u001b[1;3mclusters in queue🚀\u001b[0m\u001b[3m \u001b[0m │\n", "│ +-----------+---------+ │\n", - "│ |\u001B[1m \u001B[0m\u001B[1mName \u001B[0m\u001B[1m \u001B[0m|\u001B[1m \u001B[0m\u001B[1mStatus \u001B[0m\u001B[1m \u001B[0m| │\n", + "│ |\u001b[1m \u001b[0m\u001b[1mName \u001b[0m\u001b[1m \u001b[0m|\u001b[1m \u001b[0m\u001b[1mStatus \u001b[0m\u001b[1m \u001b[0m| │\n", "│ +===========+=========+ │\n", - "│ |\u001B[36m \u001B[0m\u001B[36mhfgputest\u001B[0m\u001B[36m \u001B[0m|\u001B[35m \u001B[0m\u001B[35mpending\u001B[0m\u001B[35m \u001B[0m| │\n", - "│ |\u001B[36m \u001B[0m\u001B[36m \u001B[0m\u001B[36m \u001B[0m|\u001B[35m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m| │\n", + "│ |\u001b[36m \u001b[0m\u001b[36mhfgputest\u001b[0m\u001b[36m \u001b[0m|\u001b[35m \u001b[0m\u001b[35mpending\u001b[0m\u001b[35m \u001b[0m| │\n", + "│ |\u001b[36m \u001b[0m\u001b[36m \u001b[0m\u001b[36m \u001b[0m|\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m| │\n", "│ +-----------+---------+ │\n", "╰─────────────────────────╯\n" ] @@ -240,22 +239,22 @@ "\n" ], "text/plain": [ - "\u001B[3m \u001B[0m\u001B[1;3m 🚀 List of CodeFlare clusters 🚀\u001B[0m\u001B[3m \u001B[0m\n", - "\u001B[1m \u001B[0m\u001B[1m \u001B[0m\u001B[1m \u001B[0m\n", + "\u001b[3m \u001b[0m\u001b[1;3m 🚀 List of CodeFlare clusters 🚀\u001b[0m\u001b[3m \u001b[0m\n", + "\u001b[1m \u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m\n", " ╭────────────────────────────────────────────────────────────────╮ \n", - " │ \u001B[1;37;42mOwner\u001B[0m │ \n", - " │ \u001B[1;4mhfgputest\u001B[0m Active ✅ │ \n", + " │ \u001b[1;37;42mOwner\u001b[0m │ \n", + " │ \u001b[1;4mhfgputest\u001b[0m Active ✅ │ \n", " │ │ \n", - " │ \u001B[1mURI:\u001B[0m ray://hfgputest-head-svc.default.svc:10001 │ \n", + " │ \u001b[1mURI:\u001b[0m ray://hfgputest-head-svc.default.svc:10001 │ \n", " │ │ \n", - " │ \u001B]8;id=552692;ray-dashboard-hfgputest-default.apps.prepfullinstall.psap.aws.rhperfscale.org\u001B\\\u001B[4;34mDashboard🔗\u001B[0m\u001B]8;;\u001B\\ │ \n", + " │ \u001b]8;id=552692;ray-dashboard-hfgputest-default.apps.prepfullinstall.psap.aws.rhperfscale.org\u001b\\\u001b[4;34mDashboard🔗\u001b[0m\u001b]8;;\u001b\\ │ \n", " │ │ \n", - " │ \u001B[3m Cluster Resources \u001B[0m │ \n", + " │ \u001b[3m Cluster Resources \u001b[0m │ \n", " │ ╭─ Workers ──╮ ╭───────── Worker specs(each) ─────────╮ │ \n", - " │ │ \u001B[1m \u001B[0m\u001B[1mMin\u001B[0m\u001B[1m \u001B[0m\u001B[1m \u001B[0m\u001B[1mMax\u001B[0m\u001B[1m \u001B[0m │ │ \u001B[1m \u001B[0m\u001B[1mMemory \u001B[0m\u001B[1m \u001B[0m\u001B[1m \u001B[0m\u001B[1mCPU \u001B[0m\u001B[1m \u001B[0m\u001B[1m \u001B[0m\u001B[1mGPU \u001B[0m\u001B[1m \u001B[0m │ │ \n", - " │ │ \u001B[36m \u001B[0m\u001B[36m \u001B[0m\u001B[36m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m │ │ \u001B[36m \u001B[0m\u001B[36m \u001B[0m\u001B[36m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m │ │ \n", - " │ │ \u001B[36m \u001B[0m\u001B[36m1 \u001B[0m\u001B[36m \u001B[0m\u001B[35m \u001B[0m\u001B[35m1 \u001B[0m\u001B[35m \u001B[0m │ │ \u001B[36m \u001B[0m\u001B[36m16G~16G \u001B[0m\u001B[36m \u001B[0m\u001B[35m \u001B[0m\u001B[35m8 \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m\u001B[35m4 \u001B[0m\u001B[35m \u001B[0m │ │ \n", - " │ │ \u001B[36m \u001B[0m\u001B[36m \u001B[0m\u001B[36m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m │ │ \u001B[36m \u001B[0m\u001B[36m \u001B[0m\u001B[36m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m │ │ \n", + " │ │ \u001b[1m \u001b[0m\u001b[1mMin\u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m\u001b[1mMax\u001b[0m\u001b[1m \u001b[0m │ │ \u001b[1m \u001b[0m\u001b[1mMemory \u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m\u001b[1mCPU \u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m\u001b[1mGPU \u001b[0m\u001b[1m \u001b[0m │ │ \n", + " │ │ \u001b[36m \u001b[0m\u001b[36m \u001b[0m\u001b[36m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m │ │ \u001b[36m \u001b[0m\u001b[36m \u001b[0m\u001b[36m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m │ │ \n", + " │ │ \u001b[36m \u001b[0m\u001b[36m1 \u001b[0m\u001b[36m \u001b[0m\u001b[35m \u001b[0m\u001b[35m1 \u001b[0m\u001b[35m \u001b[0m │ │ \u001b[36m \u001b[0m\u001b[36m16G~16G \u001b[0m\u001b[36m \u001b[0m\u001b[35m \u001b[0m\u001b[35m8 \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m4 \u001b[0m\u001b[35m \u001b[0m │ │ \n", + " │ │ \u001b[36m \u001b[0m\u001b[36m \u001b[0m\u001b[36m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m │ │ \u001b[36m \u001b[0m\u001b[36m \u001b[0m\u001b[36m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m │ │ \n", " │ ╰────────────╯ ╰──────────────────────────────────────╯ │ \n", " ╰────────────────────────────────────────────────────────────────╯ \n" ] @@ -469,7 +468,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Downloading and preparing dataset imdb/plain_text to /home/ray/.cache/huggingface/datasets/imdb/plain_text/1.0.0/2fdd8b9bcadd6e7055e742a706876ba43f19faee861df134affd7a3f60fc38a1...\n" + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Downloading and preparing dataset imdb/plain_text to /home/ray/.cache/huggingface/datasets/imdb/plain_text/1.0.0/2fdd8b9bcadd6e7055e742a706876ba43f19faee861df134affd7a3f60fc38a1...\n" ] }, { @@ -645,7 +644,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Dataset imdb downloaded and prepared to /home/ray/.cache/huggingface/datasets/imdb/plain_text/1.0.0/2fdd8b9bcadd6e7055e742a706876ba43f19faee861df134affd7a3f60fc38a1. Subsequent calls will reuse this data.\n" + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Dataset imdb downloaded and prepared to /home/ray/.cache/huggingface/datasets/imdb/plain_text/1.0.0/2fdd8b9bcadd6e7055e742a706876ba43f19faee861df134affd7a3f60fc38a1. Subsequent calls will reuse this data.\n" ] }, { @@ -767,13 +766,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m len of train Dataset({\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m features: ['text', 'label', 'input_ids', 'attention_mask'],\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m num_rows: 100\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m }) and test Dataset({\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m features: ['text', 'label', 'input_ids', 'attention_mask'],\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m num_rows: 100\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m })\n" + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m len of train Dataset({\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m features: ['text', 'label', 'input_ids', 'attention_mask'],\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m num_rows: 100\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m }) and test Dataset({\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m features: ['text', 'label', 'input_ids', 'attention_mask'],\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m num_rows: 100\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m })\n" ] }, { @@ -787,54 +786,54 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m To disable this warning, you can either:\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \t- Avoid using `tokenizers` before the fork if possible\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m == Status ==\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Current time: 2022-11-04 07:55:58 (running for 00:00:05.07)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Memory usage on this node: 6.4/240.1 GiB\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Using FIFO scheduling algorithm.\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Number of trials: 1/1 (1 RUNNING)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | Trial name | status | loc |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m |--------------------------------+----------+------------------|\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n" + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m To disable this warning, you can either:\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \t- Avoid using `tokenizers` before the fork if possible\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m == Status ==\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Current time: 2022-11-04 07:55:58 (running for 00:00:05.07)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Memory usage on this node: 6.4/240.1 GiB\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Using FIFO scheduling algorithm.\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Number of trials: 1/1 (1 RUNNING)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | Trial name | status | loc |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m |--------------------------------+----------+------------------|\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001B[0m 2022-11-04 07:56:02,047\tINFO torch.py:346 -- Setting up process group for: env:// [rank=0, world_size=4]\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=184, ip=10.129.66.16)\u001B[0m 2022-11-04 07:56:02,045\tINFO torch.py:346 -- Setting up process group for: env:// [rank=2, world_size=4]\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=183, ip=10.129.66.16)\u001B[0m 2022-11-04 07:56:02,047\tINFO torch.py:346 -- Setting up process group for: env:// [rank=1, world_size=4]\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=185, ip=10.129.66.16)\u001B[0m 2022-11-04 07:56:02,048\tINFO torch.py:346 -- Setting up process group for: env:// [rank=3, world_size=4]\n" + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001b[0m 2022-11-04 07:56:02,047\tINFO torch.py:346 -- Setting up process group for: env:// [rank=0, world_size=4]\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=184, ip=10.129.66.16)\u001b[0m 2022-11-04 07:56:02,045\tINFO torch.py:346 -- Setting up process group for: env:// [rank=2, world_size=4]\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=183, ip=10.129.66.16)\u001b[0m 2022-11-04 07:56:02,047\tINFO torch.py:346 -- Setting up process group for: env:// [rank=1, world_size=4]\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=185, ip=10.129.66.16)\u001b[0m 2022-11-04 07:56:02,048\tINFO torch.py:346 -- Setting up process group for: env:// [rank=3, world_size=4]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m == Status ==\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Current time: 2022-11-04 07:56:03 (running for 00:00:10.07)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Memory usage on this node: 7.2/240.1 GiB\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Using FIFO scheduling algorithm.\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Number of trials: 1/1 (1 RUNNING)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | Trial name | status | loc |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m |--------------------------------+----------+------------------|\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n" + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m == Status ==\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Current time: 2022-11-04 07:56:03 (running for 00:00:10.07)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Memory usage on this node: 7.2/240.1 GiB\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Using FIFO scheduling algorithm.\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Number of trials: 1/1 (1 RUNNING)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | Trial name | status | loc |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m |--------------------------------+----------+------------------|\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n" ] }, { @@ -881,20 +880,20 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m == Status ==\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Current time: 2022-11-04 07:56:08 (running for 00:00:15.07)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Memory usage on this node: 7.5/240.1 GiB\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Using FIFO scheduling algorithm.\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Number of trials: 1/1 (1 RUNNING)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | Trial name | status | loc |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m |--------------------------------+----------+------------------|\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n" + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m == Status ==\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Current time: 2022-11-04 07:56:08 (running for 00:00:15.07)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Memory usage on this node: 7.5/240.1 GiB\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Using FIFO scheduling algorithm.\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Number of trials: 1/1 (1 RUNNING)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | Trial name | status | loc |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m |--------------------------------+----------+------------------|\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n" ] }, { @@ -911,510 +910,510 @@ "Downloading: 95%|█████████▌| 255M/268M [00:04<00:00, 65.7MB/s]\n", "Downloading: 98%|█████████▊| 262M/268M [00:04<00:00, 65.8MB/s]\n", "Downloading: 100%|██████████| 268M/268M [00:04<00:00, 63.9MB/s]\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001B[0m Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_projector.bias', 'vocab_layer_norm.bias', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_transform.weight', 'vocab_projector.weight']\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001B[0m - This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001B[0m - This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001B[0m Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.bias', 'classifier.bias', 'classifier.weight', 'pre_classifier.weight']\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001B[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=184, ip=10.129.66.16)\u001B[0m Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.bias', 'vocab_projector.bias', 'vocab_projector.weight', 'vocab_layer_norm.weight']\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=184, ip=10.129.66.16)\u001B[0m - This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=184, ip=10.129.66.16)\u001B[0m - This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=184, ip=10.129.66.16)\u001B[0m Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier.bias', 'pre_classifier.weight', 'pre_classifier.bias', 'classifier.weight']\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=184, ip=10.129.66.16)\u001B[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=183, ip=10.129.66.16)\u001B[0m Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_projector.weight', 'vocab_transform.weight', 'vocab_projector.bias', 'vocab_transform.bias', 'vocab_layer_norm.bias', 'vocab_layer_norm.weight']\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=183, ip=10.129.66.16)\u001B[0m - This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=183, ip=10.129.66.16)\u001B[0m - This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=183, ip=10.129.66.16)\u001B[0m Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier.bias', 'pre_classifier.bias', 'pre_classifier.weight', 'classifier.weight']\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=183, ip=10.129.66.16)\u001B[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=185, ip=10.129.66.16)\u001B[0m Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_layer_norm.weight', 'vocab_projector.weight', 'vocab_transform.weight', 'vocab_projector.bias', 'vocab_layer_norm.bias', 'vocab_transform.bias']\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=185, ip=10.129.66.16)\u001B[0m - This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=185, ip=10.129.66.16)\u001B[0m - This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=185, ip=10.129.66.16)\u001B[0m Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight', 'pre_classifier.bias', 'pre_classifier.weight']\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=185, ip=10.129.66.16)\u001B[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001B[0m /tmp/ray/session_2022-11-04_07-51-23_507232_7/runtime_resources/pip/4a36d7bd0bbff8fccea52f9c0d942dd63707933f/virtualenv/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001B[0m warnings.warn(\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=183, ip=10.129.66.16)\u001B[0m /tmp/ray/session_2022-11-04_07-51-23_507232_7/runtime_resources/pip/4a36d7bd0bbff8fccea52f9c0d942dd63707933f/virtualenv/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=183, ip=10.129.66.16)\u001B[0m warnings.warn(\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=185, ip=10.129.66.16)\u001B[0m /tmp/ray/session_2022-11-04_07-51-23_507232_7/runtime_resources/pip/4a36d7bd0bbff8fccea52f9c0d942dd63707933f/virtualenv/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=185, ip=10.129.66.16)\u001B[0m warnings.warn(\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=184, ip=10.129.66.16)\u001B[0m /tmp/ray/session_2022-11-04_07-51-23_507232_7/runtime_resources/pip/4a36d7bd0bbff8fccea52f9c0d942dd63707933f/virtualenv/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=184, ip=10.129.66.16)\u001B[0m warnings.warn(\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001B[0m ***** Running training *****\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001B[0m Num examples = 6250\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001B[0m Num Epochs = 1\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001B[0m Instantaneous batch size per device = 16\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001B[0m Total train batch size (w. parallel, distributed & accumulation) = 64\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001B[0m Gradient Accumulation steps = 1\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001B[0m Total optimization steps = 391\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001B[0m Number of trainable parameters = 66955010\n" + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001b[0m Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_projector.bias', 'vocab_layer_norm.bias', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_transform.weight', 'vocab_projector.weight']\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001b[0m - This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001b[0m - This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001b[0m Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.bias', 'classifier.bias', 'classifier.weight', 'pre_classifier.weight']\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=184, ip=10.129.66.16)\u001b[0m Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.bias', 'vocab_projector.bias', 'vocab_projector.weight', 'vocab_layer_norm.weight']\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=184, ip=10.129.66.16)\u001b[0m - This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=184, ip=10.129.66.16)\u001b[0m - This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=184, ip=10.129.66.16)\u001b[0m Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier.bias', 'pre_classifier.weight', 'pre_classifier.bias', 'classifier.weight']\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=184, ip=10.129.66.16)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=183, ip=10.129.66.16)\u001b[0m Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_projector.weight', 'vocab_transform.weight', 'vocab_projector.bias', 'vocab_transform.bias', 'vocab_layer_norm.bias', 'vocab_layer_norm.weight']\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=183, ip=10.129.66.16)\u001b[0m - This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=183, ip=10.129.66.16)\u001b[0m - This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=183, ip=10.129.66.16)\u001b[0m Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier.bias', 'pre_classifier.bias', 'pre_classifier.weight', 'classifier.weight']\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=183, ip=10.129.66.16)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=185, ip=10.129.66.16)\u001b[0m Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_layer_norm.weight', 'vocab_projector.weight', 'vocab_transform.weight', 'vocab_projector.bias', 'vocab_layer_norm.bias', 'vocab_transform.bias']\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=185, ip=10.129.66.16)\u001b[0m - This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=185, ip=10.129.66.16)\u001b[0m - This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=185, ip=10.129.66.16)\u001b[0m Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight', 'pre_classifier.bias', 'pre_classifier.weight']\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=185, ip=10.129.66.16)\u001b[0m You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001b[0m /tmp/ray/session_2022-11-04_07-51-23_507232_7/runtime_resources/pip/4a36d7bd0bbff8fccea52f9c0d942dd63707933f/virtualenv/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001b[0m warnings.warn(\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=183, ip=10.129.66.16)\u001b[0m /tmp/ray/session_2022-11-04_07-51-23_507232_7/runtime_resources/pip/4a36d7bd0bbff8fccea52f9c0d942dd63707933f/virtualenv/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=183, ip=10.129.66.16)\u001b[0m warnings.warn(\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=185, ip=10.129.66.16)\u001b[0m /tmp/ray/session_2022-11-04_07-51-23_507232_7/runtime_resources/pip/4a36d7bd0bbff8fccea52f9c0d942dd63707933f/virtualenv/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=185, ip=10.129.66.16)\u001b[0m warnings.warn(\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=184, ip=10.129.66.16)\u001b[0m /tmp/ray/session_2022-11-04_07-51-23_507232_7/runtime_resources/pip/4a36d7bd0bbff8fccea52f9c0d942dd63707933f/virtualenv/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=184, ip=10.129.66.16)\u001b[0m warnings.warn(\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001b[0m ***** Running training *****\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001b[0m Num examples = 6250\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001b[0m Num Epochs = 1\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001b[0m Instantaneous batch size per device = 16\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001b[0m Total train batch size (w. parallel, distributed & accumulation) = 64\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001b[0m Gradient Accumulation steps = 1\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001b[0m Total optimization steps = 391\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001b[0m Number of trainable parameters = 66955010\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m == Status ==\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Current time: 2022-11-04 07:56:13 (running for 00:00:20.08)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Memory usage on this node: 12.3/240.1 GiB\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Using FIFO scheduling algorithm.\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Number of trials: 1/1 (1 RUNNING)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | Trial name | status | loc |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m |--------------------------------+----------+------------------|\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n" + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m == Status ==\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Current time: 2022-11-04 07:56:13 (running for 00:00:20.08)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Memory usage on this node: 12.3/240.1 GiB\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Using FIFO scheduling algorithm.\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Number of trials: 1/1 (1 RUNNING)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | Trial name | status | loc |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m |--------------------------------+----------+------------------|\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001B[0m [W reducer.cpp:1251] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration, which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=184, ip=10.129.66.16)\u001B[0m [W reducer.cpp:1251] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration, which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=183, ip=10.129.66.16)\u001B[0m [W reducer.cpp:1251] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration, which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=185, ip=10.129.66.16)\u001B[0m [W reducer.cpp:1251] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration, which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())\n" + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001b[0m [W reducer.cpp:1251] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration, which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=184, ip=10.129.66.16)\u001b[0m [W reducer.cpp:1251] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration, which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=183, ip=10.129.66.16)\u001b[0m [W reducer.cpp:1251] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration, which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=185, ip=10.129.66.16)\u001b[0m [W reducer.cpp:1251] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration, which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m == Status ==\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Current time: 2022-11-04 07:56:18 (running for 00:00:25.08)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Memory usage on this node: 13.7/240.1 GiB\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Using FIFO scheduling algorithm.\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Number of trials: 1/1 (1 RUNNING)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | Trial name | status | loc |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m |--------------------------------+----------+------------------|\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m == Status ==\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Current time: 2022-11-04 07:56:23 (running for 00:00:30.08)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Memory usage on this node: 13.7/240.1 GiB\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Using FIFO scheduling algorithm.\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Number of trials: 1/1 (1 RUNNING)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | Trial name | status | loc |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m |--------------------------------+----------+------------------|\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m == Status ==\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Current time: 2022-11-04 07:56:28 (running for 00:00:35.09)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Memory usage on this node: 13.7/240.1 GiB\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Using FIFO scheduling algorithm.\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Number of trials: 1/1 (1 RUNNING)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | Trial name | status | loc |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m |--------------------------------+----------+------------------|\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m == Status ==\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Current time: 2022-11-04 07:56:33 (running for 00:00:40.09)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Memory usage on this node: 13.7/240.1 GiB\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Using FIFO scheduling algorithm.\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Number of trials: 1/1 (1 RUNNING)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | Trial name | status | loc |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m |--------------------------------+----------+------------------|\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m == Status ==\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Current time: 2022-11-04 07:56:38 (running for 00:00:45.10)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Memory usage on this node: 13.7/240.1 GiB\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Using FIFO scheduling algorithm.\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Number of trials: 1/1 (1 RUNNING)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | Trial name | status | loc |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m |--------------------------------+----------+------------------|\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m == Status ==\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Current time: 2022-11-04 07:56:43 (running for 00:00:50.10)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Memory usage on this node: 13.7/240.1 GiB\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Using FIFO scheduling algorithm.\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Number of trials: 1/1 (1 RUNNING)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | Trial name | status | loc |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m |--------------------------------+----------+------------------|\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m == Status ==\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Current time: 2022-11-04 07:56:48 (running for 00:00:55.10)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Memory usage on this node: 13.7/240.1 GiB\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Using FIFO scheduling algorithm.\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Number of trials: 1/1 (1 RUNNING)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | Trial name | status | loc |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m |--------------------------------+----------+------------------|\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m == Status ==\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Current time: 2022-11-04 07:56:53 (running for 00:01:00.10)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Memory usage on this node: 13.7/240.1 GiB\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Using FIFO scheduling algorithm.\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Number of trials: 1/1 (1 RUNNING)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | Trial name | status | loc |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m |--------------------------------+----------+------------------|\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m == Status ==\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Current time: 2022-11-04 07:56:59 (running for 00:01:05.11)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Memory usage on this node: 13.7/240.1 GiB\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Using FIFO scheduling algorithm.\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Number of trials: 1/1 (1 RUNNING)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | Trial name | status | loc |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m |--------------------------------+----------+------------------|\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m == Status ==\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Current time: 2022-11-04 07:57:04 (running for 00:01:10.11)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Memory usage on this node: 13.7/240.1 GiB\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Using FIFO scheduling algorithm.\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Number of trials: 1/1 (1 RUNNING)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | Trial name | status | loc |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m |--------------------------------+----------+------------------|\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m == Status ==\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Current time: 2022-11-04 07:57:09 (running for 00:01:15.11)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Memory usage on this node: 13.7/240.1 GiB\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Using FIFO scheduling algorithm.\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Number of trials: 1/1 (1 RUNNING)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | Trial name | status | loc |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m |--------------------------------+----------+------------------|\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m == Status ==\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Current time: 2022-11-04 07:57:14 (running for 00:01:20.12)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Memory usage on this node: 13.7/240.1 GiB\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Using FIFO scheduling algorithm.\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Number of trials: 1/1 (1 RUNNING)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | Trial name | status | loc |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m |--------------------------------+----------+------------------|\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m == Status ==\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Current time: 2022-11-04 07:57:19 (running for 00:01:25.12)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Memory usage on this node: 13.7/240.1 GiB\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Using FIFO scheduling algorithm.\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Number of trials: 1/1 (1 RUNNING)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | Trial name | status | loc |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m |--------------------------------+----------+------------------|\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m == Status ==\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Current time: 2022-11-04 07:57:24 (running for 00:01:30.12)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Memory usage on this node: 13.7/240.1 GiB\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Using FIFO scheduling algorithm.\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Number of trials: 1/1 (1 RUNNING)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | Trial name | status | loc |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m |--------------------------------+----------+------------------|\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m == Status ==\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Current time: 2022-11-04 07:57:29 (running for 00:01:35.13)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Memory usage on this node: 13.7/240.1 GiB\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Using FIFO scheduling algorithm.\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Number of trials: 1/1 (1 RUNNING)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | Trial name | status | loc |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m |--------------------------------+----------+------------------|\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m == Status ==\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Current time: 2022-11-04 07:57:34 (running for 00:01:40.13)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Memory usage on this node: 13.7/240.1 GiB\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Using FIFO scheduling algorithm.\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Number of trials: 1/1 (1 RUNNING)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | Trial name | status | loc |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m |--------------------------------+----------+------------------|\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m == Status ==\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Current time: 2022-11-04 07:57:39 (running for 00:01:45.13)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Memory usage on this node: 13.7/240.1 GiB\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Using FIFO scheduling algorithm.\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Number of trials: 1/1 (1 RUNNING)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | Trial name | status | loc |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m |--------------------------------+----------+------------------|\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m == Status ==\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Current time: 2022-11-04 07:57:44 (running for 00:01:50.13)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Memory usage on this node: 13.7/240.1 GiB\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Using FIFO scheduling algorithm.\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Number of trials: 1/1 (1 RUNNING)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | Trial name | status | loc |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m |--------------------------------+----------+------------------|\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m == Status ==\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Current time: 2022-11-04 07:57:49 (running for 00:01:55.14)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Memory usage on this node: 13.7/240.1 GiB\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Using FIFO scheduling algorithm.\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Number of trials: 1/1 (1 RUNNING)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | Trial name | status | loc |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m |--------------------------------+----------+------------------|\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m == Status ==\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Current time: 2022-11-04 07:57:54 (running for 00:02:00.14)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Memory usage on this node: 13.7/240.1 GiB\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Using FIFO scheduling algorithm.\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Number of trials: 1/1 (1 RUNNING)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | Trial name | status | loc |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m |--------------------------------+----------+------------------|\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m == Status ==\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Current time: 2022-11-04 07:57:59 (running for 00:02:05.15)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Memory usage on this node: 13.7/240.1 GiB\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Using FIFO scheduling algorithm.\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Number of trials: 1/1 (1 RUNNING)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | Trial name | status | loc |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m |--------------------------------+----------+------------------|\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n" + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m == Status ==\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Current time: 2022-11-04 07:56:18 (running for 00:00:25.08)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Memory usage on this node: 13.7/240.1 GiB\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Using FIFO scheduling algorithm.\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Number of trials: 1/1 (1 RUNNING)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | Trial name | status | loc |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m |--------------------------------+----------+------------------|\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m == Status ==\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Current time: 2022-11-04 07:56:23 (running for 00:00:30.08)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Memory usage on this node: 13.7/240.1 GiB\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Using FIFO scheduling algorithm.\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Number of trials: 1/1 (1 RUNNING)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | Trial name | status | loc |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m |--------------------------------+----------+------------------|\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m == Status ==\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Current time: 2022-11-04 07:56:28 (running for 00:00:35.09)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Memory usage on this node: 13.7/240.1 GiB\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Using FIFO scheduling algorithm.\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Number of trials: 1/1 (1 RUNNING)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | Trial name | status | loc |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m |--------------------------------+----------+------------------|\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m == Status ==\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Current time: 2022-11-04 07:56:33 (running for 00:00:40.09)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Memory usage on this node: 13.7/240.1 GiB\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Using FIFO scheduling algorithm.\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Number of trials: 1/1 (1 RUNNING)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | Trial name | status | loc |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m |--------------------------------+----------+------------------|\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m == Status ==\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Current time: 2022-11-04 07:56:38 (running for 00:00:45.10)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Memory usage on this node: 13.7/240.1 GiB\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Using FIFO scheduling algorithm.\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Number of trials: 1/1 (1 RUNNING)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | Trial name | status | loc |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m |--------------------------------+----------+------------------|\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m == Status ==\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Current time: 2022-11-04 07:56:43 (running for 00:00:50.10)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Memory usage on this node: 13.7/240.1 GiB\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Using FIFO scheduling algorithm.\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Number of trials: 1/1 (1 RUNNING)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | Trial name | status | loc |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m |--------------------------------+----------+------------------|\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m == Status ==\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Current time: 2022-11-04 07:56:48 (running for 00:00:55.10)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Memory usage on this node: 13.7/240.1 GiB\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Using FIFO scheduling algorithm.\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Number of trials: 1/1 (1 RUNNING)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | Trial name | status | loc |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m |--------------------------------+----------+------------------|\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m == Status ==\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Current time: 2022-11-04 07:56:53 (running for 00:01:00.10)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Memory usage on this node: 13.7/240.1 GiB\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Using FIFO scheduling algorithm.\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Number of trials: 1/1 (1 RUNNING)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | Trial name | status | loc |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m |--------------------------------+----------+------------------|\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m == Status ==\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Current time: 2022-11-04 07:56:59 (running for 00:01:05.11)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Memory usage on this node: 13.7/240.1 GiB\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Using FIFO scheduling algorithm.\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Number of trials: 1/1 (1 RUNNING)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | Trial name | status | loc |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m |--------------------------------+----------+------------------|\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m == Status ==\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Current time: 2022-11-04 07:57:04 (running for 00:01:10.11)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Memory usage on this node: 13.7/240.1 GiB\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Using FIFO scheduling algorithm.\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Number of trials: 1/1 (1 RUNNING)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | Trial name | status | loc |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m |--------------------------------+----------+------------------|\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m == Status ==\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Current time: 2022-11-04 07:57:09 (running for 00:01:15.11)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Memory usage on this node: 13.7/240.1 GiB\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Using FIFO scheduling algorithm.\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Number of trials: 1/1 (1 RUNNING)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | Trial name | status | loc |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m |--------------------------------+----------+------------------|\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m == Status ==\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Current time: 2022-11-04 07:57:14 (running for 00:01:20.12)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Memory usage on this node: 13.7/240.1 GiB\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Using FIFO scheduling algorithm.\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Number of trials: 1/1 (1 RUNNING)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | Trial name | status | loc |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m |--------------------------------+----------+------------------|\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m == Status ==\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Current time: 2022-11-04 07:57:19 (running for 00:01:25.12)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Memory usage on this node: 13.7/240.1 GiB\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Using FIFO scheduling algorithm.\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Number of trials: 1/1 (1 RUNNING)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | Trial name | status | loc |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m |--------------------------------+----------+------------------|\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m == Status ==\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Current time: 2022-11-04 07:57:24 (running for 00:01:30.12)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Memory usage on this node: 13.7/240.1 GiB\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Using FIFO scheduling algorithm.\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Number of trials: 1/1 (1 RUNNING)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | Trial name | status | loc |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m |--------------------------------+----------+------------------|\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m == Status ==\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Current time: 2022-11-04 07:57:29 (running for 00:01:35.13)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Memory usage on this node: 13.7/240.1 GiB\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Using FIFO scheduling algorithm.\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Number of trials: 1/1 (1 RUNNING)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | Trial name | status | loc |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m |--------------------------------+----------+------------------|\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m == Status ==\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Current time: 2022-11-04 07:57:34 (running for 00:01:40.13)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Memory usage on this node: 13.7/240.1 GiB\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Using FIFO scheduling algorithm.\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Number of trials: 1/1 (1 RUNNING)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | Trial name | status | loc |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m |--------------------------------+----------+------------------|\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m == Status ==\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Current time: 2022-11-04 07:57:39 (running for 00:01:45.13)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Memory usage on this node: 13.7/240.1 GiB\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Using FIFO scheduling algorithm.\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Number of trials: 1/1 (1 RUNNING)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | Trial name | status | loc |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m |--------------------------------+----------+------------------|\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m == Status ==\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Current time: 2022-11-04 07:57:44 (running for 00:01:50.13)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Memory usage on this node: 13.7/240.1 GiB\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Using FIFO scheduling algorithm.\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Number of trials: 1/1 (1 RUNNING)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | Trial name | status | loc |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m |--------------------------------+----------+------------------|\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m == Status ==\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Current time: 2022-11-04 07:57:49 (running for 00:01:55.14)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Memory usage on this node: 13.7/240.1 GiB\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Using FIFO scheduling algorithm.\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Number of trials: 1/1 (1 RUNNING)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | Trial name | status | loc |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m |--------------------------------+----------+------------------|\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m == Status ==\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Current time: 2022-11-04 07:57:54 (running for 00:02:00.14)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Memory usage on this node: 13.7/240.1 GiB\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Using FIFO scheduling algorithm.\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Number of trials: 1/1 (1 RUNNING)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | Trial name | status | loc |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m |--------------------------------+----------+------------------|\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m == Status ==\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Current time: 2022-11-04 07:57:59 (running for 00:02:05.15)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Memory usage on this node: 13.7/240.1 GiB\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Using FIFO scheduling algorithm.\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Number of trials: 1/1 (1 RUNNING)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | Trial name | status | loc |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m |--------------------------------+----------+------------------|\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001B[0m Saving model checkpoint to /tmp/hf_imdb/test/checkpoint-391\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001B[0m Configuration saved in /tmp/hf_imdb/test/checkpoint-391/config.json\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001B[0m Model weights saved in /tmp/hf_imdb/test/checkpoint-391/pytorch_model.bin\n" + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001b[0m Saving model checkpoint to /tmp/hf_imdb/test/checkpoint-391\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001b[0m Configuration saved in /tmp/hf_imdb/test/checkpoint-391/config.json\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001b[0m Model weights saved in /tmp/hf_imdb/test/checkpoint-391/pytorch_model.bin\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Result for HuggingFaceTrainer_c7d60_00000:\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m _time_this_iter_s: 118.07144260406494\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m _timestamp: 1667573883\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m _training_iteration: 1\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m date: 2022-11-04_07-58-03\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m done: false\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m epoch: 1.0\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m experiment_id: 7bc6ab25d0414fcbb589bcb5d0f29b99\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m hostname: hfgputest-worker-small-group-hfgputest-q4758\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m iterations_since_restore: 1\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m node_ip: 10.129.66.16\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m pid: 146\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m should_checkpoint: true\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m step: 391\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m time_since_restore: 124.55581378936768\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m time_this_iter_s: 124.55581378936768\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m time_total_s: 124.55581378936768\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m timestamp: 1667573883\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m timesteps_since_restore: 0\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m train_loss: 0.2760564701636429\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m train_runtime: 109.7668\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m train_samples_per_second: 56.939\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m train_steps_per_second: 3.562\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m training_iteration: 1\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m trial_id: c7d60_00000\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m warmup_time: 0.003995656967163086\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001B[0m {'train_runtime': 109.7668, 'train_samples_per_second': 56.939, 'train_steps_per_second': 3.562, 'train_loss': 0.2760564701636429, 'epoch': 1.0}\n" + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Result for HuggingFaceTrainer_c7d60_00000:\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m _time_this_iter_s: 118.07144260406494\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m _timestamp: 1667573883\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m _training_iteration: 1\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m date: 2022-11-04_07-58-03\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m done: false\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m epoch: 1.0\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m experiment_id: 7bc6ab25d0414fcbb589bcb5d0f29b99\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m hostname: hfgputest-worker-small-group-hfgputest-q4758\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m iterations_since_restore: 1\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m node_ip: 10.129.66.16\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m pid: 146\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m should_checkpoint: true\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m step: 391\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m time_since_restore: 124.55581378936768\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m time_this_iter_s: 124.55581378936768\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m time_total_s: 124.55581378936768\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m timestamp: 1667573883\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m timesteps_since_restore: 0\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m train_loss: 0.2760564701636429\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m train_runtime: 109.7668\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m train_samples_per_second: 56.939\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m train_steps_per_second: 3.562\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m training_iteration: 1\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m trial_id: c7d60_00000\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m warmup_time: 0.003995656967163086\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001b[0m {'train_runtime': 109.7668, 'train_samples_per_second': 56.939, 'train_steps_per_second': 3.562, 'train_loss': 0.2760564701636429, 'epoch': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001B[0m \n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001B[0m \n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001B[0m Training completed. Do not forget to share your model on huggingface.co/models =)\n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001B[0m \n", - "\u001B[2m\u001B[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001B[0m \n" + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001b[0m \n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001b[0m \n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001b[0m Training completed. Do not forget to share your model on huggingface.co/models =)\n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001b[0m \n", + "\u001b[2m\u001b[36m(BaseWorkerMixin pid=182, ip=10.129.66.16)\u001b[0m \n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m == Status ==\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Current time: 2022-11-04 07:58:13 (running for 00:02:19.36)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Memory usage on this node: 16.0/240.1 GiB\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Using FIFO scheduling algorithm.\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Number of trials: 1/1 (1 RUNNING)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+--------+------------------+-----------------+----------------------------+--------------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | Trial name | status | loc | iter | total time (s) | train_runtime | train_samples_per_second | train_steps_per_second |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m |--------------------------------+----------+------------------+--------+------------------+-----------------+----------------------------+--------------------------|\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 | 1 | 124.556 | 109.767 | 56.939 | 3.562 |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+----------+------------------+--------+------------------+-----------------+----------------------------+--------------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n" + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m == Status ==\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Current time: 2022-11-04 07:58:13 (running for 00:02:19.36)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Memory usage on this node: 16.0/240.1 GiB\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Using FIFO scheduling algorithm.\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Resources requested: 5.0/10 CPUs, 4.0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Number of trials: 1/1 (1 RUNNING)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+--------+------------------+-----------------+----------------------------+--------------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | Trial name | status | loc | iter | total time (s) | train_runtime | train_samples_per_second | train_steps_per_second |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m |--------------------------------+----------+------------------+--------+------------------+-----------------+----------------------------+--------------------------|\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | HuggingFaceTrainer_c7d60_00000 | RUNNING | 10.129.66.16:146 | 1 | 124.556 | 109.767 | 56.939 | 3.562 |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+----------+------------------+--------+------------------+-----------------+----------------------------+--------------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m 2022-11-04 07:58:13,248\tWARNING util.py:214 -- The `process_trial_save` operation took 9.709 s, which may be a performance bottleneck.\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m 2022-11-04 07:58:13,248\tWARNING trial_runner.py:856 -- Consider turning off forced head-worker trial checkpoint syncs by setting sync_on_checkpoint=False. Note that this may result in faulty trial restoration if a failure occurs while the checkpoint is being synced from the worker to the head node.\n" + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m 2022-11-04 07:58:13,248\tWARNING util.py:214 -- The `process_trial_save` operation took 9.709 s, which may be a performance bottleneck.\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m 2022-11-04 07:58:13,248\tWARNING trial_runner.py:856 -- Consider turning off forced head-worker trial checkpoint syncs by setting sync_on_checkpoint=False. Note that this may result in faulty trial restoration if a failure occurs while the checkpoint is being synced from the worker to the head node.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Result for HuggingFaceTrainer_c7d60_00000:\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m _time_this_iter_s: 118.07144260406494\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m _timestamp: 1667573883\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m _training_iteration: 1\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m date: 2022-11-04_07-58-03\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m done: true\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m epoch: 1.0\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m experiment_id: 7bc6ab25d0414fcbb589bcb5d0f29b99\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m experiment_tag: '0'\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m hostname: hfgputest-worker-small-group-hfgputest-q4758\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m iterations_since_restore: 1\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m node_ip: 10.129.66.16\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m pid: 146\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m should_checkpoint: true\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m step: 391\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m time_since_restore: 124.55581378936768\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m time_this_iter_s: 124.55581378936768\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m time_total_s: 124.55581378936768\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m timestamp: 1667573883\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m timesteps_since_restore: 0\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m train_loss: 0.2760564701636429\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m train_runtime: 109.7668\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m train_samples_per_second: 56.939\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m train_steps_per_second: 3.562\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m training_iteration: 1\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m trial_id: c7d60_00000\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m warmup_time: 0.003995656967163086\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m == Status ==\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Current time: 2022-11-04 07:58:16 (running for 00:02:22.40)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Memory usage on this node: 9.1/240.1 GiB\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Using FIFO scheduling algorithm.\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Resources requested: 0/10 CPUs, 0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m Number of trials: 1/1 (1 TERMINATED)\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+------------+------------------+--------+------------------+-----------------+----------------------------+--------------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | Trial name | status | loc | iter | total time (s) | train_runtime | train_samples_per_second | train_steps_per_second |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m |--------------------------------+------------+------------------+--------+------------------+-----------------+----------------------------+--------------------------|\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m | HuggingFaceTrainer_c7d60_00000 | TERMINATED | 10.129.66.16:146 | 1 | 124.556 | 109.767 | 56.939 | 3.562 |\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m +--------------------------------+------------+------------------+--------+------------------+-----------------+----------------------------+--------------------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m \n" + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Result for HuggingFaceTrainer_c7d60_00000:\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m _time_this_iter_s: 118.07144260406494\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m _timestamp: 1667573883\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m _training_iteration: 1\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m date: 2022-11-04_07-58-03\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m done: true\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m epoch: 1.0\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m experiment_id: 7bc6ab25d0414fcbb589bcb5d0f29b99\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m experiment_tag: '0'\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m hostname: hfgputest-worker-small-group-hfgputest-q4758\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m iterations_since_restore: 1\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m node_ip: 10.129.66.16\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m pid: 146\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m should_checkpoint: true\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m step: 391\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m time_since_restore: 124.55581378936768\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m time_this_iter_s: 124.55581378936768\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m time_total_s: 124.55581378936768\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m timestamp: 1667573883\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m timesteps_since_restore: 0\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m train_loss: 0.2760564701636429\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m train_runtime: 109.7668\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m train_samples_per_second: 56.939\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m train_steps_per_second: 3.562\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m training_iteration: 1\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m trial_id: c7d60_00000\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m warmup_time: 0.003995656967163086\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m == Status ==\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Current time: 2022-11-04 07:58:16 (running for 00:02:22.40)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Memory usage on this node: 9.1/240.1 GiB\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Using FIFO scheduling algorithm.\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Resources requested: 0/10 CPUs, 0/4 GPUs, 0.0/22.35 GiB heap, 0.0/6.59 GiB objects\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2022-11-04_07-55-53\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m Number of trials: 1/1 (1 TERMINATED)\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+------------+------------------+--------+------------------+-----------------+----------------------------+--------------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | Trial name | status | loc | iter | total time (s) | train_runtime | train_samples_per_second | train_steps_per_second |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m |--------------------------------+------------+------------------+--------+------------------+-----------------+----------------------------+--------------------------|\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m | HuggingFaceTrainer_c7d60_00000 | TERMINATED | 10.129.66.16:146 | 1 | 124.556 | 109.767 | 56.939 | 3.562 |\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m +--------------------------------+------------+------------------+--------+------------------+-----------------+----------------------------+--------------------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m 2022-11-04 07:58:16,286\tWARNING util.py:214 -- The `process_trial_save` operation took 2.161 s, which may be a performance bottleneck.\n", - "\u001B[2m\u001B[36m(train_fn pid=250)\u001B[0m 2022-11-04 07:58:16,398\tINFO tune.py:747 -- Total run time: 142.70 seconds (142.40 seconds for the tuning loop).\n" + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m 2022-11-04 07:58:16,286\tWARNING util.py:214 -- The `process_trial_save` operation took 2.161 s, which may be a performance bottleneck.\n", + "\u001b[2m\u001b[36m(train_fn pid=250)\u001b[0m 2022-11-04 07:58:16,398\tINFO tune.py:747 -- Total run time: 142.70 seconds (142.40 seconds for the tuning loop).\n" ] } ], diff --git a/demo-notebooks/additional-demos/local_interactive.ipynb b/demo-notebooks/additional-demos/local_interactive.ipynb index 1d3c83ad2..8ea088347 100644 --- a/demo-notebooks/additional-demos/local_interactive.ipynb +++ b/demo-notebooks/additional-demos/local_interactive.ipynb @@ -50,10 +50,9 @@ "source": [ "# Create and submit our cluster\n", "# The SDK will try to find the name of your default local queue based on the annotation \"kueue.x-k8s.io/default-queue\": \"true\" unless you specify the local queue manually below\n", - "namespace = \"default\" # Update to your namespace\n", "cluster_name = \"hfgputest-1\"\n", "\n", - "cluster = Cluster(ClusterConfiguration(namespace=namespace,\n", + "cluster = Cluster(ClusterConfiguration(\n", " name=cluster_name,\n", " head_gpus=0, # For GPU enabled workloads set the head_gpus and num_gpus\n", " num_gpus=0,\n", @@ -118,8 +117,8 @@ "source": [ "from codeflare_sdk import generate_cert\n", "\n", - "generate_cert.generate_tls_cert(cluster_name, namespace)\n", - "generate_cert.export_env(cluster_name, namespace)" + "generate_cert.generate_tls_cert(cluster_name, cluster.config.namespace)\n", + "generate_cert.export_env(cluster_name, cluster.config.namespace)" ] }, { diff --git a/demo-notebooks/additional-demos/ray_job_client.ipynb b/demo-notebooks/additional-demos/ray_job_client.ipynb index c452fb31e..4b9434bc9 100644 --- a/demo-notebooks/additional-demos/ray_job_client.ipynb +++ b/demo-notebooks/additional-demos/ray_job_client.ipynb @@ -45,7 +45,6 @@ "# The SDK will try to find the name of your default local queue based on the annotation \"kueue.x-k8s.io/default-queue\": \"true\" unless you specify the local queue manually below\n", "cluster = Cluster(ClusterConfiguration(\n", " name='jobtest',\n", - " namespace='default', # Update to your namespace\n", " head_gpus=0, # For GPU enabled workloads set the head_gpus and num_gpus\n", " num_gpus=0,\n", " num_workers=2,\n", diff --git a/demo-notebooks/guided-demos/0_basic_ray.ipynb b/demo-notebooks/guided-demos/0_basic_ray.ipynb index 0cd3419f6..3f0f62e47 100644 --- a/demo-notebooks/guided-demos/0_basic_ray.ipynb +++ b/demo-notebooks/guided-demos/0_basic_ray.ipynb @@ -62,7 +62,6 @@ "# The SDK will try to find the name of your default local queue based on the annotation \"kueue.x-k8s.io/default-queue\": \"true\" unless you specify the local queue manually below\n", "cluster = Cluster(ClusterConfiguration(\n", " name='raytest', \n", - " namespace='default', # Update to your namespace\n", " head_gpus=0, # For GPU enabled workloads set the head_gpus and num_gpus\n", " num_gpus=0,\n", " num_workers=2,\n", diff --git a/demo-notebooks/guided-demos/1_cluster_job_client.ipynb b/demo-notebooks/guided-demos/1_cluster_job_client.ipynb index de8fafdd6..00576024a 100644 --- a/demo-notebooks/guided-demos/1_cluster_job_client.ipynb +++ b/demo-notebooks/guided-demos/1_cluster_job_client.ipynb @@ -44,7 +44,6 @@ "# The SDK will try to find the name of your default local queue based on the annotation \"kueue.x-k8s.io/default-queue\": \"true\" unless you specify the local queue manually below\n", "cluster = Cluster(ClusterConfiguration(\n", " name='jobtest',\n", - " namespace='default', # Update to your namespace\n", " head_gpus=1, # For GPU enabled workloads set the head_gpus and num_gpus\n", " num_gpus=1,\n", " num_workers=2,\n", diff --git a/demo-notebooks/guided-demos/2_basic_interactive.ipynb b/demo-notebooks/guided-demos/2_basic_interactive.ipynb index b6a13b8c4..0692caa4c 100644 --- a/demo-notebooks/guided-demos/2_basic_interactive.ipynb +++ b/demo-notebooks/guided-demos/2_basic_interactive.ipynb @@ -57,11 +57,9 @@ "source": [ "# Create and configure our cluster object\n", "# The SDK will try to find the name of your default local queue based on the annotation \"kueue.x-k8s.io/default-queue\": \"true\" unless you specify the local queue manually below\n", - "namespace = \"default\" # Update to your namespace\n", "cluster_name = \"interactivetest\"\n", "cluster = Cluster(ClusterConfiguration(\n", " name=cluster_name,\n", - " namespace=namespace,\n", " head_gpus=1, # For GPU enabled workloads set the head_gpus and num_gpus\n", " num_gpus=1,\n", " num_workers=2,\n", @@ -137,8 +135,8 @@ "source": [ "from codeflare_sdk import generate_cert\n", "# Create required TLS cert and export the environment variables to enable TLS\n", - "generate_cert.generate_tls_cert(cluster_name, namespace)\n", - "generate_cert.export_env(cluster_name, namespace)" + "generate_cert.generate_tls_cert(cluster_name, cluster.config.namespace)\n", + "generate_cert.export_env(cluster_name, cluster.config.namespace)" ] }, { diff --git a/demo-notebooks/guided-demos/notebook-ex-outputs/0_basic_ray.ipynb b/demo-notebooks/guided-demos/notebook-ex-outputs/0_basic_ray.ipynb index 646e24242..8338ce749 100644 --- a/demo-notebooks/guided-demos/notebook-ex-outputs/0_basic_ray.ipynb +++ b/demo-notebooks/guided-demos/notebook-ex-outputs/0_basic_ray.ipynb @@ -70,7 +70,6 @@ "# The SDK will try to find the name of your default local queue based on the annotation \"kueue.x-k8s.io/default-queue\": \"true\" unless you specify the local queue manually below\n", "cluster = Cluster(ClusterConfiguration(\n", " name='raytest',\n", - " namespace='default', # Update to your namespace\n", " head_gpus=0, # For GPU enabled workloads set the head_gpus and num_gpus\n", " num_gpus=0,\n", " num_workers=2,\n", @@ -134,13 +133,13 @@ ], "text/plain": [ "╭───────────────────────╮\n", - "│ \u001B[3m \u001B[0m\u001B[1;3m 🚀 Cluster Queue\u001B[0m\u001B[3m \u001B[0m │\n", - "│ \u001B[3m \u001B[0m\u001B[1;3mStatus 🚀\u001B[0m\u001B[3m \u001B[0m │\n", + "│ \u001b[3m \u001b[0m\u001b[1;3m 🚀 Cluster Queue\u001b[0m\u001b[3m \u001b[0m │\n", + "│ \u001b[3m \u001b[0m\u001b[1;3mStatus 🚀\u001b[0m\u001b[3m \u001b[0m │\n", "│ +---------+---------+ │\n", - "│ |\u001B[1m \u001B[0m\u001B[1mName \u001B[0m\u001B[1m \u001B[0m|\u001B[1m \u001B[0m\u001B[1mStatus \u001B[0m\u001B[1m \u001B[0m| │\n", + "│ |\u001b[1m \u001b[0m\u001b[1mName \u001b[0m\u001b[1m \u001b[0m|\u001b[1m \u001b[0m\u001b[1mStatus \u001b[0m\u001b[1m \u001b[0m| │\n", "│ +=========+=========+ │\n", - "│ |\u001B[36m \u001B[0m\u001B[36mraytest\u001B[0m\u001B[36m \u001B[0m|\u001B[35m \u001B[0m\u001B[35mpending\u001B[0m\u001B[35m \u001B[0m| │\n", - "│ |\u001B[36m \u001B[0m\u001B[36m \u001B[0m\u001B[36m \u001B[0m|\u001B[35m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m| │\n", + "│ |\u001b[36m \u001b[0m\u001b[36mraytest\u001b[0m\u001b[36m \u001b[0m|\u001b[35m \u001b[0m\u001b[35mpending\u001b[0m\u001b[35m \u001b[0m| │\n", + "│ |\u001b[36m \u001b[0m\u001b[36m \u001b[0m\u001b[36m \u001b[0m|\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m| │\n", "│ +---------+---------+ │\n", "╰───────────────────────╯\n" ] @@ -205,15 +204,15 @@ "\n" ], "text/plain": [ - "\u001B[3m \u001B[0m\u001B[1;3m 🚀 CodeFlare Cluster Status 🚀\u001B[0m\u001B[3m \u001B[0m\n", - "\u001B[1m \u001B[0m\u001B[1m \u001B[0m\u001B[1m \u001B[0m\n", + "\u001b[3m \u001b[0m\u001b[1;3m 🚀 CodeFlare Cluster Status 🚀\u001b[0m\u001b[3m \u001b[0m\n", + "\u001b[1m \u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m\n", " ╭──────────────────────────────────────────────────────────────╮ \n", - " │ \u001B[1;37;42mName\u001B[0m │ \n", - " │ \u001B[1;4mraytest\u001B[0m Active ✅ │ \n", + " │ \u001b[1;37;42mName\u001b[0m │ \n", + " │ \u001b[1;4mraytest\u001b[0m Active ✅ │ \n", " │ │ \n", - " │ \u001B[1mURI:\u001B[0m ray://raytest-head-svc.default.svc:10001 │ \n", + " │ \u001b[1mURI:\u001b[0m ray://raytest-head-svc.default.svc:10001 │ \n", " │ │ \n", - " │ \u001B]8;id=630217;ray-dashboard-raytest-default.apps.meyceoz-07122023.psap.aws.rhperfscale.org\u001B\\\u001B[4;34mDashboard🔗\u001B[0m\u001B]8;;\u001B\\ │ \n", + " │ \u001b]8;id=630217;ray-dashboard-raytest-default.apps.meyceoz-07122023.psap.aws.rhperfscale.org\u001b\\\u001b[4;34mDashboard🔗\u001b[0m\u001b]8;;\u001b\\ │ \n", " │ │ \n", " ╰──────────────────────────────────────────────────────────────╯ \n" ] @@ -274,22 +273,22 @@ "\n" ], "text/plain": [ - "\u001B[3m \u001B[0m\u001B[1;3m 🚀 CodeFlare Cluster Details 🚀\u001B[0m\u001B[3m \u001B[0m\n", - "\u001B[1m \u001B[0m\u001B[1m \u001B[0m\u001B[1m \u001B[0m\n", + "\u001b[3m \u001b[0m\u001b[1;3m 🚀 CodeFlare Cluster Details 🚀\u001b[0m\u001b[3m \u001b[0m\n", + "\u001b[1m \u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m\n", " ╭───────────────────────────────────────────────────────────────╮ \n", - " │ \u001B[1;37;42mName\u001B[0m │ \n", - " │ \u001B[1;4mraytest\u001B[0m Active ✅ │ \n", + " │ \u001b[1;37;42mName\u001b[0m │ \n", + " │ \u001b[1;4mraytest\u001b[0m Active ✅ │ \n", " │ │ \n", - " │ \u001B[1mURI:\u001B[0m ray://raytest-head-svc.default.svc:10001 │ \n", + " │ \u001b[1mURI:\u001b[0m ray://raytest-head-svc.default.svc:10001 │ \n", " │ │ \n", - " │ \u001B]8;id=623965;http://ray-dashboard-raytest-default.apps.meyceoz-07122023.psap.aws.rhperfscale.org\u001B\\\u001B[4;34mDashboard🔗\u001B[0m\u001B]8;;\u001B\\ │ \n", + " │ \u001b]8;id=623965;http://ray-dashboard-raytest-default.apps.meyceoz-07122023.psap.aws.rhperfscale.org\u001b\\\u001b[4;34mDashboard🔗\u001b[0m\u001b]8;;\u001b\\ │ \n", " │ │ \n", - " │ \u001B[3m Cluster Resources \u001B[0m │ \n", + " │ \u001b[3m Cluster Resources \u001b[0m │ \n", " │ ╭── Workers ──╮ ╭───────── Worker specs(each) ─────────╮ │ \n", - " │ │ \u001B[1m \u001B[0m\u001B[1m# Workers\u001B[0m\u001B[1m \u001B[0m │ │ \u001B[1m \u001B[0m\u001B[1mMemory \u001B[0m\u001B[1m \u001B[0m\u001B[1m \u001B[0m\u001B[1mCPU \u001B[0m\u001B[1m \u001B[0m\u001B[1m \u001B[0m\u001B[1mGPU \u001B[0m\u001B[1m \u001B[0m │ │ \n", - " │ │ \u001B[35m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m │ │ \u001B[36m \u001B[0m\u001B[36m \u001B[0m\u001B[36m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m │ │ \n", - " │ │ \u001B[35m \u001B[0m\u001B[35m2 \u001B[0m\u001B[35m \u001B[0m │ │ \u001B[36m \u001B[0m\u001B[36m4~4 \u001B[0m\u001B[36m \u001B[0m\u001B[35m \u001B[0m\u001B[35m1 \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m\u001B[35m0 \u001B[0m\u001B[35m \u001B[0m │ │ \n", - " │ │ \u001B[35m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m │ │ \u001B[36m \u001B[0m\u001B[36m \u001B[0m\u001B[36m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m │ │ \n", + " │ │ \u001b[1m \u001b[0m\u001b[1m# Workers\u001b[0m\u001b[1m \u001b[0m │ │ \u001b[1m \u001b[0m\u001b[1mMemory \u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m\u001b[1mCPU \u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m\u001b[1mGPU \u001b[0m\u001b[1m \u001b[0m │ │ \n", + " │ │ \u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m │ │ \u001b[36m \u001b[0m\u001b[36m \u001b[0m\u001b[36m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m │ │ \n", + " │ │ \u001b[35m \u001b[0m\u001b[35m2 \u001b[0m\u001b[35m \u001b[0m │ │ \u001b[36m \u001b[0m\u001b[36m4~4 \u001b[0m\u001b[36m \u001b[0m\u001b[35m \u001b[0m\u001b[35m1 \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m0 \u001b[0m\u001b[35m \u001b[0m │ │ \n", + " │ │ \u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m │ │ \u001b[36m \u001b[0m\u001b[36m \u001b[0m\u001b[36m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m │ │ \n", " │ ╰─────────────╯ ╰──────────────────────────────────────╯ │ \n", " ╰───────────────────────────────────────────────────────────────╯ \n" ] diff --git a/demo-notebooks/guided-demos/notebook-ex-outputs/1_cluster_job_client.ipynb b/demo-notebooks/guided-demos/notebook-ex-outputs/1_cluster_job_client.ipynb index b2e954e21..e354f36de 100644 --- a/demo-notebooks/guided-demos/notebook-ex-outputs/1_cluster_job_client.ipynb +++ b/demo-notebooks/guided-demos/notebook-ex-outputs/1_cluster_job_client.ipynb @@ -44,7 +44,6 @@ "# The SDK will try to find the name of your default local queue based on the annotation \"kueue.x-k8s.io/default-queue\": \"true\" unless you specify the local queue manually below\n", "cluster = Cluster(ClusterConfiguration(\n", " name='jobtest',\n", - " namespace='default', # Update to your namespace\n", " head_gpus=1, # For GPU enabled workloads set the head_gpus and num_gpus\n", " num_gpus=1,\n", " num_workers=2,\n", diff --git a/demo-notebooks/guided-demos/notebook-ex-outputs/2_basic_interactive.ipynb b/demo-notebooks/guided-demos/notebook-ex-outputs/2_basic_interactive.ipynb index 443ea0633..4e28e53bd 100644 --- a/demo-notebooks/guided-demos/notebook-ex-outputs/2_basic_interactive.ipynb +++ b/demo-notebooks/guided-demos/notebook-ex-outputs/2_basic_interactive.ipynb @@ -65,11 +65,9 @@ "source": [ "# Create and configure our cluster object\n", "# The SDK will try to find the name of your default local queue based on the annotation \"kueue.x-k8s.io/default-queue\": \"true\" unless you specify the local queue manually below\n", - "namespace = \"default\" # Update to your namespace\n", "cluster_name = \"interactivetest\"\n", "cluster = Cluster(ClusterConfiguration(\n", " name=cluster_name,\n", - " namespace=namespace,\n", " head_gpus=1, # For GPU enabled workloads set the head_gpus and num_gpus\n", " num_gpus=1,\n", " num_workers=2,\n", @@ -134,22 +132,22 @@ "\n" ], "text/plain": [ - "\u001B[3m \u001B[0m\u001B[1;3m 🚀 CodeFlare Cluster Details 🚀\u001B[0m\u001B[3m \u001B[0m\n", - "\u001B[1m \u001B[0m\u001B[1m \u001B[0m\u001B[1m \u001B[0m\n", + "\u001b[3m \u001b[0m\u001b[1;3m 🚀 CodeFlare Cluster Details 🚀\u001b[0m\u001b[3m \u001b[0m\n", + "\u001b[1m \u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m\n", " ╭──────────────────────────────────────────────────────────────────────╮ \n", - " │ \u001B[1;37;42mName\u001B[0m │ \n", - " │ \u001B[1;4minteractivetest\u001B[0m Active ✅ │ \n", + " │ \u001b[1;37;42mName\u001b[0m │ \n", + " │ \u001b[1;4minteractivetest\u001b[0m Active ✅ │ \n", " │ │ \n", - " │ \u001B[1mURI:\u001B[0m ray://interactivetest-head-svc.default.svc:10001 │ \n", + " │ \u001b[1mURI:\u001b[0m ray://interactivetest-head-svc.default.svc:10001 │ \n", " │ │ \n", - " │ \u001B]8;id=970589;http://ray-dashboard-interactivetest-default.apps.meyceoz-07122023.psap.aws.rhperfscale.org\u001B\\\u001B[4;34mDashboard🔗\u001B[0m\u001B]8;;\u001B\\ │ \n", + " │ \u001b]8;id=970589;http://ray-dashboard-interactivetest-default.apps.meyceoz-07122023.psap.aws.rhperfscale.org\u001b\\\u001b[4;34mDashboard🔗\u001b[0m\u001b]8;;\u001b\\ │ \n", " │ │ \n", - " │ \u001B[3m Cluster Resources \u001B[0m │ \n", + " │ \u001b[3m Cluster Resources \u001b[0m │ \n", " │ ╭── Workers ──╮ ╭───────── Worker specs(each) ─────────╮ │ \n", - " │ │ \u001B[1m \u001B[0m\u001B[1m# Workers\u001B[0m\u001B[1m \u001B[0m │ │ \u001B[1m \u001B[0m\u001B[1mMemory \u001B[0m\u001B[1m \u001B[0m\u001B[1m \u001B[0m\u001B[1mCPU \u001B[0m\u001B[1m \u001B[0m\u001B[1m \u001B[0m\u001B[1mGPU \u001B[0m\u001B[1m \u001B[0m │ │ \n", - " │ │ \u001B[35m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m │ │ \u001B[36m \u001B[0m\u001B[36m \u001B[0m\u001B[36m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m │ │ \n", - " │ │ \u001B[35m \u001B[0m\u001B[35m2 \u001B[0m\u001B[35m \u001B[0m │ │ \u001B[36m \u001B[0m\u001B[36m8~8 \u001B[0m\u001B[36m \u001B[0m\u001B[35m \u001B[0m\u001B[35m2 \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m\u001B[35m1 \u001B[0m\u001B[35m \u001B[0m │ │ \n", - " │ │ \u001B[35m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m │ │ \u001B[36m \u001B[0m\u001B[36m \u001B[0m\u001B[36m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m\u001B[35m \u001B[0m │ │ \n", + " │ │ \u001b[1m \u001b[0m\u001b[1m# Workers\u001b[0m\u001b[1m \u001b[0m │ │ \u001b[1m \u001b[0m\u001b[1mMemory \u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m\u001b[1mCPU \u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m\u001b[1mGPU \u001b[0m\u001b[1m \u001b[0m │ │ \n", + " │ │ \u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m │ │ \u001b[36m \u001b[0m\u001b[36m \u001b[0m\u001b[36m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m │ │ \n", + " │ │ \u001b[35m \u001b[0m\u001b[35m2 \u001b[0m\u001b[35m \u001b[0m │ │ \u001b[36m \u001b[0m\u001b[36m8~8 \u001b[0m\u001b[36m \u001b[0m\u001b[35m \u001b[0m\u001b[35m2 \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m1 \u001b[0m\u001b[35m \u001b[0m │ │ \n", + " │ │ \u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m │ │ \u001b[36m \u001b[0m\u001b[36m \u001b[0m\u001b[36m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m │ │ \n", " │ ╰─────────────╯ ╰──────────────────────────────────────╯ │ \n", " ╰──────────────────────────────────────────────────────────────────────╯ \n" ] @@ -221,8 +219,8 @@ "source": [ "from codeflare_sdk import generate_cert\n", "# Create required TLS cert and export the environment variables to enable TLS\n", - "generate_cert.generate_tls_cert(cluster_name, namespace)\n", - "generate_cert.export_env(cluster_name, namespace)" + "generate_cert.generate_tls_cert(cluster_name, cluster.config.namespace)\n", + "generate_cert.export_env(cluster_name, cluster.config.namespace)" ] }, { @@ -381,7 +379,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m Downloading and preparing dataset imdb/plain_text to /home/ray/.cache/huggingface/datasets/imdb/plain_text/1.0.0/d613c88cf8fa3bab83b4ded3713f1f74830d1100e171db75bbddb80b3345c9c0...\n" + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m Downloading and preparing dataset imdb/plain_text to /home/ray/.cache/huggingface/datasets/imdb/plain_text/1.0.0/d613c88cf8fa3bab83b4ded3713f1f74830d1100e171db75bbddb80b3345c9c0...\n" ] }, { @@ -494,7 +492,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m Dataset imdb downloaded and prepared to /home/ray/.cache/huggingface/datasets/imdb/plain_text/1.0.0/d613c88cf8fa3bab83b4ded3713f1f74830d1100e171db75bbddb80b3345c9c0. Subsequent calls will reuse this data.\n" + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m Dataset imdb downloaded and prepared to /home/ray/.cache/huggingface/datasets/imdb/plain_text/1.0.0/d613c88cf8fa3bab83b4ded3713f1f74830d1100e171db75bbddb80b3345c9c0. Subsequent calls will reuse this data.\n" ] }, { @@ -613,13 +611,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m len of train Dataset({\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m features: ['text', 'label', 'input_ids', 'attention_mask'],\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m num_rows: 100\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m }) and test Dataset({\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m features: ['text', 'label', 'input_ids', 'attention_mask'],\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m num_rows: 100\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m })\n" + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m len of train Dataset({\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m features: ['text', 'label', 'input_ids', 'attention_mask'],\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m num_rows: 100\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m }) and test Dataset({\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m features: ['text', 'label', 'input_ids', 'attention_mask'],\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m num_rows: 100\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m })\n" ] }, { @@ -627,106 +625,106 @@ "output_type": "stream", "text": [ " \n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m 2023-08-09 14:51:50,865\tWARNING dataset.py:253 -- \u001B[33mImportant: Ray Data requires schemas for all datasets in Ray 2.5. This means that standalone Python objects are no longer supported. In addition, the default batch format is fixed to NumPy. To revert to legacy behavior temporarily, set the environment variable RAY_DATA_STRICT_MODE=0 on all cluster processes.\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m Learn more here: https://docs.ray.io/en/master/data/faq.html#migrating-to-strict-mode\u001B[0m\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m /tmp/ipykernel_265/307576807.py:57: DeprecationWarning: `HuggingFaceTrainer`, `HuggingFacePredictor` and `HuggingFaceCheckpoint` have been renamed to `TransformersTrainer`, `TransformersPredictor` and `TransformersCheckpoint` respectively. Update your code to use the new import paths. This will raise an exception in the future.\n" + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m 2023-08-09 14:51:50,865\tWARNING dataset.py:253 -- \u001b[33mImportant: Ray Data requires schemas for all datasets in Ray 2.5. This means that standalone Python objects are no longer supported. In addition, the default batch format is fixed to NumPy. To revert to legacy behavior temporarily, set the environment variable RAY_DATA_STRICT_MODE=0 on all cluster processes.\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m Learn more here: https://docs.ray.io/en/master/data/faq.html#migrating-to-strict-mode\u001b[0m\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m /tmp/ipykernel_265/307576807.py:57: DeprecationWarning: `HuggingFaceTrainer`, `HuggingFacePredictor` and `HuggingFaceCheckpoint` have been renamed to `TransformersTrainer`, `TransformersPredictor` and `TransformersCheckpoint` respectively. Update your code to use the new import paths. This will raise an exception in the future.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m To disable this warning, you can either:\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m \t- Avoid using `tokenizers` before the fork if possible\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m \t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m == Status ==\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m Current time: 2023-08-09 14:51:51 (running for 00:00:00.12)\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m Using FIFO scheduling algorithm.\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m Logical resource usage: 0/6 CPUs, 0/2 GPUs\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2023-08-09_14-51-51\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m Number of trials: 1/1 (1 PENDING)\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m +--------------------------------+----------+-------+\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m | Trial name | status | loc |\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m |--------------------------------+----------+-------|\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m | HuggingFaceTrainer_f2621_00000 | PENDING | |\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m +--------------------------------+----------+-------+\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m \n" + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m To disable this warning, you can either:\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m \t- Avoid using `tokenizers` before the fork if possible\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m \t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m == Status ==\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m Current time: 2023-08-09 14:51:51 (running for 00:00:00.12)\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m Using FIFO scheduling algorithm.\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m Logical resource usage: 0/6 CPUs, 0/2 GPUs\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2023-08-09_14-51-51\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m Number of trials: 1/1 (1 PENDING)\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m +--------------------------------+----------+-------+\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m | Trial name | status | loc |\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m |--------------------------------+----------+-------|\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m | HuggingFaceTrainer_f2621_00000 | PENDING | |\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m +--------------------------------+----------+-------+\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "\u001B[2m\u001B[36m(HuggingFaceTrainer pid=196, ip=10.130.4.19)\u001B[0m 2023-08-09 14:51:55,978\tWARNING dataset.py:253 -- \u001B[33mImportant: Ray Data requires schemas for all datasets in Ray 2.5. This means that standalone Python objects are no longer supported. In addition, the default batch format is fixed to NumPy. To revert to legacy behavior temporarily, set the environment variable RAY_DATA_STRICT_MODE=0 on all cluster processes.\n", - "\u001B[2m\u001B[36m(HuggingFaceTrainer pid=196, ip=10.130.4.19)\u001B[0m \n", - "\u001B[2m\u001B[36m(HuggingFaceTrainer pid=196, ip=10.130.4.19)\u001B[0m Learn more here: https://docs.ray.io/en/master/data/faq.html#migrating-to-strict-mode\u001B[0m\n" + "\u001b[2m\u001b[36m(HuggingFaceTrainer pid=196, ip=10.130.4.19)\u001b[0m 2023-08-09 14:51:55,978\tWARNING dataset.py:253 -- \u001b[33mImportant: Ray Data requires schemas for all datasets in Ray 2.5. This means that standalone Python objects are no longer supported. In addition, the default batch format is fixed to NumPy. To revert to legacy behavior temporarily, set the environment variable RAY_DATA_STRICT_MODE=0 on all cluster processes.\n", + "\u001b[2m\u001b[36m(HuggingFaceTrainer pid=196, ip=10.130.4.19)\u001b[0m \n", + "\u001b[2m\u001b[36m(HuggingFaceTrainer pid=196, ip=10.130.4.19)\u001b[0m Learn more here: https://docs.ray.io/en/master/data/faq.html#migrating-to-strict-mode\u001b[0m\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m == Status ==\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m Current time: 2023-08-09 14:51:56 (running for 00:00:05.16)\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m Using FIFO scheduling algorithm.\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m Logical resource usage: 1.0/6 CPUs, 2.0/2 GPUs\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2023-08-09_14-51-51\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m Number of trials: 1/1 (1 RUNNING)\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m +--------------------------------+----------+-----------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m | Trial name | status | loc |\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m |--------------------------------+----------+-----------------|\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m | HuggingFaceTrainer_f2621_00000 | RUNNING | 10.130.4.19:196 |\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m +--------------------------------+----------+-----------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m \n" + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m == Status ==\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m Current time: 2023-08-09 14:51:56 (running for 00:00:05.16)\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m Using FIFO scheduling algorithm.\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m Logical resource usage: 1.0/6 CPUs, 2.0/2 GPUs\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2023-08-09_14-51-51\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m Number of trials: 1/1 (1 RUNNING)\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m +--------------------------------+----------+-----------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m | Trial name | status | loc |\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m |--------------------------------+----------+-----------------|\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m | HuggingFaceTrainer_f2621_00000 | RUNNING | 10.130.4.19:196 |\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m +--------------------------------+----------+-----------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "\u001B[2m\u001B[36m(HuggingFaceTrainer pid=196, ip=10.130.4.19)\u001B[0m 2023-08-09 14:51:57,260\tINFO backend_executor.py:137 -- Starting distributed worker processes: ['235 (10.130.4.19)', '232 (10.129.4.19)']\n", - "\u001B[2m\u001B[36m(HuggingFaceTrainer pid=196, ip=10.130.4.19)\u001B[0m 2023-08-09 14:51:58,957\tINFO streaming_executor.py:91 -- Executing DAG InputDataBuffer[Input] -> AllToAllOperator[RandomizeBlockOrder]\n", - "\u001B[2m\u001B[36m(HuggingFaceTrainer pid=196, ip=10.130.4.19)\u001B[0m 2023-08-09 14:51:58,957\tINFO streaming_executor.py:92 -- Execution config: ExecutionOptions(resource_limits=ExecutionResources(cpu=None, gpu=None, object_store_memory=None), locality_with_output=False, preserve_order=False, actor_locality_enabled=True, verbose_progress=False)\n", - "\u001B[2m\u001B[36m(HuggingFaceTrainer pid=196, ip=10.130.4.19)\u001B[0m 2023-08-09 14:51:58,958\tINFO streaming_executor.py:94 -- Tip: For detailed progress reporting, run `ray.data.DataContext.get_current().execution_options.verbose_progress = True`\n", - "\u001B[2m\u001B[36m(HuggingFaceTrainer pid=196, ip=10.130.4.19)\u001B[0m 2023-08-09 14:51:58,969\tINFO streaming_executor.py:149 -- Shutting down .\n", - "\u001B[2m\u001B[36m(RayTrainWorker pid=235, ip=10.130.4.19)\u001B[0m 2023-08-09 14:51:58,912\tINFO config.py:86 -- Setting up process group for: env:// [rank=0, world_size=2]\n" + "\u001b[2m\u001b[36m(HuggingFaceTrainer pid=196, ip=10.130.4.19)\u001b[0m 2023-08-09 14:51:57,260\tINFO backend_executor.py:137 -- Starting distributed worker processes: ['235 (10.130.4.19)', '232 (10.129.4.19)']\n", + "\u001b[2m\u001b[36m(HuggingFaceTrainer pid=196, ip=10.130.4.19)\u001b[0m 2023-08-09 14:51:58,957\tINFO streaming_executor.py:91 -- Executing DAG InputDataBuffer[Input] -> AllToAllOperator[RandomizeBlockOrder]\n", + "\u001b[2m\u001b[36m(HuggingFaceTrainer pid=196, ip=10.130.4.19)\u001b[0m 2023-08-09 14:51:58,957\tINFO streaming_executor.py:92 -- Execution config: ExecutionOptions(resource_limits=ExecutionResources(cpu=None, gpu=None, object_store_memory=None), locality_with_output=False, preserve_order=False, actor_locality_enabled=True, verbose_progress=False)\n", + "\u001b[2m\u001b[36m(HuggingFaceTrainer pid=196, ip=10.130.4.19)\u001b[0m 2023-08-09 14:51:58,958\tINFO streaming_executor.py:94 -- Tip: For detailed progress reporting, run `ray.data.DataContext.get_current().execution_options.verbose_progress = True`\n", + "\u001b[2m\u001b[36m(HuggingFaceTrainer pid=196, ip=10.130.4.19)\u001b[0m 2023-08-09 14:51:58,969\tINFO streaming_executor.py:149 -- Shutting down .\n", + "\u001b[2m\u001b[36m(RayTrainWorker pid=235, ip=10.130.4.19)\u001b[0m 2023-08-09 14:51:58,912\tINFO config.py:86 -- Setting up process group for: env:// [rank=0, world_size=2]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m == Status ==\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m Current time: 2023-08-09 14:52:01 (running for 00:00:10.18)\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m Using FIFO scheduling algorithm.\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m Logical resource usage: 1.0/6 CPUs, 2.0/2 GPUs\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2023-08-09_14-51-51\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m Number of trials: 1/1 (1 RUNNING)\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m +--------------------------------+----------+-----------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m | Trial name | status | loc |\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m |--------------------------------+----------+-----------------|\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m | HuggingFaceTrainer_f2621_00000 | RUNNING | 10.130.4.19:196 |\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m +--------------------------------+----------+-----------------+\n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m \n", - "\u001B[2m\u001B[36m(train_fn pid=425)\u001B[0m \n" + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m == Status ==\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m Current time: 2023-08-09 14:52:01 (running for 00:00:10.18)\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m Using FIFO scheduling algorithm.\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m Logical resource usage: 1.0/6 CPUs, 2.0/2 GPUs\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m Result logdir: /home/ray/ray_results/HuggingFaceTrainer_2023-08-09_14-51-51\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m Number of trials: 1/1 (1 RUNNING)\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m +--------------------------------+----------+-----------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m | Trial name | status | loc |\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m |--------------------------------+----------+-----------------|\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m | HuggingFaceTrainer_f2621_00000 | RUNNING | 10.130.4.19:196 |\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m +--------------------------------+----------+-----------------+\n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m \n", + "\u001b[2m\u001b[36m(train_fn pid=425)\u001b[0m \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "\u001B[2m\u001B[36m(RayTrainWorker pid=235, ip=10.130.4.19)\u001B[0m 2023-08-09 14:52:01,262\tINFO streaming_executor.py:91 -- Executing DAG InputDataBuffer[Input] -> AllToAllOperator[RandomizeBlockOrder]\n", - "\u001B[2m\u001B[36m(RayTrainWorker pid=235, ip=10.130.4.19)\u001B[0m 2023-08-09 14:52:01,262\tINFO streaming_executor.py:92 -- Execution config: ExecutionOptions(resource_limits=ExecutionResources(cpu=None, gpu=None, object_store_memory=None), locality_with_output=False, preserve_order=False, actor_locality_enabled=True, verbose_progress=False)\n", - "\u001B[2m\u001B[36m(RayTrainWorker pid=235, ip=10.130.4.19)\u001B[0m 2023-08-09 14:52:01,262\tINFO streaming_executor.py:94 -- Tip: For detailed progress reporting, run `ray.data.DataContext.get_current().execution_options.verbose_progress = True`\n", - "\u001B[2m\u001B[36m(RayTrainWorker pid=235, ip=10.130.4.19)\u001B[0m 2023-08-09 14:52:01,274\tINFO streaming_executor.py:149 -- Shutting down .\n", - "\u001B[2m\u001B[36m(RayTrainWorker pid=232, ip=10.129.4.19)\u001B[0m 2023-08-09 14:52:01,252\tINFO streaming_executor.py:91 -- Executing DAG InputDataBuffer[Input] -> AllToAllOperator[RandomizeBlockOrder]\n", - "\u001B[2m\u001B[36m(RayTrainWorker pid=232, ip=10.129.4.19)\u001B[0m 2023-08-09 14:52:01,252\tINFO streaming_executor.py:92 -- Execution config: ExecutionOptions(resource_limits=ExecutionResources(cpu=None, gpu=None, object_store_memory=None), locality_with_output=False, preserve_order=False, actor_locality_enabled=True, verbose_progress=False)\n", - "\u001B[2m\u001B[36m(RayTrainWorker pid=232, ip=10.129.4.19)\u001B[0m 2023-08-09 14:52:01,252\tINFO streaming_executor.py:94 -- Tip: For detailed progress reporting, run `ray.data.DataContext.get_current().execution_options.verbose_progress = True`\n", - "\u001B[2m\u001B[36m(RayTrainWorker pid=232, ip=10.129.4.19)\u001B[0m 2023-08-09 14:52:01,263\tINFO streaming_executor.py:149 -- Shutting down .\n", + "\u001b[2m\u001b[36m(RayTrainWorker pid=235, ip=10.130.4.19)\u001b[0m 2023-08-09 14:52:01,262\tINFO streaming_executor.py:91 -- Executing DAG InputDataBuffer[Input] -> AllToAllOperator[RandomizeBlockOrder]\n", + "\u001b[2m\u001b[36m(RayTrainWorker pid=235, ip=10.130.4.19)\u001b[0m 2023-08-09 14:52:01,262\tINFO streaming_executor.py:92 -- Execution config: ExecutionOptions(resource_limits=ExecutionResources(cpu=None, gpu=None, object_store_memory=None), locality_with_output=False, preserve_order=False, actor_locality_enabled=True, verbose_progress=False)\n", + "\u001b[2m\u001b[36m(RayTrainWorker pid=235, ip=10.130.4.19)\u001b[0m 2023-08-09 14:52:01,262\tINFO streaming_executor.py:94 -- Tip: For detailed progress reporting, run `ray.data.DataContext.get_current().execution_options.verbose_progress = True`\n", + "\u001b[2m\u001b[36m(RayTrainWorker pid=235, ip=10.130.4.19)\u001b[0m 2023-08-09 14:52:01,274\tINFO streaming_executor.py:149 -- Shutting down .\n", + "\u001b[2m\u001b[36m(RayTrainWorker pid=232, ip=10.129.4.19)\u001b[0m 2023-08-09 14:52:01,252\tINFO streaming_executor.py:91 -- Executing DAG InputDataBuffer[Input] -> AllToAllOperator[RandomizeBlockOrder]\n", + "\u001b[2m\u001b[36m(RayTrainWorker pid=232, ip=10.129.4.19)\u001b[0m 2023-08-09 14:52:01,252\tINFO streaming_executor.py:92 -- Execution config: ExecutionOptions(resource_limits=ExecutionResources(cpu=None, gpu=None, object_store_memory=None), locality_with_output=False, preserve_order=False, actor_locality_enabled=True, verbose_progress=False)\n", + "\u001b[2m\u001b[36m(RayTrainWorker pid=232, ip=10.129.4.19)\u001b[0m 2023-08-09 14:52:01,252\tINFO streaming_executor.py:94 -- Tip: For detailed progress reporting, run `ray.data.DataContext.get_current().execution_options.verbose_progress = True`\n", + "\u001b[2m\u001b[36m(RayTrainWorker pid=232, ip=10.129.4.19)\u001b[0m 2023-08-09 14:52:01,263\tINFO streaming_executor.py:149 -- Shutting down .\n", "Downloading (…)lve/main/config.json: 100%|██████████| 483/483 [00:00<00:00, 151kB/s]\n", "Downloading (…)lve/main/config.json: 100%|██████████| 483/483 [00:00<00:00, 146kB/s]\n", "Downloading model.safetensors: 0%| | 0.00/268M [00:00