From 9f57303fd1a27e4a287fc296883620e378fc337f Mon Sep 17 00:00:00 2001 From: ori-kron-wis Date: Mon, 19 Aug 2024 13:32:40 +0000 Subject: [PATCH 1/3] run tuning/autotune_scvi.ipynb --- tuning/autotune_scvi.ipynb | 174 +++++++++++++++++-------------------- 1 file changed, 81 insertions(+), 93 deletions(-) diff --git a/tuning/autotune_scvi.ipynb b/tuning/autotune_scvi.ipynb index 0666993..794287b 100644 --- a/tuning/autotune_scvi.ipynb +++ b/tuning/autotune_scvi.ipynb @@ -58,10 +58,10 @@ "execution_count": 1, "metadata": { "execution": { - "iopub.execute_input": "2024-02-12T22:35:22.024305Z", - "iopub.status.busy": "2024-02-12T22:35:22.024163Z", - "iopub.status.idle": "2024-02-12T22:35:24.282949Z", - "shell.execute_reply": "2024-02-12T22:35:24.282507Z" + "iopub.execute_input": "2024-08-19T13:30:38.572210Z", + "iopub.status.busy": "2024-08-19T13:30:38.571973Z", + "iopub.status.idle": "2024-08-19T13:30:41.407158Z", + "shell.execute_reply": "2024-08-19T13:30:41.406716Z" } }, "outputs": [ @@ -73,11 +73,20 @@ "\u001b[0m" ] }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\r\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.2\u001b[0m\r\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\r\n" + ] + }, { "name": "stderr", "output_type": "stream", "text": [ - "/env/lib/python3.11/site-packages/scvi_colab/_core.py:41: UserWarning: \n", + "/usr/local/lib/python3.11/site-packages/scvi_colab/_core.py:41: UserWarning: \n", " Not currently in Google Colab environment.\n", "\n", " Please run with `run_outside_colab=True` to override.\n", @@ -100,10 +109,10 @@ "execution_count": 2, "metadata": { "execution": { - "iopub.execute_input": "2024-02-12T22:35:24.284608Z", - "iopub.status.busy": "2024-02-12T22:35:24.284494Z", - "iopub.status.idle": "2024-02-12T22:35:27.784569Z", - "shell.execute_reply": "2024-02-12T22:35:27.784105Z" + "iopub.execute_input": "2024-08-19T13:30:41.408899Z", + "iopub.status.busy": "2024-08-19T13:30:41.408760Z", + "iopub.status.idle": "2024-08-19T13:30:52.136803Z", + "shell.execute_reply": "2024-08-19T13:30:52.136288Z" } }, "outputs": [], @@ -124,10 +133,10 @@ "execution_count": 3, "metadata": { "execution": { - "iopub.execute_input": "2024-02-12T22:35:27.787002Z", - "iopub.status.busy": "2024-02-12T22:35:27.786116Z", - "iopub.status.idle": "2024-02-12T22:35:27.790145Z", - "shell.execute_reply": "2024-02-12T22:35:27.789805Z" + "iopub.execute_input": "2024-08-19T13:30:52.138845Z", + "iopub.status.busy": "2024-08-19T13:30:52.138459Z", + "iopub.status.idle": "2024-08-19T13:30:52.142762Z", + "shell.execute_reply": "2024-08-19T13:30:52.142363Z" } }, "outputs": [ @@ -142,7 +151,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Last run with scvi-tools version: 1.1.0\n" + "Last run with scvi-tools version: 1.1.6\n" ] } ], @@ -165,10 +174,10 @@ "execution_count": 4, "metadata": { "execution": { - "iopub.execute_input": "2024-02-12T22:35:27.791523Z", - "iopub.status.busy": "2024-02-12T22:35:27.791410Z", - "iopub.status.idle": "2024-02-12T22:35:27.800544Z", - "shell.execute_reply": "2024-02-12T22:35:27.800156Z" + "iopub.execute_input": "2024-08-19T13:30:52.144345Z", + "iopub.status.busy": "2024-08-19T13:30:52.144081Z", + "iopub.status.idle": "2024-08-19T13:30:52.157872Z", + "shell.execute_reply": "2024-08-19T13:30:52.157476Z" } }, "outputs": [], @@ -196,10 +205,10 @@ "execution_count": 5, "metadata": { "execution": { - "iopub.execute_input": "2024-02-12T22:35:27.802041Z", - "iopub.status.busy": "2024-02-12T22:35:27.801843Z", - "iopub.status.idle": "2024-02-12T22:35:31.940208Z", - "shell.execute_reply": "2024-02-12T22:35:31.939775Z" + "iopub.execute_input": "2024-08-19T13:30:52.159262Z", + "iopub.status.busy": "2024-08-19T13:30:52.159150Z", + "iopub.status.idle": "2024-08-19T13:30:56.864218Z", + "shell.execute_reply": "2024-08-19T13:30:56.863836Z" } }, "outputs": [ @@ -207,7 +216,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[34mINFO \u001b[0m Downloading file at \u001b[35m/tmp/tmp6xxk_1x6/\u001b[0m\u001b[95mhca_subsampled_20k.h5ad\u001b[0m \n" + "\u001b[34mINFO \u001b[0m Downloading file at \u001b[35m/tmp/tmp5sb6wnfn/\u001b[0m\u001b[95mhca_subsampled_20k.h5ad\u001b[0m \n" ] }, { @@ -223,15 +232,7 @@ "output_type": "stream", "text": [ "\r", - "Downloading...: 6%|▋ | 4257/65714.0 [00:00<00:01, 42352.04it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\r", - "Downloading...: 25%|██▌ | 16445/65714.0 [00:00<00:00, 88999.31it/s]" + "Downloading...: 17%|█▋ | 11027/65714.0 [00:00<00:00, 110227.01it/s]" ] }, { @@ -239,7 +240,7 @@ "output_type": "stream", "text": [ "\r", - "Downloading...: 41%|████ | 26816/65714.0 [00:00<00:00, 95706.36it/s]" + "Downloading...: 35%|███▍ | 22791/65714.0 [00:00<00:00, 114526.73it/s]" ] }, { @@ -247,7 +248,7 @@ "output_type": "stream", "text": [ "\r", - "Downloading...: 57%|█████▋ | 37316/65714.0 [00:00<00:00, 99334.82it/s]" + "Downloading...: 53%|█████▎ | 34554/65714.0 [00:00<00:00, 115898.50it/s]" ] }, { @@ -255,7 +256,7 @@ "output_type": "stream", "text": [ "\r", - "Downloading...: 73%|███████▎ | 47862/65714.0 [00:00<00:00, 101523.09it/s]" + "Downloading...: 70%|███████ | 46302/65714.0 [00:00<00:00, 116486.78it/s]" ] }, { @@ -263,7 +264,7 @@ "output_type": "stream", "text": [ "\r", - "Downloading...: 89%|████████▉ | 58480/65714.0 [00:00<00:00, 103095.62it/s]" + "Downloading...: 88%|████████▊ | 58065/65714.0 [00:00<00:00, 116865.83it/s]" ] }, { @@ -271,7 +272,7 @@ "output_type": "stream", "text": [ "\r", - "Downloading...: 100%|██████████| 65714/65714.0 [00:00<00:00, 98281.68it/s] " + "Downloading...: 100%|██████████| 65714/65714.0 [00:00<00:00, 116051.10it/s]" ] }, { @@ -313,10 +314,10 @@ "execution_count": 6, "metadata": { "execution": { - "iopub.execute_input": "2024-02-12T22:35:31.941784Z", - "iopub.status.busy": "2024-02-12T22:35:31.941663Z", - "iopub.status.idle": "2024-02-12T22:35:33.369112Z", - "shell.execute_reply": "2024-02-12T22:35:33.368699Z" + "iopub.execute_input": "2024-08-19T13:30:56.865874Z", + "iopub.status.busy": "2024-08-19T13:30:56.865750Z", + "iopub.status.idle": "2024-08-19T13:30:59.314233Z", + "shell.execute_reply": "2024-08-19T13:30:59.313797Z" } }, "outputs": [ @@ -360,26 +361,13 @@ "execution_count": 7, "metadata": { "execution": { - "iopub.execute_input": "2024-02-12T22:35:33.370656Z", - "iopub.status.busy": "2024-02-12T22:35:33.370538Z", - "iopub.status.idle": "2024-02-12T22:35:33.509456Z", - "shell.execute_reply": "2024-02-12T22:35:33.509017Z" + "iopub.execute_input": "2024-08-19T13:30:59.315858Z", + "iopub.status.busy": "2024-08-19T13:30:59.315736Z", + "iopub.status.idle": "2024-08-19T13:30:59.464174Z", + "shell.execute_reply": "2024-08-19T13:30:59.463706Z" } }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - ":119: FutureWarning: SparseDataset is deprecated and will be removed in late 2024. It has been replaced by the public classes CSRDataset and CSCDataset.\n", - "\n", - "For instance checks, use `isinstance(X, (anndata.experimental.CSRDataset, anndata.experimental.CSCDataset))` instead.\n", - "\n", - "For creation, use `anndata.experimental.sparse_dataset(X)` instead.\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "model_cls = scvi.model.SCVI\n", "model_cls.setup_anndata(adata)" @@ -398,10 +386,10 @@ "execution_count": 8, "metadata": { "execution": { - "iopub.execute_input": "2024-02-12T22:35:33.511026Z", - "iopub.status.busy": "2024-02-12T22:35:33.510901Z", - "iopub.status.idle": "2024-02-12T22:35:33.517224Z", - "shell.execute_reply": "2024-02-12T22:35:33.516822Z" + "iopub.execute_input": "2024-08-19T13:30:59.465994Z", + "iopub.status.busy": "2024-08-19T13:30:59.465867Z", + "iopub.status.idle": "2024-08-19T13:30:59.472739Z", + "shell.execute_reply": "2024-08-19T13:30:59.472325Z" } }, "outputs": [], @@ -426,10 +414,10 @@ "execution_count": 9, "metadata": { "execution": { - "iopub.execute_input": "2024-02-12T22:35:33.518599Z", - "iopub.status.busy": "2024-02-12T22:35:33.518489Z", - "iopub.status.idle": "2024-02-12T22:35:33.531420Z", - "shell.execute_reply": "2024-02-12T22:35:33.531154Z" + "iopub.execute_input": "2024-08-19T13:30:59.474207Z", + "iopub.status.busy": "2024-08-19T13:30:59.474093Z", + "iopub.status.idle": "2024-08-19T13:30:59.487814Z", + "shell.execute_reply": "2024-08-19T13:30:59.487468Z" } }, "outputs": [ @@ -595,10 +583,10 @@ "execution_count": 10, "metadata": { "execution": { - "iopub.execute_input": "2024-02-12T22:35:33.532888Z", - "iopub.status.busy": "2024-02-12T22:35:33.532777Z", - "iopub.status.idle": "2024-02-12T22:35:33.534904Z", - "shell.execute_reply": "2024-02-12T22:35:33.534575Z" + "iopub.execute_input": "2024-08-19T13:30:59.489355Z", + "iopub.status.busy": "2024-08-19T13:30:59.489242Z", + "iopub.status.idle": "2024-08-19T13:30:59.491636Z", + "shell.execute_reply": "2024-08-19T13:30:59.491232Z" } }, "outputs": [], @@ -648,10 +636,10 @@ "execution_count": 11, "metadata": { "execution": { - "iopub.execute_input": "2024-02-12T22:35:33.536311Z", - "iopub.status.busy": "2024-02-12T22:35:33.536202Z", - "iopub.status.idle": "2024-02-12T22:37:57.793792Z", - "shell.execute_reply": "2024-02-12T22:37:57.793301Z" + "iopub.execute_input": "2024-08-19T13:30:59.493161Z", + "iopub.status.busy": "2024-08-19T13:30:59.492905Z", + "iopub.status.idle": "2024-08-19T13:32:34.218534Z", + "shell.execute_reply": "2024-08-19T13:32:34.218113Z" } }, "outputs": [ @@ -664,16 +652,16 @@ "

Tune Status

\n", " \n", "\n", - "\n", - "\n", - "\n", + "\n", + "\n", + "\n", "\n", "
Current time:2024-02-12 22:37:57
Running for: 00:02:21.96
Memory: 8.0/125.7 GiB
Current time:2024-08-19 13:32:34
Running for: 00:01:32.13
Memory: 5.9/125.7 GiB
\n", " \n", "
\n", "
\n", "

System Info

\n", - " Using AsyncHyperBand: num_stopped=5
Bracket: Iter 64.000: -457.03416442871094 | Iter 32.000: -457.7937927246094 | Iter 16.000: -462.5426483154297 | Iter 8.000: -470.41017150878906 | Iter 4.000: -484.6981658935547 | Iter 2.000: -516.056640625 | Iter 1.000: -592.379638671875
Logical resource usage: 10.0/20 CPUs, 1.0/1 GPUs (0.0/1.0 accelerator_type:G)\n", + " Using AsyncHyperBand: num_stopped=5
Bracket: Iter 64.000: -456.6855773925781 | Iter 32.000: -458.8435363769531 | Iter 16.000: -464.8856201171875 | Iter 8.000: -474.24481201171875 | Iter 4.000: -490.88726806640625 | Iter 2.000: -507.64378356933594 | Iter 1.000: -611.3925170898438
Logical resource usage: 10.0/20 CPUs, 1.0/1 GPUs (0.0/1.0 accelerator_type:G)\n", "
\n", " \n", " \n", @@ -685,11 +673,11 @@ "Trial name status loc n_hidden n_layers lr validation_loss\n", "\n", "\n", - "_trainable_8f41734dTERMINATED172.29.0.2:1469 128 30.00148765 457.597\n", - "_trainable_f42dd38fTERMINATED172.29.0.2:1469 64 30.000455217 748.298\n", - "_trainable_a19ea641TERMINATED172.29.0.2:1469 256 10.000388237 525.478\n", - "_trainable_619aa78dTERMINATED172.29.0.2:1469 64 20.000326625 758.515\n", - "_trainable_681ab3baTERMINATED172.29.0.2:1469 256 30.00392304 457.508\n", + "_trainable_3afda194TERMINATED172.29.0.2:1509 256 20.000639646 459.067\n", + "_trainable_a74e6edaTERMINATED172.29.0.2:1509 256 20.000551579 612.665\n", + "_trainable_92bc17edTERMINATED172.29.0.2:1509 64 10.000642845 611.393\n", + "_trainable_1cc85ac8TERMINATED172.29.0.2:1509 64 30.00985192 494.813\n", + "_trainable_21481557TERMINATED172.29.0.2:1509 128 30.000347918 708.115\n", "\n", "\n", " \n", @@ -736,14 +724,14 @@ "name": "stderr", "output_type": "stream", "text": [ - "2024-02-12 22:35:35,776\tINFO tune.py:583 -- [output] This uses the legacy output and progress reporter, as Jupyter notebooks are not supported by the new engine, yet. For more information, please see https://github.com/ray-project/ray/issues/36949\n" + "2024-08-19 13:31:01,903\tINFO tune.py:583 -- [output] This uses the legacy output and progress reporter, as Jupyter notebooks are not supported by the new engine, yet. For more information, please see https://github.com/ray-project/ray/issues/36949\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "2024-02-12 22:37:57,784\tINFO tune.py:1042 -- Total run time: 142.01 seconds (141.96 seconds for the tuning loop).\n" + "2024-08-19 13:32:34,211\tINFO tune.py:1042 -- Total run time: 92.31 seconds (92.12 seconds for the tuning loop).\n" ] } ], @@ -764,10 +752,10 @@ "execution_count": 12, "metadata": { "execution": { - "iopub.execute_input": "2024-02-12T22:37:57.795563Z", - "iopub.status.busy": "2024-02-12T22:37:57.795405Z", - "iopub.status.idle": "2024-02-12T22:37:57.798204Z", - "shell.execute_reply": "2024-02-12T22:37:57.797791Z" + "iopub.execute_input": "2024-08-19T13:32:34.220206Z", + "iopub.status.busy": "2024-08-19T13:32:34.220034Z", + "iopub.status.idle": "2024-08-19T13:32:34.222491Z", + "shell.execute_reply": "2024-08-19T13:32:34.222138Z" } }, "outputs": [ @@ -775,8 +763,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "{'n_hidden': 256, 'n_layers': 3}\n", - "{'plan_kwargs': {'lr': 0.003923035750529307}}\n" + "{'n_hidden': 256, 'n_layers': 2}\n", + "{'plan_kwargs': {'lr': 0.0006396456621967837}}\n" ] } ], @@ -850,7 +838,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.7" + "version": "3.11.9" }, "vscode": { "interpreter": { From 38cb31c3582f03abdd7f0b479666d3ddd78da858 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 13:32:59 +0000 Subject: [PATCH 2/3] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scrna/cellassign_tutorial.ipynb | 1 - 1 file changed, 1 deletion(-) diff --git a/scrna/cellassign_tutorial.ipynb b/scrna/cellassign_tutorial.ipynb index eb854d1..fa62bdc 100644 --- a/scrna/cellassign_tutorial.ipynb +++ b/scrna/cellassign_tutorial.ipynb @@ -115,7 +115,6 @@ "import gdown\n", "import matplotlib.pyplot as plt\n", "import pandas as pd\n", - "import numpy as np\n", "import scanpy as sc\n", "import scvi\n", "import seaborn as sns\n", From b6f40396145eb62109a0bdda4df53f7e88c8d1ae Mon Sep 17 00:00:00 2001 From: Ori Kronfeld Date: Tue, 27 Aug 2024 17:34:31 +0300 Subject: [PATCH 3/3] Update cellassign_tutorial.ipynb --- scrna/cellassign_tutorial.ipynb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrna/cellassign_tutorial.ipynb b/scrna/cellassign_tutorial.ipynb index fa62bdc..2e8d7e0 100644 --- a/scrna/cellassign_tutorial.ipynb +++ b/scrna/cellassign_tutorial.ipynb @@ -114,6 +114,7 @@ "\n", "import gdown\n", "import matplotlib.pyplot as plt\n", + "import numpy as np\n", "import pandas as pd\n", "import scanpy as sc\n", "import scvi\n", @@ -151,7 +152,8 @@ ], "source": [ "scvi.settings.seed = 0\n", - "print(\"Last run with scvi-tools version:\", scvi.__version__)" + "print(\"Last run with scvi-tools version:\", scvi.__version__)\n", + "print(\"Last run with numpy version:\", np.__version__)" ] }, {