From 1e51f9b181990da93fb2442060ec0ccd20fdfb06 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 22:26:52 +0000 Subject: [PATCH 1/2] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.5.0 → v0.6.3](https://github.com/astral-sh/ruff-pre-commit/compare/v0.5.0...v0.6.3) - [github.com/pre-commit/mirrors-eslint: v9.6.0 → v9.9.1](https://github.com/pre-commit/mirrors-eslint/compare/v9.6.0...v9.9.1) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 00571d74..f723f5a7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ ci: repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.0 + rev: v0.6.3 hooks: - id: ruff args: @@ -21,7 +21,7 @@ repos: - id: check-executables-have-shebangs - id: requirements-txt-fixer - repo: https://github.com/pre-commit/mirrors-eslint - rev: v9.6.0 + rev: v9.9.1 hooks: - id: eslint files: \.[jt]sx?$ # *.js, *.jsx, *.ts and *.tsx From 4acb4c3c4ef4c3ca3e699ba28f9c3abfee3c0b2f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 22:27:23 +0000 Subject: [PATCH 2/2] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- benchmarks/async_benchmark.ipynb | 101 +++++----- benchmarks/benchmark_results.ipynb | 184 ++++++++---------- benchmarks/depth_benchmark.ipynb | 46 ++--- benchmarks/profiling_initial_results.ipynb | 15 +- benchmarks/profiling_latest_results.ipynb | 15 +- benchmarks/push_benchmarks.ipynb | 101 ++++------ benchmarks/scheduler_benchmarks.ipynb | 91 ++++----- benchmarks/throughtput_benchmarks.ipynb | 92 ++++----- docs/source/examples/Cluster API.ipynb | 5 +- .../examples/Data Publication API.ipynb | 8 +- docs/source/examples/Futures.ipynb | 27 +-- .../Monitoring an MPI Simulation - 1.ipynb | 40 ++-- .../Monitoring an MPI Simulation - 2.ipynb | 25 +-- .../source/examples/Monte Carlo Options.ipynb | 38 ++-- .../examples/Parallel Decorator and map.ipynb | 9 +- docs/source/examples/Parallel Magics.ipynb | 25 +-- docs/source/examples/Using Dill.ipynb | 17 +- .../Using MPI with IPython Parallel.ipynb | 2 + .../examples/broadcast/Broadcast view.ipynb | 8 +- .../examples/broadcast/MPI Broadcast.ipynb | 14 +- .../examples/broadcast/memmap Broadcast.ipynb | 15 +- docs/source/examples/dask.ipynb | 4 +- docs/source/examples/joblib.ipynb | 4 + docs/source/examples/progress.ipynb | 3 +- docs/source/examples/visualizing-tasks.ipynb | 16 +- 25 files changed, 432 insertions(+), 473 deletions(-) diff --git a/benchmarks/async_benchmark.ipynb b/benchmarks/async_benchmark.ipynb index 68963d2d..a9ae954d 100644 --- a/benchmarks/async_benchmark.ipynb +++ b/benchmarks/async_benchmark.ipynb @@ -6,16 +6,11 @@ "metadata": {}, "outputs": [], "source": [ - "from IPython.core.interactiveshell import InteractiveShell\n", - "from IPython.display import display, Markdown, SVG, HTML\n", - "import pandas as pd\n", - "import altair as alt\n", - "import re\n", "import pickle\n", - "from utils import seconds_to_ms, ms_to_seconds\n", - "from benchmark_result import get_benchmark_results, BenchmarkType, SchedulerType, get_broadcast_source, get_async_source\n", - "from benchmarks.utils import echo\n", - "from benchmarks.throughput import make_benchmark, make_multiple_message_benchmark" + "\n", + "import altair as alt\n", + "import pandas as pd\n", + "from benchmark_result import get_async_source" ] }, { @@ -24,8 +19,7 @@ "metadata": {}, "outputs": [], "source": [ - "#benchmark_results = get_benchmark_results()\n", - "from benchmark_result import BenchmarkResult, Result \n", + "# benchmark_results = get_benchmark_results()\n", "with open('saved_results.pkl', 'rb') as saved_results:\n", " benchmark_results = pickle.load(saved_results)" ] @@ -118,9 +112,9 @@ "outputs": [], "source": [ "source = get_async_source(benchmark_results)\n", - "dview = pd.DataFrame(source['DirectView']) \n", + "dview = pd.DataFrame(source['DirectView'])\n", "dview['Scheduler name'] = 'DirectView'\n", - "dview['Speedup'] = 1\n" + "dview['Speedup'] = 1" ] }, { @@ -288,9 +282,13 @@ "source": [ "datas = []\n", "for scheduler_name, scheduler_results in source.items():\n", - " data = pd.DataFrame(scheduler_results) \n", + " data = pd.DataFrame(scheduler_results)\n", " data['Scheduler name'] = scheduler_name\n", - " data['Speedup'] = 1 if scheduler_name == 'DirectView' else dview['Duration in ms'] / data['Duration in ms']\n", + " data['Speedup'] = (\n", + " 1\n", + " if scheduler_name == 'DirectView'\n", + " else dview['Duration in ms'] / data['Duration in ms']\n", + " )\n", " datas.append(data)\n", "data = pd.concat(datas)\n", "data" @@ -386,8 +384,14 @@ "metadata": {}, "outputs": [], "source": [ - "data['Messages per engine per second'] = round(data['Number of messages'] / data['Duration in ms'] * 1000, 2)\n", - "data['Total messages per second'] = round(((data['Number of messages'] * data['Number of engines']) / data['Duration in ms']) * 1000, 2)" + "data['Messages per engine per second'] = round(\n", + " data['Number of messages'] / data['Duration in ms'] * 1000, 2\n", + ")\n", + "data['Total messages per second'] = round(\n", + " ((data['Number of messages'] * data['Number of engines']) / data['Duration in ms'])\n", + " * 1000,\n", + " 2,\n", + ")" ] }, { @@ -598,7 +602,7 @@ ], "source": [ "dview = data[data['Number of messages'] == 100]\n", - "dview['Number of engines'].unique()\n" + "dview['Number of engines'].unique()" ] }, { @@ -670,17 +674,13 @@ ], "source": [ "alt.Chart(dview).mark_line(point=True).encode(\n", - " alt.X(\n", - " 'Number of engines',\n", - " scale=alt.Scale(type='log', base=2)\n", - " ),\n", - " alt.Y(\n", - " 'Messages per engine per second',\n", - " scale=alt.Scale(type='log')\n", - " ),\n", + " alt.X('Number of engines', scale=alt.Scale(type='log', base=2)),\n", + " alt.Y('Messages per engine per second', scale=alt.Scale(type='log')),\n", " color='Scheduler name:N',\n", " tooltip='Messages per engine per second',\n", - ").configure_axis(labelFontSize=20, titleFontSize=20).properties(title='Runtime of apply using DirectView', width=1080).interactive().display(renderer='svg')" + ").configure_axis(labelFontSize=20, titleFontSize=20).properties(\n", + " title='Runtime of apply using DirectView', width=1080\n", + ").interactive().display(renderer='svg')" ] }, { @@ -752,17 +752,13 @@ ], "source": [ "alt.Chart(dview).mark_line(point=True).encode(\n", - " alt.X(\n", - " 'Number of engines',\n", - " scale=alt.Scale(type='log', base=2)\n", - " ),\n", - " alt.Y(\n", - " 'Total messages per second',\n", - " scale=alt.Scale(type='log')\n", - " ),\n", + " alt.X('Number of engines', scale=alt.Scale(type='log', base=2)),\n", + " alt.Y('Total messages per second', scale=alt.Scale(type='log')),\n", " color='Scheduler name:N',\n", " tooltip='Total messages per second',\n", - ").configure_axis(labelFontSize=20, titleFontSize=20).properties(title='Runtime of apply using DirectView', width=1080).interactive().display(renderer='svg')" + ").configure_axis(labelFontSize=20, titleFontSize=20).properties(\n", + " title='Runtime of apply using DirectView', width=1080\n", + ").interactive().display(renderer='svg')" ] }, { @@ -958,16 +954,15 @@ "for scheduler_name in data['Scheduler name'].unique():\n", " scheduler_data = data[data['Scheduler name'] == scheduler_name]\n", " alt.Chart(scheduler_data).mark_line(point=True).encode(\n", - " alt.X(\n", - " 'Number of messages',\n", - " scale=alt.Scale(type='log')\n", - " ),\n", + " alt.X('Number of messages', scale=alt.Scale(type='log')),\n", " alt.Y(\n", " 'Messages per engine per second',\n", " ),\n", " color='Number of engines:N',\n", - " tooltip='Duration in ms', \n", - " ).configure_axis(labelFontSize=20, titleFontSize=20).properties(title=scheduler_name, width=800).interactive().display(renderer='svg')" + " tooltip='Duration in ms',\n", + " ).configure_axis(labelFontSize=20, titleFontSize=20).properties(\n", + " title=scheduler_name, width=800\n", + " ).interactive().display(renderer='svg')" ] }, { @@ -1042,20 +1037,18 @@ } ], "source": [ - "data['combined']= data['Scheduler name'] + ' ' + data['Number of engines'].astype(str)\n", + "data['combined'] = data['Scheduler name'] + ' ' + data['Number of engines'].astype(str)\n", "alt.Chart(data[data['Scheduler name'] != 'DirectView']).mark_line(point=True).encode(\n", - " alt.X(\n", - " 'Number of messages',\n", - " scale=alt.Scale(type='log')\n", - " ),\n", + " alt.X('Number of messages', scale=alt.Scale(type='log')),\n", " alt.Y(\n", " 'Speedup',\n", " ),\n", " color='Number of engines:N',\n", " strokeDash=alt.StrokeDash(shorthand='Scheduler name', legend=None),\n", " tooltip='combined',\n", - "\n", - ").properties(title='schedulers vs directView scaling engines', width=800).interactive().display(renderer='svg')\n" + ").properties(\n", + " title='schedulers vs directView scaling engines', width=800\n", + ").interactive().display(renderer='svg')" ] }, { @@ -1557,14 +1550,16 @@ } ], "source": [ - "for engine in data['Number of engines'].unique(): \n", + "for engine in data['Number of engines'].unique():\n", " alt.Chart(data[data['Number of engines'] == engine]).mark_bar().encode(\n", " x='Scheduler name',\n", " y='Duration in ms',\n", " color='Scheduler name:N',\n", - " column='Number of messages:N', \n", - " tooltip='Duration in ms'\n", - " ).properties(title=f'Runtime on {engine} engines:').interactive().display(renderer='svg')" + " column='Number of messages:N',\n", + " tooltip='Duration in ms',\n", + " ).properties(title=f'Runtime on {engine} engines:').interactive().display(\n", + " renderer='svg'\n", + " )" ] } ], diff --git a/benchmarks/benchmark_results.ipynb b/benchmarks/benchmark_results.ipynb index 4f042c05..348b7da5 100644 --- a/benchmarks/benchmark_results.ipynb +++ b/benchmarks/benchmark_results.ipynb @@ -18,16 +18,21 @@ } ], "source": [ - "from IPython.core.interactiveshell import InteractiveShell\n", - "from IPython.display import display, Markdown, SVG, HTML\n", - "import pandas as pd\n", - "import altair as alt\n", - "import re\n", "import pickle\n", - "from utils import seconds_to_ms, ms_to_seconds\n", - "from benchmark_result import get_benchmark_results,get_time_n_tasks_source, get_no_delay_source, BenchmarkType, SchedulerType, get_broad_cast_source, get_echo_many_arguments_source\n", - "from benchmarks.utils import echo\n", - "from benchmarks.overhead_latency import echo_many_arguments\n", + "\n", + "import altair as alt\n", + "import pandas as pd\n", + "from benchmark_result import (\n", + " BenchmarkType,\n", + " get_broad_cast_source,\n", + " get_echo_many_arguments_source,\n", + " get_no_delay_source,\n", + " get_time_n_tasks_source,\n", + ")\n", + "from IPython.core.interactiveshell import InteractiveShell\n", + "from IPython.display import Markdown, display\n", + "from utils import ms_to_seconds\n", + "\n", "InteractiveShell.ast_node_interactivity = \"all\"" ] }, @@ -37,8 +42,7 @@ "metadata": {}, "outputs": [], "source": [ - "#benchmark_results = get_benchmark_results()\n", - "from benchmark_result import BenchmarkResult, Result \n", + "# benchmark_results = get_benchmark_results()\n", "with open('saved_results.pkl', 'rb') as saved_results:\n", " benchmark_results = pickle.load(saved_results)" ] @@ -113,41 +117,26 @@ "source = get_time_n_tasks_source(benchmark_results)\n", "for delay, result_for_delay in source.items():\n", " display(Markdown(f'### With a delay of {ms_to_seconds(delay)}s. :'))\n", - " \n", - " for core_num, results in sorted(result_for_delay.items(), key=lambda key: key[0]): \n", + "\n", + " for core_num, results in sorted(result_for_delay.items(), key=lambda key: key[0]):\n", " display(Markdown(f'#### {core_num} cores:'))\n", - " alt.Chart(\n", - " pd.DataFrame(results['direct_view'])\n", - " ).mark_line(point=True).encode(\n", - " alt.X(\n", - " 'Number of tasks',\n", - " scale=alt.Scale(type='log')\n", - " ),\n", + " alt.Chart(pd.DataFrame(results['direct_view'])).mark_line(point=True).encode(\n", + " alt.X('Number of tasks', scale=alt.Scale(type='log')),\n", " y='Duration in ms',\n", " color='Number of engines:N',\n", - " tooltip='Duration in ms' \n", - " ).properties(\n", - " title=f'DirectView',\n", - " width=800\n", - " ).interactive().display(renderer='svg')\n", - " \n", - " alt.Chart(\n", - " pd.DataFrame(results['load_balanced'])\n", - " ).mark_line(point=True).encode(\n", - " alt.X(\n", - " 'Number of tasks',\n", - " scale=alt.Scale(type='log')\n", - " ),\n", + " tooltip='Duration in ms',\n", + " ).properties(title='DirectView', width=800).interactive().display(\n", + " renderer='svg'\n", + " )\n", + "\n", + " alt.Chart(pd.DataFrame(results['load_balanced'])).mark_line(point=True).encode(\n", + " alt.X('Number of tasks', scale=alt.Scale(type='log')),\n", " y='Duration in ms',\n", " color='Number of engines:N',\n", - " tooltip='Duration in ms' \n", - " ).properties(\n", - " title=f'Load Balanced',\n", - " width=800\n", - " ).interactive().display(renderer='svg')\n", - "\n", - " \n", - " " + " tooltip='Duration in ms',\n", + " ).properties(title='Load Balanced', width=800).interactive().display(\n", + " renderer='svg'\n", + " )" ] }, { @@ -422,28 +411,28 @@ ], "source": [ "no_delay_source = get_no_delay_source(benchmark_results)\n", - "display(Markdown(f'### With no delay and 100 engines:'))\n", + "display(Markdown('### With no delay and 100 engines:'))\n", "data = pd.DataFrame(no_delay_source[BenchmarkType.TIME_N_TASKS_NO_DELAY]['direct_view'])\n", "alt.Chart(data).mark_line(point=True).encode(\n", - " alt.X(\n", - " 'Number of tasks',\n", - " scale=alt.Scale(type='log')\n", - " ),\n", + " alt.X('Number of tasks', scale=alt.Scale(type='log')),\n", " color='Number of cores:N',\n", " y='Duration in ms',\n", - " tooltip='Duration in ms', \n", - ").properties(title=f'Ran with no delay on 100 engines Direct View', width=800).interactive().display(renderer='svg')\n", + " tooltip='Duration in ms',\n", + ").properties(\n", + " title='Ran with no delay on 100 engines Direct View', width=800\n", + ").interactive().display(renderer='svg')\n", "\n", - "data = pd.DataFrame(no_delay_source[BenchmarkType.TIME_N_TASKS_NO_DELAY]['load_balanced'])\n", + "data = pd.DataFrame(\n", + " no_delay_source[BenchmarkType.TIME_N_TASKS_NO_DELAY]['load_balanced']\n", + ")\n", "alt.Chart(data).mark_line(point=True).encode(\n", - " alt.X(\n", - " 'Number of tasks',\n", - " scale=alt.Scale(type='log')\n", - " ),\n", + " alt.X('Number of tasks', scale=alt.Scale(type='log')),\n", " color='Number of cores:N',\n", " y='Duration in ms',\n", - " tooltip='Duration in ms', \n", - ").properties(title=f'Ran with no delay on 100 engines Load Balanced', width=800).interactive().display(renderer='svg')" + " tooltip='Duration in ms',\n", + ").properties(\n", + " title='Ran with no delay on 100 engines Load Balanced', width=800\n", + ").interactive().display(renderer='svg')" ] }, { @@ -721,28 +710,30 @@ } ], "source": [ - "display(Markdown(f'### With no delay and non-blocking map on 100 engines:'))\n", - "data = pd.DataFrame(no_delay_source[BenchmarkType.TIME_N_TASKS_NO_DELAY_NON_BLOCKING]['direct_view'])\n", + "display(Markdown('### With no delay and non-blocking map on 100 engines:'))\n", + "data = pd.DataFrame(\n", + " no_delay_source[BenchmarkType.TIME_N_TASKS_NO_DELAY_NON_BLOCKING]['direct_view']\n", + ")\n", "alt.Chart(data).mark_line(point=True).encode(\n", - " alt.X(\n", - " 'Number of tasks',\n", - " scale=alt.Scale(type='log')\n", - " ),\n", + " alt.X('Number of tasks', scale=alt.Scale(type='log')),\n", " color='Number of cores:N',\n", " y='Duration in ms',\n", - " tooltip='Duration in ms', \n", - ").properties(title=f'Ran with no delay on 100 engines Direct View', width=800).interactive().display(renderer='svg')\n", + " tooltip='Duration in ms',\n", + ").properties(\n", + " title='Ran with no delay on 100 engines Direct View', width=800\n", + ").interactive().display(renderer='svg')\n", "\n", - "data = pd.DataFrame(no_delay_source[BenchmarkType.TIME_N_TASKS_NO_DELAY_NON_BLOCKING]['load_balanced'])\n", + "data = pd.DataFrame(\n", + " no_delay_source[BenchmarkType.TIME_N_TASKS_NO_DELAY_NON_BLOCKING]['load_balanced']\n", + ")\n", "alt.Chart(data).mark_line(point=True).encode(\n", - " alt.X(\n", - " 'Number of tasks',\n", - " scale=alt.Scale(type='log')\n", - " ),\n", + " alt.X('Number of tasks', scale=alt.Scale(type='log')),\n", " color='Number of cores:N',\n", " y='Duration in ms',\n", - " tooltip='Duration in ms', \n", - ").properties(title=f'Ran with no delay on 100 engines Load Balanced', width=800).interactive().display(renderer='svg')" + " tooltip='Duration in ms',\n", + ").properties(\n", + " title='Ran with no delay on 100 engines Load Balanced', width=800\n", + ").interactive().display(renderer='svg')" ] }, { @@ -780,7 +771,6 @@ } ], "source": [ - "from benchmarks.throughput import NumpyArrayBroadcast\n", "NumpyArrayBroadcast.time_broadcast??" ] }, @@ -1187,14 +1177,14 @@ "for core_num, results in sorted(source.items(), key=lambda key: key[0]):\n", " data = pd.DataFrame(results)\n", " alt.Chart(data).mark_line(point=True).encode(\n", - " alt.X(\n", - " 'Number of bytes',\n", - " scale=alt.Scale(type='log')\n", - " ),\n", + " alt.X('Number of bytes', scale=alt.Scale(type='log')),\n", " y='Duration in ms',\n", " color='Number of engines:N',\n", - " tooltip='Duration in ms', \n", - " ).properties(title=f'Broadcast benchmark running on {core_num} cores with Direct View', width=800).interactive().display(renderer='svg')" + " tooltip='Duration in ms',\n", + " ).properties(\n", + " title=f'Broadcast benchmark running on {core_num} cores with Direct View',\n", + " width=800,\n", + " ).interactive().display(renderer='svg')" ] }, { @@ -1584,37 +1574,23 @@ ], "source": [ "source = get_echo_many_arguments_source(benchmark_results)\n", - "display(Markdown(f'### With non-blocking map on {source[\"number_of_engines\"]} engines:'))\n", + "display(\n", + " Markdown(f'### With non-blocking map on {source[\"number_of_engines\"]} engines:')\n", + ")\n", "display(Markdown(f'#### {core_num} cores:'))\n", - "alt.Chart(\n", - " pd.DataFrame(source['direct_view'])\n", - ").mark_line(point=True).encode(\n", - " alt.X(\n", - " 'Number of arguments',\n", - " scale=alt.Scale(type='log')\n", - " ),\n", + "alt.Chart(pd.DataFrame(source['direct_view'])).mark_line(point=True).encode(\n", + " alt.X('Number of arguments', scale=alt.Scale(type='log')),\n", " y='Duration in ms',\n", " color='Number of cores:N',\n", - " tooltip='Duration in ms' \n", - ").properties(\n", - " title=f'DirectView',\n", - " width=800\n", - ").interactive().display(renderer='svg') \n", + " tooltip='Duration in ms',\n", + ").properties(title='DirectView', width=800).interactive().display(renderer='svg')\n", "\n", - "alt.Chart(\n", - " pd.DataFrame(source['load_balanced'])\n", - ").mark_line(point=True).encode(\n", - " alt.X(\n", - " 'Number of arguments',\n", - " scale=alt.Scale(type='log')\n", - " ),\n", - " y='Duration in ms',\n", - " color='Number of cores:N',\n", - " tooltip='Duration in ms' \n", - " ).properties(\n", - " title=f'Load Balanced',\n", - " width=800\n", - " ).interactive().display(renderer='svg')" + "alt.Chart(pd.DataFrame(source['load_balanced'])).mark_line(point=True).encode(\n", + " alt.X('Number of arguments', scale=alt.Scale(type='log')),\n", + " y='Duration in ms',\n", + " color='Number of cores:N',\n", + " tooltip='Duration in ms',\n", + ").properties(title='Load Balanced', width=800).interactive().display(renderer='svg')" ] } ], diff --git a/benchmarks/depth_benchmark.ipynb b/benchmarks/depth_benchmark.ipynb index bed82621..9cd64c64 100644 --- a/benchmarks/depth_benchmark.ipynb +++ b/benchmarks/depth_benchmark.ipynb @@ -6,16 +6,11 @@ "metadata": {}, "outputs": [], "source": [ - "from IPython.core.interactiveshell import InteractiveShell\n", - "from IPython.display import display, Markdown, SVG, HTML\n", - "import pandas as pd\n", - "import altair as alt\n", - "import re\n", "import pickle\n", - "from utils import seconds_to_ms, ms_to_seconds\n", - "from benchmark_result import get_benchmark_results, BenchmarkType, SchedulerType, get_depth_testing_source\n", - "from benchmarks.utils import echo\n", - "from benchmarks.throughput import make_benchmark, DepthTestingSuite" + "\n", + "import altair as alt\n", + "import pandas as pd\n", + "from benchmark_result import get_depth_testing_source" ] }, { @@ -28,8 +23,7 @@ }, "outputs": [], "source": [ - "#benchmark_results = get_benchmark_results()\n", - "from benchmark_result import BenchmarkResult, Result \n", + "# benchmark_results = get_benchmark_results()\n", "with open('saved_results.pkl', 'rb') as saved_results:\n", " benchmark_results = pickle.load(saved_results)" ] @@ -124,7 +118,7 @@ }, "outputs": [], "source": [ - "source = get_depth_testing_source(benchmark_results)\n" + "source = get_depth_testing_source(benchmark_results)" ] }, { @@ -680,7 +674,9 @@ ], "source": [ "data = pd.DataFrame(source['Depth'])\n", - "data['Scheduler name'] = data.apply(lambda row: 'Coalescing' if row['Is coalescing'] else 'NonCoalescing', axis=1)\n", + "data['Scheduler name'] = data.apply(\n", + " lambda row: 'Coalescing' if row['Is coalescing'] else 'NonCoalescing', axis=1\n", + ")\n", "data" ] }, @@ -816,19 +812,11 @@ "for scheduler_name in data['Scheduler name'].unique():\n", " scheduler_data = data[data['Scheduler name'] == scheduler_name]\n", " alt.Chart(scheduler_data).mark_line(point=True).encode(\n", - " alt.X(\n", - " 'Number of engines',\n", - " scale=alt.Scale(type='log')\n", - " ),\n", - " alt.Y(\n", - " 'Duration in ms',\n", - " scale=alt.Scale(type='log')\n", - "\n", - " ),\n", + " alt.X('Number of engines', scale=alt.Scale(type='log')),\n", + " alt.Y('Duration in ms', scale=alt.Scale(type='log')),\n", " color='Depth:N',\n", " tooltip='Duration in ms',\n", - " \n", - " ).properties(title=scheduler_name, width=800).interactive().display(renderer='svg')\n" + " ).properties(title=scheduler_name, width=800).interactive().display(renderer='svg')" ] }, { @@ -1269,14 +1257,16 @@ } ], "source": [ - "for engine in data['Number of engines'].unique(): \n", + "for engine in data['Number of engines'].unique():\n", " alt.Chart(data[data['Number of engines'] == engine]).mark_bar().encode(\n", " x='Scheduler name',\n", " y='Duration in ms',\n", " color='Scheduler name:N',\n", - " column='Depth:N', \n", - " tooltip='Duration in ms'\n", - " ).properties(title=f'Runtime on {engine} engines:').interactive().display(renderer='svg')" + " column='Depth:N',\n", + " tooltip='Duration in ms',\n", + " ).properties(title=f'Runtime on {engine} engines:').interactive().display(\n", + " renderer='svg'\n", + " )" ] }, { diff --git a/benchmarks/profiling_initial_results.ipynb b/benchmarks/profiling_initial_results.ipynb index c708ff48..9b79da1c 100644 --- a/benchmarks/profiling_initial_results.ipynb +++ b/benchmarks/profiling_initial_results.ipynb @@ -7,11 +7,14 @@ "outputs": [], "source": [ "import os\n", + "\n", "from IPython.core.interactiveshell import InteractiveShell\n", - "from IPython.display import display, Markdown, SVG, HTML, IFrame\n", - "from profiling import profiling_code\n", - "from profiling.view_profiling_results import get_latest_results_dir, get_initial_results_dir\n", - "from profiling.profiling_code_runner import profiling_tasks\n", + "from IPython.display import IFrame, display\n", + "from profiling.view_profiling_results import (\n", + " get_initial_results_dir,\n", + " get_latest_results_dir,\n", + ")\n", + "\n", "InteractiveShell.ast_node_interactivity = \"all\"" ] }, @@ -38,15 +41,19 @@ "def get_results(dir_name, file_name):\n", " return os.path.join(dir_name, file_name)\n", "\n", + "\n", "def get_file_name(task_name, is_client):\n", " return f'{task_name}_{\"client\" if is_client else \"scheduler\"}.svg'\n", "\n", + "\n", "def get_latest_results(task_name, is_client=True):\n", " return get_results(get_latest_results_dir(), get_file_name(task_name, is_client))\n", "\n", + "\n", "def get_initial_results(task_name, is_client=True):\n", " return get_results(get_initial_results_dir(), get_file_name(task_name, is_client))\n", "\n", + "\n", "def display_svg(svg_file_path):\n", " display(IFrame(src=svg_file_path, width='100%', height=800))" ] diff --git a/benchmarks/profiling_latest_results.ipynb b/benchmarks/profiling_latest_results.ipynb index c0932c94..2c417c46 100644 --- a/benchmarks/profiling_latest_results.ipynb +++ b/benchmarks/profiling_latest_results.ipynb @@ -7,11 +7,14 @@ "outputs": [], "source": [ "import os\n", + "\n", "from IPython.core.interactiveshell import InteractiveShell\n", - "from IPython.display import display, Markdown, SVG, HTML, IFrame\n", - "from profiling import profiling_code\n", - "from profiling.view_profiling_results import get_latest_results_dir, get_initial_results_dir\n", - "from profiling.profiling_code_runner import profiling_tasks\n", + "from IPython.display import IFrame, display\n", + "from profiling.view_profiling_results import (\n", + " get_initial_results_dir,\n", + " get_latest_results_dir,\n", + ")\n", + "\n", "InteractiveShell.ast_node_interactivity = \"all\"" ] }, @@ -38,15 +41,19 @@ "def get_results(dir_name, file_name):\n", " return os.path.join(dir_name, file_name)\n", "\n", + "\n", "def get_file_name(task_name, is_client):\n", " return f'{task_name}_{\"client\" if is_client else \"scheduler\"}.svg'\n", "\n", + "\n", "def get_latest_results(task_name, is_client=True):\n", " return get_results(get_latest_results_dir(), get_file_name(task_name, is_client))\n", "\n", + "\n", "def get_initial_results(task_name, is_client=True):\n", " return get_results(get_initial_results_dir(), get_file_name(task_name, is_client))\n", "\n", + "\n", "def display_svg(svg_file_path):\n", " display(IFrame(src=svg_file_path, width='100%', height=800))" ] diff --git a/benchmarks/push_benchmarks.ipynb b/benchmarks/push_benchmarks.ipynb index 167d6221..7ce329cb 100644 --- a/benchmarks/push_benchmarks.ipynb +++ b/benchmarks/push_benchmarks.ipynb @@ -26,16 +26,12 @@ "metadata": {}, "outputs": [], "source": [ - "from IPython.core.interactiveshell import InteractiveShell\n", - "from IPython.display import display, Markdown, SVG, HTML\n", - "import pandas as pd\n", - "import altair as alt\n", - "import re\n", "import pickle\n", - "from utils import seconds_to_ms, ms_to_seconds\n", - "from benchmark_result import get_benchmark_results, BenchmarkType, SchedulerType, get_broadcast_source, get_async_source, get_push_source\n", - "from benchmarks.utils import echo\n", - "from benchmarks.throughput import make_benchmark, make_multiple_message_benchmark, make_push_benchmark" + "\n", + "import altair as alt\n", + "import pandas as pd\n", + "from benchmark_result import get_push_source\n", + "from IPython.display import Markdown, display" ] }, { @@ -44,8 +40,7 @@ "metadata": {}, "outputs": [], "source": [ - "#benchmark_results = get_benchmark_results()\n", - "from benchmark_result import BenchmarkResult, Result \n", + "# benchmark_results = get_benchmark_results()\n", "with open('saved_results.pkl', 'rb') as saved_results:\n", " benchmark_results = pickle.load(saved_results)" ] @@ -114,7 +109,7 @@ } ], "source": [ - "??make_push_benchmark\n" + "??make_push_benchmark" ] }, { @@ -140,7 +135,7 @@ }, "outputs": [], "source": [ - "dview = pd.DataFrame(source['DirectView']) \n", + "dview = pd.DataFrame(source['DirectView'])\n", "dview['Scheduler name'] = 'DirectView'\n", "dview['Speedup'] = 1" ] @@ -314,7 +309,7 @@ "source": [ "datas = []\n", "for scheduler_name, scheduler_results in source.items():\n", - " data = pd.DataFrame(scheduler_results) \n", + " data = pd.DataFrame(scheduler_results)\n", " data['Scheduler name'] = scheduler_name\n", " data['Speedup'] = dview['Duration in ms'] / data['Duration in ms']\n", " datas.append(data)\n", @@ -392,17 +387,13 @@ "source": [ "ldata = data[data['Number of bytes'] == 2000_000]\n", "alt.Chart(ldata).mark_line(point=True).encode(\n", - " alt.X(\n", - " 'Number of engines',\n", - " scale=alt.Scale(type='log', base=2)\n", - " ),\n", - " alt.Y(\n", - " 'Duration in ms',\n", - " scale=alt.Scale(type='log')\n", - " ),\n", + " alt.X('Number of engines', scale=alt.Scale(type='log', base=2)),\n", + " alt.Y('Duration in ms', scale=alt.Scale(type='log')),\n", " color='Scheduler name:N',\n", " tooltip='Duration in ms',\n", - ").configure_axis(labelFontSize=20, titleFontSize=20).properties(title='Runtime of apply using DirectView', width=1080).interactive().display(renderer='svg')" + ").configure_axis(labelFontSize=20, titleFontSize=20).properties(\n", + " title='Runtime of apply using DirectView', width=1080\n", + ").interactive().display(renderer='svg')" ] }, { @@ -802,19 +793,11 @@ "for scheduler_name in data['Scheduler name'].unique():\n", " scheduler_data = data[data['Scheduler name'] == scheduler_name]\n", " alt.Chart(scheduler_data).mark_line(point=True).encode(\n", - " alt.X(\n", - " 'Number of bytes',\n", - " scale=alt.Scale(type='log')\n", - " ),\n", - " alt.Y(\n", - " 'Duration in ms',\n", - " scale=alt.Scale(type='log')\n", - "\n", - " ),\n", + " alt.X('Number of bytes', scale=alt.Scale(type='log')),\n", + " alt.Y('Duration in ms', scale=alt.Scale(type='log')),\n", " color='Number of engines:N',\n", " tooltip='Duration in ms',\n", - " \n", - " ).properties(title=scheduler_name, width=800).interactive().display(renderer='svg')\n" + " ).properties(title=scheduler_name, width=800).interactive().display(renderer='svg')" ] }, { @@ -887,17 +870,13 @@ "source": [ "ldata = data[data['Number of bytes'] == 1000]\n", "alt.Chart(ldata).mark_line(point=True).encode(\n", - " alt.X(\n", - " 'Number of engines',\n", - " scale=alt.Scale(type='log', base=2)\n", - " ),\n", - " alt.Y(\n", - " 'Duration in ms',\n", - " scale=alt.Scale(type='log')\n", - " ),\n", + " alt.X('Number of engines', scale=alt.Scale(type='log', base=2)),\n", + " alt.Y('Duration in ms', scale=alt.Scale(type='log')),\n", " color='Scheduler name:N',\n", " tooltip='Duration in ms',\n", - ").configure_axis(labelFontSize=20, titleFontSize=20).properties(title='Runtime of apply using DirectView', width=1080).interactive().display(renderer='svg')" + ").configure_axis(labelFontSize=20, titleFontSize=20).properties(\n", + " title='Runtime of apply using DirectView', width=1080\n", + ").interactive().display(renderer='svg')" ] }, { @@ -1041,21 +1020,19 @@ } ], "source": [ - "display(Markdown(f'## Results for duration[DirectView]/duration[scheduler]'))\n", + "display(Markdown('## Results for duration[DirectView]/duration[scheduler]'))\n", "for scheduler_name in data['Scheduler name'].unique():\n", " if scheduler_name == 'DirectView':\n", " continue\n", - " alt.Chart(data[data['Scheduler name'] == scheduler_name]).mark_line(point=True).encode(\n", - " alt.X(\n", - " 'Number of bytes',\n", - " scale=alt.Scale(type='log')\n", - " ),\n", + " alt.Chart(data[data['Scheduler name'] == scheduler_name]).mark_line(\n", + " point=True\n", + " ).encode(\n", + " alt.X('Number of bytes', scale=alt.Scale(type='log')),\n", " alt.Y(\n", - " 'Speedup',\n", + " 'Speedup',\n", " ),\n", " color='Number of engines:N',\n", " tooltip='Number of engines',\n", - " \n", " ).properties(title=scheduler_name, width=800).interactive().display(renderer='svg')" ] }, @@ -1131,20 +1108,18 @@ } ], "source": [ - "data['combined']= data['Scheduler name'] + ' ' + data['Number of bytes'].astype(str)\n", + "data['combined'] = data['Scheduler name'] + ' ' + data['Number of bytes'].astype(str)\n", "alt.Chart(data[data['Scheduler name'] != 'DirectView']).mark_line(point=True).encode(\n", - " alt.X(\n", - " 'Number of engines',\n", - " scale=alt.Scale(type='log')\n", - " ),\n", + " alt.X('Number of engines', scale=alt.Scale(type='log')),\n", " alt.Y(\n", " 'Speedup',\n", " ),\n", " color='Number of bytes:N',\n", " strokeDash=alt.StrokeDash(shorthand='Scheduler name', legend=None),\n", " tooltip='combined',\n", - "\n", - ").properties(title='schedulers vs directView scaling engines', width=800).interactive().display(renderer='svg')\n" + ").properties(\n", + " title='schedulers vs directView scaling engines', width=800\n", + ").interactive().display(renderer='svg')" ] }, { @@ -1646,14 +1621,16 @@ } ], "source": [ - "for engine in data['Number of engines'].unique(): \n", + "for engine in data['Number of engines'].unique():\n", " alt.Chart(data[data['Number of engines'] == engine]).mark_bar().encode(\n", " x='Scheduler name',\n", " y='Duration in ms',\n", " color='Scheduler name:N',\n", - " column='Number of bytes:N', \n", - " tooltip='Duration in ms'\n", - " ).properties(title=f'Runtime on {engine} engines:').interactive().display(renderer='svg')" + " column='Number of bytes:N',\n", + " tooltip='Duration in ms',\n", + " ).properties(title=f'Runtime on {engine} engines:').interactive().display(\n", + " renderer='svg'\n", + " )" ] } ], diff --git a/benchmarks/scheduler_benchmarks.ipynb b/benchmarks/scheduler_benchmarks.ipynb index 3b4ef73a..10200309 100644 --- a/benchmarks/scheduler_benchmarks.ipynb +++ b/benchmarks/scheduler_benchmarks.ipynb @@ -6,16 +6,19 @@ "metadata": {}, "outputs": [], "source": [ - "from IPython.core.interactiveshell import InteractiveShell\n", - "from IPython.display import display, Markdown, SVG, HTML\n", - "import pandas as pd\n", - "import altair as alt\n", - "import re\n", "import pickle\n", - "from utils import seconds_to_ms, ms_to_seconds\n", - "from benchmark_result import get_benchmark_results,get_time_n_tasks_source, get_no_delay_source, BenchmarkType, SchedulerType, get_broad_cast_source, get_echo_many_arguments_source\n", - "from benchmarks.utils import echo\n", - "from benchmarks.overhead_latency import echo_many_arguments\n", + "\n", + "import altair as alt\n", + "import pandas as pd\n", + "from benchmark_result import (\n", + " BenchmarkType,\n", + " get_no_delay_source,\n", + " get_time_n_tasks_source,\n", + ")\n", + "from IPython.core.interactiveshell import InteractiveShell\n", + "from IPython.display import Markdown, display\n", + "from utils import ms_to_seconds\n", + "\n", "InteractiveShell.ast_node_interactivity = \"all\"" ] }, @@ -25,8 +28,7 @@ "metadata": {}, "outputs": [], "source": [ - "#benchmark_results = get_benchmark_results()\n", - "from benchmark_result import BenchmarkResult, Result \n", + "# benchmark_results = get_benchmark_results()\n", "with open('saved_results.pkl', 'rb') as saved_results:\n", " benchmark_results = pickle.load(saved_results)" ] @@ -55,20 +57,20 @@ { "data": { "text/plain": [ - "\u001B[0;31mSignature:\u001B[0m \u001B[0mecho\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mdelay\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;36m0\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", - "\u001B[0;31mDocstring:\u001B[0m \n", - "\u001B[0;31mSource:\u001B[0m \n", - "\u001B[0;32mdef\u001B[0m \u001B[0mecho\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mdelay\u001B[0m\u001B[0;34m=\u001B[0m\u001B[0;36m0\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\n", - "\u001B[0;34m\u001B[0m \u001B[0;32mdef\u001B[0m \u001B[0minner_echo\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mx\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0;34m**\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\n", - "\u001B[0;34m\u001B[0m \u001B[0;32mimport\u001B[0m \u001B[0mtime\u001B[0m\u001B[0;34m\u001B[0m\n", - "\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\n", - "\u001B[0;34m\u001B[0m \u001B[0;32mif\u001B[0m \u001B[0mdelay\u001B[0m\u001B[0;34m:\u001B[0m\u001B[0;34m\u001B[0m\n", - "\u001B[0;34m\u001B[0m \u001B[0mtime\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0msleep\u001B[0m\u001B[0;34m(\u001B[0m\u001B[0mdelay\u001B[0m\u001B[0;34m)\u001B[0m\u001B[0;34m\u001B[0m\n", - "\u001B[0;34m\u001B[0m \u001B[0;32mreturn\u001B[0m \u001B[0mx\u001B[0m\u001B[0;34m\u001B[0m\n", - "\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\n", - "\u001B[0;34m\u001B[0m \u001B[0;32mreturn\u001B[0m \u001B[0minner_echo\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n", - "\u001B[0;31mFile:\u001B[0m ~/ipyparallel_master_project/benchmarks/utils.py\n", - "\u001B[0;31mType:\u001B[0m function\n" + "\u001b[0;31mSignature:\u001b[0m \u001b[0mecho\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdelay\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mDocstring:\u001b[0m \n", + "\u001b[0;31mSource:\u001b[0m \n", + "\u001b[0;32mdef\u001b[0m \u001b[0mecho\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdelay\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0minner_echo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mdelay\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdelay\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0minner_echo\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mFile:\u001b[0m ~/ipyparallel_master_project/benchmarks/utils.py\n", + "\u001b[0;31mType:\u001b[0m function\n" ] }, "metadata": {}, @@ -1187,25 +1189,18 @@ ], "source": [ "source = get_time_n_tasks_source(benchmark_results)\n", - "display(Markdown(f'## With 16 cores:'))\n", + "display(Markdown('## With 16 cores:'))\n", "for delay, result_for_delay in source.items():\n", " display(Markdown(f'### With a delay of {ms_to_seconds(delay)}s. :'))\n", " for scheduler_name, results in result_for_delay[16].items():\n", - " alt.Chart(\n", - " pd.DataFrame(results)\n", - " ).mark_line(point=True).encode(\n", - " alt.X(\n", - " 'Number of tasks',\n", - " scale=alt.Scale(type='log')\n", - " ),\n", + " alt.Chart(pd.DataFrame(results)).mark_line(point=True).encode(\n", + " alt.X('Number of tasks', scale=alt.Scale(type='log')),\n", " y='Duration in ms',\n", " color='Number of engines:N',\n", - " tooltip='Duration in ms' \n", - " ).properties(\n", - " title=f'{scheduler_name}',\n", - " width=800\n", - " ).interactive().display(renderer='svg')\n", - " " + " tooltip='Duration in ms',\n", + " ).properties(title=f'{scheduler_name}', width=800).interactive().display(\n", + " renderer='svg'\n", + " )" ] }, { @@ -1737,19 +1732,19 @@ ], "source": [ "no_delay_source = get_no_delay_source(benchmark_results)\n", - "display(Markdown(f'### With no delay and 100 engines:'))\n", - "for scheduler_name, results in no_delay_source[BenchmarkType.TIME_N_TASKS_NO_DELAY].items():\n", + "display(Markdown('### With no delay and 100 engines:'))\n", + "for scheduler_name, results in no_delay_source[\n", + " BenchmarkType.TIME_N_TASKS_NO_DELAY\n", + "].items():\n", " data = pd.DataFrame(results)\n", " alt.Chart(data).mark_line(point=True).encode(\n", - " alt.X(\n", - " 'Number of tasks',\n", - " scale=alt.Scale(type='log')\n", - " ),\n", + " alt.X('Number of tasks', scale=alt.Scale(type='log')),\n", " color='Number of cores:N',\n", " y='Duration in ms',\n", - " tooltip='Duration in ms', \n", - " ).properties(title=f'Ran with no delay on 100 engines {scheduler_name}', width=800).interactive().display(renderer='svg')\n", - "\n" + " tooltip='Duration in ms',\n", + " ).properties(\n", + " title=f'Ran with no delay on 100 engines {scheduler_name}', width=800\n", + " ).interactive().display(renderer='svg')" ] } ], diff --git a/benchmarks/throughtput_benchmarks.ipynb b/benchmarks/throughtput_benchmarks.ipynb index ad573d8b..67c21003 100644 --- a/benchmarks/throughtput_benchmarks.ipynb +++ b/benchmarks/throughtput_benchmarks.ipynb @@ -6,16 +6,12 @@ "metadata": {}, "outputs": [], "source": [ - "from IPython.core.interactiveshell import InteractiveShell\n", - "from IPython.display import display, Markdown, SVG, HTML\n", - "import pandas as pd\n", - "import altair as alt\n", - "import re\n", "import pickle\n", - "from utils import seconds_to_ms, ms_to_seconds\n", - "from benchmark_result import get_benchmark_results, BenchmarkType, SchedulerType, get_broadcast_source, get_async_source\n", - "from benchmarks.utils import echo\n", - "from benchmarks.throughput import make_benchmark, make_multiple_message_benchmark" + "\n", + "import altair as alt\n", + "import pandas as pd\n", + "from benchmark_result import get_broadcast_source\n", + "from IPython.display import Markdown, display" ] }, { @@ -24,8 +20,7 @@ "metadata": {}, "outputs": [], "source": [ - "#benchmark_results = get_benchmark_results()\n", - "from benchmark_result import BenchmarkResult, Result \n", + "# benchmark_results = get_benchmark_results()\n", "with open('saved_results.pkl', 'rb') as saved_results:\n", " benchmark_results = pickle.load(saved_results)" ] @@ -163,7 +158,7 @@ "metadata": {}, "outputs": [], "source": [ - "dview = pd.DataFrame(source['DirectView']) \n", + "dview = pd.DataFrame(source['DirectView'])\n", "dview['Scheduler name'] = 'DirectView'\n", "dview['Speedup'] = 1" ] @@ -333,7 +328,7 @@ "source": [ "datas = []\n", "for scheduler_name, scheduler_results in source.items():\n", - " data = pd.DataFrame(scheduler_results) \n", + " data = pd.DataFrame(scheduler_results)\n", " data['Scheduler name'] = scheduler_name\n", " data['Speedup'] = dview['Duration in ms'] / data['Duration in ms']\n", " datas.append(data)\n", @@ -411,17 +406,13 @@ "source": [ "ldata = data[data['Number of bytes'] == 2000_000]\n", "alt.Chart(ldata).mark_line(point=True).encode(\n", - " alt.X(\n", - " 'Number of engines',\n", - " scale=alt.Scale(type='log', base=2)\n", - " ),\n", - " alt.Y(\n", - " 'Duration in ms',\n", - " scale=alt.Scale(type='log')\n", - " ),\n", + " alt.X('Number of engines', scale=alt.Scale(type='log', base=2)),\n", + " alt.Y('Duration in ms', scale=alt.Scale(type='log')),\n", " color='Scheduler name:N',\n", " tooltip='Duration in ms',\n", - ").configure_axis(labelFontSize=20, titleFontSize=20).properties(title='Runtime of apply using DirectView', width=1080).interactive().display(renderer='svg')" + ").configure_axis(labelFontSize=20, titleFontSize=20).properties(\n", + " title='Runtime of apply using DirectView', width=1080\n", + ").interactive().display(renderer='svg')" ] }, { @@ -617,19 +608,11 @@ "for scheduler_name in data['Scheduler name'].unique():\n", " scheduler_data = data[data['Scheduler name'] == scheduler_name]\n", " alt.Chart(scheduler_data).mark_line(point=True).encode(\n", - " alt.X(\n", - " 'Number of bytes',\n", - " scale=alt.Scale(type='log')\n", - " ),\n", - " alt.Y(\n", - " 'Duration in ms',\n", - " scale=alt.Scale(type='log')\n", - "\n", - " ),\n", + " alt.X('Number of bytes', scale=alt.Scale(type='log')),\n", + " alt.Y('Duration in ms', scale=alt.Scale(type='log')),\n", " color='Number of engines:N',\n", " tooltip='Duration in ms',\n", - " \n", - " ).properties(title=scheduler_name, width=800).interactive().display(renderer='svg')\n" + " ).properties(title=scheduler_name, width=800).interactive().display(renderer='svg')" ] }, { @@ -777,21 +760,19 @@ } ], "source": [ - "display(Markdown(f'## Results for duration[DirectView]/duration[scheduler]'))\n", + "display(Markdown('## Results for duration[DirectView]/duration[scheduler]'))\n", "for scheduler_name in data['Scheduler name'].unique():\n", " if scheduler_name == 'DirectView':\n", " continue\n", - " alt.Chart(data[data['Scheduler name'] == scheduler_name]).mark_line(point=True).encode(\n", - " alt.X(\n", - " 'Number of bytes',\n", - " scale=alt.Scale(type='log')\n", - " ),\n", + " alt.Chart(data[data['Scheduler name'] == scheduler_name]).mark_line(\n", + " point=True\n", + " ).encode(\n", + " alt.X('Number of bytes', scale=alt.Scale(type='log')),\n", " alt.Y(\n", - " 'Speedup',\n", + " 'Speedup',\n", " ),\n", " color='Number of engines:N',\n", " tooltip='Number of engines',\n", - " \n", " ).properties(title=scheduler_name, width=800).interactive().display(renderer='svg')" ] }, @@ -1390,20 +1371,23 @@ } ], "source": [ - "\n", "for engine in data['Number of engines'].unique():\n", " display(Markdown(f'### Running on {engine} engines'))\n", - " alt.Chart(data[(data['Number of engines'] == engine) & (data['Scheduler name'] != 'DirectView')]).mark_line(point=True).encode(\n", - " alt.X(\n", - " 'Number of bytes',\n", - " scale=alt.Scale(type='log')\n", - " ),\n", + " alt.Chart(\n", + " data[\n", + " (data['Number of engines'] == engine)\n", + " & (data['Scheduler name'] != 'DirectView')\n", + " ]\n", + " ).mark_line(point=True).encode(\n", + " alt.X('Number of bytes', scale=alt.Scale(type='log')),\n", " alt.Y(\n", " 'Speedup',\n", " ),\n", " color='Scheduler name:N',\n", " tooltip='Duration in ms',\n", - " ).properties(title='schedulers vs directView', width=800).interactive().display(renderer='svg')" + " ).properties(title='schedulers vs directView', width=800).interactive().display(\n", + " renderer='svg'\n", + " )" ] }, { @@ -1478,20 +1462,18 @@ } ], "source": [ - "data['combined']= data['Scheduler name'] + ' ' + data['Number of bytes'].astype(str)\n", + "data['combined'] = data['Scheduler name'] + ' ' + data['Number of bytes'].astype(str)\n", "alt.Chart(data[data['Scheduler name'] != 'DirectView']).mark_line(point=True).encode(\n", - " alt.X(\n", - " 'Number of engines',\n", - " scale=alt.Scale(type='log')\n", - " ),\n", + " alt.X('Number of engines', scale=alt.Scale(type='log')),\n", " alt.Y(\n", " 'Speedup',\n", " ),\n", " color='Number of bytes:N',\n", " strokeDash=alt.StrokeDash(shorthand='Scheduler name', legend=None),\n", " tooltip='combined',\n", - "\n", - ").properties(title='schedulers vs directView scaling engines', width=800).interactive().display(renderer='svg')\n" + ").properties(\n", + " title='schedulers vs directView scaling engines', width=800\n", + ").interactive().display(renderer='svg')" ] }, { diff --git a/docs/source/examples/Cluster API.ipynb b/docs/source/examples/Cluster API.ipynb index 48535daa..6a58b499 100644 --- a/docs/source/examples/Cluster API.ipynb +++ b/docs/source/examples/Cluster API.ipynb @@ -489,6 +489,7 @@ ], "source": [ "import os\n", + "\n", "os.environ[\"OMPI_MCA_rmaps_base_oversubscribe\"] = \"1\"\n", "\n", "cluster = Cluster(n=4, engines='MPI')\n", @@ -547,7 +548,6 @@ ], "source": [ "def uhoh():\n", - " import time\n", " from mpi4py import MPI\n", "\n", " rank = MPI.COMM_WORLD.rank\n", @@ -580,6 +580,7 @@ "outputs": [], "source": [ "import signal\n", + "\n", "await cluster.signal_engines(signal.SIGINT)\n", "ar.get(timeout=2)" ] @@ -674,6 +675,7 @@ "\n", " return MPI.COMM_WORLD.rank\n", "\n", + "\n", "rank_map = rc[:].apply_async(get_rank).get_dict()\n", "rank_map" ] @@ -738,6 +740,7 @@ "outputs": [], "source": [ "import ipyparallel as ipp\n", + "\n", "cluster = ipp.Cluster.from_file()" ] }, diff --git a/docs/source/examples/Data Publication API.ipynb b/docs/source/examples/Data Publication API.ipynb index 2a04cb71..27be94c9 100644 --- a/docs/source/examples/Data Publication API.ipynb +++ b/docs/source/examples/Data Publication API.ipynb @@ -91,6 +91,7 @@ "source": [ "def publish_it():\n", " from ipyparallel.datapub import publish_data\n", + "\n", " publish_data(dict(a='hi'))" ] }, @@ -165,9 +166,12 @@ "outputs": [], "source": [ "def simulation_loop():\n", - " from ipyparallel.datapub import publish_data\n", " import time\n", + "\n", " import numpy as np\n", + "\n", + " from ipyparallel.datapub import publish_data\n", + "\n", " for i in range(10):\n", " publish_data(dict(a=np.random.rand(20), i=i))\n", " time.sleep(1)" @@ -239,7 +243,7 @@ "source": [ "data = ar.data\n", "for i, d in enumerate(data):\n", - " plt.plot(d['a'], label='engine: '+str(i))\n", + " plt.plot(d['a'], label='engine: ' + str(i))\n", "plt.title('Data published at time step: ' + str(data[0]['i']))\n", "plt.legend()" ] diff --git a/docs/source/examples/Futures.ipynb b/docs/source/examples/Futures.ipynb index 3164e58b..7de49b36 100644 --- a/docs/source/examples/Futures.ipynb +++ b/docs/source/examples/Futures.ipynb @@ -50,6 +50,7 @@ ], "source": [ "import ipyparallel as ipp\n", + "\n", "rc = ipp.Cluster(n=4).start_and_connect_sync()\n", "\n", "dv = rc[:]\n", @@ -88,7 +89,7 @@ "def random_norm(n):\n", " \"\"\"Generates a 1xN array and computes its 2-norm\"\"\"\n", " A = numpy.random.random(n)\n", - " return norm(A, 2)\n" + " return norm(A, 2)" ] }, { @@ -205,6 +206,7 @@ ], "source": [ "import os\n", + "\n", "f = rc[-1].apply(os.getpid)\n", "f.add_done_callback(lambda _: print(\"I got PID: %i\" % _.result()))\n", "f.result()" @@ -235,30 +237,33 @@ ], "source": [ "import asyncio\n", - "from tornado.ioloop import IOLoop\n", "import sys\n", "\n", + "\n", "def sleep_task(t):\n", " time.sleep(t)\n", " return os.getpid()\n", "\n", + "\n", "async def background():\n", " \"\"\"A backgorund coroutine to demonstrate that we aren't blocking\"\"\"\n", " while True:\n", " await asyncio.sleep(1)\n", " print('.', end=' ')\n", - " sys.stdout.flush() # not needed after ipykernel 4.3\n", + " sys.stdout.flush() # not needed after ipykernel 4.3\n", + "\n", "\n", "async def work():\n", " \"\"\"Submit some work and print the results when complete\"\"\"\n", - " for t in [ 1, 2, 3, 4 ]:\n", - " ar = rc[:].apply(sleep_task, t)\n", - " result = await asyncio.wrap_future(ar) # this waits\n", + " for t in [1, 2, 3, 4]:\n", + " ar = rc[:].apply(sleep_task, t)\n", + " result = await asyncio.wrap_future(ar) # this waits\n", " print(result)\n", "\n", + "\n", "bg = asyncio.Task(background())\n", "await work()\n", - "bg.cancel();\n" + "bg.cancel();" ] }, { @@ -434,6 +439,7 @@ " \"\"\"Generates a 1xN array and computes its 2-norm\"\"\"\n", " import numpy\n", " from numpy.linalg import norm\n", + "\n", " A = numpy.ones(n)\n", " return norm(A, 2)" ] @@ -444,8 +450,7 @@ "metadata": {}, "outputs": [], "source": [ - "from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor\n", - "import distributed\n", + "from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor\n", "\n", "distributed_client = rc.become_dask()\n", "dist_ex = distributed_client.get_executor()\n", @@ -453,7 +458,7 @@ "N = 4\n", "ip_ex = rc.executor(targets=range(N))\n", "thread_ex = ThreadPoolExecutor(N)\n", - "process_ex = ProcessPoolExecutor(N)\n" + "process_ex = ProcessPoolExecutor(N)" ] }, { @@ -485,7 +490,7 @@ "for executor in executors:\n", " print(executor.__class__.__name__)\n", " it = executor.map(str, range(5))\n", - " print(list(it))\n" + " print(list(it))" ] }, { diff --git a/docs/source/examples/Monitoring an MPI Simulation - 1.ipynb b/docs/source/examples/Monitoring an MPI Simulation - 1.ipynb index fdb075a0..379eec45 100644 --- a/docs/source/examples/Monitoring an MPI Simulation - 1.ipynb +++ b/docs/source/examples/Monitoring an MPI Simulation - 1.ipynb @@ -29,10 +29,9 @@ }, "outputs": [], "source": [ - "import numpy as np\n", "import matplotlib.pyplot as plt\n", - "\n", - "from IPython.display import display\n" + "import numpy as np\n", + "from IPython.display import display" ] }, { @@ -75,6 +74,7 @@ ], "source": [ "import ipyparallel as ipp\n", + "\n", "n = 4\n", "\n", "rc = ipp.Cluster(engines=\"mpi\", n=n).start_and_connect_sync()\n", @@ -154,6 +154,7 @@ "ranks = view['rank']\n", "rank_indices = np.argsort(ranks)\n", "\n", + "\n", "def mpi_order(seq):\n", " \"\"\"Return elements of a sequence ordered by MPI rank.\n", "\n", @@ -272,20 +273,21 @@ "source": [ "from IPython.display import clear_output\n", "\n", + "\n", "def plot_current_results(in_place=True):\n", " \"\"\"Makes a blocking call to retrieve remote data and displays the solution mesh\n", " as a contour plot.\n", - " \n", + "\n", " Parameters\n", " ----------\n", " in_place : bool\n", " By default it calls clear_output so that new plots replace old ones. Set\n", " to False to allow keeping of all previous outputs.\n", " \"\"\"\n", - " \n", - " # We make a blocking call to load the remote data from the simulation into simple named \n", + "\n", + " # We make a blocking call to load the remote data from the simulation into simple named\n", " # variables we can read from the engine namespaces\n", - " #view.apply_sync(load_simulation_globals)\n", + " # view.apply_sync(load_simulation_globals)\n", " # And now we can use the view to read these variables from all the engines. Then we\n", " # concatenate all of them into single arrays for local plotting\n", " try:\n", @@ -294,11 +296,11 @@ " print(\"dimension mismatch in Z, not plotting\")\n", " ax = plt.gca()\n", " return ax.figure\n", - " \n", + "\n", " nx, nyt, j, nsteps = view.pull(['nx', 'nyt', 'j', 'nsteps'], targets=0, block=True)\n", " fig, ax = plt.subplots()\n", " ax.contourf(Z)\n", - " ax.set_title('Mesh: %i x %i, step %i/%i' % (nx, nyt, j+1, nsteps))\n", + " ax.set_title('Mesh: %i x %i, step %i/%i' % (nx, nyt, j + 1, nsteps))\n", " plt.axis('off')\n", " # We clear the notebook output before plotting this if in-place plot updating is requested\n", " if in_place:\n", @@ -329,9 +331,8 @@ "outputs": [], "source": [ "def simulation_alive():\n", - " \"\"\"Return True if the simulation thread is still running on any engine.\n", - " \"\"\"\n", - " return any(view.apply_sync(lambda : simulation_thread.is_alive()))" + " \"\"\"Return True if the simulation thread is still running on any engine.\"\"\"\n", + " return any(view.apply_sync(lambda: simulation_thread.is_alive()))" ] }, { @@ -358,14 +359,14 @@ "def monitor_simulation(refresh=5.0, plots_in_place=True):\n", " \"\"\"Monitor the simulation progress and call plotting routine.\n", "\n", - " Supress KeyboardInterrupt exception if interrupted, ensure that the last \n", + " Supress KeyboardInterrupt exception if interrupted, ensure that the last\n", " figure is always displayed and provide basic timing and simulation status.\n", "\n", " Parameters\n", " ----------\n", " refresh : float\n", " Refresh interval between calls to retrieve and plot data. The default\n", - " is 5s, adjust depending on the desired refresh rate, but be aware that \n", + " is 5s, adjust depending on the desired refresh rate, but be aware that\n", " very short intervals will start having a significant impact.\n", "\n", " plots_in_place : bool\n", @@ -373,21 +374,22 @@ " animation effect in the notebook. If false, all frames are plotted\n", " in sequence and appended in the output area.\n", " \"\"\"\n", - " import datetime as dt, time\n", - " \n", + " import datetime as dt\n", + " import time\n", + "\n", " if not simulation_alive():\n", " plot_current_results(in_place=plots_in_place)\n", " plt.close('all')\n", " print('Simulation has already finished, no monitoring to do.')\n", " return\n", - " \n", + "\n", " t0 = dt.datetime.now()\n", " fig = None\n", " try:\n", " while simulation_alive():\n", " fig = plot_current_results(in_place=plots_in_place)\n", - " plt.close('all') # prevent re-plot of old figures\n", - " time.sleep(refresh) # so we don't hammer the server too fast\n", + " plt.close('all') # prevent re-plot of old figures\n", + " time.sleep(refresh) # so we don't hammer the server too fast\n", " except (KeyboardInterrupt, TimeoutError):\n", " msg = 'Monitoring interrupted, simulation is ongoing!'\n", " else:\n", diff --git a/docs/source/examples/Monitoring an MPI Simulation - 2.ipynb b/docs/source/examples/Monitoring an MPI Simulation - 2.ipynb index ab884112..f19e764f 100644 --- a/docs/source/examples/Monitoring an MPI Simulation - 2.ipynb +++ b/docs/source/examples/Monitoring an MPI Simulation - 2.ipynb @@ -67,8 +67,8 @@ "outputs": [], "source": [ "%matplotlib inline\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt" + "import matplotlib.pyplot as plt\n", + "import numpy as np" ] }, { @@ -137,6 +137,7 @@ "ranks = view['rank']\n", "engine_mpi = np.argsort(ranks)\n", "\n", + "\n", "def mpi_order(seq):\n", " \"\"\"Return elements of a sequence ordered by MPI rank.\n", "\n", @@ -236,7 +237,7 @@ "metadata": {}, "outputs": [], "source": [ - "from IPython.display import display, clear_output\n", + "from IPython.display import clear_output, display\n", "\n", "\n", "def plot_current_results(ar, in_place=True):\n", @@ -255,7 +256,7 @@ " data = ar.data[engine_mpi[0]]\n", "\n", " try:\n", - " nx, nyt, j, nsteps = [data[k] for k in [\"nx\", \"nyt\", \"j\", \"nsteps\"]]\n", + " nx, nyt, j, nsteps = (data[k] for k in [\"nx\", \"nyt\", \"j\", \"nsteps\"])\n", " Z = data[\"Z\"]\n", " except KeyError:\n", " # This can happen if we read from the engines so quickly that the data\n", @@ -266,7 +267,6 @@ " display(fig)\n", " return fig\n", " else:\n", - "\n", " fig, ax = plt.subplots()\n", " ax.contourf(Z)\n", " ax.set_title(\"Mesh: %i x %i, step %i/%i\" % (nx, nyt, j + 1, nsteps))\n", @@ -296,7 +296,7 @@ "def monitor_simulation(ar, refresh=5.0, plots_in_place=True):\n", " \"\"\"Monitor the simulation progress and call plotting routine.\n", "\n", - " Supress KeyboardInterrupt exception if interrupted, ensure that the last \n", + " Supress KeyboardInterrupt exception if interrupted, ensure that the last\n", " figure is always displayed and provide basic timing and simulation status.\n", "\n", " Parameters\n", @@ -305,7 +305,7 @@ "\n", " refresh : float\n", " Refresh interval between calls to retrieve and plot data. The default\n", - " is 5s, adjust depending on the desired refresh rate, but be aware that \n", + " is 5s, adjust depending on the desired refresh rate, but be aware that\n", " very short intervals will start having a significant impact.\n", "\n", " plots_in_place : bool\n", @@ -313,8 +313,9 @@ " animation effect in the notebook. If false, all frames are plotted\n", " in sequence and appended in the output area.\n", " \"\"\"\n", - " import datetime as dt, time\n", - " \n", + " import datetime as dt\n", + " import time\n", + "\n", " if ar.ready():\n", " ar.get()\n", " ar.wait_for_output()\n", @@ -322,13 +323,13 @@ " plt.close('all')\n", " print('Simulation has already finished, no monitoring to do.')\n", " return\n", - " \n", + "\n", " t0 = dt.datetime.now()\n", " fig = None\n", " try:\n", " while not ar.ready():\n", " fig = plot_current_results(ar, in_place=plots_in_place)\n", - " plt.close('all') # prevent re-plot of old figures\n", + " plt.close('all') # prevent re-plot of old figures\n", " time.sleep(refresh)\n", " except (KeyboardInterrupt, ipp.error.TimeoutError):\n", " msg = 'Monitoring interrupted, simulation is ongoing!'\n", @@ -365,7 +366,7 @@ "source": [ "# And now we call on all available nodes our simulation routine,\n", "# as an asynchronous task\n", - "ar = view.apply_async(lambda : simulation(nsteps=100, delay=0.1))" + "ar = view.apply_async(lambda: simulation(nsteps=100, delay=0.1))" ] }, { diff --git a/docs/source/examples/Monte Carlo Options.ipynb b/docs/source/examples/Monte Carlo Options.ipynb index 5135bbcb..e89a5d9e 100644 --- a/docs/source/examples/Monte Carlo Options.ipynb +++ b/docs/source/examples/Monte Carlo Options.ipynb @@ -39,8 +39,6 @@ }, "outputs": [], "source": [ - "import sys\n", - "import time\n", "import numpy as np" ] }, @@ -98,7 +96,6 @@ } ], "source": [ - "from __future__ import print_function # legacy Python support\n", "print(\"Strike prices: \", strike_vals)\n", "print(\"Volatilities: \", sigma_vals)" ] @@ -146,25 +143,26 @@ " -------\n", " A tuple of (E. call, E. put, A. call, A. put) option prices.\n", " \"\"\"\n", + " from math import exp, sqrt\n", + "\n", " import numpy as np\n", - " from math import exp,sqrt\n", - " \n", - " h = 1.0/days\n", - " const1 = exp((r-0.5*sigma**2)*h)\n", - " const2 = sigma*sqrt(h)\n", - " stock_price = S*np.ones(paths, dtype='float64')\n", + "\n", + " h = 1.0 / days\n", + " const1 = exp((r - 0.5 * sigma**2) * h)\n", + " const2 = sigma * sqrt(h)\n", + " stock_price = S * np.ones(paths, dtype='float64')\n", " stock_price_sum = np.zeros(paths, dtype='float64')\n", " for j in range(days):\n", - " growth_factor = const1*np.exp(const2*np.random.standard_normal(paths))\n", - " stock_price = stock_price*growth_factor\n", + " growth_factor = const1 * np.exp(const2 * np.random.standard_normal(paths))\n", + " stock_price = stock_price * growth_factor\n", " stock_price_sum = stock_price_sum + stock_price\n", - " stock_price_avg = stock_price_sum/days\n", + " stock_price_avg = stock_price_sum / days\n", " zeros = np.zeros(paths, dtype='float64')\n", - " r_factor = exp(-r*h*days)\n", - " euro_put = r_factor*np.mean(np.maximum(zeros, K-stock_price))\n", - " asian_put = r_factor*np.mean(np.maximum(zeros, K-stock_price_avg))\n", - " euro_call = r_factor*np.mean(np.maximum(zeros, stock_price-K))\n", - " asian_call = r_factor*np.mean(np.maximum(zeros, stock_price_avg-K))\n", + " r_factor = exp(-r * h * days)\n", + " euro_put = r_factor * np.mean(np.maximum(zeros, K - stock_price))\n", + " asian_put = r_factor * np.mean(np.maximum(zeros, K - stock_price_avg))\n", + " euro_call = r_factor * np.mean(np.maximum(zeros, stock_price - K))\n", + " asian_call = r_factor * np.mean(np.maximum(zeros, stock_price_avg - K))\n", " return (euro_call, euro_put, asian_call, asian_put)" ] }, @@ -230,6 +228,7 @@ "outputs": [], "source": [ "import ipyparallel as ipp\n", + "\n", "rc = ipp.Client()" ] }, @@ -356,8 +355,9 @@ }, "outputs": [], "source": [ - "prices = np.empty(n_strikes*n_sigmas,\n", - " dtype=[('ecall',float),('eput',float),('acall',float),('aput',float)]\n", + "prices = np.empty(\n", + " n_strikes * n_sigmas,\n", + " dtype=[('ecall', float), ('eput', float), ('acall', float), ('aput', float)],\n", ")\n", "\n", "for i, price in enumerate(results):\n", diff --git a/docs/source/examples/Parallel Decorator and map.ipynb b/docs/source/examples/Parallel Decorator and map.ipynb index d69c55e5..a0e09cbd 100644 --- a/docs/source/examples/Parallel Decorator and map.ipynb +++ b/docs/source/examples/Parallel Decorator and map.ipynb @@ -15,7 +15,6 @@ }, "outputs": [], "source": [ - "from __future__ import print_function\n", "import ipyparallel as ipp" ] }, @@ -43,7 +42,7 @@ } ], "source": [ - "result = v.map(lambda x: 2*x, range(10))\n", + "result = v.map(lambda x: 2 * x, range(10))\n", "print(\"Simple, default map: \", list(result))" ] }, @@ -62,7 +61,7 @@ } ], "source": [ - "ar = v.map_async(lambda x: 2*x, range(10))\n", + "ar = v.map_async(lambda x: 2 * x, range(10))\n", "print(\"Submitted tasks, got ids: \", ar.msg_ids)\n", "result = ar.get()\n", "print(\"Using a mapper: \", result)" @@ -83,7 +82,9 @@ ], "source": [ "@v.parallel(block=True)\n", - "def f(x): return 2*x\n", + "def f(x):\n", + " return 2 * x\n", + "\n", "\n", "result = f.map(range(10))\n", "print(\"Using a parallel function: \", result)" diff --git a/docs/source/examples/Parallel Magics.ipynb b/docs/source/examples/Parallel Magics.ipynb index 8c05c0ed..d3aeedda 100644 --- a/docs/source/examples/Parallel Magics.ipynb +++ b/docs/source/examples/Parallel Magics.ipynb @@ -290,7 +290,7 @@ ], "source": [ "with dv.sync_imports():\n", - " import sys" + " pass" ] }, { @@ -1623,29 +1623,30 @@ }, "outputs": [], "source": [ - "from __future__ import print_function\n", - "\n", "def generate_output():\n", " \"\"\"function for testing output\n", - " \n", + "\n", " publishes two outputs of each type, and returns something\n", " \"\"\"\n", - " \n", - " import sys,os\n", - " from IPython.display import display, HTML, Math\n", - " \n", + "\n", + " import os\n", + " import sys\n", + "\n", + " from IPython.display import HTML, Math, display\n", + "\n", " print(\"stdout\")\n", " print(\"stderr\", file=sys.stderr)\n", - " \n", + "\n", " display(HTML(\"HTML\"))\n", - " \n", + "\n", " print(\"stdout2\")\n", " print(\"stderr2\", file=sys.stderr)\n", - " \n", + "\n", " display(Math(r\"\\alpha=\\beta\"))\n", - " \n", + "\n", " return os.getpid()\n", "\n", + "\n", "dv['generate_output'] = generate_output" ] }, diff --git a/docs/source/examples/Using Dill.ipynb b/docs/source/examples/Using Dill.ipynb index 76fd3965..f1d33e68 100644 --- a/docs/source/examples/Using Dill.ipynb +++ b/docs/source/examples/Using Dill.ipynb @@ -39,12 +39,15 @@ "def make_closure(a):\n", " \"\"\"make a weird function with a closure on an open file, and return it\"\"\"\n", " import os\n", + "\n", " f = open('/tmp/dilltest', 'a')\n", + "\n", " def has_closure(b):\n", " product = a * b\n", " f.write(\"%i: %g\\n\" % (os.getpid(), product))\n", " f.flush()\n", " return product\n", + "\n", " return has_closure" ] }, @@ -92,7 +95,7 @@ } ], "source": [ - "cat /tmp/dilltest" + "cat / tmp / dilltest" ] }, { @@ -334,7 +337,7 @@ } ], "source": [ - "cat /tmp/dilltest" + "cat / tmp / dilltest" ] }, { @@ -385,7 +388,7 @@ } ], "source": [ - "cat /tmp/dilltest" + "cat / tmp / dilltest" ] }, { @@ -407,7 +410,9 @@ " def inner(b):\n", " def inner_again(c):\n", " return c * b * a\n", + "\n", " return inner_again\n", + "\n", " return inner" ] }, @@ -438,7 +443,7 @@ } ], "source": [ - "view.apply_sync(lambda f: f(3),view.apply_sync(outer, 1)(2))" + "view.apply_sync(lambda f: f(3), view.apply_sync(outer, 1)(2))" ] }, { @@ -475,7 +480,7 @@ "%px foo = 5\n", "\n", "print(rc[:]['foo'])\n", - "rc[:]['bar'] = lambda : 2 * foo\n", + "rc[:]['bar'] = lambda: 2 * foo\n", "rc[:].apply_sync(ipp.Reference('bar'))" ] }, @@ -541,7 +546,7 @@ "source": [ "v = rc[-1]\n", "v['D'] = testdill.D\n", - "d = v.apply_sync(lambda : D())\n", + "d = v.apply_sync(lambda: D())\n", "print(d.a, d.b)" ] }, diff --git a/docs/source/examples/Using MPI with IPython Parallel.ipynb b/docs/source/examples/Using MPI with IPython Parallel.ipynb index c8180fe8..983585b7 100644 --- a/docs/source/examples/Using MPI with IPython Parallel.ipynb +++ b/docs/source/examples/Using MPI with IPython Parallel.ipynb @@ -42,6 +42,7 @@ "outputs": [], "source": [ "import ipyparallel as ipp\n", + "\n", "rc = ipp.Client()\n", "view = rc[:]" ] @@ -67,6 +68,7 @@ "@view.remote(block=True)\n", "def mpi_rank():\n", " from mpi4py import MPI\n", + "\n", " comm = MPI.COMM_WORLD\n", " return comm.Get_rank()" ] diff --git a/docs/source/examples/broadcast/Broadcast view.ipynb b/docs/source/examples/broadcast/Broadcast view.ipynb index 0f15a739..e197af29 100644 --- a/docs/source/examples/broadcast/Broadcast view.ipynb +++ b/docs/source/examples/broadcast/Broadcast view.ipynb @@ -288,13 +288,13 @@ } ], "source": [ - "import pandas as pd\n", "import matplotlib.pyplot as plt\n", + "import pandas as pd\n", "\n", "df = pd.DataFrame({\"n\": n_list, \"direct view\": dview_times, \"broadcast\": bcast_times})\n", "df[\"broadcast speedup\"] = df[\"direct view\"] / df.broadcast\n", "# MB is the total number of megabytes delivered (n * n * 8 * number of engines bytes)\n", - "df[\"MB\"] = df[\"n\"] ** 2 * 8 * len(rc) / (1024 * 1024) \n", + "df[\"MB\"] = df[\"n\"] ** 2 * 8 * len(rc) / (1024 * 1024)\n", "df" ] }, @@ -333,7 +333,7 @@ ], "source": [ "df.plot(x='MB', y=['direct view', 'broadcast'], style='-o', loglog=True)\n", - "plt.ylabel(\"seconds\")\n" + "plt.ylabel(\"seconds\")" ] }, { @@ -420,9 +420,11 @@ "\n", "dview_sends = 2 * N\n", "\n", + "\n", "def bcast_sends(N, depth):\n", " return 1 + (2**depth) + (N / (2**depth))\n", "\n", + "\n", "plt.loglog(N, dview_sends, '--', label=\"direct view\")\n", "for depth in range(5):\n", " plt.loglog(N, bcast_sends(N, depth), label=f\"broadcast (d={depth})\")\n", diff --git a/docs/source/examples/broadcast/MPI Broadcast.ipynb b/docs/source/examples/broadcast/MPI Broadcast.ipynb index 03931e05..e35d6679 100644 --- a/docs/source/examples/broadcast/MPI Broadcast.ipynb +++ b/docs/source/examples/broadcast/MPI Broadcast.ipynb @@ -22,9 +22,6 @@ "metadata": {}, "outputs": [], "source": [ - "import socket\n", - "import os, sys, re\n", - "\n", "import numpy as np\n", "\n", "import ipyparallel as ipp" @@ -194,8 +191,8 @@ } ], "source": [ - "mpi_ranks = eall.apply_async(lambda : MPI.Get_rank()).get_dict()\n", - "root_rank = root.apply_sync(lambda : MPI.Get_rank())\n", + "mpi_ranks = eall.apply_async(lambda: MPI.Get_rank()).get_dict()\n", + "root_rank = root.apply_sync(lambda: MPI.Get_rank())\n", "mpi_ranks" ] }, @@ -253,7 +250,7 @@ "source": [ "%%time \n", "ar = eall.push({'data': data}, block=False)\n", - "ar.wait_interactive()\n" + "ar.wait_interactive()" ] }, { @@ -270,13 +267,14 @@ " obj = MPI.bcast(obj, root_rank)\n", " g[key] = obj\n", "\n", + "\n", "def broadcast(key, obj, dv, root, root_rank):\n", " \"\"\"More efficient broadcast by doing push to root,\n", " and MPI broadcast to other engines.\n", - " \n", + "\n", " Still O(N) messages, but all but one message is always small.\n", " \"\"\"\n", - " root.push({key : obj}, block=False)\n", + " root.push({key: obj}, block=False)\n", " return dv.apply_async(_bcast, key, root_rank)" ] }, diff --git a/docs/source/examples/broadcast/memmap Broadcast.ipynb b/docs/source/examples/broadcast/memmap Broadcast.ipynb index c415b767..d2ecb83f 100644 --- a/docs/source/examples/broadcast/memmap Broadcast.ipynb +++ b/docs/source/examples/broadcast/memmap Broadcast.ipynb @@ -29,7 +29,6 @@ "outputs": [], "source": [ "import socket\n", - "import os, sys, re\n", "\n", "import numpy as np\n", "\n", @@ -273,7 +272,6 @@ "metadata": {}, "outputs": [], "source": [ - "import numpy as np\n", "data = np.random.random((512, 512))" ] }, @@ -345,6 +343,7 @@ "def array_to_file(A, name):\n", " \"\"\"write an array to a temporary file, return its filename\"\"\"\n", " import tempfile\n", + "\n", " with tempfile.NamedTemporaryFile(suffix='.np', delete=False) as tf:\n", " np.save(tf, A)\n", " data_path = tf.name\n", @@ -373,21 +372,23 @@ "source": [ "def bcast_memmap(data, name, client, host_engines):\n", " \"\"\"broadcast a numpy array efficiently\n", - " \n", + "\n", " - sends data to each remote host only once\n", " - loads with memmap everywhere\n", " \"\"\"\n", "\n", " # actually push the data, just once to each machine\n", " memmap_path_name = f\"_bcast_array_{name}\"\n", - " \n", - " one_per_host = rc.broadcast_view([engines[0] for engines in host_engines.values()], coalescing=True)\n", + "\n", + " one_per_host = rc.broadcast_view(\n", + " [engines[0] for engines in host_engines.values()], coalescing=True\n", + " )\n", " send_ar = one_per_host.apply_async(array_to_file, data, name=memmap_path_name)\n", - " \n", + "\n", " # load the data on all engines into a memmapped array\n", " async_results = []\n", " e_all = rc.broadcast_view(coalescing=True)\n", - " return e_all.apply_async(load_memmap, name, ipp.Reference(memmap_path_name))\n" + " return e_all.apply_async(load_memmap, name, ipp.Reference(memmap_path_name))" ] }, { diff --git a/docs/source/examples/dask.ipynb b/docs/source/examples/dask.ipynb index aefa64a0..156c9290 100644 --- a/docs/source/examples/dask.ipynb +++ b/docs/source/examples/dask.ipynb @@ -372,7 +372,7 @@ "\n", "\n", "def square(x):\n", - " return x ** 2\n", + " return x**2\n", "\n", "\n", "def neg(x):\n", @@ -923,8 +923,8 @@ } ], "source": [ - "from IPython.display import display\n", "from distributed import progress\n", + "from IPython.display import display\n", "\n", "\n", "def shift5(x):\n", diff --git a/docs/source/examples/joblib.ipynb b/docs/source/examples/joblib.ipynb index e0844240..b5231c46 100644 --- a/docs/source/examples/joblib.ipynb +++ b/docs/source/examples/joblib.ipynb @@ -117,9 +117,11 @@ "source": [ "from joblib import Parallel, delayed\n", "\n", + "\n", "def neg(x):\n", " return -x\n", "\n", + "\n", "Parallel(backend='ipyparallel')(delayed(neg)(i) for i in range(32))" ] }, @@ -186,6 +188,7 @@ ], "source": [ "import os\n", + "\n", "Parallel()(delayed(os.getpid)() for i in range(10))" ] }, @@ -217,6 +220,7 @@ "even.register_joblib_backend('even')\n", "\n", "from joblib import parallel_backend\n", + "\n", "with parallel_backend('even'):\n", " result = Parallel()(delayed(os.getpid)() for i in range(5))\n", "result" diff --git a/docs/source/examples/progress.ipynb b/docs/source/examples/progress.ipynb index 786ca618..5133d854 100644 --- a/docs/source/examples/progress.ipynb +++ b/docs/source/examples/progress.ipynb @@ -56,6 +56,7 @@ ], "source": [ "import ipyparallel as ipp\n", + "\n", "rc = ipp.Cluster(n=4).start_and_connect_sync()\n", "rc.activate()" ] @@ -108,7 +109,7 @@ " progress.description = f\"Step {progress.value}/{progress.max}\"\n", " time.sleep(0.1)\n", "\n", - "progress.bar_style = \"success\" # change color when it's done" + "progress.bar_style = \"success\" # change color when it's done" ] }, { diff --git a/docs/source/examples/visualizing-tasks.ipynb b/docs/source/examples/visualizing-tasks.ipynb index c03bf31d..5ef28ecf 100644 --- a/docs/source/examples/visualizing-tasks.ipynb +++ b/docs/source/examples/visualizing-tasks.ipynb @@ -48,7 +48,7 @@ } ], "source": [ - "rc = await ipp.Cluster().start_and_connect(n=8)\n" + "rc = await ipp.Cluster().start_and_connect(n=8)" ] }, { @@ -84,7 +84,6 @@ "import datetime\n", "import random\n", "\n", - "\n", "v = rc.load_balanced_view()\n", "\n", "\n", @@ -166,9 +165,10 @@ "metadata": {}, "outputs": [], "source": [ - "import pandas as pd\n", "import datetime\n", "\n", + "import pandas as pd\n", + "\n", "data = {\n", " \"engine_id\": ar.engine_id,\n", "}\n", @@ -181,7 +181,7 @@ "df['prepare'] = df['submitted'] - df['submitted'].min()\n", "df['schedule'] = df['started'] - df['submitted']\n", "df['compute'] = df['completed'] - df['started']\n", - "df['reply'] = df['received'] - df['completed']\n" + "df['reply'] = df['received'] - df['completed']" ] }, { @@ -223,7 +223,7 @@ "df.plot.barh(y=['prepare', 'schedule', 'compute', 'reply'], stacked=True, width=1)\n", "plt.grid((True, False))\n", "yticks = [0, len(df) // 2, len(df)]\n", - "plt.yticks(yticks, yticks);\n", + "plt.yticks(yticks, yticks)\n", "plt.xlabel(\"seconds\")\n", "plt.ylabel(\"tasks\")\n", "# plt.ylim([0, len(df)])" @@ -310,9 +310,9 @@ "height = np.ones(len(df))\n", "\n", "import matplotlib as mpl\n", + "from matplotlib import cm\n", "from matplotlib.collections import PatchCollection\n", "from matplotlib.patches import Rectangle\n", - "from matplotlib import cm\n", "\n", "color_map = mpl.cm.ScalarMappable(\n", " norm=mpl.colors.Normalize(vmin=0, vmax=df.compute.max()),\n", @@ -597,7 +597,6 @@ " remaining = df\n", " working = []\n", " for t in timeline:\n", - "\n", " remaining = remaining[remaining.completed >= t]\n", " active = remaining[remaining.started < t]\n", " working.append(len(active))\n", @@ -612,7 +611,8 @@ " plt.xlabel(\"seconds\")\n", " plt.ylabel(\"busy engines\")\n", " plt.xlim(0, timeline[-1])\n", - " \n", + "\n", + "\n", "plot_load(df)" ] }