From 15ea1ddf3589dc52dd4a832bae8ee0a0953561e6 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sat, 1 Apr 2023 06:34:08 +0200 Subject: [PATCH 1/7] Add profile --- .../console/tests/test_console_kernel.py | 30 ++++++ spyder_kernels/customize/code_runner.py | 97 ++++++++++++++++++- 2 files changed, 125 insertions(+), 2 deletions(-) diff --git a/spyder_kernels/console/tests/test_console_kernel.py b/spyder_kernels/console/tests/test_console_kernel.py index ab8f665a..065b53a1 100644 --- a/spyder_kernels/console/tests/test_console_kernel.py +++ b/spyder_kernels/console/tests/test_console_kernel.py @@ -1406,5 +1406,35 @@ def test_django_settings(kernel): assert "'settings':" in nsview +@flaky(max_runs=3) +def test_running_namespace_profile(tmpdir): + """ + Test that profile can get variables from running namespace. + """ + # Command to start the kernel + cmd = "from spyder_kernels.console import start; start.main()" + + with setup_kernel(cmd) as client: + # Remove all variables + client.execute_interactive("%reset -f", timeout=TIMEOUT) + + # Write defined variable code to a file + code = "result = 10\n%profile print(result)\nsucess=True" + d = tmpdir.join("defined-test.ipy") + d.write(code) + + # Run code file `d` + client.execute_interactive("%runfile {}" + .format(repr(str(d))), timeout=TIMEOUT) + + # Verify that `result` is defined in the current namespace + client.inspect('sucess') + msg = client.get_shell_msg(timeout=TIMEOUT) + while "found" not in msg['content']: + msg = client.get_shell_msg(timeout=TIMEOUT) + content = msg['content'] + assert content['found'] + + if __name__ == "__main__": pytest.main() diff --git a/spyder_kernels/customize/code_runner.py b/spyder_kernels/customize/code_runner.py index f4321eaa..7d675c02 100644 --- a/spyder_kernels/customize/code_runner.py +++ b/spyder_kernels/customize/code_runner.py @@ -14,10 +14,13 @@ import bdb import builtins from contextlib import contextmanager +import cProfile +from functools import partial import io import logging import os import pdb +import tempfile import shlex import sys import time @@ -30,6 +33,8 @@ ) from IPython.core.magic import ( needs_local_scope, + no_var_expand, + line_cell_magic, magics_class, Magics, line_magic, @@ -37,11 +42,11 @@ from IPython.core import magic_arguments # Local imports -from spyder_kernels.comms.frontendcomm import frontend_request +from spyder_kernels.comms.frontendcomm import frontend_request, CommError from spyder_kernels.customize.namespace_manager import NamespaceManager from spyder_kernels.customize.spyderpdb import SpyderPdb from spyder_kernels.customize.umr import UserModuleReloader -from spyder_kernels.customize.utils import capture_last_Expr, canonic +from spyder_kernels.customize.utils import capture_last_Expr, canonic, create_pathlist # For logging @@ -192,6 +197,29 @@ def debugfile(self, line, local_ns=None): context_locals=local_ns, ) + @runfile_arguments + @needs_local_scope + @line_magic + def profilefile(self, line, local_ns=None): + """ + Profile a file. + """ + args, local_ns = self._parse_runfile_argstring( + self.profilefile, line, local_ns) + + with self._profile_exec() as prof_exec: + self._exec_file( + filename=args.filename, + canonic_filename=args.canonic_filename, + wdir=args.wdir, + current_namespace=args.current_namespace, + args=args.args, + exec_fun=prof_exec, + post_mortem=args.post_mortem, + context_globals=args.namespace, + context_locals=local_ns, + ) + @runcell_arguments @needs_local_scope @line_magic @@ -230,6 +258,36 @@ def debugcell(self, line, local_ns=None): context_locals=local_ns, ) + @runcell_arguments + @needs_local_scope + @line_magic + def profilecell(self, line, local_ns=None): + """ + Profile a code cell from an editor. + """ + args = self._parse_runcell_argstring(self.profilecell, line) + + with self._profile_exec() as prof_exec: + return self._exec_cell( + cell_id=args.cell_id, + filename=args.filename, + canonic_filename=args.canonic_filename, + exec_fun=prof_exec, + post_mortem=args.post_mortem, + context_globals=self.shell.user_ns, + context_locals=local_ns, + ) + + @no_var_expand + @needs_local_scope + @line_cell_magic + def profile(self, line, cell=None, local_ns=None): + """Profile the given line.""" + if cell is not None: + line += "\n" + cell + with self._profile_exec() as prof_exec: + return prof_exec(line, self.shell.user_ns, local_ns) + @contextmanager def _debugger_exec(self, filename, continue_if_has_breakpoints): """Get an exec function to use for debugging.""" @@ -251,6 +309,41 @@ def debug_exec(code, glob, loc): # Enter recursive debugger yield debug_exec + @contextmanager + def _profile_exec(self): + """Get an exec function for profiling.""" + with tempfile.TemporaryDirectory() as tempdir: + # Reset the tracing function in case we are debugging + trace_fun = sys.gettrace() + sys.settrace(None) + # Get a file to save the results + profile_filename = os.path.join(tempdir, "profile.prof") + try: + if self.shell.is_debugging(): + def prof_exec(code, glob, loc): + # if we are debugging (tracing), call_tracing is + # necessary for profiling + return sys.call_tracing(cProfile.runctx, ( + code, glob, loc, profile_filename + )) + yield prof_exec + else: + yield partial(cProfile.runctx, filename=profile_filename) + finally: + # Resect tracing function + sys.settrace(trace_fun) + if os.path.isfile(profile_filename): + # Send result to frontend + with open(profile_filename, "br") as f: + profile_result = f.read() + try: + frontend_request(blocking=False).show_profile_file( + profile_result, create_pathlist() + ) + except CommError: + logger.debug( + "Could not send profile result to the frontend.") + def _exec_file( self, filename=None, From 8eb7bc97e93425a2fca12e9df6d1d91e07206b87 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 2 Apr 2023 22:43:42 +0200 Subject: [PATCH 2/7] Apply suggestions from code review Co-authored-by: Carlos Cordoba --- .../console/tests/test_console_kernel.py | 6 ++-- spyder_kernels/customize/code_runner.py | 28 +++++++++++-------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/spyder_kernels/console/tests/test_console_kernel.py b/spyder_kernels/console/tests/test_console_kernel.py index 065b53a1..a2676c51 100644 --- a/spyder_kernels/console/tests/test_console_kernel.py +++ b/spyder_kernels/console/tests/test_console_kernel.py @@ -1409,7 +1409,7 @@ def test_django_settings(kernel): @flaky(max_runs=3) def test_running_namespace_profile(tmpdir): """ - Test that profile can get variables from running namespace. + Test that profile can get variables from the running namespace. """ # Command to start the kernel cmd = "from spyder_kernels.console import start; start.main()" @@ -1424,8 +1424,8 @@ def test_running_namespace_profile(tmpdir): d.write(code) # Run code file `d` - client.execute_interactive("%runfile {}" - .format(repr(str(d))), timeout=TIMEOUT) + client.execute_interactive( + "%runfile {}".format(repr(str(d))), timeout=TIMEOUT) # Verify that `result` is defined in the current namespace client.inspect('sucess') diff --git a/spyder_kernels/customize/code_runner.py b/spyder_kernels/customize/code_runner.py index 7d675c02..15dd62f2 100644 --- a/spyder_kernels/customize/code_runner.py +++ b/spyder_kernels/customize/code_runner.py @@ -46,7 +46,8 @@ from spyder_kernels.customize.namespace_manager import NamespaceManager from spyder_kernels.customize.spyderpdb import SpyderPdb from spyder_kernels.customize.umr import UserModuleReloader -from spyder_kernels.customize.utils import capture_last_Expr, canonic, create_pathlist +from spyder_kernels.customize.utils import ( + capture_last_Expr, canonic, create_pathlist) # For logging @@ -201,9 +202,7 @@ def debugfile(self, line, local_ns=None): @needs_local_scope @line_magic def profilefile(self, line, local_ns=None): - """ - Profile a file. - """ + """Profile a file.""" args, local_ns = self._parse_runfile_argstring( self.profilefile, line, local_ns) @@ -262,9 +261,7 @@ def debugcell(self, line, local_ns=None): @needs_local_scope @line_magic def profilecell(self, line, local_ns=None): - """ - Profile a code cell from an editor. - """ + """Profile a code cell.""" args = self._parse_runcell_argstring(self.profilecell, line) with self._profile_exec() as prof_exec: @@ -316,24 +313,30 @@ def _profile_exec(self): # Reset the tracing function in case we are debugging trace_fun = sys.gettrace() sys.settrace(None) + # Get a file to save the results profile_filename = os.path.join(tempdir, "profile.prof") + try: if self.shell.is_debugging(): def prof_exec(code, glob, loc): - # if we are debugging (tracing), call_tracing is - # necessary for profiling + """ + If we are debugging (tracing), call_tracing is + necessary for profiling. + """ return sys.call_tracing(cProfile.runctx, ( code, glob, loc, profile_filename )) + yield prof_exec else: yield partial(cProfile.runctx, filename=profile_filename) finally: - # Resect tracing function + # Reset tracing function sys.settrace(trace_fun) + + # Send result to frontend if os.path.isfile(profile_filename): - # Send result to frontend with open(profile_filename, "br") as f: profile_result = f.read() try: @@ -342,7 +345,8 @@ def prof_exec(code, glob, loc): ) except CommError: logger.debug( - "Could not send profile result to the frontend.") + "Could not send profile result to the frontend." + ) def _exec_file( self, From 6fcc786a5673d930cab5895cb776ab7787d366ab Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 2 Apr 2023 22:46:27 +0200 Subject: [PATCH 3/7] remove old test --- .../console/tests/test_console_kernel.py | 30 ------------------- 1 file changed, 30 deletions(-) diff --git a/spyder_kernels/console/tests/test_console_kernel.py b/spyder_kernels/console/tests/test_console_kernel.py index a2676c51..ab8f665a 100644 --- a/spyder_kernels/console/tests/test_console_kernel.py +++ b/spyder_kernels/console/tests/test_console_kernel.py @@ -1406,35 +1406,5 @@ def test_django_settings(kernel): assert "'settings':" in nsview -@flaky(max_runs=3) -def test_running_namespace_profile(tmpdir): - """ - Test that profile can get variables from the running namespace. - """ - # Command to start the kernel - cmd = "from spyder_kernels.console import start; start.main()" - - with setup_kernel(cmd) as client: - # Remove all variables - client.execute_interactive("%reset -f", timeout=TIMEOUT) - - # Write defined variable code to a file - code = "result = 10\n%profile print(result)\nsucess=True" - d = tmpdir.join("defined-test.ipy") - d.write(code) - - # Run code file `d` - client.execute_interactive( - "%runfile {}".format(repr(str(d))), timeout=TIMEOUT) - - # Verify that `result` is defined in the current namespace - client.inspect('sucess') - msg = client.get_shell_msg(timeout=TIMEOUT) - while "found" not in msg['content']: - msg = client.get_shell_msg(timeout=TIMEOUT) - content = msg['content'] - assert content['found'] - - if __name__ == "__main__": pytest.main() From 26c9dad0eb4cfd37bcc32e65e8ec08ee8b18dd67 Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Mon, 3 Apr 2023 06:42:33 +0200 Subject: [PATCH 4/7] add xdg --- spyder_kernels/customize/code_runner.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/spyder_kernels/customize/code_runner.py b/spyder_kernels/customize/code_runner.py index 15dd62f2..411d4b85 100644 --- a/spyder_kernels/customize/code_runner.py +++ b/spyder_kernels/customize/code_runner.py @@ -309,7 +309,16 @@ def debug_exec(code, glob, loc): @contextmanager def _profile_exec(self): """Get an exec function for profiling.""" - with tempfile.TemporaryDirectory() as tempdir: + tmp_dir = None + if sys.platform.startswith('linux'): + # Do not use /tmp for temporary files + try: + from xdg.BaseDirectory import xdg_data_home + tmp_dir = xdg_data_home + os.makedirs(tmp_dir, exist_ok=True) + except Exception: + tmp_dir = None + with tempfile.TemporaryDirectory(dir=tmp_dir) as tempdir: # Reset the tracing function in case we are debugging trace_fun = sys.gettrace() sys.settrace(None) From 2bb93c215c03d4052bb98fc94a472e452c23980d Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Wed, 5 Jul 2023 20:27:26 +0200 Subject: [PATCH 5/7] Update spyder_kernels/customize/code_runner.py Co-authored-by: Carlos Cordoba --- spyder_kernels/customize/code_runner.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spyder_kernels/customize/code_runner.py b/spyder_kernels/customize/code_runner.py index 411d4b85..680787fa 100644 --- a/spyder_kernels/customize/code_runner.py +++ b/spyder_kernels/customize/code_runner.py @@ -318,6 +318,7 @@ def _profile_exec(self): os.makedirs(tmp_dir, exist_ok=True) except Exception: tmp_dir = None + with tempfile.TemporaryDirectory(dir=tmp_dir) as tempdir: # Reset the tracing function in case we are debugging trace_fun = sys.gettrace() From 281be2ba8b74e7808cb054fb011a2a223a73db0d Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 17 Mar 2024 10:28:39 +0100 Subject: [PATCH 6/7] fix required locals --- spyder_kernels/customize/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spyder_kernels/customize/utils.py b/spyder_kernels/customize/utils.py index fff18581..23e3c37e 100644 --- a/spyder_kernels/customize/utils.py +++ b/spyder_kernels/customize/utils.py @@ -206,7 +206,7 @@ def exec_encapsulate_locals( exec_fun = exec if filename is None: filename = "" - exec_fun(compile(code_ast, filename, "exec"), globals) + exec_fun(compile(code_ast, filename, "exec"), globals, None) finally: if use_locals_hack: # Cleanup code From acfd317ac5c578476120fc46f383b91927855fcc Mon Sep 17 00:00:00 2001 From: Quentin Peter Date: Sun, 29 Sep 2024 20:19:45 +0200 Subject: [PATCH 7/7] add None defaults --- spyder_kernels/customize/code_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spyder_kernels/customize/code_runner.py b/spyder_kernels/customize/code_runner.py index e8485a55..ae8ae4ef 100644 --- a/spyder_kernels/customize/code_runner.py +++ b/spyder_kernels/customize/code_runner.py @@ -330,7 +330,7 @@ def _profile_exec(self): try: if self.shell.is_debugging(): - def prof_exec(code, glob, loc): + def prof_exec(code, glob=None, loc=None): """ If we are debugging (tracing), call_tracing is necessary for profiling.