From 8569063668bf939cfe6347534314fe8f1581ca75 Mon Sep 17 00:00:00 2001 From: Evgeniy Krysanov Date: Sun, 9 Dec 2018 18:28:06 +0300 Subject: [PATCH 1/4] Add conditional profiling. --- README.rst | 35 +++++++++--- speedinfo/conditions/__init__.py | 34 +++++++++++ speedinfo/conditions/base.py | 22 ++++++++ speedinfo/conditions/exclude_urls.py | 37 ++++++++++++ speedinfo/middleware.py | 84 +++++++++++++++++----------- speedinfo/settings.py | 4 ++ tests/tests.py | 6 +- 7 files changed, 177 insertions(+), 45 deletions(-) create mode 100644 speedinfo/conditions/__init__.py create mode 100644 speedinfo/conditions/base.py create mode 100644 speedinfo/conditions/exclude_urls.py diff --git a/README.rst b/README.rst index 7914213..735242a 100644 --- a/README.rst +++ b/README.rst @@ -54,15 +54,6 @@ to control profiler state. Press ``Reset`` button to reset all data. Configuration ============= -To exclude some urls from profiling add it to the ``SPEEDINFO_EXCLUDE_URLS`` list. -``SpeedInfo`` uses re.match internally to test requested url. Example:: - - SPEEDINFO_EXCLUDE_URLS = [ - r'/admin/', - r'/news/$', - r'/movie/\d+/$', - ] - ``SpeedInfo`` automatically detects when using Django per-site caching via ``UpdateCacheMiddleware`` and ``FetchFromCacheMiddleware`` middlewares or per-view caching via ``cache_page`` decorator and counts cache hit @@ -93,6 +84,32 @@ Default value:: ) +Profiling conditions +==================== + +``SPEEDINFO_PROFILING_CONDITIONS`` settings allows to declare a list of imported by path classes +to define the conditions for profiling the processed view. By default, the only condition is enabled:: + + SPEEDINFO_PROFILING_CONDITIONS = [ + 'speedinfo.conditions.exclude_urls.ExcludeURLCondition', + ] + + +``ExcludeURLCondition`` allows you to exclude some urls from profiling by adding them to +the ``SPEEDINFO_EXCLUDE_URLS`` list. ``ExcludeURLCondition`` uses ``re.match`` internally to test +requested url. Example:: + + SPEEDINFO_EXCLUDE_URLS = [ + r'/admin/', + r'/news/$', + r'/movie/\d+/$', + ] + + +To define your own condition class, you must inherit from the base class ``speedinfo.conditions.base.Condition`` +and implement all abstract methods. See ``ExcludeURLCondition`` source code for implementation example. + + Notice ====== diff --git a/speedinfo/conditions/__init__.py b/speedinfo/conditions/__init__.py new file mode 100644 index 0000000..462d78b --- /dev/null +++ b/speedinfo/conditions/__init__.py @@ -0,0 +1,34 @@ +# coding: utf-8 + +from importlib import import_module + +from speedinfo.settings import speedinfo_settings + + +class ConditionsDispatcher: + def __init__(self): + self.conditions = None + + def import_conditions(self): + self.conditions = [] + + for module_path in speedinfo_settings.SPEEDINFO_PROFILING_CONDITIONS: + path, class_name = module_path.rsplit('.', 1) + + try: + module = import_module(path) + self.conditions.append( + getattr(module, class_name)() + ) + except (AttributeError, ImportError) as e: + msg = 'Could not import "{}". {}: {}.'.format(module_path, e.__class__.__name__, e) + raise ImportError(msg) + + def get_conditions(self): + if self.conditions is None: + self.import_conditions() + + return self.conditions + + +conditions_dispatcher = ConditionsDispatcher() diff --git a/speedinfo/conditions/base.py b/speedinfo/conditions/base.py new file mode 100644 index 0000000..381e8f3 --- /dev/null +++ b/speedinfo/conditions/base.py @@ -0,0 +1,22 @@ +# coding: utf-8 + + +class Condition: + """ + Base class for user-defined profiling condition classes + """ + def process_request(self, request): + """ + :type request: :class:`django.http.HttpRequest` + :return: False if requested page should be excluded from profiling + :rtype: bool + """ + raise NotImplementedError + + def process_response(self, response): + """ + :type response: :class:`django.http.HttpResponse` + :return: False if requested page should be excluded from profiling + :rtype: bool + """ + raise NotImplementedError diff --git a/speedinfo/conditions/exclude_urls.py b/speedinfo/conditions/exclude_urls.py new file mode 100644 index 0000000..efd8143 --- /dev/null +++ b/speedinfo/conditions/exclude_urls.py @@ -0,0 +1,37 @@ +# coding: utf-8 + +import re + +from speedinfo.conditions.base import Condition +from speedinfo.settings import speedinfo_settings + + +class ExcludeURLCondition(Condition): + """ + Plugin for conditional profiling based on the list of + urls specified in SPEEDINFO_EXCLUDE_URLS settings + """ + def __init__(self): + self.patterns = None + + def get_patterns(self): + if (self.patterns is None) or speedinfo_settings.SPEEDINFO_TESTS: + self.patterns = [re.compile(pattern) for pattern in speedinfo_settings.SPEEDINFO_EXCLUDE_URLS] + + return self.patterns + + def process_request(self, request): + """Checks requested page url against the list of excluded urls. + + :type request: :class:`django.http.HttpRequest` + :return: False if path matches to any of the exclude urls + :rtype: bool + """ + for pattern in self.get_patterns(): + if pattern.match(request.path): + return False + + return True + + def process_response(self, response): + return True diff --git a/speedinfo/middleware.py b/speedinfo/middleware.py index 5ef0cd0..938e93a 100644 --- a/speedinfo/middleware.py +++ b/speedinfo/middleware.py @@ -1,12 +1,11 @@ # coding: utf-8 -import re - from itertools import islice from timeit import default_timer from django.db import connection from speedinfo import profiler +from speedinfo.conditions import conditions_dispatcher from speedinfo.settings import speedinfo_settings try: @@ -26,20 +25,6 @@ def __init__(self, get_response=None): self.force_debug_cursor = False self.start_time = 0 self.existing_sql_count = 0 - self.exclude_urls_re = [re.compile(pattern) for pattern in speedinfo_settings.SPEEDINFO_EXCLUDE_URLS] - - def match_exclude_urls(self, path): - """Looks for a match requested page url to the exclude urls list. - - :param str path: Path to the requested page, not including the scheme or domain - :return: True if path matches to any of the exclude urls - :rtype: bool - """ - for url_re in self.exclude_urls_re: - if url_re.match(path): - return True - - return False def get_view_name(self, request): """Returns full view name from request, eg. 'app.module.view_name'. @@ -54,17 +39,47 @@ def get_view_name(self, request): except Resolver404: return None + def can_process_request(self, request): + """Checks conditions to start profiling the request + + :type request: :class:`django.http.HttpRequest` + :return: True if request can be processed + :rtype: bool + """ + result = profiler.is_on and hasattr(request, 'user') and self.get_view_name(request) + + for condition in conditions_dispatcher.get_conditions(): + result = result and condition.process_request(request) + + if not result: + break + + return result + + def can_process_response(self, response): + """Checks conditions to finish profiling the request + + :type response: :class:`django.http.HttpResponse` + :return: True if request can be processed + :rtype: bool + """ + result = True + + for condition in conditions_dispatcher.get_conditions(): + result = result and condition.process_response(response) + + if not result: + break + + return result + def process_view(self, request, *args, **kwargs): """Initialize statistics variables and environment. :return: Response object or None :rtype: :class:`django.http.HttpResponse` or None """ - # Checks conditions to start profiling the request - self.is_active = profiler.is_on and \ - hasattr(request, 'user') and \ - self.get_view_name(request) and \ - not self.match_exclude_urls(request.path) + self.is_active = self.can_process_request(request) if self.is_active: # Force DB connection to debug mode to get sql time and number of queries @@ -85,21 +100,24 @@ def process_response(self, request, response): :rtype: :class:`django.http.HttpResponse` or :class:`django.http.StreamingHttpResponse` """ if self.is_active: - view_execution_time = default_timer() - self.start_time + if self.can_process_response(response): + view_execution_time = default_timer() - self.start_time - # Calculate the execution time and the number of queries. - # Exclude queries made before the call of our middleware (e.g. in SessionMiddleware). - sql_count = max(len(connection.queries) - self.existing_sql_count, 0) - sql_time = sum(float(q['time']) for q in islice(connection.queries, self.existing_sql_count, None)) - connection.force_debug_cursor = self.force_debug_cursor + # Calculate the execution time and the number of queries. + # Exclude queries made before the call of our middleware (e.g. in SessionMiddleware). + sql_count = max(len(connection.queries) - self.existing_sql_count, 0) + sql_time = sum(float(q['time']) for q in islice(connection.queries, self.existing_sql_count, None)) + + # Collects request and response params + view_name = self.get_view_name(request) + is_anon_call = request.user.is_anonymous() if callable(request.user.is_anonymous) else request.user.is_anonymous + is_cache_hit = getattr(response, speedinfo_settings.SPEEDINFO_CACHED_RESPONSE_ATTR_NAME, False) - # Collects request and response params - view_name = self.get_view_name(request) - is_anon_call = request.user.is_anonymous() if callable(request.user.is_anonymous) else request.user.is_anonymous - is_cache_hit = getattr(response, speedinfo_settings.SPEEDINFO_CACHED_RESPONSE_ATTR_NAME, False) + # Saves profiler data + profiler.data.add(view_name, request.method, is_anon_call, is_cache_hit, sql_time, sql_count, view_execution_time) - # Saves profiler data - profiler.data.add(view_name, request.method, is_anon_call, is_cache_hit, sql_time, sql_count, view_execution_time) + # Rollback debug cursor value even if process response condition is disabled + connection.force_debug_cursor = self.force_debug_cursor return response diff --git a/speedinfo/settings.py b/speedinfo/settings.py index dc3a5b5..3258038 100644 --- a/speedinfo/settings.py +++ b/speedinfo/settings.py @@ -7,7 +7,11 @@ ReportColumnFormat = namedtuple('ReportColumnFormat', ['name', 'format', 'attr_name', 'order_field']) DEFAULTS = { + 'SPEEDINFO_TESTS': False, 'SPEEDINFO_CACHED_RESPONSE_ATTR_NAME': 'is_cached', + 'SPEEDINFO_PROFILING_CONDITIONS': [ + 'speedinfo.conditions.exclude_urls.ExcludeURLCondition', + ], 'SPEEDINFO_EXCLUDE_URLS': [], 'SPEEDINFO_REPORT_COLUMNS': ( 'view_name', 'method', 'anon_calls_ratio', 'cache_hits_ratio', diff --git a/tests/tests.py b/tests/tests.py index e2e5d45..f8d3e3e 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -17,7 +17,7 @@ from django.core.urlresolvers import reverse -@override_settings(SPEEDINFO_EXCLUDE_URLS=[]) +@override_settings(SPEEDINFO_EXCLUDE_URLS=[], SPEEDINFO_TESTS=True) class ProfilerTest(TestCase): def setUp(self): cache.clear() @@ -159,7 +159,7 @@ def test_reset(self): self.assertFalse(ViewProfiler.objects.exists()) -@override_settings(SPEEDINFO_EXCLUDE_URLS=[reverse('func-view')]) +@override_settings(SPEEDINFO_EXCLUDE_URLS=[reverse('func-view')], SPEEDINFO_TESTS=True) class ExcludeURLConditionTest(TestCase): def setUp(self): cache.clear() @@ -180,7 +180,7 @@ def test_exclude_urls(self): self.assertTrue(ViewProfiler.objects.exists()) -@override_settings(SPEEDINFO_EXCLUDE_URLS=[reverse('admin:index')]) +@override_settings(SPEEDINFO_EXCLUDE_URLS=[reverse('admin:index')], SPEEDINFO_TESTS=True) class ProfilerAdminTest(TestCase): def setUp(self): cache.clear() From 90f981f9ed59edc8ef34f4a8b81e38686a2c7b09 Mon Sep 17 00:00:00 2001 From: Evgeniy Krysanov Date: Sun, 9 Dec 2018 22:54:00 +0300 Subject: [PATCH 2/4] Update README. --- README.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 735242a..a785ea7 100644 --- a/README.rst +++ b/README.rst @@ -75,7 +75,7 @@ Example:: setattr(response, SPEEDINFO_CACHED_RESPONSE_ATTR_NAME, True) return response -Change ``SPEEDINFO_REPORT_COLUMNS`` settings to customize Django admin profiler columns. +Change ``SPEEDINFO_REPORT_COLUMNS`` setting to customize Django admin profiler columns. Default value:: SPEEDINFO_REPORT_COLUMNS = ( @@ -87,7 +87,7 @@ Default value:: Profiling conditions ==================== -``SPEEDINFO_PROFILING_CONDITIONS`` settings allows to declare a list of imported by path classes +``SPEEDINFO_PROFILING_CONDITIONS`` setting allows to declare a list of imported by path classes to define the conditions for profiling the processed view. By default, the only condition is enabled:: SPEEDINFO_PROFILING_CONDITIONS = [ @@ -107,7 +107,9 @@ requested url. Example:: To define your own condition class, you must inherit from the base class ``speedinfo.conditions.base.Condition`` -and implement all abstract methods. See ``ExcludeURLCondition`` source code for implementation example. +and implement all abstract methods. See ``ExcludeURLCondition`` source code for implementation example. Then add +full path to your class to ``SPEEDINFO_PROFILING_CONDITIONS`` list as shown above. Conditions in mentioned list +are executed in a top-down order. The first condition returning ``False`` interrupts the further check. Notice From 447897d5468e06236333969b020b1730e5b5ec8c Mon Sep 17 00:00:00 2001 From: Evgeniy Krysanov Date: Tue, 11 Dec 2018 14:01:34 +0300 Subject: [PATCH 3/4] Update setup.py. --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index a65fd31..a0267a3 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name='django-speedinfo', version='1.3.7', - packages=['speedinfo', 'speedinfo.migrations'], + packages=['speedinfo', 'speedinfo.migrations', 'speedinfo.conditions'], include_package_data=True, license='MIT', description='Live profiling tool for Django framework to measure views performance', @@ -19,7 +19,6 @@ author='Evgeniy Krysanov', author_email='evgeniy.krysanov@gmail.com', classifiers=[ - 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'Environment :: Web Environment', 'Framework :: Django', From 80ab2d6fab184d57f685166b4ab7da6fd259bbe2 Mon Sep 17 00:00:00 2001 From: Evgeniy Krysanov Date: Tue, 11 Dec 2018 14:01:52 +0300 Subject: [PATCH 4/4] Bump version number to 1.4.0. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a0267a3..5a7bd45 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ setup( name='django-speedinfo', - version='1.3.7', + version='1.4.0', packages=['speedinfo', 'speedinfo.migrations', 'speedinfo.conditions'], include_package_data=True, license='MIT',