From 7fbf5fbb82bc0c120b0a3f6f0fb4e88ba761e9e2 Mon Sep 17 00:00:00 2001 From: Alex Ehlke Date: Wed, 4 Jun 2014 16:33:00 -0400 Subject: [PATCH 01/13] refactor JS compressors to take JS lazily --- pipeline/compressors/__init__.py | 22 ++++++++++++---------- pipeline/compressors/closure.py | 4 ++-- pipeline/compressors/jsmin.py | 4 ++-- pipeline/compressors/uglifyjs.py | 4 ++-- pipeline/compressors/yuglify.py | 4 ++-- pipeline/compressors/yui.py | 4 ++-- 6 files changed, 22 insertions(+), 20 deletions(-) diff --git a/pipeline/compressors/__init__.py b/pipeline/compressors/__init__.py index 920e0191..e0abfd27 100644 --- a/pipeline/compressors/__init__.py +++ b/pipeline/compressors/__init__.py @@ -53,16 +53,18 @@ def css_compressor(self): def compress_js(self, paths, templates=None, **kwargs): """Concatenate and compress JS files""" - js = self.concatenate(paths) - if templates: - js = js + self.compile_templates(templates) - - if not settings.PIPELINE_DISABLE_WRAPPER: - js = "(function() { %s }).call(this);" % js - - compressor = self.js_compressor - if compressor: - js = getattr(compressor(verbose=self.verbose), 'compress_js')(js) + def get_js(): + js = self.concatenate(paths) + if templates: + js = js + self.compile_templates(templates) + + if not settings.PIPELINE_DISABLE_WRAPPER: + js = "(function() { %s }).call(this);" % js + + compressor_cls = self.js_compressor + if compressor_cls: + compressor = compressor_cls(verbose=self.verbose) + js = getattr(compressor, 'compress_js')(get_js, paths) return js diff --git a/pipeline/compressors/closure.py b/pipeline/compressors/closure.py index 521c7a28..c6f93693 100644 --- a/pipeline/compressors/closure.py +++ b/pipeline/compressors/closure.py @@ -5,6 +5,6 @@ class ClosureCompressor(SubProcessCompressor): - def compress_js(self, js): + def compress_js(self, get_js, *args, **kwargs): command = '%s %s' % (settings.PIPELINE_CLOSURE_BINARY, settings.PIPELINE_CLOSURE_ARGUMENTS) - return self.execute_command(command, js) + return self.execute_command(command, get_js()) diff --git a/pipeline/compressors/jsmin.py b/pipeline/compressors/jsmin.py index dd1fc731..f105f589 100644 --- a/pipeline/compressors/jsmin.py +++ b/pipeline/compressors/jsmin.py @@ -8,6 +8,6 @@ class JSMinCompressor(CompressorBase): JS compressor based on the Python library jsmin (http://pypi.python.org/pypi/jsmin/). """ - def compress_js(self, js): + def compress_js(self, get_js, *args, **kwargs): from jsmin import jsmin - return jsmin(js) + return jsmin(get_js()) diff --git a/pipeline/compressors/uglifyjs.py b/pipeline/compressors/uglifyjs.py index 4ed2e8c2..f92d9d73 100644 --- a/pipeline/compressors/uglifyjs.py +++ b/pipeline/compressors/uglifyjs.py @@ -5,8 +5,8 @@ class UglifyJSCompressor(SubProcessCompressor): - def compress_js(self, js): + def compress_js(self, get_js, args, **kwargs): command = '%s %s' % (settings.PIPELINE_UGLIFYJS_BINARY, settings.PIPELINE_UGLIFYJS_ARGUMENTS) if self.verbose: command += ' --verbose' - return self.execute_command(command, js) + return self.execute_command(command, get_js()) diff --git a/pipeline/compressors/yuglify.py b/pipeline/compressors/yuglify.py index a6e73763..99361d3b 100644 --- a/pipeline/compressors/yuglify.py +++ b/pipeline/compressors/yuglify.py @@ -9,8 +9,8 @@ def compress_common(self, content, compress_type, arguments): command = '%s --type=%s %s' % (settings.PIPELINE_YUGLIFY_BINARY, compress_type, arguments) return self.execute_command(command, content) - def compress_js(self, js): - return self.compress_common(js, 'js', settings.PIPELINE_YUGLIFY_JS_ARGUMENTS) + def compress_js(self, get_js, *args, **kwargs): + return self.compress_common(get_js(), 'js', settings.PIPELINE_YUGLIFY_JS_ARGUMENTS) def compress_css(self, css): return self.compress_common(css, 'css', settings.PIPELINE_YUGLIFY_CSS_ARGUMENTS) diff --git a/pipeline/compressors/yui.py b/pipeline/compressors/yui.py index 0771c9e4..656301fa 100644 --- a/pipeline/compressors/yui.py +++ b/pipeline/compressors/yui.py @@ -9,8 +9,8 @@ def compress_common(self, content, compress_type, arguments): command = '%s --type=%s %s' % (settings.PIPELINE_YUI_BINARY, compress_type, arguments) return self.execute_command(command, content) - def compress_js(self, js): - return self.compress_common(js, 'js', settings.PIPELINE_YUI_JS_ARGUMENTS) + def compress_js(self, get_js, *args, **kwargs): + return self.compress_common(get_js(), 'js', settings.PIPELINE_YUI_JS_ARGUMENTS) def compress_css(self, css): return self.compress_common(css, 'css', settings.PIPELINE_YUI_CSS_ARGUMENTS) From c832dce8c426b03cb4daa6cd79316961b3340857 Mon Sep 17 00:00:00 2001 From: Alex Ehlke Date: Wed, 4 Jun 2014 16:45:11 -0400 Subject: [PATCH 02/13] add about: to list of non-rewritable URLs --- pipeline/compressors/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/compressors/__init__.py b/pipeline/compressors/__init__.py index e0abfd27..5e26a043 100644 --- a/pipeline/compressors/__init__.py +++ b/pipeline/compressors/__init__.py @@ -16,7 +16,7 @@ URL_DETECTOR = r'url\([\'"]?([^\s)]+\.[a-z]+[^\'"\s]*)[\'"]?\)' URL_REPLACER = r'url\(__EMBED__(.+?)(\?\d+)?\)' -NON_REWRITABLE_URL = re.compile(r'^(http:|https:|data:|//)') +NON_REWRITABLE_URL = re.compile(r'^(http:|https:|data:|about:|//)') DEFAULT_TEMPLATE_FUNC = "template" TEMPLATE_FUNC = r"""var template = function(str){var fn = new Function('obj', 'var __p=[],print=function(){__p.push.apply(__p,arguments);};with(obj||{}){__p.push(\''+str.replace(/\\/g, '\\\\').replace(/'/g, "\\'").replace(/<%=([\s\S]+?)%>/g,function(match,code){return "',"+code.replace(/\\'/g, "'")+",'";}).replace(/<%([\s\S]+?)%>/g,function(match,code){return "');"+code.replace(/\\'/g, "'").replace(/[\r\n\t]/g,' ')+"__p.push('";}).replace(/\r/g,'\\r').replace(/\n/g,'\\n').replace(/\t/g,'\\t')+"');}return __p.join('');");return fn;};""" From 9c6d5ec9b1da91240b1b61b15c4d5afd51ac0afd Mon Sep 17 00:00:00 2001 From: Alex Ehlke Date: Wed, 4 Jun 2014 17:09:13 -0400 Subject: [PATCH 03/13] skip about: urls --- pipeline/storage.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pipeline/storage.py b/pipeline/storage.py index ff659f5f..75f3b254 100644 --- a/pipeline/storage.py +++ b/pipeline/storage.py @@ -99,7 +99,12 @@ class NonPackagingPipelineStorage(NonPackagingMixin, PipelineStorage): class PipelineCachedStorage(PipelineMixin, CachedStaticFilesStorage): - pass + patterns = ( + ("*.css", ( + r"""(url\(['"]{0,1}\s*(.*?)["']{0,1}\))""", + (r"""(@import\s*["']\s*(?!\s*about:)(.*?)["'])""", """@import url("%s")"""), + )), + ) class NonPackagingPipelineCachedStorage(NonPackagingMixin, PipelineCachedStorage): From 7547bd248652df648521cb9ab44560a9a8cc1e30 Mon Sep 17 00:00:00 2001 From: Alex Ehlke Date: Wed, 4 Jun 2014 17:17:47 -0400 Subject: [PATCH 04/13] fix about: ignore --- pipeline/storage.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/pipeline/storage.py b/pipeline/storage.py index 75f3b254..ff0a1ddb 100644 --- a/pipeline/storage.py +++ b/pipeline/storage.py @@ -99,12 +99,23 @@ class NonPackagingPipelineStorage(NonPackagingMixin, PipelineStorage): class PipelineCachedStorage(PipelineMixin, CachedStaticFilesStorage): - patterns = ( - ("*.css", ( - r"""(url\(['"]{0,1}\s*(.*?)["']{0,1}\))""", - (r"""(@import\s*["']\s*(?!\s*about:)(.*?)["'])""", """@import url("%s")"""), - )), - ) + def url_converter(self, name, template=None): + """ + Returns the custom URL converter for the given file name. + """ + django_converter = super(PipelineCachedStorage, self).url_converter( + name, template=template) + + def converter(matchobj): + matched, url = matchobj.groups() + # Completely ignore http(s) prefixed URLs, + # fragments and data-uri URLs + if url.startswith(('about:')): + return matched + + return django_converter(matchobj) + + return converter class NonPackagingPipelineCachedStorage(NonPackagingMixin, PipelineCachedStorage): From 9e418e9653f9adc777f4633fa5f2a922f567fe66 Mon Sep 17 00:00:00 2001 From: Alex Ehlke Date: Wed, 4 Jun 2014 18:27:15 -0400 Subject: [PATCH 05/13] fix get_js --- pipeline/compressors/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pipeline/compressors/__init__.py b/pipeline/compressors/__init__.py index 5e26a043..1829b1c3 100644 --- a/pipeline/compressors/__init__.py +++ b/pipeline/compressors/__init__.py @@ -61,6 +61,8 @@ def get_js(): if not settings.PIPELINE_DISABLE_WRAPPER: js = "(function() { %s }).call(this);" % js + return js + compressor_cls = self.js_compressor if compressor_cls: compressor = compressor_cls(verbose=self.verbose) From 62113405ab2322a8d3566777a9dfb239a22a2c57 Mon Sep 17 00:00:00 2001 From: Alex Ehlke Date: Wed, 4 Jun 2014 18:57:46 -0400 Subject: [PATCH 06/13] add source map support --- pipeline/compressors/__init__.py | 15 ++++++++++----- pipeline/packager.py | 7 ++++++- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/pipeline/compressors/__init__.py b/pipeline/compressors/__init__.py index 1829b1c3..3f602620 100644 --- a/pipeline/compressors/__init__.py +++ b/pipeline/compressors/__init__.py @@ -61,14 +61,19 @@ def get_js(): if not settings.PIPELINE_DISABLE_WRAPPER: js = "(function() { %s }).call(this);" % js - return js + return js, None compressor_cls = self.js_compressor if compressor_cls: compressor = compressor_cls(verbose=self.verbose) - js = getattr(compressor, 'compress_js')(get_js, paths) + if hasattr(compressor, 'compress_js_with_source_map'): + return getattr(compressor, + 'compress_js_with_source_map')(get_js, paths) + else: + js = getattr(compressor, 'compress_js')(get_js, paths) + return js, None - return js + return js, None def compress_css(self, paths, output_filename, variant=None, **kwargs): """Concatenate and compress CSS files""" @@ -77,9 +82,9 @@ def compress_css(self, paths, output_filename, variant=None, **kwargs): if compressor: css = getattr(compressor(verbose=self.verbose), 'compress_css')(css) if not variant: - return css + return css, None elif variant == "datauri": - return self.with_data_uri(css) + return self.with_data_uri(css), None else: raise CompressorError("\"%s\" is not a valid variant" % variant) diff --git a/pipeline/packager.py b/pipeline/packager.py index 5acda214..a30d4cff 100644 --- a/pipeline/packager.py +++ b/pipeline/packager.py @@ -101,7 +101,12 @@ def pack(self, package, compress, signal, **kwargs): if self.verbose: print("Saving: %s" % output_filename) paths = self.compile(package.paths, force=True) - content = compress(paths, **kwargs) + content, source_map = compress(paths, **kwargs) + if source_map is not None: + source_map_output_filename = output_filename + '.map' + self.save_file(source_map_output_filename, source_map) + content = content + '\n/*# sourceMappingURL={} */'.format( + source_map_output_filename) self.save_file(output_filename, content) signal.send(sender=self, package=package, **kwargs) return output_filename From 1575b25122001a4fbff9c819f715cf000a37c589 Mon Sep 17 00:00:00 2001 From: Alex Ehlke Date: Thu, 5 Jun 2014 12:48:04 -0400 Subject: [PATCH 07/13] fix sourceMappingURL --- pipeline/packager.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pipeline/packager.py b/pipeline/packager.py index a30d4cff..e298ad55 100644 --- a/pipeline/packager.py +++ b/pipeline/packager.py @@ -1,4 +1,5 @@ from __future__ import unicode_literals +import os.path from django.contrib.staticfiles.finders import find from django.core.files.base import ContentFile @@ -90,13 +91,14 @@ def individual_url(self, filename): def pack_stylesheets(self, package, **kwargs): return self.pack(package, self.compressor.compress_css, css_compressed, + '/*# sourceMappingURL={} */', output_filename=package.output_filename, variant=package.variant, **kwargs) def compile(self, paths, force=False): return self.compiler.compile(paths, force=force) - def pack(self, package, compress, signal, **kwargs): + def pack(self, package, compress, signal, source_mapping_template, **kwargs): output_filename = package.output_filename if self.verbose: print("Saving: %s" % output_filename) @@ -105,14 +107,16 @@ def pack(self, package, compress, signal, **kwargs): if source_map is not None: source_map_output_filename = output_filename + '.map' self.save_file(source_map_output_filename, source_map) - content = content + '\n/*# sourceMappingURL={} */'.format( - source_map_output_filename) + content = content + '\n' + source_mapping_template.format( + os.path.basename(source_map_output_filename)) self.save_file(output_filename, content) signal.send(sender=self, package=package, **kwargs) return output_filename def pack_javascripts(self, package, **kwargs): - return self.pack(package, self.compressor.compress_js, js_compressed, templates=package.templates, **kwargs) + return self.pack(package, self.compressor.compress_js, js_compressed, + '//# sourceMappingURL={}', + templates=package.templates, **kwargs) def pack_templates(self, package): return self.compressor.compile_templates(package.templates) From 90fedc8f36cc6feeded757f86b59d02ac72a1ea3 Mon Sep 17 00:00:00 2001 From: Alex Ehlke Date: Thu, 5 Jun 2014 14:11:44 -0400 Subject: [PATCH 08/13] fix get_js --- pipeline/compressors/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/compressors/__init__.py b/pipeline/compressors/__init__.py index 3f602620..6d6b51b4 100644 --- a/pipeline/compressors/__init__.py +++ b/pipeline/compressors/__init__.py @@ -61,7 +61,7 @@ def get_js(): if not settings.PIPELINE_DISABLE_WRAPPER: js = "(function() { %s }).call(this);" % js - return js, None + return js compressor_cls = self.js_compressor if compressor_cls: From 138754b129fb4e488eec412df295135b0fa48458 Mon Sep 17 00:00:00 2001 From: Alex Ehlke Date: Mon, 9 Jun 2014 12:44:01 -0400 Subject: [PATCH 09/13] add -sourcemaps to version string for now --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7a7848fa..b4560a1c 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name='django-pipeline', - version='1.3.24', + version='1.3.24-sourcemaps', description='Pipeline is an asset packaging library for Django.', long_description=io.open('README.rst', encoding='utf-8').read() + '\n\n' + io.open('HISTORY.rst', encoding='utf-8').read(), From b1b15d09e20d1887c7a7de7e677f52a94e35520a Mon Sep 17 00:00:00 2001 From: Alex Ehlke Date: Tue, 8 Jul 2014 14:11:41 -0400 Subject: [PATCH 10/13] fix .map files being left out of post-processing, stop passing get_js around --- pipeline/compressors/__init__.py | 4 ++-- pipeline/compressors/closure.py | 4 ++-- pipeline/compressors/jsmin.py | 4 ++-- pipeline/compressors/uglifyjs.py | 4 ++-- pipeline/compressors/yuglify.py | 4 ++-- pipeline/compressors/yui.py | 4 ++-- pipeline/packager.py | 5 ++++- pipeline/storage.py | 19 ++++++++++++------- setup.py | 2 +- 9 files changed, 29 insertions(+), 21 deletions(-) diff --git a/pipeline/compressors/__init__.py b/pipeline/compressors/__init__.py index 6d6b51b4..2e805f32 100644 --- a/pipeline/compressors/__init__.py +++ b/pipeline/compressors/__init__.py @@ -68,9 +68,9 @@ def get_js(): compressor = compressor_cls(verbose=self.verbose) if hasattr(compressor, 'compress_js_with_source_map'): return getattr(compressor, - 'compress_js_with_source_map')(get_js, paths) + 'compress_js_with_source_map')(paths) else: - js = getattr(compressor, 'compress_js')(get_js, paths) + js = getattr(compressor, 'compress_js')(get_js()) return js, None return js, None diff --git a/pipeline/compressors/closure.py b/pipeline/compressors/closure.py index c6f93693..521c7a28 100644 --- a/pipeline/compressors/closure.py +++ b/pipeline/compressors/closure.py @@ -5,6 +5,6 @@ class ClosureCompressor(SubProcessCompressor): - def compress_js(self, get_js, *args, **kwargs): + def compress_js(self, js): command = '%s %s' % (settings.PIPELINE_CLOSURE_BINARY, settings.PIPELINE_CLOSURE_ARGUMENTS) - return self.execute_command(command, get_js()) + return self.execute_command(command, js) diff --git a/pipeline/compressors/jsmin.py b/pipeline/compressors/jsmin.py index f105f589..dd1fc731 100644 --- a/pipeline/compressors/jsmin.py +++ b/pipeline/compressors/jsmin.py @@ -8,6 +8,6 @@ class JSMinCompressor(CompressorBase): JS compressor based on the Python library jsmin (http://pypi.python.org/pypi/jsmin/). """ - def compress_js(self, get_js, *args, **kwargs): + def compress_js(self, js): from jsmin import jsmin - return jsmin(get_js()) + return jsmin(js) diff --git a/pipeline/compressors/uglifyjs.py b/pipeline/compressors/uglifyjs.py index f92d9d73..4ed2e8c2 100644 --- a/pipeline/compressors/uglifyjs.py +++ b/pipeline/compressors/uglifyjs.py @@ -5,8 +5,8 @@ class UglifyJSCompressor(SubProcessCompressor): - def compress_js(self, get_js, args, **kwargs): + def compress_js(self, js): command = '%s %s' % (settings.PIPELINE_UGLIFYJS_BINARY, settings.PIPELINE_UGLIFYJS_ARGUMENTS) if self.verbose: command += ' --verbose' - return self.execute_command(command, get_js()) + return self.execute_command(command, js) diff --git a/pipeline/compressors/yuglify.py b/pipeline/compressors/yuglify.py index 99361d3b..a6e73763 100644 --- a/pipeline/compressors/yuglify.py +++ b/pipeline/compressors/yuglify.py @@ -9,8 +9,8 @@ def compress_common(self, content, compress_type, arguments): command = '%s --type=%s %s' % (settings.PIPELINE_YUGLIFY_BINARY, compress_type, arguments) return self.execute_command(command, content) - def compress_js(self, get_js, *args, **kwargs): - return self.compress_common(get_js(), 'js', settings.PIPELINE_YUGLIFY_JS_ARGUMENTS) + def compress_js(self, js): + return self.compress_common(js, 'js', settings.PIPELINE_YUGLIFY_JS_ARGUMENTS) def compress_css(self, css): return self.compress_common(css, 'css', settings.PIPELINE_YUGLIFY_CSS_ARGUMENTS) diff --git a/pipeline/compressors/yui.py b/pipeline/compressors/yui.py index 656301fa..0771c9e4 100644 --- a/pipeline/compressors/yui.py +++ b/pipeline/compressors/yui.py @@ -9,8 +9,8 @@ def compress_common(self, content, compress_type, arguments): command = '%s --type=%s %s' % (settings.PIPELINE_YUI_BINARY, compress_type, arguments) return self.execute_command(command, content) - def compress_js(self, get_js, *args, **kwargs): - return self.compress_common(get_js(), 'js', settings.PIPELINE_YUI_JS_ARGUMENTS) + def compress_js(self, js): + return self.compress_common(js, 'js', settings.PIPELINE_YUI_JS_ARGUMENTS) def compress_css(self, css): return self.compress_common(css, 'css', settings.PIPELINE_YUI_CSS_ARGUMENTS) diff --git a/pipeline/packager.py b/pipeline/packager.py index e298ad55..cbba21b0 100644 --- a/pipeline/packager.py +++ b/pipeline/packager.py @@ -106,12 +106,15 @@ def pack(self, package, compress, signal, source_mapping_template, **kwargs): content, source_map = compress(paths, **kwargs) if source_map is not None: source_map_output_filename = output_filename + '.map' + if self.verbose: + print("Saving: %s" % source_map_output_filename) self.save_file(source_map_output_filename, source_map) content = content + '\n' + source_mapping_template.format( os.path.basename(source_map_output_filename)) + yield source_map_output_filename self.save_file(output_filename, content) signal.send(sender=self, package=package, **kwargs) - return output_filename + yield output_filename def pack_javascripts(self, package, **kwargs): return self.pack(package, self.compressor.compress_js, js_compressed, diff --git a/pipeline/storage.py b/pipeline/storage.py index ff0a1ddb..c8a0b1b8 100644 --- a/pipeline/storage.py +++ b/pipeline/storage.py @@ -29,16 +29,21 @@ def post_process(self, paths, dry_run=False, **options): package = packager.package_for('css', package_name) output_file = package.output_filename if self.packing: - packager.pack_stylesheets(package) - paths[output_file] = (self, output_file) - yield output_file, output_file, True + processor = packager.pack_stylesheets(package): + else: + processor = [package.output_filename] + for output_filename in processor: + paths[output_filename] = (self, output_filename) + yield output_filename, output_filename, True for package_name in packager.packages['js']: package = packager.package_for('js', package_name) - output_file = package.output_filename if self.packing: - packager.pack_javascripts(package) - paths[output_file] = (self, output_file) - yield output_file, output_file, True + processor = packager.pack_javascripts(package): + else: + processor = [package.output_filename] + for output_filename in processor: + paths[output_filename] = (self, output_filename) + yield output_filename, output_filename, True super_class = super(PipelineMixin, self) if hasattr(super_class, 'post_process'): diff --git a/setup.py b/setup.py index b4560a1c..14153136 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name='django-pipeline', - version='1.3.24-sourcemaps', + version='1.3.24-sourcemaps-rev2', description='Pipeline is an asset packaging library for Django.', long_description=io.open('README.rst', encoding='utf-8').read() + '\n\n' + io.open('HISTORY.rst', encoding='utf-8').read(), From 1311214ee2b5dc8724f11c4b397b483aaca5b2b7 Mon Sep 17 00:00:00 2001 From: Alex Ehlke Date: Tue, 8 Jul 2014 15:05:10 -0400 Subject: [PATCH 11/13] syntax error --- pipeline/storage.py | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pipeline/storage.py b/pipeline/storage.py index c8a0b1b8..e053330b 100644 --- a/pipeline/storage.py +++ b/pipeline/storage.py @@ -29,7 +29,7 @@ def post_process(self, paths, dry_run=False, **options): package = packager.package_for('css', package_name) output_file = package.output_filename if self.packing: - processor = packager.pack_stylesheets(package): + processor = packager.pack_stylesheets(package) else: processor = [package.output_filename] for output_filename in processor: @@ -38,7 +38,7 @@ def post_process(self, paths, dry_run=False, **options): for package_name in packager.packages['js']: package = packager.package_for('js', package_name) if self.packing: - processor = packager.pack_javascripts(package): + processor = packager.pack_javascripts(package) else: processor = [package.output_filename] for output_filename in processor: diff --git a/setup.py b/setup.py index 14153136..f644106c 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name='django-pipeline', - version='1.3.24-sourcemaps-rev2', + version='1.3.24-sourcemaps-rev3', description='Pipeline is an asset packaging library for Django.', long_description=io.open('README.rst', encoding='utf-8').read() + '\n\n' + io.open('HISTORY.rst', encoding='utf-8').read(), From 79508aa9f03cf022605aabaaf288c532880f25fb Mon Sep 17 00:00:00 2001 From: Brian Peiris Date: Thu, 13 Nov 2014 17:39:09 -0500 Subject: [PATCH 12/13] Fix unassigned variable so that tests pass. --- pipeline/compressors/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipeline/compressors/__init__.py b/pipeline/compressors/__init__.py index 2e805f32..0effda6b 100644 --- a/pipeline/compressors/__init__.py +++ b/pipeline/compressors/__init__.py @@ -73,7 +73,7 @@ def get_js(): js = getattr(compressor, 'compress_js')(get_js()) return js, None - return js, None + return None, None def compress_css(self, paths, output_filename, variant=None, **kwargs): """Concatenate and compress CSS files""" From 23dea511b76fbe7adaa80c71aee14a66340fa101 Mon Sep 17 00:00:00 2001 From: Brian Peiris Date: Fri, 14 Nov 2014 11:41:46 -0500 Subject: [PATCH 13/13] Add new concurrency setting with tests and docs. --- docs/configuration.rst | 8 ++++++ pipeline/compilers/__init__.py | 8 +++++- pipeline/conf.py | 1 + tests/tests/test_compiler.py | 51 ++++++++++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 1 deletion(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index f10bbdb7..8d70f6cb 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -118,6 +118,14 @@ Other settings Defaults to ``not settings.DEBUG``. +``PIPELINE_COMPILER_CONCURRENCY`` +................................. + + If set, overrides the number of threads used to compile assets. Otherwise the + compiler will attempt to use as many threads as there are available cores. + + Defaults to ``None``. + ``PIPELINE_CSS_COMPRESSOR`` ............................ diff --git a/pipeline/compilers/__init__.py b/pipeline/compilers/__init__.py index 60091408..5d189a35 100644 --- a/pipeline/compilers/__init__.py +++ b/pipeline/compilers/__init__.py @@ -51,7 +51,13 @@ def _compile(input_path): except ImportError: return list(map(_compile, paths)) else: - with futures.ThreadPoolExecutor(max_workers=multiprocessing.cpu_count()) as executor: + max_workers = ( + settings.PIPELINE_COMPILER_CONCURRENCY or + multiprocessing.cpu_count()) + + with futures.ThreadPoolExecutor( + max_workers=max_workers + ) as executor: return list(executor.map(_compile, paths)) def output_path(self, path, extension): diff --git a/pipeline/conf.py b/pipeline/conf.py index 9f7bc6ce..a454a6f3 100644 --- a/pipeline/conf.py +++ b/pipeline/conf.py @@ -16,6 +16,7 @@ 'PIPELINE_CSS_COMPRESSOR': 'pipeline.compressors.yuglify.YuglifyCompressor', 'PIPELINE_JS_COMPRESSOR': 'pipeline.compressors.yuglify.YuglifyCompressor', 'PIPELINE_COMPILERS': [], + 'PIPELINE_COMPILER_CONCURRENCY': None, 'PIPELINE_CSS': {}, 'PIPELINE_JS': {}, diff --git a/tests/tests/test_compiler.py b/tests/tests/test_compiler.py index 4096ef2e..0cf7e274 100644 --- a/tests/tests/test_compiler.py +++ b/tests/tests/test_compiler.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals from django.test import TestCase +from mock import MagicMock, patch from pipeline.conf import settings from pipeline.compilers import Compiler, CompilerBase @@ -39,5 +40,55 @@ def test_compile(self): ]) self.assertEqual([_('pipeline/js/dummy.js'), _('pipeline/js/application.js')], list(paths)) + def _get_mocked_concurrency_packages(self, mock_cpu_count=4): + multiprocessing_mock = MagicMock() + multiprocessing_mock.cpu_count.return_value = mock_cpu_count + + concurrent_mock = MagicMock() + thread_pool_executor_mock = concurrent_mock.futures.ThreadPoolExecutor + thread_pool_executor_mock.return_value.__exit__.return_value = False + + modules = { + 'multiprocessing': multiprocessing_mock, + 'concurrent': concurrent_mock, + 'concurrent.futures': concurrent_mock.futures, + } + return modules, thread_pool_executor_mock + + def test_concurrency_setting(self): + ''' + Setting PIPELINE_COMPILER_CONCURRENCY should override the default + CPU count. + ''' + modules, thread_pool_executor_mock = ( + self._get_mocked_concurrency_packages()) + + settings.PIPELINE_COMPILER_CONCURRENCY = 2 + + with patch.dict('sys.modules', modules): + self.compiler.compile([]) + + thread_pool_executor_mock.assert_called_once_with( + max_workers=settings.PIPELINE_COMPILER_CONCURRENCY + ) + + settings.PIPELINE_COMPILER_CONCURRENCY = None + + def test_empty_concurrency_setting(self): + ''' + Compiler should use cpu_count() if PIPELINE_COMPILER_CONCURRENCY is + not set. + ''' + MOCK_CPU_COUNT = 4 + modules, thread_pool_executor_mock = ( + self._get_mocked_concurrency_packages(MOCK_CPU_COUNT)) + + with patch.dict('sys.modules', modules): + self.compiler.compile([]) + + thread_pool_executor_mock.assert_called_once_with( + max_workers=MOCK_CPU_COUNT + ) + def tearDown(self): settings.PIPELINE_COMPILERS = self.old_compilers