From 5ac0ded46820b191ec253e2c448322caab419e54 Mon Sep 17 00:00:00 2001 From: Vincent Bernat Date: Sat, 7 May 2011 02:34:12 +0200 Subject: [PATCH 1/5] Add a plugin to combine files. This new plugin allow to combine multiple files into one. Here is an example: ``` --- combine: files: luffy.*.js where: top --- luffy.effects(); luffy.search(); ``` All luffy.*.js files will be combined and added to the top of the file. --- hyde/ext/plugins/combine.py | 74 +++++++++++++++++++++++++++++++++++++ hyde/generator.py | 4 +- 2 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 hyde/ext/plugins/combine.py diff --git a/hyde/ext/plugins/combine.py b/hyde/ext/plugins/combine.py new file mode 100644 index 0000000..fcf426b --- /dev/null +++ b/hyde/ext/plugins/combine.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +""" +Contains classes to combine files into one +""" + +from fnmatch import fnmatch + +from hyde.model import Expando +from hyde.plugin import Plugin + +class CombinePlugin(Plugin): + """ + To use this combine, the following configuration should be added + to meta data:: + combine: + files: + - ns1.*.js + - ns2.*.js + where: top + + `files` is a list of resources (or just a resource) that should be + combined. Globbing is performed. `where` indicate where the + combination should be done. This could be `top` or `bottom` of the + file. + """ + + def __init__(self, site): + super(CombinePlugin, self).__init__(site) + + def begin_text_resource(self, resource, text): + """ + When generating a resource, add combined file if needed. + """ + # Grab configuration + try: + config = resource.meta.combine + except AttributeError: + return + # Grab file list + try: + files = config.files + except AttributeError: + raise "No resources to combine for [%s]" % resource + if type(files) is str: + files = [ files ] + where = "bottom" + try: + where = config.where + except AttributeError: + pass + + if where not in [ "top", "bottom" ]: + raise ValueError("%r should be either `top` or `bottom`" % where) + + resources = [] + for r in resource.node.resources: + for f in files: + if fnmatch(r.name, f): + resources.append(r) + break + if not resources: + self.logger.debug("No resources to combine for [%s]" % resource) + return + self.logger.debug( + "Combining %d resources for [%s]" % (len(resources), + resource)) + if not hasattr(resource, 'depends'): + resource.depends = [] + resource.depends.extend([r.relative_path for r in resources + if r.relative_path not in resource.depends]) + if where == "top": + return "".join([r.source.read_all() for r in resources] + [text]) + else: + return "".join([text] + [r.source.read_all() for r in resources]) diff --git a/hyde/generator.py b/hyde/generator.py index f7bb8de..8da9d8e 100644 --- a/hyde/generator.py +++ b/hyde/generator.py @@ -161,7 +161,7 @@ class Generator(object): if not target.exists or target.older_than(resource.source_file): logger.debug("Found changes in %s" % resource) return True - if resource.source_file.is_binary or not resource.uses_template: + if resource.source_file.is_binary: logger.debug("No Changes found in %s" % resource) return False deps = self.get_dependencies(resource) @@ -302,7 +302,6 @@ class Generator(object): logger.debug("Processing [%s]", resource) with self.context_for_resource(resource) as context: if resource.source_file.is_text: - self.update_deps(resource) if resource.uses_template: logger.debug("Rendering [%s]", resource) try: @@ -322,6 +321,7 @@ class Generator(object): resource.relative_deploy_path)) target.parent.make() target.write(text) + self.update_deps(resource) else: logger.debug("Copying binary file [%s]", resource) self.events.begin_binary_resource(resource) From 8a2a524a73633616f56ad122487090ef1c710a70 Mon Sep 17 00:00:00 2001 From: Vincent Bernat Date: Sat, 7 May 2011 11:50:04 +0200 Subject: [PATCH 2/5] Add the ability to remove combined resources from processing. A new keyword `remove` allows to not deploy resources that have been combined. Small rework on how the plugin work to remove one change from hyde generator about dependencies handling. --- hyde/ext/plugins/combine.py | 70 +++++++++++++++++++++++++++---------- hyde/generator.py | 2 +- 2 files changed, 53 insertions(+), 19 deletions(-) diff --git a/hyde/ext/plugins/combine.py b/hyde/ext/plugins/combine.py index fcf426b..9303201 100644 --- a/hyde/ext/plugins/combine.py +++ b/hyde/ext/plugins/combine.py @@ -17,41 +17,34 @@ class CombinePlugin(Plugin): - ns1.*.js - ns2.*.js where: top + remove: yes `files` is a list of resources (or just a resource) that should be combined. Globbing is performed. `where` indicate where the combination should be done. This could be `top` or `bottom` of the - file. + file. `remove` tell if we should remove resources that have been + combined into the resource. """ def __init__(self, site): super(CombinePlugin, self).__init__(site) - def begin_text_resource(self, resource, text): + def _combined(self, resource): """ - When generating a resource, add combined file if needed. + Return the list of resources to combine to build this one. """ - # Grab configuration try: config = resource.meta.combine except AttributeError: - return - # Grab file list + return [] # Not a combined resource try: files = config.files except AttributeError: raise "No resources to combine for [%s]" % resource if type(files) is str: files = [ files ] - where = "bottom" - try: - where = config.where - except AttributeError: - pass - - if where not in [ "top", "bottom" ]: - raise ValueError("%r should be either `top` or `bottom`" % where) + # Grab resources to combine resources = [] for r in resource.node.resources: for f in files: @@ -60,14 +53,55 @@ class CombinePlugin(Plugin): break if not resources: self.logger.debug("No resources to combine for [%s]" % resource) + return [] + + return resources + + def begin_site(self): + """ + Initialize the plugin and search for the combined resources + """ + for node in self.site.content.walk(): + for resource in node.resources: + resources = self._combined(resource) + if not resources: + continue + + # Build depends + if not hasattr(resource, 'depends'): + resource.depends = [] + resource.depends.extend( + [r.relative_path for r in resources + if r.relative_path not in resource.depends]) + + # Remove combined resources if needed + if hasattr(resource.meta.combine, "remove") and \ + resource.meta.combine.remove: + for r in resources: + self.logger.debug( + "Resource [%s] removed because combined" % r) + r.is_processable = False + + def begin_text_resource(self, resource, text): + """ + When generating a resource, add combined file if needed. + """ + resources = self._combined(resource) + if not resources: return + + where = "bottom" + try: + where = resource.meta.combine.where + except AttributeError: + pass + + if where not in [ "top", "bottom" ]: + raise ValueError("%r should be either `top` or `bottom`" % where) + self.logger.debug( "Combining %d resources for [%s]" % (len(resources), resource)) - if not hasattr(resource, 'depends'): - resource.depends = [] - resource.depends.extend([r.relative_path for r in resources - if r.relative_path not in resource.depends]) if where == "top": return "".join([r.source.read_all() for r in resources] + [text]) else: diff --git a/hyde/generator.py b/hyde/generator.py index 8da9d8e..df3d06b 100644 --- a/hyde/generator.py +++ b/hyde/generator.py @@ -302,6 +302,7 @@ class Generator(object): logger.debug("Processing [%s]", resource) with self.context_for_resource(resource) as context: if resource.source_file.is_text: + self.update_deps(resource) if resource.uses_template: logger.debug("Rendering [%s]", resource) try: @@ -321,7 +322,6 @@ class Generator(object): resource.relative_deploy_path)) target.parent.make() target.write(text) - self.update_deps(resource) else: logger.debug("Copying binary file [%s]", resource) self.events.begin_binary_resource(resource) From 12efa56ce25001b8c2872aa468c3f23c83279c4a Mon Sep 17 00:00:00 2001 From: Vincent Bernat Date: Sat, 7 May 2011 12:22:08 +0200 Subject: [PATCH 3/5] Add tests for combine plugin. --- hyde/tests/ext/combine/script.1.js | 1 + hyde/tests/ext/combine/script.2.js | 1 + hyde/tests/ext/combine/script.3.js | 1 + hyde/tests/ext/test_combine.py | 82 ++++++++++++++++++++++++++++++ 4 files changed, 85 insertions(+) create mode 100644 hyde/tests/ext/combine/script.1.js create mode 100644 hyde/tests/ext/combine/script.2.js create mode 100644 hyde/tests/ext/combine/script.3.js create mode 100644 hyde/tests/ext/test_combine.py diff --git a/hyde/tests/ext/combine/script.1.js b/hyde/tests/ext/combine/script.1.js new file mode 100644 index 0000000..2e644cd --- /dev/null +++ b/hyde/tests/ext/combine/script.1.js @@ -0,0 +1 @@ +var a = 1 + 2; diff --git a/hyde/tests/ext/combine/script.2.js b/hyde/tests/ext/combine/script.2.js new file mode 100644 index 0000000..6e74290 --- /dev/null +++ b/hyde/tests/ext/combine/script.2.js @@ -0,0 +1 @@ +var b = a + 3; diff --git a/hyde/tests/ext/combine/script.3.js b/hyde/tests/ext/combine/script.3.js new file mode 100644 index 0000000..ef4b9de --- /dev/null +++ b/hyde/tests/ext/combine/script.3.js @@ -0,0 +1 @@ +var c = a + 5; diff --git a/hyde/tests/ext/test_combine.py b/hyde/tests/ext/test_combine.py new file mode 100644 index 0000000..4cff807 --- /dev/null +++ b/hyde/tests/ext/test_combine.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +""" +Use nose +`$ pip install nose` +`$ nosetests` +""" +from hyde.fs import File, Folder +from hyde.model import Expando +from hyde.generator import Generator +from hyde.site import Site + +COMBINE_SOURCE = File(__file__).parent.child_folder('combine') +TEST_SITE = File(__file__).parent.parent.child_folder('_test') + + +class TestCombine(object): + + def setUp(self): + TEST_SITE.make() + TEST_SITE.parent.child_folder( + 'sites/test_jinja').copy_contents_to(TEST_SITE) + TEST_SITE.child_folder('content/media/js').make() + COMBINE_SOURCE.copy_contents_to(TEST_SITE.child('content/media/js')) + + def tearDown(self): + TEST_SITE.delete() + + def _test_combine(self, content): + s = Site(TEST_SITE) + s.config.plugins = [ + 'hyde.ext.plugins.meta.MetaPlugin', + 'hyde.ext.plugins.combine.CombinePlugin'] + source = TEST_SITE.child('content/media/js/script.js') + target = File(Folder(s.config.deploy_root_path).child('media/js/script.js')) + File(source).write(content) + + gen = Generator(s) + gen.generate_resource_at_path(source) + + assert target.exists + text = target.read_all() + return text, s + + def test_combine_top(self): + text, _ = self._test_combine(""" +--- +combine: + files: script.*.js + where: top +--- + +Last line""") + assert len(text.split("\n")) == 4 + assert text.split("\n")[-1] == "Last line" + return + + def test_combine_bottom(self): + text, _ = self._test_combine(""" +--- +combine: + files: script.*.js + where: bottom +--- + +First line +""") + assert len(text.split("\n")) == 4 + assert text.split("\n")[0] == "First line" + return + + def test_combine_remove(self): + _, s = self._test_combine(""" +--- +combine: + files: script.*.js + remove: yes +--- + +First line""") + for i in range(1,4): + assert not File(Folder(s.config.deploy_root_path). + child('media/js/script.%d.js' % i)).exists From 16b763887abeed6485a50ccc7cd33a038c4f2808 Mon Sep 17 00:00:00 2001 From: Vincent Bernat Date: Sat, 7 May 2011 15:39:58 +0200 Subject: [PATCH 4/5] Fix exception raising. --- hyde/ext/plugins/combine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hyde/ext/plugins/combine.py b/hyde/ext/plugins/combine.py index 9303201..b005108 100644 --- a/hyde/ext/plugins/combine.py +++ b/hyde/ext/plugins/combine.py @@ -40,7 +40,7 @@ class CombinePlugin(Plugin): try: files = config.files except AttributeError: - raise "No resources to combine for [%s]" % resource + raise AttributeError("No resources to combine for [%s]" % resource) if type(files) is str: files = [ files ] From cc814a394045d2fde32ddaa28149c1fe9258c5d6 Mon Sep 17 00:00:00 2001 From: Vincent Bernat Date: Sat, 7 May 2011 15:54:05 +0200 Subject: [PATCH 5/5] Sort resources by name before combining. --- hyde/ext/plugins/combine.py | 2 +- hyde/tests/ext/test_combine.py | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/hyde/ext/plugins/combine.py b/hyde/ext/plugins/combine.py index b005108..c4b2a82 100644 --- a/hyde/ext/plugins/combine.py +++ b/hyde/ext/plugins/combine.py @@ -55,7 +55,7 @@ class CombinePlugin(Plugin): self.logger.debug("No resources to combine for [%s]" % resource) return [] - return resources + return sorted(resources, key=lambda r: r.name) def begin_site(self): """ diff --git a/hyde/tests/ext/test_combine.py b/hyde/tests/ext/test_combine.py index 4cff807..2d3dbd8 100644 --- a/hyde/tests/ext/test_combine.py +++ b/hyde/tests/ext/test_combine.py @@ -50,8 +50,10 @@ combine: --- Last line""") - assert len(text.split("\n")) == 4 - assert text.split("\n")[-1] == "Last line" + assert text == """var a = 1 + 2; +var b = a + 3; +var c = a + 5; +Last line""" return def test_combine_bottom(self): @@ -64,8 +66,10 @@ combine: First line """) - assert len(text.split("\n")) == 4 - assert text.split("\n")[0] == "First line" + assert text == """First line +var a = 1 + 2; +var b = a + 3; +var c = a + 5;""" return def test_combine_remove(self):