From 5367261cae74cdd8f9dde32661914839fe9da75b Mon Sep 17 00:00:00 2001 From: Lakshmi Vyasarajan Date: Tue, 8 Mar 2011 16:30:00 +0530 Subject: [PATCH] Added support for incremental generation --- hyde/engine.py | 4 +- hyde/generator.py | 69 +++++++++++-------- hyde/model.py | 21 ++++++ hyde/server.py | 17 ++--- .../ext/stylus/expected-site-compressed.css | 4 ++ 5 files changed, 74 insertions(+), 41 deletions(-) create mode 100644 hyde/tests/ext/stylus/expected-site-compressed.css diff --git a/hyde/engine.py b/hyde/engine.py index 54947ca..286d8f5 100644 --- a/hyde/engine.py +++ b/hyde/engine.py @@ -70,6 +70,8 @@ class Engine(Application): help='The configuration used to generate the site') @store('-d', '--deploy-path', dest='deploy', default=None, help='Where should the site be generated?') + @true('-i', '--incremental', dest='incremental', default=False, + help='Only process changed files') def gen(self, args): """ The generate command. Generates the site at the given @@ -79,7 +81,7 @@ class Engine(Application): site = self.make_site(args.sitepath, args.config, args.deploy) from hyde.generator import Generator gen = Generator(site) - gen.generate_all() + gen.generate_all(incremental=args.incremental) @subcommand('serve', help='Serve the website') @store('-a', '--address', default='localhost', dest='address', diff --git a/hyde/generator.py b/hyde/generator.py index 69fad92..344c60e 100644 --- a/hyde/generator.py +++ b/hyde/generator.py @@ -4,7 +4,7 @@ The generator class and related utility functions. """ from hyde.exceptions import HydeException from hyde.fs import File, Folder -from hyde.model import Context +from hyde.model import Context, Dependents from hyde.plugin import Plugin from hyde.template import Template @@ -23,6 +23,7 @@ class Generator(object): super(Generator, self).__init__() self.site = site self.generated_once = False + self.deps = Dependents(site.sitepath) self.__context__ = dict(site=site) if hasattr(site.config, 'context'): site.context = Context.load(site.sitepath, site.config.context) @@ -112,17 +113,22 @@ class Generator(object): """ Gets the dependencies for a given resource. """ + rel_path = resource.relative_path deps = [] - if hasattr(resource, 'depends'): - user_deps = resource.depends - for dep in user_deps: - deps.append(dep) - deps.extend(self.template.get_dependencies(dep)) - - deps.extend(self.template.get_dependencies(resource.relative_path)) - deps = list(set(deps)) - if None in deps: - deps.remove(None) + if not rel_path in self.deps: + if hasattr(resource, 'depends'): + user_deps = resource.depends + for dep in user_deps: + deps.append(dep) + deps.extend(self.template.get_dependencies(dep)) + + deps.extend(self.template.get_dependencies(resource.relative_path)) + deps = list(set(deps)) + if None in deps: + deps.remove(None) + self.deps[rel_path] = deps + else: + deps = self.deps[rel_path] return deps def has_resource_changed(self, resource): @@ -130,8 +136,6 @@ class Generator(object): Checks if the given resource has changed since the last generation. """ - if not self.generated_once: - return True logger.debug("Checking for changes in %s" % resource) self.load_site_if_needed() self.load_template_if_needed() @@ -163,7 +167,7 @@ class Generator(object): logger.debug("No changes found in %s" % resource) return False - def generate_all(self): + def generate_all(self, incremental=False): """ Generates the entire website """ @@ -174,31 +178,31 @@ class Generator(object): self.events.begin_site() logger.info("Generating site to [%s]" % self.site.config.deploy_root_path) - self.__generate_node__(self.site.content) + self.__generate_node__(self.site.content, incremental) self.events.site_complete() self.finalize() self.generated_once = True - def generate_node_at_path(self, node_path=None): + def generate_node_at_path(self, node_path=None, incremental=False): """ Generates a single node. If node_path is non-existent or empty, generates the entire site. """ - if not self.generated_once: + if not self.generated_once and not incremental: return self.generate_all() self.load_template_if_needed() self.load_site_if_needed() node = None if node_path: node = self.site.content.node_from_path(node_path) - self.generate_node(node) + self.generate_node(node, incremental) - def generate_node(self, node=None): + def generate_node(self, node=None, incremental=False): """ Generates the given node. If node is invalid, empty or non-existent, generates the entire website. """ - if not node or not self.generated_once: + if not node or not self.generated_once and not incremental: return self.generate_all() self.load_template_if_needed() @@ -206,17 +210,19 @@ class Generator(object): self.load_site_if_needed() try: - self.__generate_node__(node) + self.__generate_node__(node, incremental) self.finalize() except HydeException: self.generate_all() - def generate_resource_at_path(self, resource_path=None): + def generate_resource_at_path(self, + resource_path=None, + incremental=False): """ Generates a single resource. If resource_path is non-existent or empty, generats the entire website. """ - if not self.generated_once: + if not self.generated_once and not incremental: return self.generate_all() self.load_template_if_needed() @@ -224,14 +230,14 @@ class Generator(object): resource = None if resource_path: resource = self.site.content.resource_from_path(resource_path) - self.generate_resource(resource) + self.generate_resource(resource, incremental) - def generate_resource(self, resource=None): + def generate_resource(self, resource=None, incremental=False): """ Generates the given resource. If resource is invalid, empty or non-existent, generates the entire website. """ - if not resource or not self.generated_once: + if not resource or not self.generated_once and not incremental: return self.generate_all() self.load_template_if_needed() @@ -239,24 +245,27 @@ class Generator(object): self.load_site_if_needed() try: - self.__generate_resource__(resource) + self.__generate_resource__(resource, incremental) self.finalize() except HydeException: self.generate_all() - def __generate_node__(self, node): + def __generate_node__(self, node, incremental=False): for node in node.walk(): logger.debug("Generating Node [%s]", node) self.events.begin_node(node) for resource in node.resources: - self.__generate_resource__(resource) + self.__generate_resource__(resource, incremental) self.events.node_complete(node) - def __generate_resource__(self, resource): + def __generate_resource__(self, resource, incremental=False): if not resource.is_processable: logger.debug("Skipping [%s]", resource) return + if incremental and not self.has_resource_changed(resource): + logger.debug("No changes found. Skipping resource [%s]", resource) + return logger.debug("Processing [%s]", resource) with self.context_for_resource(resource) as context: if resource.source_file.is_text: diff --git a/hyde/model.py b/hyde/model.py index e1f652c..5e73212 100644 --- a/hyde/model.py +++ b/hyde/model.py @@ -3,9 +3,11 @@ Contains data structures and utilities for hyde. """ from hyde.fs import File, Folder + import codecs import yaml +from UserDict import IterableUserDict from hyde.util import getLoggerWithNullHandler logger = getLoggerWithNullHandler('hyde.engine') @@ -90,6 +92,25 @@ class Context(object): pass return context +class Dependents(IterableUserDict): + """ + Represents the dependency graph for hyde. + """ + + def __init__(self, sitepath, depends_file_name='.hyde_deps'): + self.sitepath = Folder(sitepath) + self.deps_file = File(self.sitepath.child(depends_file_name)) + self.data = {} + if self.deps_file.exists: + self.data = yaml.load(self.deps_file.read_all()) + import atexit + atexit.register(self.save) + + def save(self): + """ + Saves the dependency graph (just a dict for now). + """ + self.deps_file.write(yaml.dump(self.data)) class Config(Expando): """ diff --git a/hyde/server.py b/hyde/server.py index 14c874f..493b2e6 100644 --- a/hyde/server.py +++ b/hyde/server.py @@ -153,13 +153,10 @@ class HydeWebServer(HTTPServer): """ Regenerates the given resource. """ - target = self.site.config.deploy_root_path.child( - resource.relative_deploy_path) - if self.generator.has_resource_changed(resource): - try: - logger.info('Generating resource [%s]' % resource) - self.generator.generate_resource(resource) - except Exception, exception: - logger.error( - 'Error [%s] occured when generating the resource [%s]' - % (repr(exception), resource)) + try: + logger.info('Generating resource [%s]' % resource) + self.generator.generate_resource(resource, incremental=True) + except Exception, exception: + logger.error( + 'Error [%s] occured when generating the resource [%s]' + % (repr(exception), resource)) diff --git a/hyde/tests/ext/stylus/expected-site-compressed.css b/hyde/tests/ext/stylus/expected-site-compressed.css new file mode 100644 index 0000000..3d9d58f --- /dev/null +++ b/hyde/tests/ext/stylus/expected-site-compressed.css @@ -0,0 +1,4 @@ +*{border:0;padding:0;margin:0} +#header{color:#333;border-left:1px;border-right:2px} +#footer{color:#444} +#content{-webkit-border-radius:10px;-moz-border-radius:10px;border-radius:10px} \ No newline at end of file