Merge pull request #87 from gfuchedzhy/hyde

---

if path of the resource processed by jinja2 contains non-ascii characters or if some template references some resource which contains non-ascii characters in its path hyde fails to generate site throwing jinjaUnicodeDecodeError.

So I replaced all regular strings I found in hyde sources with unicode ones. Now it generates site in described conditions correctly. All existing regression tests passed ok. So I think it is safe to include this changes into hyde.

Note that if you pull this request and some other requests introducing some new plugins youll have to look through their sources and replace str(...) occurences with unicode(...).

**update:**
Youll also have to remove .hyde_deps after applying this patch.

hyde/ext/plugins/jpegoptim.py

@@ -51,7 +51,7 @@ class JPEGOptimPlugin(CLTransformer):
         target = File(self.site.config.deploy_root_path.child(
                                 resource.relative_deploy_path))
         jpegoptim = self.app
-        args = [str(jpegoptim)]
+        args = [unicode(jpegoptim)]
         args.extend(self.process_args(supported))
-        args.extend(["-q", str(target)])
+        args.extend(["-q", unicode(target)])
         self.call_app(args)

hyde/ext/plugins/less.py

@@ -84,9 +84,9 @@ class LessCSSPlugin(CLTransformer):
         less = self.app
         source = File.make_temp(text)
         target = File.make_temp('')
-        args = [str(less)]
+        args = [unicode(less)]
         args.extend(self.process_args(supported))
-        args.extend([str(source), str(target)])
+        args.extend([unicode(source), unicode(target)])
         try:
             self.call_app(args)
         except subprocess.CalledProcessError:

hyde/ext/plugins/optipng.py

@@ -64,7 +64,7 @@ class OptiPNGPlugin(CLTransformer):
         target = File(self.site.config.deploy_root_path.child(
                                 resource.relative_deploy_path))
         optipng = self.app
-        args = [str(optipng)]
+        args = [unicode(optipng)]
         args.extend(self.process_args(supported))
-        args.extend([str(target)])
+        args.extend([unicode(target)])
         self.call_app(args)

hyde/ext/plugins/stylus.py

@@ -104,9 +104,9 @@ class StylusPlugin(CLTransformer):
         target = source
         supported = [("compress", "c"), ("include", "I")]
 
-        args = [str(stylus)]
+        args = [unicode(stylus)]
         args.extend(self.process_args(supported))
-        args.append(str(source))
+        args.append(unicode(source))
         try:
             self.call_app(args)
         except subprocess.CalledProcessError, e:

hyde/ext/plugins/tagger.py

@@ -55,7 +55,7 @@ def get_tagger_sort_method(site):
     return walker
 
 def walk_resources_tagged_with(node, tag):
-    tags = set(str(tag).split('+'))
+    tags = set(unicode(tag).split('+'))
     walker = get_tagger_sort_method(node.site)
     for resource in walker():
         try:

hyde/ext/plugins/uglify.py

@@ -62,9 +62,9 @@ class UglifyPlugin(CLTransformer):
         uglify = self.app
         source = File.make_temp(text)
         target = File.make_temp('')
-        args = [str(uglify)]
+        args = [unicode(uglify)]
         args.extend(self.process_args(supported))
-        args.extend(["-o", str(target), str(source)])
+        args.extend(["-o", unicode(target), unicode(source)])
 
         self.call_app(args)
         out = target.read_all()

hyde/ext/publishers/dvcs.py

@@ -59,7 +59,7 @@ class Git(DVCS):
 
     def add(self, path="."):
         cmd = Popen('git add "%s"' % path,
-                        cwd=str(self.path), stdout=PIPE, shell=True)
+                        cwd=unicode(self.path), stdout=PIPE, shell=True)
         cmdresult = cmd.communicate()[0]
         if cmd.returncode:
             raise Exception(cmdresult)
@@ -67,7 +67,7 @@ class Git(DVCS):
     def pull(self):
         self.switch(self.branch)
         cmd = Popen("git pull origin %s" % self.branch,
-                    cwd=str(self.path),
+                    cwd=unicode(self.path),
                     stdout=PIPE,
                     shell=True)
         cmdresult = cmd.communicate()[0]
@@ -76,7 +76,7 @@ class Git(DVCS):
 
     def push(self):
         cmd = Popen("git push origin %s" % self.branch,
-                    cwd=str(self.path), stdout=PIPE,
+                    cwd=unicode(self.path), stdout=PIPE,
                     shell=True)
         cmdresult = cmd.communicate()[0]
         if cmd.returncode:
@@ -85,7 +85,7 @@ class Git(DVCS):
 
     def commit(self, message):
         cmd = Popen('git commit -a -m"%s"' % message,
-                    cwd=str(self.path), stdout=PIPE, shell=True)
+                    cwd=unicode(self.path), stdout=PIPE, shell=True)
         cmdresult = cmd.communicate()[0]
         if cmd.returncode:
             raise Exception(cmdresult)
@@ -93,14 +93,14 @@ class Git(DVCS):
     def switch(self, branch):
         self.branch = branch
         cmd = Popen('git checkout %s' % branch,
-                    cwd=str(self.path), stdout=PIPE, shell=True)
+                    cwd=unicode(self.path), stdout=PIPE, shell=True)
         cmdresult = cmd.communicate()[0]
         if cmd.returncode:
             raise Exception(cmdresult)
 
     def merge(self, branch):
         cmd = Popen('git merge %s' % branch,
-                    cwd=str(self.path), stdout=PIPE, shell=True)
+                    cwd=unicode(self.path), stdout=PIPE, shell=True)
         cmdresult = cmd.communicate()[0]
         if cmd.returncode:
             raise Exception(cmdresult)

hyde/ext/templates/jinja.py

@@ -551,11 +551,11 @@ class HydeLoader(FileSystemLoader):
         config = site.config if hasattr(site, 'config') else None
         if config:
             super(HydeLoader, self).__init__([
-                            str(config.content_root_path),
-                            str(config.layout_root_path),
+                            unicode(config.content_root_path),
+                            unicode(config.layout_root_path),
                         ])
         else:
-            super(HydeLoader, self).__init__(str(sitepath))
+            super(HydeLoader, self).__init__(unicode(sitepath))
 
         self.site = site
         self.preprocessor = preprocessor

hyde/fs.py

@@ -36,7 +36,7 @@ class FS(object):
             self.path = path
         else:
             self.path = os.path.expandvars(os.path.expanduser(
-                        str(path).strip().rstrip(os.sep)))
+                        unicode(path).strip().rstrip(os.sep)))
 
     def __str__(self):
         return self.path
@@ -45,10 +45,10 @@ class FS(object):
         return self.path
 
     def __eq__(self, other):
-        return str(self) == str(other)
+        return unicode(self) == unicode(other)
 
     def __ne__(self, other):
-        return str(self) != str(other)
+        return unicode(self) != unicode(other)
 
     @property
     def fully_expanded_path(self):
@@ -144,7 +144,7 @@ class FS(object):
         """
         Returns a File or Folder object that would represent the given path.
         """
-        target = str(path)
+        target = unicode(path)
         return Folder(target) if os.path.isdir(target) else File(target)
 
     def __get_destination__(self, destination):
@@ -152,7 +152,7 @@ class FS(object):
         Returns a File or Folder object that would represent this entity
         if it were copied or moved to `destination`.
         """
-        if isinstance(destination, File) or os.path.isfile(str(destination)):
+        if isinstance(destination, File) or os.path.isfile(unicode(destination)):
             return destination
         else:
             return FS.file_or_folder(Folder(destination).child(self.name))
@@ -251,7 +251,7 @@ class File(FS):
         determine age.
 
         """
-        return self.last_modified < File(str(another_file)).last_modified
+        return self.last_modified < File(unicode(another_file)).last_modified
 
     @staticmethod
     def make_temp(text):
@@ -290,7 +290,7 @@ class File(FS):
         """
         target = self.__get_destination__(destination)
         logger.info("Copying %s to %s" % (self, target))
-        shutil.copy(self.path, str(destination))
+        shutil.copy(self.path, unicode(destination))
         return target
 
     def delete(self):
@@ -539,7 +539,7 @@ class Folder(FS):
         """
         target = self.__get_destination__(destination)
         logger.info("Copying %s to %s" % (self, target))
-        shutil.copytree(self.path, str(target))
+        shutil.copytree(self.path, unicode(target))
         return target
 
     def move_to(self, destination):
@@ -549,7 +549,7 @@ class Folder(FS):
         """
         target = self.__get_destination__(destination)
         logger.info("Move %s to %s" % (self, target))
-        shutil.move(self.path, str(target))
+        shutil.move(self.path, unicode(target))
         return target
 
     def rename_to(self, destination_name):
@@ -559,7 +559,7 @@ class Folder(FS):
         """
         target = self.parent.child_folder(destination_name)
         logger.info("Rename %s to %s" % (self, target))
-        shutil.move(self.path, str(target))
+        shutil.move(self.path, unicode(target))
         return target
 
     def _create_target_tree(self, target):
@@ -588,7 +588,7 @@ class Folder(FS):
         target = Folder(destination)
         target.make()
         self._create_target_tree(target)
-        dir_util.copy_tree(self.path, str(target))
+        dir_util.copy_tree(self.path, unicode(target))
         return target
 
     def get_walker(self, pattern=None):

hyde/layout.py

@@ -38,6 +38,6 @@ class Layout(object):
         Finds the layout folder from the given root folder.
         If it does not exist, return None
         """
-        layouts_folder = Folder(str(root)).child_folder(LAYOUTS)
+        layouts_folder = Folder(unicode(root)).child_folder(LAYOUTS)
         layout_folder = layouts_folder.child_folder(layout_name)
         return layout_folder if layout_folder.exists else None

hyde/model.py

@@ -45,7 +45,7 @@ class Expando(object):
         Sets the expando attribute after
         transforming the value.
         """
-        setattr(self, str(key).encode('utf-8'), self.transform(value))
+        setattr(self, unicode(key).encode('utf-8'), self.transform(value))
 
 
     def transform(self, primitive):
@@ -62,7 +62,7 @@ class Expando(object):
             return primitive
 
     def __repr__(self):
-        return str(self.to_dict())
+        return unicode(self.to_dict())
 
     def to_dict(self):
         """

hyde/plugin.py

@@ -311,7 +311,7 @@ class CLTransformer(Plugin):
         try:
             self.logger.debug(
                 "Calling executable [%s] with arguments %s" %
-                    (args[0], str(args[1:])))
+                    (args[0], unicode(args[1:])))
             subprocess.check_call(args)
         except subprocess.CalledProcessError, error:
             self.logger.error(traceback.format_exc())

hyde/site.py

@@ -17,7 +17,7 @@ from hyde.util import getLoggerWithNullHandler
 def path_normalized(f):
     @wraps(f)
     def wrapper(self, path):
-        return f(self, str(path).replace('/', os.sep))
+        return f(self, unicode(path).replace('/', os.sep))
     return wrapper
 
 logger = getLoggerWithNullHandler('hyde.engine')
@@ -120,7 +120,7 @@ class Node(Processable):
         self.root = self
         self.module = None
         self.site = None
-        self.source_folder = Folder(str(source_folder))
+        self.source_folder = Folder(unicode(source_folder))
         self.parent = parent
         if parent:
             self.root = self.parent.root
@@ -228,7 +228,7 @@ class RootNode(Node):
         """
         if Folder(path) == self.source_folder:
             return self
-        return self.node_map.get(str(Folder(path)), None)
+        return self.node_map.get(unicode(Folder(path)), None)
 
     @path_normalized
     def node_from_relative_path(self, relative_path):
@@ -237,7 +237,7 @@ class RootNode(Node):
         If no match is found it returns None.
         """
         return self.node_from_path(
-                    self.source_folder.child(str(relative_path)))
+                    self.source_folder.child(unicode(relative_path)))
 
     @path_normalized
     def resource_from_path(self, path):
@@ -245,7 +245,7 @@ class RootNode(Node):
         Gets the resource that maps to the given path.
         If no match is found it returns None.
         """
-        return self.resource_map.get(str(File(path)), None)
+        return self.resource_map.get(unicode(File(path)), None)
 
     @path_normalized
     def resource_from_relative_path(self, relative_path):
@@ -254,14 +254,14 @@ class RootNode(Node):
         If no match is found it returns None.
         """
         return self.resource_from_path(
-                    self.source_folder.child(str(relative_path)))
+                    self.source_folder.child(relative_path))
 
     def resource_deploy_path_changed(self, resource):
         """
         Handles the case where the relative deploy path of a
         resource has changed.
         """
-        self.resource_deploy_map[str(resource.relative_deploy_path)] = resource
+        self.resource_deploy_map[unicode(resource.relative_deploy_path)] = resource
 
     @path_normalized
     def resource_from_relative_deploy_path(self, relative_deploy_path):
@@ -302,7 +302,7 @@ class RootNode(Node):
         node = parent if parent else self
         for h_folder in hierarchy:
             node = node.add_child_node(h_folder)
-            self.node_map[str(h_folder)] = node
+            self.node_map[unicode(h_folder)] = node
             logger.debug("Added node [%s] to [%s]" % (
                             node.relative_path, self.source_folder))
 
@@ -332,7 +332,7 @@ class RootNode(Node):
             node = self.add_node(afile.parent)
 
         resource = node.add_child_resource(afile)
-        self.resource_map[str(afile)] = resource
+        self.resource_map[unicode(afile)] = resource
         logger.debug("Added resource [%s] to [%s]" %
                     (resource.relative_path, self.source_folder))
         return resource

hyde/tests/test_fs.py

@@ -15,7 +15,7 @@ from nose.tools import raises, with_setup, nottest
 def test_representation():
     f = FS(__file__)
     assert f.path == __file__
-    assert str(f) == __file__
+    assert unicode(f) == __file__
     assert repr(f) == __file__
 
 def test_name():
@@ -81,7 +81,7 @@ def test_parent():
     f = File(__file__)
     p = f.parent
     assert hasattr(p, 'child_folder')
-    assert str(p) == os.path.dirname(__file__)
+    assert unicode(p) == os.path.dirname(__file__)
 
 def test_child():
     p = File(__file__).parent
@@ -92,7 +92,7 @@ def test_child_folder():
     p = File(__file__).parent
     c = p.child_folder('data')
     assert hasattr(c, 'child_folder')
-    assert str(c) == os.path.join(os.path.dirname(__file__), 'data')
+    assert unicode(c) == os.path.join(os.path.dirname(__file__), 'data')
 
 def test_exists():
     p = FS(__file__)
@@ -113,7 +113,7 @@ def test_create_folder():
     assert not c.exists
     c.make()
     assert c.exists
-    shutil.rmtree(str(c))
+    shutil.rmtree(unicode(c))
     assert not c.exists
 
 def test_remove_folder():

hyde/tests/test_initialize.py

@@ -36,56 +36,56 @@ def delete_test_site_at_user():
 def test_ensure_exception_when_site_yaml_exists():
     e = Engine(raise_exceptions=True)
     File(TEST_SITE.child('site.yaml')).write("Hey")
-    e.run(e.parse(['-s', str(TEST_SITE), 'create']))
+    e.run(e.parse(['-s', unicode(TEST_SITE), 'create']))
 
 @raises(HydeException)
 @with_setup(create_test_site, delete_test_site)
 def test_ensure_exception_when_content_folder_exists():
     e = Engine(raise_exceptions=True)
     TEST_SITE.child_folder('content').make()
-    e.run(e.parse(['-s', str(TEST_SITE), 'create']))
+    e.run(e.parse(['-s', unicode(TEST_SITE), 'create']))
 
 @raises(HydeException)
 @with_setup(create_test_site, delete_test_site)
 def test_ensure_exception_when_layout_folder_exists():
     e = Engine(raise_exceptions=True)
     TEST_SITE.child_folder('layout').make()
-    e.run(e.parse(['-s', str(TEST_SITE), 'create']))
+    e.run(e.parse(['-s', unicode(TEST_SITE), 'create']))
 
 @with_setup(create_test_site, delete_test_site)
 def test_ensure_no_exception_when_empty_site_exists():
     e = Engine(raise_exceptions=True)
-    e.run(e.parse(['-s', str(TEST_SITE), 'create']))
+    e.run(e.parse(['-s', unicode(TEST_SITE), 'create']))
     verify_site_contents(TEST_SITE, Layout.find_layout())
 
 @with_setup(create_test_site, delete_test_site)
 def test_ensure_no_exception_when_forced():
     e = Engine(raise_exceptions=True)
     TEST_SITE.child_folder('layout').make()
-    e.run(e.parse(['-s', str(TEST_SITE), 'create', '-f']))
+    e.run(e.parse(['-s', unicode(TEST_SITE), 'create', '-f']))
     verify_site_contents(TEST_SITE, Layout.find_layout())
     TEST_SITE.delete()
     TEST_SITE.child_folder('content').make()
-    e.run(e.parse(['-s', str(TEST_SITE), 'create', '-f']))
+    e.run(e.parse(['-s', unicode(TEST_SITE), 'create', '-f']))
     verify_site_contents(TEST_SITE, Layout.find_layout())
     TEST_SITE.delete()
     TEST_SITE.make()
     File(TEST_SITE.child('site.yaml')).write("Hey")
-    e.run(e.parse(['-s', str(TEST_SITE), 'create', '-f']))
+    e.run(e.parse(['-s', unicode(TEST_SITE), 'create', '-f']))
     verify_site_contents(TEST_SITE, Layout.find_layout())
 
 @with_setup(create_test_site, delete_test_site)
 def test_ensure_no_exception_when_sitepath_does_not_exist():
     e = Engine(raise_exceptions=True)
     TEST_SITE.delete()
-    e.run(e.parse(['-s', str(TEST_SITE), 'create', '-f']))
+    e.run(e.parse(['-s', unicode(TEST_SITE), 'create', '-f']))
     verify_site_contents(TEST_SITE, Layout.find_layout())
 
 @with_setup(create_test_site_at_user, delete_test_site_at_user)
 def test_ensure_can_create_site_at_user():
     e = Engine(raise_exceptions=True)
     TEST_SITE_AT_USER.delete()
-    e.run(e.parse(['-s', str(TEST_SITE_AT_USER), 'create', '-f']))
+    e.run(e.parse(['-s', unicode(TEST_SITE_AT_USER), 'create', '-f']))
     verify_site_contents(TEST_SITE_AT_USER, Layout.find_layout())
 
 @nottest
@@ -107,5 +107,5 @@ def verify_site_contents(site, layout):
 @with_setup(create_test_site, delete_test_site)
 def test_ensure_exception_when_layout_is_invalid():
     e = Engine(raise_exceptions=True)
-    e.run(e.parse(['-s', str(TEST_SITE), 'create', '-l', 'junk']))
+    e.run(e.parse(['-s', unicode(TEST_SITE), 'create', '-l', 'junk']))
 

hyde/tests/test_layout.py

@@ -31,7 +31,7 @@ def test_find_layout_from_env_var():
     f = Layout.find_layout()
     LAYOUT_ROOT.make()
     f.copy_to(LAYOUT_ROOT)
-    os.environ[HYDE_DATA] = str(DATA_ROOT)
+    os.environ[HYDE_DATA] = unicode(DATA_ROOT)
     f = Layout.find_layout()
     assert f.parent == LAYOUT_ROOT
     assert f.name == 'basic'