From 3f59caf9e9c6e1a987e6b790c840ecb3f3dd4b6e Mon Sep 17 00:00:00 2001 From: John-Mark Gurney Date: Mon, 3 Feb 2020 15:22:18 -0800 Subject: [PATCH] add support for aliases, and loading package specific aliases... --- README.md | 8 +- casimport/__init__.py | 162 +++++++++++++++++++++++++++++++---- fixtures/randpkg/__init__.py | 4 + 3 files changed, 156 insertions(+), 18 deletions(-) create mode 100644 fixtures/randpkg/__init__.py diff --git a/README.md b/README.md index caded82..d0141cc 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,10 @@ print(repr(hello('Alice'))) ``` Defintion of hash: -v__ +v__ Currently v1 is defined, and has the following types: - f The hash is the value of the Python source file. - Generated via: shasum -a 256 hello.py + f The arg is the hash of the Python source file. + Example generated via: shasum -a 256 hello.py + a An alias. Any aliases must be loaded before being + processed. This is a short name that could be used. diff --git a/casimport/__init__.py b/casimport/__init__.py index a515726..5f413d4 100644 --- a/casimport/__init__.py +++ b/casimport/__init__.py @@ -25,12 +25,13 @@ import contextlib import glob import hashlib -import importlib +import importlib.resources import os.path import pathlib import shutil import sys import tempfile +import urllib from importlib.abc import MetaPathFinder, Loader from importlib.machinery import ModuleSpec @@ -56,6 +57,36 @@ def tempset(obj, key, value): finally: obj[key] = oldvalue +@contextlib.contextmanager +def tempattrset(obj, key, value): + '''A context (with) manager for changing the value of an attribute + of an object, and restoring it after the with block. + + If the attribute does not exist, it will be deleted afterward. + + Example usage: + ``` + with tempattrset(someobj, 'a', 15): + print(repr(someobj.a) + print(repr(someobj.a) + ``` + ''' + + try: + dodelattr = False + if hasattr(obj, key): + oldvalue = getattr(obj, key) + else: + dodelattr = True + + setattr(obj, key, value) + yield + finally: + if not dodelattr: + setattr(obj, key, oldvalue) + else: + delattr(obj, key) + class FileDirCAS(object): '''A file loader for CAS that operates on a directory. It looks at files, caches their hash, and loads them upon request.''' @@ -89,21 +120,25 @@ class FileDirCAS(object): return False - def exec_module(self, hash, module): - '''Give the hash and module, load the code associated - with the hash, and exec it in the module's context.''' + def fetch_data(self, url): + '''Given the URL (must be a hash URL), return the code for it.''' self.refresh_dir() - parts = hash.split('_', 2) - fname = self._hashes[parts[2]] + hashurl = url + + if hashurl.scheme != 'hash' or hashurl.netloc != 'sha256': + raise ValueError('invalid hash url') + + hash = hashurl.path[1:] + fname = self._hashes[hash] data, fhash = self.read_hash_file(fname) - if fhash != parts[2]: + if fhash != hash: raise ValueError('file no longer matches hash on disk') - exec(data, module.__dict__) + return data class CASFinder(MetaPathFinder, Loader): '''Overall class for using Content Addressable Storage to load @@ -112,6 +147,7 @@ class CASFinder(MetaPathFinder, Loader): def __init__(self): self._loaders = [] + self._aliases = {} if [ x for x in sys.meta_path if isinstance(x, self.__class__) ]: raise RuntimeError('cannot register more than on CASFinder') @@ -124,6 +160,12 @@ class CASFinder(MetaPathFinder, Loader): def __exit__(self, exc_type, exc_value, traceback): self.disconnect() + def load_aliases(self, name): + '''Load the aliases from the module with the passed in name.''' + + aliases = importlib.resources.read_text(sys.modules[name], 'cas_aliases.txt') + self._aliases.update(self._parsealiases(aliases)) + @staticmethod def _parsealiases(data): ret = {} @@ -167,13 +209,26 @@ class CASFinder(MetaPathFinder, Loader): ms = ModuleSpec(fullname, self, is_package=True) else: parts = fullname.split('.') - for l in self._loaders: - ispkg = l.is_package(parts[1]) - break + ver, typ, arg = parts[1].split('_') + if typ == 'f': + # make hash url: + hashurl = 'hash://sha256/%s' % bytes.fromhex(arg).hex() + hashurl = urllib.parse.urlparse(hashurl) + for l in self._loaders: + ispkg = l.is_package(hashurl) + break + else: + return None else: - return None + # an alias + for i in self._aliases[arg]: + hashurl = urllib.parse.urlparse(i) + if hashurl.scheme == 'hash': + break + else: + raise ValueError('unable to find bash hash url for alias %s' % repr(arg)) - ms = ModuleSpec(fullname, self, is_package=True, loader_state=(parts[1], l)) + ms = ModuleSpec(fullname, self, is_package=False, loader_state=(hashurl,)) return ms @@ -185,8 +240,17 @@ class CASFinder(MetaPathFinder, Loader): if module.__name__ == 'cas': pass else: - hash, load = module.__spec__.loader_state - load.exec_module(hash, module) + (url,) = module.__spec__.loader_state + for load in self._loaders: + try: + data = load.fetch_data(url) + break + except: + pass + else: + raise ValueError('unable to find loader for url %s' % repr(urllib.parse.urlunparse(url))) + + exec(data, module.__dict__) def defaultinit(casf): cachedir = pathlib.Path.home() / '.casimport_cache' @@ -196,10 +260,50 @@ def defaultinit(casf): # The global version _casfinder = CASFinder() +load_aliases = _casfinder.load_aliases defaultinit(_casfinder) import unittest +class TestHelpers(unittest.TestCase): + def test_testset(self): + origobj = object() + d = dict(a=origobj, b=10) + + # that when we temporarily set it + with tempset(d, 'a', 15): + # the new value is there + self.assertEqual(d['a'], 15) + + # and that the original object is restored + self.assertIs(d['a'], origobj) + + def test_testattrset(self): + class TestObj(object): + pass + + testobj = TestObj() + + # that when we temporarily set it + with tempattrset(testobj, 'a', 15): + # the new value is there + self.assertEqual(testobj.a, 15) + + # and that there is no object + self.assertFalse(hasattr(testobj, 'a')) + + origobj = object() + newobj = object() + testobj.b = origobj + + # that when we temporarily set it + with tempattrset(testobj, 'b', newobj): + # the new value is there + self.assertIs(testobj.b, newobj) + + # and the original value is restored + self.assertIs(testobj.b, origobj) + class Test(unittest.TestCase): def setUp(self): # clear out the default casfinder if there is one @@ -301,6 +405,34 @@ class Test(unittest.TestCase): ] }) + def test_aliasimports(self): + # setup the cache + temphome = self.tempdir / 'home' + temphome.mkdir() + cachedir = temphome / '.casimport_cache' + + # add the test module's path + fixdir = str(self.fixtures) + sys.path.append(fixdir) + + with tempset(os.environ, 'HOME', str(temphome)): + try: + with CASFinder() as f, \ + tempattrset(sys.modules[__name__], 'load_aliases', + f.load_aliases): + defaultinit(f) + + # and that hello.py is in the cache + shutil.copy(self.fixtures / 'hello.py', cachedir) + + # that the import is successful + import randpkg + + # and pulled in the method + self.assertTrue(hasattr(randpkg, 'hello')) + finally: + sys.path.remove(fixdir) + def test_loaderpriority(self): # XXX - write test to allow you to specify the priority of # a loader, to ensure that cache stays at top. diff --git a/fixtures/randpkg/__init__.py b/fixtures/randpkg/__init__.py new file mode 100644 index 0000000..66d6e74 --- /dev/null +++ b/fixtures/randpkg/__init__.py @@ -0,0 +1,4 @@ +import casimport +casimport.load_aliases(__name__) + +from cas.v1_a_hello import hello