From 356be8332b0c2a1849b332eec4a0e93607ce54e9 Mon Sep 17 00:00:00 2001 From: John-Mark Gurney Date: Sat, 3 Sep 2022 11:37:22 -0700 Subject: [PATCH] support mappings, that is one host has the same files as another... this fleshes out hostuuid better.. add a more useful debug print.. --- ui/fixtures/cmd.hosts.json | 21 ----- ui/fixtures/cmd.mapping.json | 100 ++++++++++++++++++++++++ ui/medashare/cli.py | 147 ++++++++++++++++++++++++++++++----- 3 files changed, 229 insertions(+), 39 deletions(-) delete mode 100644 ui/fixtures/cmd.hosts.json create mode 100644 ui/fixtures/cmd.mapping.json diff --git a/ui/fixtures/cmd.hosts.json b/ui/fixtures/cmd.hosts.json deleted file mode 100644 index a950d88..0000000 --- a/ui/fixtures/cmd.hosts.json +++ /dev/null @@ -1,21 +0,0 @@ -[ -{ - "title": "gen ident", - "cmd": [ "genident", "name=A Test User" ] -}, -{ - "special": "set hostid", - "comment": "and that a modified hostid", - "hostid": "ceaa4862-dd00-41ba-9787-7480ec1b2679" -}, -{ - "title": "that hosts lists itself", - "cmd": [ "hosts" ], - "stdout_re": ".*\tceaa4862-dd00-41ba-9787-7480ec1b2679\n" -}, -{ - "special": "verify store object cnt", - "comment": "and that the host object was created", - "count": 1 -} -] diff --git a/ui/fixtures/cmd.mapping.json b/ui/fixtures/cmd.mapping.json new file mode 100644 index 0000000..3379275 --- /dev/null +++ b/ui/fixtures/cmd.mapping.json @@ -0,0 +1,100 @@ +[ +{ + "title": "gen ident", + "cmd": [ "genident", "name=A Test User" ] +}, +{ + "special": "set hostid", + "comment": "and that a modified hostid", + "hostid": "ceaa4862-dd00-41ba-9787-7480ec1b2679" +}, +{ + "title": "that hosts lists itself", + "cmd": [ "hosts" ], + "stdout_re": ".*\tceaa4862-dd00-41ba-9787-7480ec1b2679\n" +}, +{ + "special": "verify store object cnt", + "comment": "and that the host object was created", + "count": 1 +}, +{ + "special": "set hostid", + "comment": "and that a different hostid", + "hostid": "efdb5d9c-d123-4b30-aaa8-45a9ea8f6053" +}, +{ + "title": "that hosts lists itself first", + "cmd": [ "hosts" ], + "stdout_re": ".*\tefdb5d9c-d123-4b30-aaa8-45a9ea8f6053\n.*\tceaa4862-dd00-41ba-9787-7480ec1b2679\n" +}, +{ + "special": "verify store object cnt", + "comment": "and that the host object was created", + "count": 2 +}, +{ + "title": "that a mapping with unknown host errors", + "cmd": [ "mapping", "--create", "ceaa4862-dd00-41ba-9787-7480ec1b267a:/foo", "efdb5d9c-d123-4b30-aaa8-45a9ea8f6053:/bar" ], + "exit": 1, + "stderr": "ERROR: Unable to find host 'ceaa4862-dd00-41ba-9787-7480ec1b267a'\n" +}, +{ + "title": "that a host mapping that isn't absolute errors", + "cmd": [ "mapping", "--create", ".", "efdb5d9c-d123-4b30-aaa8-45a9ea8f6053:." ], + "stderr": "ERROR: host path must be absolute, is '.'.\n", + "exit": 1 +}, +{ + "special": "setup mapping paths" +}, +{ + "title": "that a host mapping works", + "cmd": [ "mapping", "--create", "mapa", "ceaa4862-dd00-41ba-9787-7480ec1b2679:{mappathb}" ], + "format": [ "cmd" ] +}, +{ + "special": "verify store object cnt", + "comment": "and that the host object was created", + "count": 3 +}, +{ + "title": "that it was created w/ correct values", + "cmd": [ "dump" ], + "format": [ "stdout_re" ], + "stdout_re": ".*\n.*\n.*mapping.*efdb5d9c-d123-4b30-aaa8-45a9ea8f6053:{mappatha}.*ceaa4862-dd00-41ba-9787-7480ec1b2679:{mappathb}.*type.*mapping.*\n" +}, +{ + "title": "that a file on one mapping can have metadata", + "cmd": [ "modify", "+sometag=value", "mapa/text.txt" ] +}, +{ + "special": "verify store object cnt", + "comment": "and that the file and metadata was created", + "count": 5 +}, +{ + "special": "set hostid", + "comment": "that the other host", + "hostid": "ceaa4862-dd00-41ba-9787-7480ec1b2679" +}, +{ + "special": "delete files", + "format": [ "files" ], + "files": [ "{mappatha}/text.txt" ] +}, +{ + "title": "can see the other host's metadata", + "cmd": [ "list", "mapb/text.txt" ], + "stdout_re": ".*\n.*\nsometag:\tvalue\n" +}, +{ + "title": "and add it's own value", + "cmd": [ "modify", "+sometag=anothervalue", "mapb/text.txt" ] +}, +{ + "special": "verify store object cnt", + "comment": "but didn't create an additional FileObject", + "count": 5 +} +] diff --git a/ui/medashare/cli.py b/ui/medashare/cli.py index 936ffe2..cce22b5 100644 --- a/ui/medashare/cli.py +++ b/ui/medashare/cli.py @@ -44,6 +44,18 @@ _NAMESPACE_MEDASHARE_CONTAINER = uuid.UUID('890a9d5c-0626-4de1-ab05-9e14947391eb # useful for debugging when stderr is redirected/captured _real_stderr = sys.stderr +def _debprint(*args): # pragma: no cover + import traceback, sys, os.path + st = traceback.extract_stack(limit=2)[0] + + sep = '' + if args: + sep = ':' + + print('%s:%d%s' % (os.path.basename(st.filename), st.lineno, sep), + *args, file=_real_stderr) + sys.stderr.flush() + _defaulthash = 'sha512' _validhashes = set([ 'sha256', 'sha512' ]) _hashlengths = { len(getattr(hashlib, x)().hexdigest()): x for x in @@ -333,6 +345,11 @@ class Persona(object): return self.sign(Host(*args, **kwargs)) + def Mapping(self, *args, **kwargs): + kwargs['created_by_ref'] = self.uuid + + return self.sign(Mapping(*args, **kwargs)) + def Container(self, *args, **kwargs): kwargs['created_by_ref'] = self.uuid @@ -478,6 +495,7 @@ class ObjectStore(object): self._uuids = {} self._hashes = {} self._hostuuids = {} + self._hostmappings = [] def get_host(self, hostuuid): return self._hostuuids[hostuuid] @@ -569,7 +587,15 @@ class ObjectStore(object): elif obj.type == 'host': self._uuids[obj.hostuuid] = obj self._hostuuids[obj.hostuuid] = obj - + elif obj.type == 'mapping': + hostid = _makeuuid(hostuuid()) + + maps = [ (lambda a, b: (uuid.UUID(a), pathlib.Path(b).resolve()))(*x.split(':', 1)) for x in obj.mapping ] + for idx, (id, path) in enumerate(maps): + if hostid == id: + # add other to mapping + other = tuple(maps[(idx + 1) % 2]) + self._hostmappings.append((path, ) + other) try: hashes = obj.hashes except AttributeError: @@ -623,7 +649,7 @@ class ObjectStore(object): h = self.makehash(hash, strict=False) return self._hashes[h] - def get_metadata(self, fname, persona): + def get_metadata(self, fname, persona, create_metadata=True): '''Get all MetaData objects for fname, or create one if not found. @@ -632,6 +658,9 @@ class ObjectStore(object): A Persona must be passed in to create the FileObject and MetaData objects as needed. + A MetaData object will be created if create_metadata is + True, which is the default. + Note: if a new MetaData object is created, it is not stored in the database automatically. It is expected that it will be modified and then saved, so call ObjectStore.loadobj @@ -640,12 +669,8 @@ class ObjectStore(object): try: fobj = self.by_file(fname, ('file',))[0] - #print('x:', repr(objs), file=_real_stderr) except KeyError: - #print('b:', repr(fname), file=_real_stderr) - fobj = persona.by_file(fname) - #print('c:', repr(fobj), file=_real_stderr) self.loadobj(fobj) @@ -653,13 +678,19 @@ class ObjectStore(object): try: objs = self.by_file(fname) except KeyError: - objs = [ persona.MetaData(hashes=fobj.hashes) ] + if create_metadata: + objs = [ persona.MetaData(hashes=fobj.hashes) ] + else: + objs = [ ] return objs def by_file(self, fname, types=('metadata', )): '''Return a metadata object for the file named fname. + Will check the mapping database to get hashes, and possibly + return that FileObject if requested. + Will raise a KeyError if this file does not exist in the database. @@ -670,8 +701,26 @@ class ObjectStore(object): fid = FileObject.make_id(fname) #print('bf:', repr(fid), file=_real_stderr) - fobj = self.by_id(fid) - fobj.verify() + try: + fobj = self.by_id(fid) + lclfile = None + except KeyError: + # check mappings + fname = pathlib.Path(fname).resolve() + for lclpath, hostid, rempath in self._hostmappings: + if fname.parts[:len(lclpath.parts)] == lclpath.parts: + try: + rempath = pathlib.Path(*rempath.parts + fname.parts[len(lclpath.parts):]) + fid = FileObject.make_id(rempath, hostid) + fobj = self.by_id(fid) + lclfile = fname + break + except KeyError: + continue + else: + raise + + fobj.verify(lclfile) for i in fobj.hashes: j = self.by_hash(i) @@ -703,6 +752,9 @@ def _hashfile(fname): class Host(MDBase): _type = 'host' +class Mapping(MDBase): + _type = 'mapping' + class FileObject(MDBase): _type = 'file' @@ -713,14 +765,17 @@ class FileObject(MDBase): } @staticmethod - def make_id(fname): + def make_id(fname, hostid=None): '''Take a local file name, and make the id for it. Note that converts from the local path separator to a forward slash so that it will be the same between Windows and Unix systems.''' + if hostid is None: + hostid = hostuuid() + fname = os.path.realpath(fname) return uuid.uuid5(_NAMESPACE_MEDASHARE_PATH, - str(hostuuid()) + '/'.join(os.path.split(fname))) + str(hostid) + '/'.join(os.path.split(fname))) @classmethod def from_file(cls, filename, created_by_ref): @@ -741,13 +796,17 @@ class FileObject(MDBase): return cls(obj) - def verify(self, complete=False): + def verify(self, lclfile=None): '''Verify that this FileObject is still valid. It will by default, only do a mtime verification. It will raise a ValueError if the file does not match.''' - s = os.stat(os.path.join(self.dir, self.filename)) + if lclfile is None: + s = os.stat(os.path.join(self.dir, self.filename)) + else: + s = os.stat(lclfile) + mtimets = datetime.datetime.fromtimestamp(s.st_mtime, tz=datetime.timezone.utc).timestamp() @@ -919,6 +978,36 @@ def cmd_modify(options): def printhost(host): print('%s\t%s' % (host.name, host.hostuuid)) +def cmd_mapping(options): + persona, objstr = get_objstore(options) + + if options.mapping is not None: + parts = [ x.split(':', 1) for x in options.mapping ] + + if len(parts[0]) == 1: + parts[0] = [ hostuuid(), parts[0][0] ] + + if parts[0][0] == hostuuid(): + parts[0][1] = str(pathlib.Path(parts[0][1]).resolve()) + + if parts[1][1][0] != '/': + print('ERROR: host path must be absolute, is %s.' % + repr(parts[1][1][0]), file=sys.stderr) + sys.exit(1) + + try: + [ objstr.get_host(x[0]) for x in parts ] + except KeyError as e: + print('ERROR: Unable to find host %s' % + repr(e.args[0]), file=sys.stderr) + sys.exit(1) + + m = persona.Mapping(mapping=[ ':'.join(x) for x in parts ]) + + objstr.loadobj(m) + + write_objstore(options, objstr) + def cmd_hosts(options): persona, objstr = get_objstore(options) @@ -987,11 +1076,6 @@ def cmd_list(options): except (FileNotFoundError, KeyError) as e: print('ERROR: file not found: %s' % repr(i), file=sys.stderr) sys.exit(1) - except FileNotFoundError: - # XXX - tell the difference? - print('ERROR: file not found: %s' % repr(i), - file=sys.stderr) - sys.exit(1) for j in objstr.by_file(i): for k, v in _iterdictlist(j): @@ -1149,6 +1233,11 @@ def main(): parser_hosts = subparsers.add_parser('hosts', help='dump all the hosts, self is always first') parser_hosts.set_defaults(func=cmd_hosts) + parser_mapping = subparsers.add_parser('mapping', help='list mappings, or create a mapping') + parser_mapping.add_argument('--create', dest='mapping', nargs=2, + help='mapping to add, host|hostuuid:path host|hostuuid:path') + parser_mapping.set_defaults(func=cmd_mapping) + parser_dump = subparsers.add_parser('dump', help='dump all the objects') parser_dump.set_defaults(func=cmd_dump) @@ -1633,6 +1722,13 @@ class _TestCases(unittest.TestCase): except KeyError: pass + for i in cmd.get('format', []): + if i in { 'cmd', 'files' }: + vars = locals() + cmd[i] = [ x.format(**vars) for x in cmd[i] ] + else: + cmd[i] = cmd[i].format(**locals()) + try: special = cmd['special'] except KeyError: @@ -1673,6 +1769,21 @@ class _TestCases(unittest.TestCase): sd.mkdir(exist_ok=True) bttestcase.make_files(sd, btfiles) + elif special == 'setup mapping paths': + mappatha = self.tempdir / 'mapa' + mappatha.mkdir() + + mappathb = self.tempdir / 'mapb' + mappathb.mkdir() + + filea = mappatha / 'text.txt' + filea.write_text('abc123\n') + fileb = mappathb / 'text.txt' + shutil.copyfile(filea, fileb) + shutil.copystat(filea, fileb) + elif special == 'delete files': + for i in cmd['files']: + os.unlink(i) else: # pragma: no cover raise ValueError('unhandled special: %s' % repr(special))