support mappings, that is one host has the same files as another...

this fleshes out hostuuid better..
add a more useful debug print..

ui/fixtures/cmd.hosts.json

@@ -1,21 +0,0 @@
-[
-{
-	"title": "gen ident",
-	"cmd": [ "genident", "name=A Test User" ]
-},
-{
-	"special": "set hostid",
-	"comment": "and that a modified hostid",
-	"hostid": "ceaa4862-dd00-41ba-9787-7480ec1b2679"
-},
-{
-	"title": "that hosts lists itself",
-	"cmd": [ "hosts" ],
-	"stdout_re": ".*\tceaa4862-dd00-41ba-9787-7480ec1b2679\n"
-},
-{
-	"special": "verify store object cnt",
-	"comment": "and that the host object was created",
-	"count": 1
-}
-]

ui/fixtures/cmd.mapping.json

@@ -0,0 +1,100 @@
+[
+{
+	"title": "gen ident",
+	"cmd": [ "genident", "name=A Test User" ]
+},
+{
+	"special": "set hostid",
+	"comment": "and that a modified hostid",
+	"hostid": "ceaa4862-dd00-41ba-9787-7480ec1b2679"
+},
+{
+	"title": "that hosts lists itself",
+	"cmd": [ "hosts" ],
+	"stdout_re": ".*\tceaa4862-dd00-41ba-9787-7480ec1b2679\n"
+},
+{
+	"special": "verify store object cnt",
+	"comment": "and that the host object was created",
+	"count": 1
+},
+{
+	"special": "set hostid",
+	"comment": "and that a different hostid",
+	"hostid": "efdb5d9c-d123-4b30-aaa8-45a9ea8f6053"
+},
+{
+	"title": "that hosts lists itself first",
+	"cmd": [ "hosts" ],
+	"stdout_re": ".*\tefdb5d9c-d123-4b30-aaa8-45a9ea8f6053\n.*\tceaa4862-dd00-41ba-9787-7480ec1b2679\n"
+},
+{
+	"special": "verify store object cnt",
+	"comment": "and that the host object was created",
+	"count": 2
+},
+{
+	"title": "that a mapping with unknown host errors",
+	"cmd": [ "mapping", "--create", "ceaa4862-dd00-41ba-9787-7480ec1b267a:/foo", "efdb5d9c-d123-4b30-aaa8-45a9ea8f6053:/bar" ],
+	"exit": 1,
+	"stderr": "ERROR: Unable to find host 'ceaa4862-dd00-41ba-9787-7480ec1b267a'\n"
+},
+{
+	"title": "that a host mapping that isn't absolute errors",
+	"cmd": [ "mapping", "--create", ".", "efdb5d9c-d123-4b30-aaa8-45a9ea8f6053:." ],
+	"stderr": "ERROR: host path must be absolute, is '.'.\n",
+	"exit": 1
+},
+{
+	"special": "setup mapping paths"
+},
+{
+	"title": "that a host mapping works",
+	"cmd": [ "mapping", "--create", "mapa", "ceaa4862-dd00-41ba-9787-7480ec1b2679:{mappathb}" ],
+	"format": [ "cmd" ]
+},
+{
+	"special": "verify store object cnt",
+	"comment": "and that the host object was created",
+	"count": 3
+},
+{
+	"title": "that it was created w/ correct values",
+	"cmd": [ "dump" ],
+	"format": [ "stdout_re" ],
+	"stdout_re": ".*\n.*\n.*mapping.*efdb5d9c-d123-4b30-aaa8-45a9ea8f6053:{mappatha}.*ceaa4862-dd00-41ba-9787-7480ec1b2679:{mappathb}.*type.*mapping.*\n"
+},
+{
+	"title": "that a file on one mapping can have metadata",
+	"cmd": [ "modify", "+sometag=value", "mapa/text.txt" ]
+},
+{
+	"special": "verify store object cnt",
+	"comment": "and that the file and metadata was created",
+	"count": 5
+},
+{
+	"special": "set hostid",
+	"comment": "that the other host",
+	"hostid": "ceaa4862-dd00-41ba-9787-7480ec1b2679"
+},
+{
+	"special": "delete files",
+	"format": [ "files" ],
+	"files": [ "{mappatha}/text.txt" ]
+},
+{
+	"title": "can see the other host's metadata",
+	"cmd": [ "list", "mapb/text.txt" ],
+	"stdout_re": ".*\n.*\nsometag:\tvalue\n"
+},
+{
+	"title": "and add it's own value",
+	"cmd": [ "modify", "+sometag=anothervalue", "mapb/text.txt" ]
+},
+{
+	"special": "verify store object cnt",
+	"comment": "but didn't create an additional FileObject",
+	"count": 5
+}
+]

ui/medashare/cli.py

@@ -44,6 +44,18 @@ _NAMESPACE_MEDASHARE_CONTAINER = uuid.UUID('890a9d5c-0626-4de1-ab05-9e14947391eb
 # useful for debugging when stderr is redirected/captured
 _real_stderr = sys.stderr
 
+def _debprint(*args): # pragma: no cover
+	import traceback, sys, os.path
+	st = traceback.extract_stack(limit=2)[0]
+
+	sep = ''
+	if args:
+		sep = ':'
+
+	print('%s:%d%s' % (os.path.basename(st.filename), st.lineno, sep),
+	    *args, file=_real_stderr)
+	sys.stderr.flush()
+
 _defaulthash = 'sha512'
 _validhashes = set([ 'sha256', 'sha512' ])
 _hashlengths = { len(getattr(hashlib, x)().hexdigest()): x for x in
@@ -333,6 +345,11 @@ class Persona(object):
 
 		return self.sign(Host(*args, **kwargs))
 
+	def Mapping(self, *args, **kwargs):
+		kwargs['created_by_ref'] = self.uuid
+
+		return self.sign(Mapping(*args, **kwargs))
+
 	def Container(self, *args, **kwargs):
 		kwargs['created_by_ref'] = self.uuid
 
@@ -478,6 +495,7 @@ class ObjectStore(object):
 		self._uuids = {}
 		self._hashes = {}
 		self._hostuuids = {}
+		self._hostmappings = []
 
 	def get_host(self, hostuuid):
 		return self._hostuuids[hostuuid]
@@ -569,7 +587,15 @@ class ObjectStore(object):
 		elif obj.type == 'host':
 			self._uuids[obj.hostuuid] = obj
 			self._hostuuids[obj.hostuuid] = obj
-
+		elif obj.type == 'mapping':
+			hostid = _makeuuid(hostuuid())
+
+			maps = [ (lambda a, b: (uuid.UUID(a), pathlib.Path(b).resolve()))(*x.split(':', 1)) for x in obj.mapping ]
+			for idx, (id, path) in enumerate(maps):
+				if hostid == id:
+					# add other to mapping
+					other = tuple(maps[(idx + 1) % 2])
+					self._hostmappings.append((path, ) + other)
 		try:
 			hashes = obj.hashes
 		except AttributeError:
@@ -623,7 +649,7 @@ class ObjectStore(object):
 		h = self.makehash(hash, strict=False)
 		return self._hashes[h]
 
-	def get_metadata(self, fname, persona):
+	def get_metadata(self, fname, persona, create_metadata=True):
 		'''Get all MetaData objects for fname, or create one if
 		not found.
 
@@ -632,6 +658,9 @@ class ObjectStore(object):
 		A Persona must be passed in to create the FileObject and
 		MetaData objects as needed.
 
+		A MetaData object will be created if create_metadata is
+		True, which is the default.
+
 		Note: if a new MetaData object is created, it is not
 		stored in the database automatically.  It is expected that
 		it will be modified and then saved, so call ObjectStore.loadobj
@@ -640,12 +669,8 @@ class ObjectStore(object):
 
 		try:
 			fobj = self.by_file(fname, ('file',))[0]
-			#print('x:', repr(objs), file=_real_stderr)
 		except KeyError:
-			#print('b:', repr(fname), file=_real_stderr)
-
 			fobj = persona.by_file(fname)
-			#print('c:', repr(fobj), file=_real_stderr)
 
 			self.loadobj(fobj)
 
@@ -653,13 +678,19 @@ class ObjectStore(object):
 		try:
 			objs = self.by_file(fname)
 		except KeyError:
-			objs = [ persona.MetaData(hashes=fobj.hashes) ]
+			if create_metadata:
+				objs = [ persona.MetaData(hashes=fobj.hashes) ]
+			else:
+				objs = [ ]
 
 		return objs
 
 	def by_file(self, fname, types=('metadata', )):
 		'''Return a metadata object for the file named fname.
 
+		Will check the mapping database to get hashes, and possibly
+		return that FileObject if requested.
+
 		Will raise a KeyError if this file does not exist in
 		the database.
 
@@ -670,8 +701,26 @@ class ObjectStore(object):
 		fid = FileObject.make_id(fname)
 
 		#print('bf:', repr(fid), file=_real_stderr)
-		fobj = self.by_id(fid)
-		fobj.verify()
+		try:
+			fobj = self.by_id(fid)
+			lclfile = None
+		except KeyError:
+			# check mappings
+			fname = pathlib.Path(fname).resolve()
+			for lclpath, hostid, rempath in self._hostmappings:
+				if fname.parts[:len(lclpath.parts)] == lclpath.parts:
+					try:
+						rempath = pathlib.Path(*rempath.parts + fname.parts[len(lclpath.parts):])
+						fid = FileObject.make_id(rempath, hostid)
+						fobj = self.by_id(fid)
+						lclfile = fname
+						break
+					except KeyError:
+						continue
+			else:
+				raise
+
+		fobj.verify(lclfile)
 
 		for i in fobj.hashes:
 			j = self.by_hash(i)
@@ -703,6 +752,9 @@ def _hashfile(fname):
 class Host(MDBase):
 	_type = 'host'
 
+class Mapping(MDBase):
+	_type = 'mapping'
+
 class FileObject(MDBase):
 	_type = 'file'
 
@@ -713,14 +765,17 @@ class FileObject(MDBase):
 	}
 
 	@staticmethod
-	def make_id(fname):
+	def make_id(fname, hostid=None):
 		'''Take a local file name, and make the id for it.  Note that
 		converts from the local path separator to a forward slash so
 		that it will be the same between Windows and Unix systems.'''
 
+		if hostid is None:
+			hostid = hostuuid()
+
 		fname = os.path.realpath(fname)
 		return uuid.uuid5(_NAMESPACE_MEDASHARE_PATH,
-		    str(hostuuid()) + '/'.join(os.path.split(fname)))
+		    str(hostid) + '/'.join(os.path.split(fname)))
 
 	@classmethod
 	def from_file(cls, filename, created_by_ref):
@@ -741,13 +796,17 @@ class FileObject(MDBase):
 
 		return cls(obj)
 
-	def verify(self, complete=False):
+	def verify(self, lclfile=None):
 		'''Verify that this FileObject is still valid.  It will
 		by default, only do a mtime verification.
 
 		It will raise a ValueError if the file does not match.'''
 
-		s = os.stat(os.path.join(self.dir, self.filename))
+		if lclfile is None:
+			s = os.stat(os.path.join(self.dir, self.filename))
+		else:
+			s = os.stat(lclfile)
+
 		mtimets = datetime.datetime.fromtimestamp(s.st_mtime,
 		    tz=datetime.timezone.utc).timestamp()
 
@@ -919,6 +978,36 @@ def cmd_modify(options):
 def printhost(host):
 	print('%s\t%s' % (host.name, host.hostuuid))
 
+def cmd_mapping(options):
+	persona, objstr = get_objstore(options)
+
+	if options.mapping is not None:
+		parts = [ x.split(':', 1) for x in options.mapping ]
+
+		if len(parts[0]) == 1:
+			parts[0] = [ hostuuid(), parts[0][0] ]
+
+		if parts[0][0] == hostuuid():
+			parts[0][1] = str(pathlib.Path(parts[0][1]).resolve())
+
+		if parts[1][1][0] != '/':
+			print('ERROR: host path must be absolute, is %s.' %
+			    repr(parts[1][1][0]), file=sys.stderr)
+			sys.exit(1)
+
+		try:
+			[ objstr.get_host(x[0]) for x in parts ]
+		except KeyError as e:
+			print('ERROR: Unable to find host %s' %
+			    repr(e.args[0]), file=sys.stderr)
+			sys.exit(1)
+
+		m = persona.Mapping(mapping=[ ':'.join(x) for x in parts ])
+
+	objstr.loadobj(m)
+
+	write_objstore(options, objstr)
+
 def cmd_hosts(options):
 	persona, objstr = get_objstore(options)
 
@@ -987,11 +1076,6 @@ def cmd_list(options):
 			except (FileNotFoundError, KeyError) as e:
 				print('ERROR: file not found: %s' % repr(i), file=sys.stderr)
 				sys.exit(1)
-		except FileNotFoundError:
-			# XXX - tell the difference?
-			print('ERROR: file not found: %s' % repr(i),
-			    file=sys.stderr)
-			sys.exit(1)
 
 		for j in objstr.by_file(i):
 			for k, v in _iterdictlist(j):
@@ -1149,6 +1233,11 @@ def main():
 	parser_hosts = subparsers.add_parser('hosts', help='dump all the hosts, self is always first')
 	parser_hosts.set_defaults(func=cmd_hosts)
 
+	parser_mapping = subparsers.add_parser('mapping', help='list mappings, or create a mapping')
+	parser_mapping.add_argument('--create', dest='mapping', nargs=2,
+	    help='mapping to add, host|hostuuid:path host|hostuuid:path')
+	parser_mapping.set_defaults(func=cmd_mapping)
+
 	parser_dump = subparsers.add_parser('dump', help='dump all the objects')
 	parser_dump.set_defaults(func=cmd_dump)
 
@@ -1633,6 +1722,13 @@ class _TestCases(unittest.TestCase):
 			except KeyError:
 				pass
 
+			for i in cmd.get('format', []):
+				if i in { 'cmd', 'files' }:
+					vars = locals()
+					cmd[i] = [ x.format(**vars) for x in cmd[i] ]
+				else:
+					cmd[i] = cmd[i].format(**locals())
+
 			try:
 				special = cmd['special']
 			except KeyError:
@@ -1673,6 +1769,21 @@ class _TestCases(unittest.TestCase):
 					sd.mkdir(exist_ok=True)
 
 					bttestcase.make_files(sd, btfiles)
+				elif special == 'setup mapping paths':
+					mappatha = self.tempdir / 'mapa'
+					mappatha.mkdir()
+
+					mappathb = self.tempdir / 'mapb'
+					mappathb.mkdir()
+
+					filea = mappatha / 'text.txt'
+					filea.write_text('abc123\n')
+					fileb = mappathb / 'text.txt'
+					shutil.copyfile(filea, fileb)
+					shutil.copystat(filea, fileb)
+				elif special == 'delete files':
+					for i in cmd['files']:
+						os.unlink(i)
 				else: # pragma: no cover
 					raise ValueError('unhandled special: %s' % repr(special))