add support for missing metadata objects, improve queries a bit..

Some queries weren't using the full power of SQL and doing some of
the selection in the code, fix that (by_hash and host mapping).

ui/fixtures/cmd.search.container.json

@@ -0,0 +1,25 @@
+[
+{
+	"title": "gen ident",
+	"cmd": [ "genident", "name=A Test User" ],
+	"exit": 0
+},
+{
+        "special": "setup bittorrent files",
+        "complete": false
+},
+{
+        "title": "add metadata before import",
+        "cmd": [ "modify", "+other=bar", "somedir/fileb.txt" ]
+},
+{
+        "title": "import partial container",
+        "cmd": [ "container", "somedir.torrent" ],
+        "stderr": "Warning, incomple/invalid files, not added for 'somedir.torrent':\n\t'filea.txt'\n\t'filec.txt'\n\t'filee.txt'\n"
+},
+{
+	"title": "search no other with container",
+	"cmd": [ "search", "file", "-other" ],
+	"stdout_re": "^[^\n]*/somedir/filed.txt\n[^\n]*/somedir/filef/filef.txt\n$"
+}
+]

ui/medashare/cli.py

@@ -518,19 +518,22 @@ class ObjectStore(object):
 
 		return res.data
 
-	def by_hash(self, hash):
-		'''Look up an object by it's hash value.'''
+	def by_hash(self, hash, types=None):
+		'''Look up an object by it's hash value.
+
+		types is either a list of types, or None meaning all.'''
 
 		h = self.makehash(hash, strict=False)
 
-		r = []
+		types = True if types is None else \
+		    orm.MetaDataObject.type.in_(types)
+
+		sel = select(orm.MetaDataObject.data).where(
+		    orm.MetaDataObject.uuid == orm.HashTable.uuid,
+		     orm.HashTable.hash == h, types)
+
 		with self._ses() as session:
-			# XXX - convert to union/join query
-			for i in session.scalars(select(
-			    orm.HashTable.uuid).where(orm.HashTable.hash == h)):
-				v = self._by_id(i, session)
-				if v is not None:
-					r.append(v)
+			r = list(session.scalars(sel))
 
 		return r
 
@@ -576,13 +579,14 @@ class ObjectStore(object):
 
 		hostid = _makeuuid(hostuuid())
 
+		sel = select(orm.MetaDataObject.data).where(
+		    orm.HostMapping.hostid == hostid,
+		    orm.HostMapping.objid == orm.MetaDataObject.uuid)
+
 		res = []
 		with self._ses() as session:
 			# XXX - view
-			for i in session.scalars(select(orm.HostMapping).where(
-			    orm.HostMapping.hostid == hostid)):
-				obj = self._by_id(i.objid, session)
-
+			for obj in session.scalars(sel):
 				maps = [ (lambda a, b: (uuid.UUID(a),
 				    pathlib.Path(b).resolve()))(*x.split(':',
 				    1)) for x in obj.mapping ]
@@ -1424,6 +1428,17 @@ def cmd_search(options, persona, objstr, cache):
 	#print(repr(searches), file=_real_stderr)
 
 	def testfun(x, s=searches):
+		try:
+			x = objstr.by_hash(x['hashes'][0], ('metadata',))[0]
+		except IndexError:
+			# no metadata object
+
+			# if we need anything, it's not present
+			if any(x[0] == '+' for x in s):
+				return False
+
+			return True
+
 		try:
 			for i in s:
 				try:
@@ -1452,7 +1467,7 @@ def cmd_search(options, persona, objstr, cache):
 		except KeyError:
 			return False
 
-	r = [ x for x in objstr if x.type == 'metadata' and testfun(x) ]
+	r = [ x for x in objstr if x.type == 'file' and testfun(x) ]
 
 	if _type == 'file':
 		r = [ objstr.by_hash(x['hashes'][0]) for x in r ]