update pasn1 to UTC support, add missing test files...

fix up code to pass all the tests...

ui/fixtures/cmd.basic.json

@@ -0,0 +1,140 @@
+[
+{
+	"title": "Test no ident",
+	"cmd": [ "list", "afile" ],
+	"exit": 1,
+	"stderr": "ERROR: Identity not created, create w/ -g.\n"
+},
+{
+	"title": "gen ident",
+	"cmd": [ "genident", "name=A Test User" ],
+	"exit": 0
+},
+{
+	"title": "print ident",
+	"cmd": [ "ident" ],
+	"comment": "XXX - expand modified, pubkey and sig, and fix to base58 encode",
+	"stdout_re": "^type:\tidentity\nuuid:\t[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\nmodified:\t[0-9]{4,}-[01][0-9]-[0-3][0-9] .*\nname:\tA Test User\npubkey:\t.*\nsig:\t.*\n",
+	"exit": 0
+},
+{
+	"title": "update ident",
+	"cmd": [ "ident", "name=Changed Name" ],
+	"exit": 0
+},
+{
+	"title": "ident updated",
+	"cmd": [ "ident" ],
+	"comment": "create vars store bindings between tests?",
+	"stdout_re": "^type:\tidentity\nuuid:\t[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\nmodified:\t[0-9]{4,}-[01][0-9]-[0-3][0-9] .*\nname:\tChanged Name\npubkey:\t.*\nsig:\t.*\n",
+	"exit": 0
+},
+{
+	"title": "pub key is base58 encoded",
+	"cmd": [ "pubkey" ],
+	"stdout_re": "^[1-9A-HJ-NP-Za-km-z]+\n$",
+	"exit": 0
+},
+{
+	"title": "Test file with no tag",
+	"cmd": [ "list", "newfile.txt" ],
+	"exit": 1,
+	"stderr": "ERROR: file not found: 'newfile.txt'\n"
+},
+{
+	"title": "invalid tag",
+	"cmd": [ "modify", "+tag", "newfile.txt" ],
+	"exit": 1,
+	"stderr": "ERROR: invalid tag, needs an \"=\".\n"
+},
+{
+	"title": "add tag",
+	"cmd": [ "modify", "+tag=", "+dc:creator=John-Mark Gurney", "newfile.txt" ]
+},
+{
+	"special": "verify store object cnt",
+	"comment": "should only have one file and one metadata",
+	"count": 2
+},
+{
+	"title": "verify first tags are present",
+	"cmd": [ "list", "newfile.txt" ],
+	"stdout_re": "dc:creator:\tJohn-Mark Gurney\nhashes:\tsha512:90f8342520f0ac57fb5a779f5d331c2fa87aa40f8799940257f9ba619940951e67143a8d746535ed0284924b2b7bc1478f095198800ba96d01847d7b56ca465c\nsig:\t.*\ntag:\t\n"
+},
+{
+	"title": "add tag",
+	"cmd": [ "modify", "+dc:creator=Another user", "newfile.txt" ]
+},
+{
+	"title": "another dc:creator tag is present",
+	"cmd": [ "list", "newfile.txt" ],
+	"stdout_re": "dc:creator:\tAnother user\ndc:creator:\tJohn-Mark Gurney\nhashes:\tsha512:90f8342520f0ac57fb5a779f5d331c2fa87aa40f8799940257f9ba619940951e67143a8d746535ed0284924b2b7bc1478f095198800ba96d01847d7b56ca465c\nsig:\t.*\ntag:\t\n"
+},
+{
+	"title": "remove specific tag",
+	"cmd": [ "modify", "-dc:creator=Another user", "newfile.txt" ]
+},
+{
+	"title": "another dc:creator tag is present",
+	"cmd": [ "list", "newfile.txt" ],
+	"stdout_re": "dc:creator:\tJohn-Mark Gurney\nhashes:\tsha512:90f8342520f0ac57fb5a779f5d331c2fa87aa40f8799940257f9ba619940951e67143a8d746535ed0284924b2b7bc1478f095198800ba96d01847d7b56ca465c\nsig:\t.*\ntag:\t\n"
+},
+{
+	"title": "remove all dc:creator tags",
+	"cmd": [ "modify", "-dc:creator", "newfile.txt" ]
+},
+{
+	"title": "that all dc:creator tags are removed",
+	"cmd": [ "list", "newfile.txt" ],
+	"stdout_re": "hashes:\tsha512:90f8342520f0ac57fb5a779f5d331c2fa87aa40f8799940257f9ba619940951e67143a8d746535ed0284924b2b7bc1478f095198800ba96d01847d7b56ca465c\nsig:\t.*\ntag:\t\n"
+},
+{
+	"title": "tag value w/ equals",
+	"cmd": [ "modify", "+foo=bar=baz", "newfile.txt" ]
+},
+{
+	"title": "print file",
+	"cmd": [ "list", "newfile.txt" ],
+	"stdout_re": "foo:\tbar=baz\nhashes:\tsha512:90f8342520f0ac57fb5a779f5d331c2fa87aa40f8799940257f9ba619940951e67143a8d746535ed0284924b2b7bc1478f095198800ba96d01847d7b56ca465c\nsig:\t.*\ntag:\t\n"
+},
+{
+	"title": "test file is not present",
+	"cmd": [ "list", "test.txt" ],
+	"exit": 1,
+	"stderr": "ERROR: file not found: 'test.txt'\n"
+},
+{
+	"special": "copy newfile.txt to test.txt"
+},
+{
+	"title": "copied file now has same metadata",
+	"cmd": [ "list", "test.txt" ],
+	"stdout_re": "foo:\tbar=baz\nhashes:\tsha512:90f8342520f0ac57fb5a779f5d331c2fa87aa40f8799940257f9ba619940951e67143a8d746535ed0284924b2b7bc1478f095198800ba96d01847d7b56ca465c\nsig:\t.*\ntag:\t\n"
+},
+{
+	"special": "change newfile.txt"
+},
+{
+	"title": "newfile file is now not present",
+	"cmd": [ "list", "newfile.txt" ],
+	"exit": 1,
+	"stderr": "ERROR: file not found: 'newfile.txt'\n"
+},
+{
+	"title": "old file still has same metadata",
+	"cmd": [ "list", "test.txt" ],
+	"stdout_re": "foo:\tbar=baz\nhashes:\tsha512:90f8342520f0ac57fb5a779f5d331c2fa87aa40f8799940257f9ba619940951e67143a8d746535ed0284924b2b7bc1478f095198800ba96d01847d7b56ca465c\nsig:\t.*\ntag:\t\n"
+},
+{
+	"title": "a common tag is disallowed",
+	"cmd": [ "modify", "+modified=foo", "test.txt" ],
+	"exit": 1,
+	"stderr": "ERROR: invalid tag: ['modified'].\n"
+},
+{
+	"title": "must be a + or -",
+	"cmd": [ "modify", "modified=foo", "test.txt" ],
+	"exit": 1,
+	"stderr": "ERROR: tag needs to start with a \"+\" (add) or a \"-\" (remove).\n"
+}
+]

ui/fixtures/testfiles/newfile.txt

@@ -0,0 +1 @@
+this is a new file

ui/medashare/cli.py

@@ -30,6 +30,9 @@ import uuid
 # The UUID for the namespace representing the path to a file
 _NAMESPACE_MEDASHARE_PATH = uuid.UUID('f6f36b62-3770-4a68-bc3d-dc3e31e429e6')
 
+# useful for debugging when stderr is redirected/captured
+_real_stderr = sys.stderr
+
 _defaulthash = 'sha512'
 _validhashes = set([ 'sha256', 'sha512' ])
 _hashlengths = { len(getattr(hashlib, x)().hexdigest()): x for x in
@@ -70,7 +73,8 @@ class MDBase(object):
 
 	_generated_properties = {
 		'uuid': uuid.uuid4,
-		'modified': datetime.datetime.utcnow
+		'modified': lambda: datetime.datetime.now(
+		    tz=datetime.timezone.utc),
 	}
 
 	# When decoding, the decoded value should be passed to this function
@@ -368,7 +372,7 @@ class Persona(object):
 		return True
 
 	def by_file(self, fname):
-		'''Return a metadata object for the file named fname.'''
+		'''Return a file object for the file named fname.'''
 
 		fobj = FileObject.from_file(fname, self._created_by_ref)
 
@@ -421,14 +425,20 @@ class ObjectStore(object):
 	def __len__(self):
 		return len(self._uuids)
 
+	def __iter__(self):
+		return iter(self._uuids.values())
+
 	def store(self, fname):
 		'''Write out the objects in the store to the file named
 		fname.'''
 
+		# eliminate objs stored by multiple uuids (FileObjects)
+		objs = { id(x): x for x in self._uuids.values() }
+
 		with open(fname, 'wb') as fp:
 			obj = {
 				'created_by_ref': self._created_by_ref,
-				'objects': list(self._uuids.values()),
+				'objects': list(objs.values()),
 			}
 			fp.write(_asn1coder.dumps(obj))
 
@@ -438,6 +448,10 @@ class ObjectStore(object):
 		obj = MDBase.create_obj(obj)
 
 		self._uuids[obj.uuid] = obj
+
+		if obj.type == 'file':
+			self._uuids[_makeuuid(obj.id)] = obj
+
 		for j in obj.hashes:
 			h = self.makehash(j)
 			self._hashes.setdefault(h, []).append(obj)
@@ -479,14 +493,10 @@ class ObjectStore(object):
 		'''Return a metadata object for the file named fname.'''
 
 		fid = FileObject.make_id(fname)
-		try:
-			fobj = self.by_id(fid)
-		except KeyError:
-			# unable to find it
-			fobj = FileObject.from_file(fname, self._created_by_ref)
-			self.loadobj(fobj)
 
-		# XXX - does not verify
+		fobj = self.by_id(fid)
+		fobj.verify()
+
 		for i in fobj.hashes:
 			j = self.by_hash(i)
 
@@ -536,14 +546,27 @@ class FileObject(MDBase):
 			'created_by_ref': created_by_ref,
 			'filename': os.path.basename(filename),
 			'id': cls.make_id(filename),
-			'mtime': datetime.datetime.utcfromtimestamp(s.st_mtime),
+			'mtime': datetime.datetime.fromtimestamp(s.st_mtime,
+			    tz=datetime.timezone.utc),
 			'size': s.st_size,
 			'hashes': [ _hashfile(filename), ],
 		}
 
-
 		return cls(obj)
 
+	def verify(self, complete=False):
+		'''Verify that this FileObject is still valid.  It will
+		by default, only do a mtime verification.'''
+
+		s = os.stat(os.path.join(self.dir, self.filename))
+		mtimets = datetime.datetime.fromtimestamp(s.st_mtime,
+		    tz=datetime.timezone.utc).timestamp()
+
+		if self.mtime.timestamp() != mtimets or \
+		    self.size != s.st_size:
+			raise ValueError('file %s has changed' %
+			    repr(self.filename))
+
 def enumeratedir(_dir, created_by_ref):
 	'''Enumerate all the files and directories (not recursive) in _dir.
 
@@ -671,21 +694,40 @@ def cmd_modify(options):
 
 	write_objstore(options, objstr)
 
+def cmd_dump(options):
+	persona, objstr = get_objstore(options)
+
+	for i in objstr:
+		print(repr(i))
+
 def cmd_list(options):
 	persona, objstr = get_objstore(options)
 
 	for i in options.files:
 		try:
-			for j in objstr.by_file(i):
-				#print >>sys.stderr, `j._obj`
-				for k, v in _iterdictlist(j):
-					print('%s:\t%s' % (k, v))
-		except (KeyError, FileNotFoundError):
+			objs = objstr.by_file(i)
+		except (ValueError, KeyError):
+			# create the file, it may have the same hash
+			# as something else
+			try:
+				fobj = persona.by_file(i)
+				objstr.loadobj(fobj)
+
+				objs = objstr.by_file(i)
+			except (FileNotFoundError, KeyError):
+				print('ERROR: file not found: %s' % repr(i), file=sys.stderr)
+				sys.exit(1)
+
+		except FileNotFoundError:
 			# XXX - tell the difference?
 			print('ERROR: file not found: %s' % repr(i),
 			    file=sys.stderr)
 			sys.exit(1)
 
+		for j in objstr.by_file(i):
+			#print >>sys.stderr, `j._obj`
+			for k, v in _iterdictlist(j):
+				print('%s:\t%s' % (k, v))
 def main():
 	import argparse
 
@@ -723,6 +765,9 @@ def main():
 	    help='files to modify')
 	parser_list.set_defaults(func=cmd_list)
 
+	parser_dump = subparsers.add_parser('dump', help='dump all the objects')
+	parser_dump.set_defaults(func=cmd_dump)
+
 	options = parser.parse_args()
 
 	fun = options.func
@@ -875,7 +920,8 @@ class _TestCases(unittest.TestCase):
 		self.assertEqual(ftest.id, uuid.uuid5(_NAMESPACE_MEDASHARE_PATH,
 		    '/'.join(os.path.split(self.tempdir) +
 		    ( fname, ))))
-		self.assertEqual(ftest.mtime, datetime.datetime(2019, 5, 20, 21, 47, 36))
+		self.assertEqual(ftest.mtime, datetime.datetime(2019, 5, 20,
+		    21, 47, 36, tzinfo=datetime.timezone.utc))
 		self.assertEqual(ftest.size, 15)
 		self.assertIn('sha512:7d5768d47b6bc27dc4fa7e9732cfa2de506ca262a2749cb108923e5dddffde842bbfee6cb8d692fb43aca0f12946c521cce2633887914ca1f96898478d10ad3f', ftest.hashes)
 
@@ -972,6 +1018,8 @@ class _TestCases(unittest.TestCase):
 		# has the correct created_by_ref
 		self.assertEqual(testobj.created_by_ref, idobj.uuid)
 
+		self.assertEqual(testobj.type, 'file')
+
 		# and has a signature
 		self.assertIn('sig', testobj)
 
@@ -1026,17 +1074,15 @@ class _TestCases(unittest.TestCase):
 		# and that it can be verified
 		persona.verify(mdobj)
 
-		# that when round tripped through pasn1.
-		a = mdobj.encode()
-		b = MDBase.decode(a)
-
 	def test_objectstore(self):
+		persona = Persona.load(os.path.join('fixtures', 'sample.persona.pasn1'))
 		objst = ObjectStore.load(os.path.join('fixtures', 'sample.data.pasn1'))
 
 		objst.loadobj({
 			'type': 'metadata',
 			'uuid': uuid.UUID('c9a1d1e2-3109-4efd-8948-577dc15e44e7'),
-			'modified': datetime.datetime(2019, 5, 31, 14, 3, 10),
+			'modified': datetime.datetime(2019, 5, 31, 14, 3, 10,
+			    tzinfo=datetime.timezone.utc),
 			'created_by_ref': self.created_by_ref,
 			'hashes': [ 'sha256:91751cee0a1ab8414400238a761411daa29643ab4b8243e9a91649e25be53ada' ],
 			'lang': 'en',
@@ -1055,6 +1101,7 @@ class _TestCases(unittest.TestCase):
 		self.assertEqual(r.uuid, uuid.UUID('3e466e06-45de-4ecc-84ba-2d2a3d970e96'))
 		self.assertEqual(r['dc:creator'], [ 'John-Mark Gurney' ])
 
+		# test storing the object store
 		fname = 'testfile.pasn1'
 		objst.store(fname)
 
@@ -1067,12 +1114,19 @@ class _TestCases(unittest.TestCase):
 
 		self.assertEqual(objs['created_by_ref'], self.created_by_ref.bytes)
 
+		# make sure that the read back data matches
 		for i in objs['objects']:
 			i['created_by_ref'] = uuid.UUID(bytes=i['created_by_ref'])
 			i['uuid'] = uuid.UUID(bytes=i['uuid'])
 			self.assertEqual(objst.by_id(i['uuid']), i)
 
+		# that a file
 		testfname = os.path.join(self.tempdir, 'test.txt')
+
+		# when registered
+		objst.loadobj(persona.by_file(testfname))
+
+		# can be found
 		self.assertEqual(objst.by_file(testfname), [ byid ])
 		self.assertEqual(objst.by_file(testfname), [ byid ])
 
@@ -1113,6 +1167,13 @@ class _TestCases(unittest.TestCase):
 				elif special == 'change newfile.txt':
 					with open(newtestfname, 'w') as fp:
 						fp.write('some new contents')
+				elif special == 'verify store object cnt':
+					with open(storefname, 'rb') as fp:
+						pasn1obj = pasn1.loads(fp.read())
+						objcnt = len(pasn1obj['objects'])
+					self.assertEqual(objcnt, cmd['count'])
+				else: # pragma: no cover
+					raise ValueError('unhandled special: %s' % repr(special))
 
 				continue
 
@@ -1172,8 +1233,6 @@ class _TestCases(unittest.TestCase):
 
 		import itertools
 
-		real_stderr = sys.stderr
-
 		with mock.patch('os.path.expanduser', side_effect=expandusermock) \
 		    as eu, mock.patch('medashare.cli.open') as op:
 			# that when opening the store and identity fails

ui/setup.py

@@ -27,7 +27,7 @@ setup(
         'httpx',
         'hypercorn', # option, for server only?
         'orm',
-	'pasn1 @ git+https://www.funkthat.com/gitea/jmg/pasn1.git@01d8efffd7bc3037dcb894ea44dbe959035948c6#egg=pasn1',
+	'pasn1 @ git+https://www.funkthat.com/gitea/jmg/pasn1.git@c6c64510b42292557ace2b77272eb32cb647399d#egg=pasn1',
         'pydantic[dotenv]',
     ],
     extras_require = {