From cda0102972ecd8527654d20682d6b1dc9e71fb4d Mon Sep 17 00:00:00 2001 From: John-Mark Gurney Date: Thu, 4 Aug 2022 15:35:46 -0700 Subject: [PATCH] update pasn1 to UTC support, add missing test files... fix up code to pass all the tests... --- ui/fixtures/cmd.basic.json | 140 ++++++++++++++++++++++++++++++ ui/fixtures/testfiles/newfile.txt | 1 + ui/medashare/cli.py | 109 +++++++++++++++++------ ui/setup.py | 2 +- 4 files changed, 226 insertions(+), 26 deletions(-) create mode 100644 ui/fixtures/cmd.basic.json create mode 100644 ui/fixtures/testfiles/newfile.txt diff --git a/ui/fixtures/cmd.basic.json b/ui/fixtures/cmd.basic.json new file mode 100644 index 0000000..c1f473f --- /dev/null +++ b/ui/fixtures/cmd.basic.json @@ -0,0 +1,140 @@ +[ +{ + "title": "Test no ident", + "cmd": [ "list", "afile" ], + "exit": 1, + "stderr": "ERROR: Identity not created, create w/ -g.\n" +}, +{ + "title": "gen ident", + "cmd": [ "genident", "name=A Test User" ], + "exit": 0 +}, +{ + "title": "print ident", + "cmd": [ "ident" ], + "comment": "XXX - expand modified, pubkey and sig, and fix to base58 encode", + "stdout_re": "^type:\tidentity\nuuid:\t[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\nmodified:\t[0-9]{4,}-[01][0-9]-[0-3][0-9] .*\nname:\tA Test User\npubkey:\t.*\nsig:\t.*\n", + "exit": 0 +}, +{ + "title": "update ident", + "cmd": [ "ident", "name=Changed Name" ], + "exit": 0 +}, +{ + "title": "ident updated", + "cmd": [ "ident" ], + "comment": "create vars store bindings between tests?", + "stdout_re": "^type:\tidentity\nuuid:\t[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\nmodified:\t[0-9]{4,}-[01][0-9]-[0-3][0-9] .*\nname:\tChanged Name\npubkey:\t.*\nsig:\t.*\n", + "exit": 0 +}, +{ + "title": "pub key is base58 encoded", + "cmd": [ "pubkey" ], + "stdout_re": "^[1-9A-HJ-NP-Za-km-z]+\n$", + "exit": 0 +}, +{ + "title": "Test file with no tag", + "cmd": [ "list", "newfile.txt" ], + "exit": 1, + "stderr": "ERROR: file not found: 'newfile.txt'\n" +}, +{ + "title": "invalid tag", + "cmd": [ "modify", "+tag", "newfile.txt" ], + "exit": 1, + "stderr": "ERROR: invalid tag, needs an \"=\".\n" +}, +{ + "title": "add tag", + "cmd": [ "modify", "+tag=", "+dc:creator=John-Mark Gurney", "newfile.txt" ] +}, +{ + "special": "verify store object cnt", + "comment": "should only have one file and one metadata", + "count": 2 +}, +{ + "title": "verify first tags are present", + "cmd": [ "list", "newfile.txt" ], + "stdout_re": "dc:creator:\tJohn-Mark Gurney\nhashes:\tsha512:90f8342520f0ac57fb5a779f5d331c2fa87aa40f8799940257f9ba619940951e67143a8d746535ed0284924b2b7bc1478f095198800ba96d01847d7b56ca465c\nsig:\t.*\ntag:\t\n" +}, +{ + "title": "add tag", + "cmd": [ "modify", "+dc:creator=Another user", "newfile.txt" ] +}, +{ + "title": "another dc:creator tag is present", + "cmd": [ "list", "newfile.txt" ], + "stdout_re": "dc:creator:\tAnother user\ndc:creator:\tJohn-Mark Gurney\nhashes:\tsha512:90f8342520f0ac57fb5a779f5d331c2fa87aa40f8799940257f9ba619940951e67143a8d746535ed0284924b2b7bc1478f095198800ba96d01847d7b56ca465c\nsig:\t.*\ntag:\t\n" +}, +{ + "title": "remove specific tag", + "cmd": [ "modify", "-dc:creator=Another user", "newfile.txt" ] +}, +{ + "title": "another dc:creator tag is present", + "cmd": [ "list", "newfile.txt" ], + "stdout_re": "dc:creator:\tJohn-Mark Gurney\nhashes:\tsha512:90f8342520f0ac57fb5a779f5d331c2fa87aa40f8799940257f9ba619940951e67143a8d746535ed0284924b2b7bc1478f095198800ba96d01847d7b56ca465c\nsig:\t.*\ntag:\t\n" +}, +{ + "title": "remove all dc:creator tags", + "cmd": [ "modify", "-dc:creator", "newfile.txt" ] +}, +{ + "title": "that all dc:creator tags are removed", + "cmd": [ "list", "newfile.txt" ], + "stdout_re": "hashes:\tsha512:90f8342520f0ac57fb5a779f5d331c2fa87aa40f8799940257f9ba619940951e67143a8d746535ed0284924b2b7bc1478f095198800ba96d01847d7b56ca465c\nsig:\t.*\ntag:\t\n" +}, +{ + "title": "tag value w/ equals", + "cmd": [ "modify", "+foo=bar=baz", "newfile.txt" ] +}, +{ + "title": "print file", + "cmd": [ "list", "newfile.txt" ], + "stdout_re": "foo:\tbar=baz\nhashes:\tsha512:90f8342520f0ac57fb5a779f5d331c2fa87aa40f8799940257f9ba619940951e67143a8d746535ed0284924b2b7bc1478f095198800ba96d01847d7b56ca465c\nsig:\t.*\ntag:\t\n" +}, +{ + "title": "test file is not present", + "cmd": [ "list", "test.txt" ], + "exit": 1, + "stderr": "ERROR: file not found: 'test.txt'\n" +}, +{ + "special": "copy newfile.txt to test.txt" +}, +{ + "title": "copied file now has same metadata", + "cmd": [ "list", "test.txt" ], + "stdout_re": "foo:\tbar=baz\nhashes:\tsha512:90f8342520f0ac57fb5a779f5d331c2fa87aa40f8799940257f9ba619940951e67143a8d746535ed0284924b2b7bc1478f095198800ba96d01847d7b56ca465c\nsig:\t.*\ntag:\t\n" +}, +{ + "special": "change newfile.txt" +}, +{ + "title": "newfile file is now not present", + "cmd": [ "list", "newfile.txt" ], + "exit": 1, + "stderr": "ERROR: file not found: 'newfile.txt'\n" +}, +{ + "title": "old file still has same metadata", + "cmd": [ "list", "test.txt" ], + "stdout_re": "foo:\tbar=baz\nhashes:\tsha512:90f8342520f0ac57fb5a779f5d331c2fa87aa40f8799940257f9ba619940951e67143a8d746535ed0284924b2b7bc1478f095198800ba96d01847d7b56ca465c\nsig:\t.*\ntag:\t\n" +}, +{ + "title": "a common tag is disallowed", + "cmd": [ "modify", "+modified=foo", "test.txt" ], + "exit": 1, + "stderr": "ERROR: invalid tag: ['modified'].\n" +}, +{ + "title": "must be a + or -", + "cmd": [ "modify", "modified=foo", "test.txt" ], + "exit": 1, + "stderr": "ERROR: tag needs to start with a \"+\" (add) or a \"-\" (remove).\n" +} +] diff --git a/ui/fixtures/testfiles/newfile.txt b/ui/fixtures/testfiles/newfile.txt new file mode 100644 index 0000000..3b2aed8 --- /dev/null +++ b/ui/fixtures/testfiles/newfile.txt @@ -0,0 +1 @@ +this is a new file diff --git a/ui/medashare/cli.py b/ui/medashare/cli.py index 6f8c27e..38c4cc7 100644 --- a/ui/medashare/cli.py +++ b/ui/medashare/cli.py @@ -30,6 +30,9 @@ import uuid # The UUID for the namespace representing the path to a file _NAMESPACE_MEDASHARE_PATH = uuid.UUID('f6f36b62-3770-4a68-bc3d-dc3e31e429e6') +# useful for debugging when stderr is redirected/captured +_real_stderr = sys.stderr + _defaulthash = 'sha512' _validhashes = set([ 'sha256', 'sha512' ]) _hashlengths = { len(getattr(hashlib, x)().hexdigest()): x for x in @@ -70,7 +73,8 @@ class MDBase(object): _generated_properties = { 'uuid': uuid.uuid4, - 'modified': datetime.datetime.utcnow + 'modified': lambda: datetime.datetime.now( + tz=datetime.timezone.utc), } # When decoding, the decoded value should be passed to this function @@ -368,7 +372,7 @@ class Persona(object): return True def by_file(self, fname): - '''Return a metadata object for the file named fname.''' + '''Return a file object for the file named fname.''' fobj = FileObject.from_file(fname, self._created_by_ref) @@ -421,14 +425,20 @@ class ObjectStore(object): def __len__(self): return len(self._uuids) + def __iter__(self): + return iter(self._uuids.values()) + def store(self, fname): '''Write out the objects in the store to the file named fname.''' + # eliminate objs stored by multiple uuids (FileObjects) + objs = { id(x): x for x in self._uuids.values() } + with open(fname, 'wb') as fp: obj = { 'created_by_ref': self._created_by_ref, - 'objects': list(self._uuids.values()), + 'objects': list(objs.values()), } fp.write(_asn1coder.dumps(obj)) @@ -438,6 +448,10 @@ class ObjectStore(object): obj = MDBase.create_obj(obj) self._uuids[obj.uuid] = obj + + if obj.type == 'file': + self._uuids[_makeuuid(obj.id)] = obj + for j in obj.hashes: h = self.makehash(j) self._hashes.setdefault(h, []).append(obj) @@ -479,14 +493,10 @@ class ObjectStore(object): '''Return a metadata object for the file named fname.''' fid = FileObject.make_id(fname) - try: - fobj = self.by_id(fid) - except KeyError: - # unable to find it - fobj = FileObject.from_file(fname, self._created_by_ref) - self.loadobj(fobj) - # XXX - does not verify + fobj = self.by_id(fid) + fobj.verify() + for i in fobj.hashes: j = self.by_hash(i) @@ -536,14 +546,27 @@ class FileObject(MDBase): 'created_by_ref': created_by_ref, 'filename': os.path.basename(filename), 'id': cls.make_id(filename), - 'mtime': datetime.datetime.utcfromtimestamp(s.st_mtime), + 'mtime': datetime.datetime.fromtimestamp(s.st_mtime, + tz=datetime.timezone.utc), 'size': s.st_size, 'hashes': [ _hashfile(filename), ], } - return cls(obj) + def verify(self, complete=False): + '''Verify that this FileObject is still valid. It will + by default, only do a mtime verification.''' + + s = os.stat(os.path.join(self.dir, self.filename)) + mtimets = datetime.datetime.fromtimestamp(s.st_mtime, + tz=datetime.timezone.utc).timestamp() + + if self.mtime.timestamp() != mtimets or \ + self.size != s.st_size: + raise ValueError('file %s has changed' % + repr(self.filename)) + def enumeratedir(_dir, created_by_ref): '''Enumerate all the files and directories (not recursive) in _dir. @@ -671,21 +694,40 @@ def cmd_modify(options): write_objstore(options, objstr) +def cmd_dump(options): + persona, objstr = get_objstore(options) + + for i in objstr: + print(repr(i)) + def cmd_list(options): persona, objstr = get_objstore(options) for i in options.files: try: - for j in objstr.by_file(i): - #print >>sys.stderr, `j._obj` - for k, v in _iterdictlist(j): - print('%s:\t%s' % (k, v)) - except (KeyError, FileNotFoundError): + objs = objstr.by_file(i) + except (ValueError, KeyError): + # create the file, it may have the same hash + # as something else + try: + fobj = persona.by_file(i) + objstr.loadobj(fobj) + + objs = objstr.by_file(i) + except (FileNotFoundError, KeyError): + print('ERROR: file not found: %s' % repr(i), file=sys.stderr) + sys.exit(1) + + except FileNotFoundError: # XXX - tell the difference? print('ERROR: file not found: %s' % repr(i), file=sys.stderr) sys.exit(1) + for j in objstr.by_file(i): + #print >>sys.stderr, `j._obj` + for k, v in _iterdictlist(j): + print('%s:\t%s' % (k, v)) def main(): import argparse @@ -723,6 +765,9 @@ def main(): help='files to modify') parser_list.set_defaults(func=cmd_list) + parser_dump = subparsers.add_parser('dump', help='dump all the objects') + parser_dump.set_defaults(func=cmd_dump) + options = parser.parse_args() fun = options.func @@ -875,7 +920,8 @@ class _TestCases(unittest.TestCase): self.assertEqual(ftest.id, uuid.uuid5(_NAMESPACE_MEDASHARE_PATH, '/'.join(os.path.split(self.tempdir) + ( fname, )))) - self.assertEqual(ftest.mtime, datetime.datetime(2019, 5, 20, 21, 47, 36)) + self.assertEqual(ftest.mtime, datetime.datetime(2019, 5, 20, + 21, 47, 36, tzinfo=datetime.timezone.utc)) self.assertEqual(ftest.size, 15) self.assertIn('sha512:7d5768d47b6bc27dc4fa7e9732cfa2de506ca262a2749cb108923e5dddffde842bbfee6cb8d692fb43aca0f12946c521cce2633887914ca1f96898478d10ad3f', ftest.hashes) @@ -972,6 +1018,8 @@ class _TestCases(unittest.TestCase): # has the correct created_by_ref self.assertEqual(testobj.created_by_ref, idobj.uuid) + self.assertEqual(testobj.type, 'file') + # and has a signature self.assertIn('sig', testobj) @@ -1026,17 +1074,15 @@ class _TestCases(unittest.TestCase): # and that it can be verified persona.verify(mdobj) - # that when round tripped through pasn1. - a = mdobj.encode() - b = MDBase.decode(a) - def test_objectstore(self): + persona = Persona.load(os.path.join('fixtures', 'sample.persona.pasn1')) objst = ObjectStore.load(os.path.join('fixtures', 'sample.data.pasn1')) objst.loadobj({ 'type': 'metadata', 'uuid': uuid.UUID('c9a1d1e2-3109-4efd-8948-577dc15e44e7'), - 'modified': datetime.datetime(2019, 5, 31, 14, 3, 10), + 'modified': datetime.datetime(2019, 5, 31, 14, 3, 10, + tzinfo=datetime.timezone.utc), 'created_by_ref': self.created_by_ref, 'hashes': [ 'sha256:91751cee0a1ab8414400238a761411daa29643ab4b8243e9a91649e25be53ada' ], 'lang': 'en', @@ -1055,6 +1101,7 @@ class _TestCases(unittest.TestCase): self.assertEqual(r.uuid, uuid.UUID('3e466e06-45de-4ecc-84ba-2d2a3d970e96')) self.assertEqual(r['dc:creator'], [ 'John-Mark Gurney' ]) + # test storing the object store fname = 'testfile.pasn1' objst.store(fname) @@ -1067,12 +1114,19 @@ class _TestCases(unittest.TestCase): self.assertEqual(objs['created_by_ref'], self.created_by_ref.bytes) + # make sure that the read back data matches for i in objs['objects']: i['created_by_ref'] = uuid.UUID(bytes=i['created_by_ref']) i['uuid'] = uuid.UUID(bytes=i['uuid']) self.assertEqual(objst.by_id(i['uuid']), i) + # that a file testfname = os.path.join(self.tempdir, 'test.txt') + + # when registered + objst.loadobj(persona.by_file(testfname)) + + # can be found self.assertEqual(objst.by_file(testfname), [ byid ]) self.assertEqual(objst.by_file(testfname), [ byid ]) @@ -1113,6 +1167,13 @@ class _TestCases(unittest.TestCase): elif special == 'change newfile.txt': with open(newtestfname, 'w') as fp: fp.write('some new contents') + elif special == 'verify store object cnt': + with open(storefname, 'rb') as fp: + pasn1obj = pasn1.loads(fp.read()) + objcnt = len(pasn1obj['objects']) + self.assertEqual(objcnt, cmd['count']) + else: # pragma: no cover + raise ValueError('unhandled special: %s' % repr(special)) continue @@ -1172,8 +1233,6 @@ class _TestCases(unittest.TestCase): import itertools - real_stderr = sys.stderr - with mock.patch('os.path.expanduser', side_effect=expandusermock) \ as eu, mock.patch('medashare.cli.open') as op: # that when opening the store and identity fails diff --git a/ui/setup.py b/ui/setup.py index a4f2219..dd5590f 100644 --- a/ui/setup.py +++ b/ui/setup.py @@ -27,7 +27,7 @@ setup( 'httpx', 'hypercorn', # option, for server only? 'orm', - 'pasn1 @ git+https://www.funkthat.com/gitea/jmg/pasn1.git@01d8efffd7bc3037dcb894ea44dbe959035948c6#egg=pasn1', + 'pasn1 @ git+https://www.funkthat.com/gitea/jmg/pasn1.git@c6c64510b42292557ace2b77272eb32cb647399d#egg=pasn1', 'pydantic[dotenv]', ], extras_require = {