From 6c521cbcf5baa8accc96c7df46dd56abda438ea1 Mon Sep 17 00:00:00 2001 From: John-Mark Gurney Date: Fri, 23 Jul 2021 17:11:54 -0700 Subject: [PATCH] these changes are broken, but about to clean stuff up so checkpoint.. This was partial work to go to python3, but I've now since learned that going twisted is not the way I want to work for web frameworks, so a large amount of this code will be dropped (kleintest.py) or rewritten.. --- ui/cli.py | 130 +++++++++++++++---------------------- ui/fixtures/genfixtures.py | 6 +- ui/kleintest.py | 25 +++---- ui/requirements.txt | 4 +- ui/server.py | 10 +-- 5 files changed, 76 insertions(+), 99 deletions(-) diff --git a/ui/cli.py b/ui/cli.py index e7aafb6..4c1ee7a 100644 --- a/ui/cli.py +++ b/ui/cli.py @@ -21,8 +21,6 @@ import tempfile import unittest import uuid -from contextlib import nested - # The UUID for the namespace representing the path to a file _NAMESPACE_MEDASHARE_PATH = uuid.UUID('f6f36b62-3770-4a68-bc3d-dc3e31e429e6') @@ -31,7 +29,7 @@ _validhashes = set([ 'sha256', 'sha512' ]) _hashlengths = { len(getattr(hashlib, x)().hexdigest()): x for x in _validhashes } def _iterdictlist(obj): - itms = obj.items() + itms = list(obj.items()) itms.sort() for k, v in itms: if isinstance(v, list): @@ -72,7 +70,7 @@ class MDBase(object): _common_properties = [ 'type', 'created_by_ref' ] # XXX - add lang? _common_optional = set(('parent_refs', 'sig')) - _common_names = set(_common_properties + _generated_properties.keys()) + _common_names = set(_common_properties + list(_generated_properties.keys())) def __init__(self, obj={}, **kwargs): obj = copy.deepcopy(obj) @@ -84,20 +82,20 @@ class MDBase(object): if 'type' in obj and obj['type'] != self._type: raise ValueError( 'trying to create the wrong type of object, got: %s, expected: %s' % - (`obj['type']`, `self._type`)) + (repr(obj['type']), repr(self._type))) if 'type' not in obj: obj['type'] = self._type for x in self._common_properties: if x not in obj: - raise ValueError('common property %s not present' % `x`) + raise ValueError('common property %s not present' % repr(x)) - for x, fun in self._instance_properties.iteritems(): + for x, fun in self._instance_properties.items(): if x in obj: obj[x] = fun(obj[x]) - for x, fun in self._generated_properties.iteritems(): + for x, fun in self._generated_properties.items(): if x not in obj: obj[x] = fun() @@ -120,7 +118,7 @@ class MDBase(object): return i(obj) else: raise ValueError('Unable to find class for type %s' % - `ty`) + repr(ty)) def new_version(self, *args): '''For each k, v pari, add the property k as an additional one @@ -140,7 +138,7 @@ class MDBase(object): return self.create_obj(obj) def __repr__(self): # pragma: no cover - return '%s(%s)' % (self.__class__.__name__, `self._obj`) + return '%s(%s)' % (self.__class__.__name__, repr(self._obj)) def __getattr__(self, k): try: @@ -161,13 +159,13 @@ class MDBase(object): return self._obj def __eq__(self, o): - return cmp(self._obj, o) == 0 + return self._obj == o def __contains__(self, k): return k in self._obj def items(self, skipcommon=True): - return [ (k, v) for k, v in self._obj.iteritems() if + return [ (k, v) for k, v in self._obj.items() if not skipcommon or k not in self._common_names ] def encode(self): @@ -188,7 +186,7 @@ class Identity(MDBase): 'created_by_ref' ] _common_optional = set([ x for x in MDBase._common_optional if x != 'parent_refs' ] + [ 'name', 'pubkey' ]) - _common_names = set(_common_properties + MDBase._generated_properties.keys()) + _common_names = set(_common_properties + list(MDBase._generated_properties.keys())) def _trytodict(o): if isinstance(o, uuid.UUID): @@ -196,13 +194,13 @@ def _trytodict(o): try: return 'dict', o.__to_dict__() except Exception: # pragma: no cover - raise TypeError('unable to find __to_dict__ on %s: %s' % (type(o), `o`)) + raise TypeError('unable to find __to_dict__ on %s: %s' % (type(o), repr(o))) class CanonicalCoder(pasn1.ASN1DictCoder): def enc_dict(self, obj, **kwargs): class FakeIter: - def iteritems(self): - itms = obj.items() + def items(self): + itms = list(obj.items()) itms.sort() return iter(itms) @@ -245,7 +243,7 @@ class Persona(object): def __repr__(self): # pragma: no cover r = '' % \ (self._key is not None, self._pubkey is not None, - `self._identity`) + repr(self._identity)) return r @@ -284,7 +282,7 @@ class Persona(object): '''Store the Persona to a file. If there is a private key associated w/ the Persona, it will be saved as well.''' - with open(fname, 'w') as fp: + with open(fname, 'wb') as fp: obj = { 'identity': self._identity, } @@ -299,7 +297,7 @@ class Persona(object): def load(cls, fname): '''Load the Persona from the provided file.''' - with open(fname) as fp: + with open(fname, 'rb') as fp: objs = _asn1coder.loads(fp.read()) kwargs = {} @@ -393,7 +391,8 @@ class ObjectStore(object): hash = _hashlengths[len(hashstr)] value = hashstr - if strict and len(str(value).translate(None, string.hexdigits.lower())) != 0: + bvalue = value.encode('ascii') + if strict and len(bvalue.translate(None, string.hexdigits.lower().encode('ascii'))) != 0: raise ValueError('value has invalid hex digits (must be lower case)', value) if hash in _validhashes: @@ -408,10 +407,10 @@ class ObjectStore(object): '''Write out the objects in the store to the file named fname.''' - with open(fname, 'w') as fp: + with open(fname, 'wb') as fp: obj = { 'created_by_ref': self._created_by_ref, - 'objects': self._uuids.values(), + 'objects': list(self._uuids.values()), } fp.write(_asn1coder.dumps(obj)) @@ -433,7 +432,7 @@ class ObjectStore(object): The objects will be accessible via other methods.''' - with open(fname) as fp: + with open(fname, 'rb') as fp: objs = _asn1coder.loads(fp.read()) obj = cls(objs['created_by_ref']) @@ -481,7 +480,7 @@ class ObjectStore(object): def _hashfile(fname): hash = getattr(hashlib, _defaulthash)() - with open(fname) as fp: + with open(fname, 'rb') as fp: r = fp.read() hash.update(r) @@ -521,8 +520,7 @@ def enumeratedir(_dir, created_by_ref): Returned is a list of FileObjects.''' - return map(lambda x: FileObject.from_file(os.path.join(_dir, x), created_by_ref), - os.listdir(_dir)) + return [FileObject.from_file(os.path.join(_dir, x), created_by_ref) for x in os.listdir(_dir)] def main(): from optparse import OptionParser @@ -545,13 +543,13 @@ def main(): options, args = parser.parse_args() # this is shared between generateident and add - addprops = map(lambda x: x.split('=', 1), options.add) + addprops = [x.split('=', 1) for x in options.add] if options.generateident or options.updateident or options.printpub: identfname = os.path.expanduser('~/.medashare_identity.pasn1') if options.generateident and os.path.exists(identfname): - print >>sys.stderr, 'Error: Identity already created.' + print('Error: Identity already created.', file=sys.stderr) sys.exit(1) if options.generateident: @@ -561,7 +559,7 @@ def main(): persona = Persona.load(identfname) if options.printpub: - print persona.get_pubkey() + print(persona.get_pubkey()) return persona.new_version(*addprops) @@ -578,7 +576,7 @@ def main(): for j in objstr.by_file(i): #print >>sys.stderr, `j._obj` for k, v in _iterdictlist(j): - print '%s:\t%s' % (k, v) + print('%s:\t%s' % (k, v)) elif options.add: for i in args: for j in objstr.by_file(i): @@ -607,14 +605,14 @@ if __name__ == '__main__': # pragma: no cover class _TestCononicalCoder(unittest.TestCase): def test_con(self): obja = { 'foo': 23984732, 'a': 5, 'b': 6, 'something': '2398472398723498273dfasdfjlaksdfj' } - objaitems = obja.items() + objaitems = list(obja.items()) objaitems.sort() objb = dict(objaitems) self.assertEqual(obja, objb) # This is to make sure that item order changed - self.assertNotEqual(obja.items(), objb.items()) + self.assertNotEqual(list(obja.items()), list(objb.items())) astr = pasn1.dumps(obja) bstr = pasn1.dumps(objb) @@ -709,7 +707,7 @@ class _TestCases(unittest.TestCase): eobj = _asn1coder.loads(coded) # the uuid property is a str instance - self.assertIsInstance(eobj['uuid'], str) + self.assertIsInstance(eobj['uuid'], bytes) # and has the length of 16 self.assertEqual(len(eobj['uuid']), 16) @@ -782,7 +780,7 @@ class _TestCases(unittest.TestCase): # and that when you get it's properties oobj = objst.by_id(oid) - odict = dict(oobj.items()) + odict = dict(list(oobj.items())) # that is has the overlays property self.assertEqual(odict['parent_refs'], [ bid ]) @@ -811,7 +809,7 @@ class _TestCases(unittest.TestCase): # that the pubkey property is present idobj = persona.get_identity() - self.assertIsInstance(idobj['pubkey'], str) + self.assertIsInstance(idobj['pubkey'], bytes) # that get_pubkey returns the correct thing pubstr = _asn1coder.dumps([ idobj.uuid, idobj['pubkey'] ]) @@ -819,7 +817,7 @@ class _TestCases(unittest.TestCase): base58.b58encode_check(pubstr)) # and that there is a signature - self.assertIsInstance(idobj['sig'], str) + self.assertIsInstance(idobj['sig'], bytes) # and that it can verify itself persona.verify(idobj) @@ -917,12 +915,12 @@ class _TestCases(unittest.TestCase): r = byid self.assertEqual(r.uuid, uuid.UUID('3e466e06-45de-4ecc-84ba-2d2a3d970e96')) - self.assertEqual(r['dc:creator'], [ u'John-Mark Gurney' ]) + self.assertEqual(r['dc:creator'], [ 'John-Mark Gurney' ]) fname = 'testfile.pasn1' objst.store(fname) - with open(fname) as fp: + with open(fname, 'rb') as fp: objs = _asn1coder.loads(fp.read()) os.unlink(fname) @@ -968,15 +966,13 @@ class _TestCases(unittest.TestCase): testfname = os.path.join(self.tempdir, 'test.txt') import sys - import StringIO + import io import itertools with mock.patch('os.path.expanduser', side_effect=expandusermock) \ as eu: # that generating a new identity - with nested(mock.patch('sys.stdout', - StringIO.StringIO()), mock.patch('sys.argv', - [ 'progname', '-g', '-a', 'name=A Test User' ])) as (stdout, argv): + with mock.patch('sys.stdout', io.StringIO()) as stdout, mock.patch('sys.argv', [ 'progname', '-g', '-a', 'name=A Test User' ]) as argv: main() # does not output anything @@ -993,15 +989,13 @@ class _TestCases(unittest.TestCase): self.assertEqual(pident.name, 'A Test User') # that when generating an identity when one already exists - with nested(mock.patch('sys.stderr', - StringIO.StringIO()), mock.patch('sys.argv', - [ 'progname', '-g', '-a', 'name=A Test User' ])) as (stderr, argv): + with mock.patch('sys.stderr', io.StringIO()) as stderr, mock.patch('sys.argv', [ 'progname', '-g', '-a', 'name=A Test User' ]) as argv: # that it exits with self.assertRaises(SystemExit) as cm: main() # with error code 1 - self.assertEqual(cm.exception[0], 1) + self.assertEqual(cm.exception.code, 1) # and outputs an error message self.assertEqual(stderr.getvalue(), @@ -1011,9 +1005,7 @@ class _TestCases(unittest.TestCase): eu.assert_called_with('~/.medashare_identity.pasn1') # that when updating the identity - with nested(mock.patch('sys.stdout', - StringIO.StringIO()), mock.patch('sys.argv', - [ 'progname', '-i', '-a', 'name=Changed Name' ])) as (stdout, argv): + with mock.patch('sys.stdout', io.StringIO()) as stdout, mock.patch('sys.argv', [ 'progname', '-i', '-a', 'name=Changed Name' ]) as argv: main() # it doesn't output anything @@ -1038,9 +1030,7 @@ class _TestCases(unittest.TestCase): self.assertTrue(persona.verify(nident)) # that when asked to print the public key - with nested(mock.patch('sys.stdout', - StringIO.StringIO()), mock.patch('sys.argv', - [ 'progname', '-p' ])) as (stdout, argv): + with mock.patch('sys.stdout', io.StringIO()) as stdout, mock.patch('sys.argv', [ 'progname', '-p' ]) as argv: main() # the correct key is printed @@ -1050,58 +1040,40 @@ class _TestCases(unittest.TestCase): # and looked up the correct file eu.assert_called_with('~/.medashare_identity.pasn1') - with nested(mock.patch('sys.stdout', - StringIO.StringIO()), mock.patch('sys.argv', - [ 'progname', '-l', testfname ])) as (stdout, argv): + with mock.patch('sys.stdout', io.StringIO()) as stdout, mock.patch('sys.argv', [ 'progname', '-l', testfname ]) as argv: main() self.assertEqual(stdout.getvalue(), 'dc:creator:\tJohn-Mark Gurney\nhashes:\tsha256:91751cee0a1ab8414400238a761411daa29643ab4b8243e9a91649e25be53ada\nhashes:\tsha512:7d5768d47b6bc27dc4fa7e9732cfa2de506ca262a2749cb108923e5dddffde842bbfee6cb8d692fb43aca0f12946c521cce2633887914ca1f96898478d10ad3f\nlang:\ten\n') eu.assert_called_with('~/.medashare_store.pasn1') - with nested(mock.patch('sys.stdout', - StringIO.StringIO()), mock.patch('sys.argv', - [ 'progname', '-a', 'dc:creator=Another user', '-a', 'foo=bar=baz', testfname ])) as (stdout, argv): + with mock.patch('sys.stdout', io.StringIO()) as stdout, mock.patch('sys.argv', [ 'progname', '-a', 'dc:creator=Another user', '-a', 'foo=bar=baz', testfname ]) as argv: main() - with nested(mock.patch('sys.stdout', - StringIO.StringIO()), mock.patch('sys.argv', - [ 'progname', '-l', testfname ])) as (stdout, argv): + with mock.patch('sys.stdout', io.StringIO()) as stdout, mock.patch('sys.argv', [ 'progname', '-l', testfname ]) as argv: main() self.assertEqual(stdout.getvalue(), 'dc:creator:\tAnother user\ndc:creator:\tJohn-Mark Gurney\nfoo:\tbar=baz\nhashes:\tsha256:91751cee0a1ab8414400238a761411daa29643ab4b8243e9a91649e25be53ada\nhashes:\tsha512:7d5768d47b6bc27dc4fa7e9732cfa2de506ca262a2749cb108923e5dddffde842bbfee6cb8d692fb43aca0f12946c521cce2633887914ca1f96898478d10ad3f\nlang:\ten\n') - with nested(mock.patch('sys.stdout', - StringIO.StringIO()), mock.patch('sys.argv', - [ 'progname', '-d', 'dc:creator', testfname ])) as (stdout, argv): + with mock.patch('sys.stdout', io.StringIO()) as stdout, mock.patch('sys.argv', [ 'progname', '-d', 'dc:creator', testfname ]) as argv: main() - with nested(mock.patch('sys.stdout', - StringIO.StringIO()), mock.patch('sys.argv', - [ 'progname', '-l', testfname ])) as (stdout, argv): + with mock.patch('sys.stdout', io.StringIO()) as stdout, mock.patch('sys.argv', [ 'progname', '-l', testfname ]) as argv: main() self.assertEqual(stdout.getvalue(), 'foo:\tbar=baz\nhashes:\tsha256:91751cee0a1ab8414400238a761411daa29643ab4b8243e9a91649e25be53ada\nhashes:\tsha512:7d5768d47b6bc27dc4fa7e9732cfa2de506ca262a2749cb108923e5dddffde842bbfee6cb8d692fb43aca0f12946c521cce2633887914ca1f96898478d10ad3f\nlang:\ten\n') - with nested(mock.patch('sys.stdout', - StringIO.StringIO()), mock.patch('sys.argv', - [ 'progname', '-a', 'foo=bleh', testfname ])) as (stdout, argv): + with mock.patch('sys.stdout', io.StringIO()) as stdout, mock.patch('sys.argv', [ 'progname', '-a', 'foo=bleh', testfname ]) as argv: main() - with nested(mock.patch('sys.stdout', - StringIO.StringIO()), mock.patch('sys.argv', - [ 'progname', '-l', testfname ])) as (stdout, argv): + with mock.patch('sys.stdout', io.StringIO()) as stdout, mock.patch('sys.argv', [ 'progname', '-l', testfname ]) as argv: main() self.assertEqual(stdout.getvalue(), 'foo:\tbar=baz\nfoo:\tbleh\nhashes:\tsha256:91751cee0a1ab8414400238a761411daa29643ab4b8243e9a91649e25be53ada\nhashes:\tsha512:7d5768d47b6bc27dc4fa7e9732cfa2de506ca262a2749cb108923e5dddffde842bbfee6cb8d692fb43aca0f12946c521cce2633887914ca1f96898478d10ad3f\nlang:\ten\n') - with nested(mock.patch('sys.stdout', - StringIO.StringIO()), mock.patch('sys.argv', - [ 'progname', '-d', 'foo=bar=baz', testfname ])) as (stdout, argv): + with mock.patch('sys.stdout', io.StringIO()) as stdout, mock.patch('sys.argv', [ 'progname', '-d', 'foo=bar=baz', testfname ]) as argv: main() - with nested(mock.patch('sys.stdout', - StringIO.StringIO()), mock.patch('sys.argv', - [ 'progname', '-l', testfname ])) as (stdout, argv): + with mock.patch('sys.stdout', io.StringIO()) as stdout, mock.patch('sys.argv', [ 'progname', '-l', testfname ]) as argv: main() self.assertEqual(stdout.getvalue(), 'foo:\tbleh\nhashes:\tsha256:91751cee0a1ab8414400238a761411daa29643ab4b8243e9a91649e25be53ada\nhashes:\tsha512:7d5768d47b6bc27dc4fa7e9732cfa2de506ca262a2749cb108923e5dddffde842bbfee6cb8d692fb43aca0f12946c521cce2633887914ca1f96898478d10ad3f\nlang:\ten\n') diff --git a/ui/fixtures/genfixtures.py b/ui/fixtures/genfixtures.py index d6424b1..3de1e6b 100644 --- a/ui/fixtures/genfixtures.py +++ b/ui/fixtures/genfixtures.py @@ -7,19 +7,19 @@ persona = cli.Persona() persona.generate_key() cbr = persona.get_identity().uuid objst = cli.ObjectStore(cbr) -map(objst.loadobj, +list(map(objst.loadobj, [ { 'type': 'metadata', 'uuid': uuid.UUID('3e466e06-45de-4ecc-84ba-2d2a3d970e96'), 'created_by_ref': cbr, 'modified': datetime.datetime(2019, 5, 31, 14, 5, 3), - 'dc:creator': [ u'John-Mark Gurney' ], + 'dc:creator': [ 'John-Mark Gurney' ], 'hashes': [ 'sha256:91751cee0a1ab8414400238a761411daa29643ab4b8243e9a91649e25be53ada', 'sha512:7d5768d47b6bc27dc4fa7e9732cfa2de506ca262a2749cb108923e5dddffde842bbfee6cb8d692fb43aca0f12946c521cce2633887914ca1f96898478d10ad3f' ], 'lang': 'en' } ] -) +)) objst.store('sample.data.pasn1') persona.store('sample.persona.pasn1') diff --git a/ui/kleintest.py b/ui/kleintest.py index b612717..f954262 100644 --- a/ui/kleintest.py +++ b/ui/kleintest.py @@ -5,15 +5,15 @@ from klein.interfaces import IKleinRequest from twisted.internet.defer import Deferred from twisted.trial import unittest from twisted.web.http_headers import Headers -from zope.interface import implements -from StringIO import StringIO +from zope.interface import implementer +from io import StringIO from requests.structures import CaseInsensitiveDict __all__ = [ 'FakeRequests', ] +# https://github.com/twisted/twisted/blob/twisted-19.7.0/src/twisted/web/http.py#L664 +@implementer(IKleinRequest) class FakeHTTPRequest(object): - # https://github.com/twisted/twisted/blob/twisted-19.7.0/src/twisted/web/http.py#L664 - implements(IKleinRequest) code = 200 @@ -26,7 +26,7 @@ class FakeHTTPRequest(object): self.path = uri self.prepath = [] - self.postpath = uri.split('/') + self.postpath = uri.split(b'/') self.method = meth self.notifications = [] self.finished = False @@ -38,10 +38,10 @@ class FakeHTTPRequest(object): self.code = code def getRequestHostname(self): - return '' + return b'' def getHost(self): - return '' + return b'' def isSecure(self): return False @@ -74,6 +74,7 @@ class FakeRequestsResponse(object): self._req = req self._io = StringIO() req.write = self.write + self.status_code = 500 def _finished(self, arg): if arg is not None: # pragma: no cover @@ -118,7 +119,7 @@ class FakeRequests(object): self._res = app.resource() def _makerequest(self, method, url, data=''): - if url[0] != '/': + if url[0:1] != b'/': raise ValueError('url must be absolute (start w/ a slash)') req = FakeHTTPRequest('GET', url, data) @@ -133,6 +134,7 @@ class FakeRequests(object): def get(self, url): '''Return a response for the passed in url.''' + print('ktg:', repr(url)) return self._makerequest('GET', url) def put(self, url, data=''): @@ -150,7 +152,7 @@ class TestFakeRequests(unittest.TestCase): def home(request): request.setHeader('x-testing', 'value') - return 'hello' + return b'hello' @app.route('/500') def causeerror(request): @@ -171,10 +173,11 @@ class TestFakeRequests(unittest.TestCase): self.requests = FakeRequests(app) def test_bad(self): - self.assertRaises(ValueError, self.requests.get, 'foobar') + self.assertRaises(ValueError, self.requests.get, b'foobar') def test_basic(self): - r = self.requests.get('/') + r = self.requests.get(b'/') + print(repr(r)) self.assertEqual(r.status_code, 200) self.assertEqual(r.text, 'hello') self.assertEqual(r.headers['X-testing'], 'value') diff --git a/ui/requirements.txt b/ui/requirements.txt index 81ca6d5..dab9871 100644 --- a/ui/requirements.txt +++ b/ui/requirements.txt @@ -1,7 +1,9 @@ urwid --e git://github.com/jmgurney/pasn1@27ff594a2609c07205753ce24f74d8f45a7ea418#egg=pasn1 +-e git+https://www.funkthat.com/gitea/jmg/pasn1.git@01d8efffd7bc3037dcb894ea44dbe959035948c6#egg=pasn1 coverage mock klein cryptography base58 +# for kleintest +requests diff --git a/ui/server.py b/ui/server.py index 5bc6c7c..a1101fd 100644 --- a/ui/server.py +++ b/ui/server.py @@ -9,7 +9,6 @@ # Going Async from Flask to Twisted Klein: https://crossbario.com/blog/Going-Asynchronous-from-Flask-to-Twisted-Klein/ # Klein POST docs: https://klein.readthedocs.io/en/latest/examples/handlingpost.html -from contextlib import nested from klein import Klein from kleintest import * from twisted.trial import unittest @@ -216,9 +215,9 @@ class _TestCases(_BaseServerTest): persona = Persona() persona.generate_key() - with nested(mock.patch('server.MEDAServer.addpubkey'), + with mock.patch('server.MEDAServer.addpubkey') as addpub, \ mock.patch('sys.argv', [ 'progname', '-a', - persona.get_pubkey() ])) as (addpub, argv): + persona.get_pubkey() ]) as argv: main() addpub.assert_called_with(persona.get_pubkey()) @@ -254,10 +253,11 @@ class _TestPostConfig(_BaseServerTest): def test_hashlookup(self): # that the hash of a file - h = hashlib.sha256(open('fixtures/testfiles/test.txt').read()).hexdigest() + h = hashlib.sha256(open('fixtures/testfiles/test.txt', 'rb').read()).hexdigest() # when looked up - r = self.requests.get('/lookup/%s' % h) + print('hl') + r = self.requests.get(('/lookup/%s' % h).encode('ascii')) # returns a 404 self.assertEqual(r.status_code, 404)