Browse Source

these changes are broken, but about to clean stuff up so checkpoint..

This was partial work to go to python3, but I've now since learned that
going twisted is not the way I want to work for web frameworks, so a
large amount of this code will be dropped (kleintest.py) or rewritten..
main
John-Mark Gurney 3 years ago
parent
commit
6c521cbcf5
5 changed files with 76 additions and 99 deletions
  1. +51
    -79
      ui/cli.py
  2. +3
    -3
      ui/fixtures/genfixtures.py
  3. +14
    -11
      ui/kleintest.py
  4. +3
    -1
      ui/requirements.txt
  5. +5
    -5
      ui/server.py

+ 51
- 79
ui/cli.py View File

@@ -21,8 +21,6 @@ import tempfile
import unittest
import uuid

from contextlib import nested

# The UUID for the namespace representing the path to a file
_NAMESPACE_MEDASHARE_PATH = uuid.UUID('f6f36b62-3770-4a68-bc3d-dc3e31e429e6')

@@ -31,7 +29,7 @@ _validhashes = set([ 'sha256', 'sha512' ])
_hashlengths = { len(getattr(hashlib, x)().hexdigest()): x for x in _validhashes }

def _iterdictlist(obj):
itms = obj.items()
itms = list(obj.items())
itms.sort()
for k, v in itms:
if isinstance(v, list):
@@ -72,7 +70,7 @@ class MDBase(object):

_common_properties = [ 'type', 'created_by_ref' ] # XXX - add lang?
_common_optional = set(('parent_refs', 'sig'))
_common_names = set(_common_properties + _generated_properties.keys())
_common_names = set(_common_properties + list(_generated_properties.keys()))

def __init__(self, obj={}, **kwargs):
obj = copy.deepcopy(obj)
@@ -84,20 +82,20 @@ class MDBase(object):
if 'type' in obj and obj['type'] != self._type:
raise ValueError(
'trying to create the wrong type of object, got: %s, expected: %s' %
(`obj['type']`, `self._type`))
(repr(obj['type']), repr(self._type)))

if 'type' not in obj:
obj['type'] = self._type

for x in self._common_properties:
if x not in obj:
raise ValueError('common property %s not present' % `x`)
raise ValueError('common property %s not present' % repr(x))

for x, fun in self._instance_properties.iteritems():
for x, fun in self._instance_properties.items():
if x in obj:
obj[x] = fun(obj[x])

for x, fun in self._generated_properties.iteritems():
for x, fun in self._generated_properties.items():
if x not in obj:
obj[x] = fun()

@@ -120,7 +118,7 @@ class MDBase(object):
return i(obj)
else:
raise ValueError('Unable to find class for type %s' %
`ty`)
repr(ty))

def new_version(self, *args):
'''For each k, v pari, add the property k as an additional one
@@ -140,7 +138,7 @@ class MDBase(object):
return self.create_obj(obj)

def __repr__(self): # pragma: no cover
return '%s(%s)' % (self.__class__.__name__, `self._obj`)
return '%s(%s)' % (self.__class__.__name__, repr(self._obj))

def __getattr__(self, k):
try:
@@ -161,13 +159,13 @@ class MDBase(object):
return self._obj

def __eq__(self, o):
return cmp(self._obj, o) == 0
return self._obj == o

def __contains__(self, k):
return k in self._obj

def items(self, skipcommon=True):
return [ (k, v) for k, v in self._obj.iteritems() if
return [ (k, v) for k, v in self._obj.items() if
not skipcommon or k not in self._common_names ]

def encode(self):
@@ -188,7 +186,7 @@ class Identity(MDBase):
'created_by_ref' ]
_common_optional = set([ x for x in MDBase._common_optional if x !=
'parent_refs' ] + [ 'name', 'pubkey' ])
_common_names = set(_common_properties + MDBase._generated_properties.keys())
_common_names = set(_common_properties + list(MDBase._generated_properties.keys()))

def _trytodict(o):
if isinstance(o, uuid.UUID):
@@ -196,13 +194,13 @@ def _trytodict(o):
try:
return 'dict', o.__to_dict__()
except Exception: # pragma: no cover
raise TypeError('unable to find __to_dict__ on %s: %s' % (type(o), `o`))
raise TypeError('unable to find __to_dict__ on %s: %s' % (type(o), repr(o)))

class CanonicalCoder(pasn1.ASN1DictCoder):
def enc_dict(self, obj, **kwargs):
class FakeIter:
def iteritems(self):
itms = obj.items()
def items(self):
itms = list(obj.items())
itms.sort()

return iter(itms)
@@ -245,7 +243,7 @@ class Persona(object):
def __repr__(self): # pragma: no cover
r = '<Persona: has key: %s, has pubkey: %s, identity: %s>' % \
(self._key is not None, self._pubkey is not None,
`self._identity`)
repr(self._identity))

return r

@@ -284,7 +282,7 @@ class Persona(object):
'''Store the Persona to a file. If there is a private
key associated w/ the Persona, it will be saved as well.'''

with open(fname, 'w') as fp:
with open(fname, 'wb') as fp:
obj = {
'identity': self._identity,
}
@@ -299,7 +297,7 @@ class Persona(object):
def load(cls, fname):
'''Load the Persona from the provided file.'''

with open(fname) as fp:
with open(fname, 'rb') as fp:
objs = _asn1coder.loads(fp.read())

kwargs = {}
@@ -393,7 +391,8 @@ class ObjectStore(object):
hash = _hashlengths[len(hashstr)]
value = hashstr

if strict and len(str(value).translate(None, string.hexdigits.lower())) != 0:
bvalue = value.encode('ascii')
if strict and len(bvalue.translate(None, string.hexdigits.lower().encode('ascii'))) != 0:
raise ValueError('value has invalid hex digits (must be lower case)', value)

if hash in _validhashes:
@@ -408,10 +407,10 @@ class ObjectStore(object):
'''Write out the objects in the store to the file named
fname.'''

with open(fname, 'w') as fp:
with open(fname, 'wb') as fp:
obj = {
'created_by_ref': self._created_by_ref,
'objects': self._uuids.values(),
'objects': list(self._uuids.values()),
}
fp.write(_asn1coder.dumps(obj))

@@ -433,7 +432,7 @@ class ObjectStore(object):

The objects will be accessible via other methods.'''

with open(fname) as fp:
with open(fname, 'rb') as fp:
objs = _asn1coder.loads(fp.read())

obj = cls(objs['created_by_ref'])
@@ -481,7 +480,7 @@ class ObjectStore(object):

def _hashfile(fname):
hash = getattr(hashlib, _defaulthash)()
with open(fname) as fp:
with open(fname, 'rb') as fp:
r = fp.read()
hash.update(r)

@@ -521,8 +520,7 @@ def enumeratedir(_dir, created_by_ref):

Returned is a list of FileObjects.'''

return map(lambda x: FileObject.from_file(os.path.join(_dir, x), created_by_ref),
os.listdir(_dir))
return [FileObject.from_file(os.path.join(_dir, x), created_by_ref) for x in os.listdir(_dir)]

def main():
from optparse import OptionParser
@@ -545,13 +543,13 @@ def main():
options, args = parser.parse_args()

# this is shared between generateident and add
addprops = map(lambda x: x.split('=', 1), options.add)
addprops = [x.split('=', 1) for x in options.add]

if options.generateident or options.updateident or options.printpub:
identfname = os.path.expanduser('~/.medashare_identity.pasn1')

if options.generateident and os.path.exists(identfname):
print >>sys.stderr, 'Error: Identity already created.'
print('Error: Identity already created.', file=sys.stderr)
sys.exit(1)

if options.generateident:
@@ -561,7 +559,7 @@ def main():
persona = Persona.load(identfname)

if options.printpub:
print persona.get_pubkey()
print(persona.get_pubkey())
return

persona.new_version(*addprops)
@@ -578,7 +576,7 @@ def main():
for j in objstr.by_file(i):
#print >>sys.stderr, `j._obj`
for k, v in _iterdictlist(j):
print '%s:\t%s' % (k, v)
print('%s:\t%s' % (k, v))
elif options.add:
for i in args:
for j in objstr.by_file(i):
@@ -607,14 +605,14 @@ if __name__ == '__main__': # pragma: no cover
class _TestCononicalCoder(unittest.TestCase):
def test_con(self):
obja = { 'foo': 23984732, 'a': 5, 'b': 6, 'something': '2398472398723498273dfasdfjlaksdfj' }
objaitems = obja.items()
objaitems = list(obja.items())
objaitems.sort()
objb = dict(objaitems)

self.assertEqual(obja, objb)

# This is to make sure that item order changed
self.assertNotEqual(obja.items(), objb.items())
self.assertNotEqual(list(obja.items()), list(objb.items()))

astr = pasn1.dumps(obja)
bstr = pasn1.dumps(objb)
@@ -709,7 +707,7 @@ class _TestCases(unittest.TestCase):
eobj = _asn1coder.loads(coded)

# the uuid property is a str instance
self.assertIsInstance(eobj['uuid'], str)
self.assertIsInstance(eobj['uuid'], bytes)

# and has the length of 16
self.assertEqual(len(eobj['uuid']), 16)
@@ -782,7 +780,7 @@ class _TestCases(unittest.TestCase):

# and that when you get it's properties
oobj = objst.by_id(oid)
odict = dict(oobj.items())
odict = dict(list(oobj.items()))

# that is has the overlays property
self.assertEqual(odict['parent_refs'], [ bid ])
@@ -811,7 +809,7 @@ class _TestCases(unittest.TestCase):

# that the pubkey property is present
idobj = persona.get_identity()
self.assertIsInstance(idobj['pubkey'], str)
self.assertIsInstance(idobj['pubkey'], bytes)

# that get_pubkey returns the correct thing
pubstr = _asn1coder.dumps([ idobj.uuid, idobj['pubkey'] ])
@@ -819,7 +817,7 @@ class _TestCases(unittest.TestCase):
base58.b58encode_check(pubstr))

# and that there is a signature
self.assertIsInstance(idobj['sig'], str)
self.assertIsInstance(idobj['sig'], bytes)

# and that it can verify itself
persona.verify(idobj)
@@ -917,12 +915,12 @@ class _TestCases(unittest.TestCase):
r = byid

self.assertEqual(r.uuid, uuid.UUID('3e466e06-45de-4ecc-84ba-2d2a3d970e96'))
self.assertEqual(r['dc:creator'], [ u'John-Mark Gurney' ])
self.assertEqual(r['dc:creator'], [ 'John-Mark Gurney' ])

fname = 'testfile.pasn1'
objst.store(fname)

with open(fname) as fp:
with open(fname, 'rb') as fp:
objs = _asn1coder.loads(fp.read())

os.unlink(fname)
@@ -968,15 +966,13 @@ class _TestCases(unittest.TestCase):
testfname = os.path.join(self.tempdir, 'test.txt')

import sys
import StringIO
import io
import itertools

with mock.patch('os.path.expanduser', side_effect=expandusermock) \
as eu:
# that generating a new identity
with nested(mock.patch('sys.stdout',
StringIO.StringIO()), mock.patch('sys.argv',
[ 'progname', '-g', '-a', 'name=A Test User' ])) as (stdout, argv):
with mock.patch('sys.stdout', io.StringIO()) as stdout, mock.patch('sys.argv', [ 'progname', '-g', '-a', 'name=A Test User' ]) as argv:
main()

# does not output anything
@@ -993,15 +989,13 @@ class _TestCases(unittest.TestCase):
self.assertEqual(pident.name, 'A Test User')

# that when generating an identity when one already exists
with nested(mock.patch('sys.stderr',
StringIO.StringIO()), mock.patch('sys.argv',
[ 'progname', '-g', '-a', 'name=A Test User' ])) as (stderr, argv):
with mock.patch('sys.stderr', io.StringIO()) as stderr, mock.patch('sys.argv', [ 'progname', '-g', '-a', 'name=A Test User' ]) as argv:
# that it exits
with self.assertRaises(SystemExit) as cm:
main()

# with error code 1
self.assertEqual(cm.exception[0], 1)
self.assertEqual(cm.exception.code, 1)

# and outputs an error message
self.assertEqual(stderr.getvalue(),
@@ -1011,9 +1005,7 @@ class _TestCases(unittest.TestCase):
eu.assert_called_with('~/.medashare_identity.pasn1')

# that when updating the identity
with nested(mock.patch('sys.stdout',
StringIO.StringIO()), mock.patch('sys.argv',
[ 'progname', '-i', '-a', 'name=Changed Name' ])) as (stdout, argv):
with mock.patch('sys.stdout', io.StringIO()) as stdout, mock.patch('sys.argv', [ 'progname', '-i', '-a', 'name=Changed Name' ]) as argv:
main()

# it doesn't output anything
@@ -1038,9 +1030,7 @@ class _TestCases(unittest.TestCase):
self.assertTrue(persona.verify(nident))

# that when asked to print the public key
with nested(mock.patch('sys.stdout',
StringIO.StringIO()), mock.patch('sys.argv',
[ 'progname', '-p' ])) as (stdout, argv):
with mock.patch('sys.stdout', io.StringIO()) as stdout, mock.patch('sys.argv', [ 'progname', '-p' ]) as argv:
main()

# the correct key is printed
@@ -1050,58 +1040,40 @@ class _TestCases(unittest.TestCase):
# and looked up the correct file
eu.assert_called_with('~/.medashare_identity.pasn1')

with nested(mock.patch('sys.stdout',
StringIO.StringIO()), mock.patch('sys.argv',
[ 'progname', '-l', testfname ])) as (stdout, argv):
with mock.patch('sys.stdout', io.StringIO()) as stdout, mock.patch('sys.argv', [ 'progname', '-l', testfname ]) as argv:
main()
self.assertEqual(stdout.getvalue(),
'dc:creator:\tJohn-Mark Gurney\nhashes:\tsha256:91751cee0a1ab8414400238a761411daa29643ab4b8243e9a91649e25be53ada\nhashes:\tsha512:7d5768d47b6bc27dc4fa7e9732cfa2de506ca262a2749cb108923e5dddffde842bbfee6cb8d692fb43aca0f12946c521cce2633887914ca1f96898478d10ad3f\nlang:\ten\n')
eu.assert_called_with('~/.medashare_store.pasn1')

with nested(mock.patch('sys.stdout',
StringIO.StringIO()), mock.patch('sys.argv',
[ 'progname', '-a', 'dc:creator=Another user', '-a', 'foo=bar=baz', testfname ])) as (stdout, argv):
with mock.patch('sys.stdout', io.StringIO()) as stdout, mock.patch('sys.argv', [ 'progname', '-a', 'dc:creator=Another user', '-a', 'foo=bar=baz', testfname ]) as argv:
main()

with nested(mock.patch('sys.stdout',
StringIO.StringIO()), mock.patch('sys.argv',
[ 'progname', '-l', testfname ])) as (stdout, argv):
with mock.patch('sys.stdout', io.StringIO()) as stdout, mock.patch('sys.argv', [ 'progname', '-l', testfname ]) as argv:
main()
self.assertEqual(stdout.getvalue(),
'dc:creator:\tAnother user\ndc:creator:\tJohn-Mark Gurney\nfoo:\tbar=baz\nhashes:\tsha256:91751cee0a1ab8414400238a761411daa29643ab4b8243e9a91649e25be53ada\nhashes:\tsha512:7d5768d47b6bc27dc4fa7e9732cfa2de506ca262a2749cb108923e5dddffde842bbfee6cb8d692fb43aca0f12946c521cce2633887914ca1f96898478d10ad3f\nlang:\ten\n')

with nested(mock.patch('sys.stdout',
StringIO.StringIO()), mock.patch('sys.argv',
[ 'progname', '-d', 'dc:creator', testfname ])) as (stdout, argv):
with mock.patch('sys.stdout', io.StringIO()) as stdout, mock.patch('sys.argv', [ 'progname', '-d', 'dc:creator', testfname ]) as argv:
main()

with nested(mock.patch('sys.stdout',
StringIO.StringIO()), mock.patch('sys.argv',
[ 'progname', '-l', testfname ])) as (stdout, argv):
with mock.patch('sys.stdout', io.StringIO()) as stdout, mock.patch('sys.argv', [ 'progname', '-l', testfname ]) as argv:
main()
self.assertEqual(stdout.getvalue(),
'foo:\tbar=baz\nhashes:\tsha256:91751cee0a1ab8414400238a761411daa29643ab4b8243e9a91649e25be53ada\nhashes:\tsha512:7d5768d47b6bc27dc4fa7e9732cfa2de506ca262a2749cb108923e5dddffde842bbfee6cb8d692fb43aca0f12946c521cce2633887914ca1f96898478d10ad3f\nlang:\ten\n')

with nested(mock.patch('sys.stdout',
StringIO.StringIO()), mock.patch('sys.argv',
[ 'progname', '-a', 'foo=bleh', testfname ])) as (stdout, argv):
with mock.patch('sys.stdout', io.StringIO()) as stdout, mock.patch('sys.argv', [ 'progname', '-a', 'foo=bleh', testfname ]) as argv:
main()

with nested(mock.patch('sys.stdout',
StringIO.StringIO()), mock.patch('sys.argv',
[ 'progname', '-l', testfname ])) as (stdout, argv):
with mock.patch('sys.stdout', io.StringIO()) as stdout, mock.patch('sys.argv', [ 'progname', '-l', testfname ]) as argv:
main()
self.assertEqual(stdout.getvalue(),
'foo:\tbar=baz\nfoo:\tbleh\nhashes:\tsha256:91751cee0a1ab8414400238a761411daa29643ab4b8243e9a91649e25be53ada\nhashes:\tsha512:7d5768d47b6bc27dc4fa7e9732cfa2de506ca262a2749cb108923e5dddffde842bbfee6cb8d692fb43aca0f12946c521cce2633887914ca1f96898478d10ad3f\nlang:\ten\n')

with nested(mock.patch('sys.stdout',
StringIO.StringIO()), mock.patch('sys.argv',
[ 'progname', '-d', 'foo=bar=baz', testfname ])) as (stdout, argv):
with mock.patch('sys.stdout', io.StringIO()) as stdout, mock.patch('sys.argv', [ 'progname', '-d', 'foo=bar=baz', testfname ]) as argv:
main()

with nested(mock.patch('sys.stdout',
StringIO.StringIO()), mock.patch('sys.argv',
[ 'progname', '-l', testfname ])) as (stdout, argv):
with mock.patch('sys.stdout', io.StringIO()) as stdout, mock.patch('sys.argv', [ 'progname', '-l', testfname ]) as argv:
main()
self.assertEqual(stdout.getvalue(),
'foo:\tbleh\nhashes:\tsha256:91751cee0a1ab8414400238a761411daa29643ab4b8243e9a91649e25be53ada\nhashes:\tsha512:7d5768d47b6bc27dc4fa7e9732cfa2de506ca262a2749cb108923e5dddffde842bbfee6cb8d692fb43aca0f12946c521cce2633887914ca1f96898478d10ad3f\nlang:\ten\n')

+ 3
- 3
ui/fixtures/genfixtures.py View File

@@ -7,19 +7,19 @@ persona = cli.Persona()
persona.generate_key()
cbr = persona.get_identity().uuid
objst = cli.ObjectStore(cbr)
map(objst.loadobj,
list(map(objst.loadobj,
[
{
'type': 'metadata',
'uuid': uuid.UUID('3e466e06-45de-4ecc-84ba-2d2a3d970e96'),
'created_by_ref': cbr,
'modified': datetime.datetime(2019, 5, 31, 14, 5, 3),
'dc:creator': [ u'John-Mark Gurney' ],
'dc:creator': [ 'John-Mark Gurney' ],
'hashes': [ 'sha256:91751cee0a1ab8414400238a761411daa29643ab4b8243e9a91649e25be53ada', 'sha512:7d5768d47b6bc27dc4fa7e9732cfa2de506ca262a2749cb108923e5dddffde842bbfee6cb8d692fb43aca0f12946c521cce2633887914ca1f96898478d10ad3f' ],
'lang': 'en'
}
]
)
))

objst.store('sample.data.pasn1')
persona.store('sample.persona.pasn1')

+ 14
- 11
ui/kleintest.py View File

@@ -5,15 +5,15 @@ from klein.interfaces import IKleinRequest
from twisted.internet.defer import Deferred
from twisted.trial import unittest
from twisted.web.http_headers import Headers
from zope.interface import implements
from StringIO import StringIO
from zope.interface import implementer
from io import StringIO
from requests.structures import CaseInsensitiveDict

__all__ = [ 'FakeRequests', ]

# https://github.com/twisted/twisted/blob/twisted-19.7.0/src/twisted/web/http.py#L664
@implementer(IKleinRequest)
class FakeHTTPRequest(object):
# https://github.com/twisted/twisted/blob/twisted-19.7.0/src/twisted/web/http.py#L664
implements(IKleinRequest)

code = 200

@@ -26,7 +26,7 @@ class FakeHTTPRequest(object):

self.path = uri
self.prepath = []
self.postpath = uri.split('/')
self.postpath = uri.split(b'/')
self.method = meth
self.notifications = []
self.finished = False
@@ -38,10 +38,10 @@ class FakeHTTPRequest(object):
self.code = code

def getRequestHostname(self):
return ''
return b''

def getHost(self):
return ''
return b''

def isSecure(self):
return False
@@ -74,6 +74,7 @@ class FakeRequestsResponse(object):
self._req = req
self._io = StringIO()
req.write = self.write
self.status_code = 500

def _finished(self, arg):
if arg is not None: # pragma: no cover
@@ -118,7 +119,7 @@ class FakeRequests(object):
self._res = app.resource()

def _makerequest(self, method, url, data=''):
if url[0] != '/':
if url[0:1] != b'/':
raise ValueError('url must be absolute (start w/ a slash)')

req = FakeHTTPRequest('GET', url, data)
@@ -133,6 +134,7 @@ class FakeRequests(object):
def get(self, url):
'''Return a response for the passed in url.'''

print('ktg:', repr(url))
return self._makerequest('GET', url)

def put(self, url, data=''):
@@ -150,7 +152,7 @@ class TestFakeRequests(unittest.TestCase):
def home(request):
request.setHeader('x-testing', 'value')

return 'hello'
return b'hello'

@app.route('/500')
def causeerror(request):
@@ -171,10 +173,11 @@ class TestFakeRequests(unittest.TestCase):
self.requests = FakeRequests(app)

def test_bad(self):
self.assertRaises(ValueError, self.requests.get, 'foobar')
self.assertRaises(ValueError, self.requests.get, b'foobar')

def test_basic(self):
r = self.requests.get('/')
r = self.requests.get(b'/')
print(repr(r))
self.assertEqual(r.status_code, 200)
self.assertEqual(r.text, 'hello')
self.assertEqual(r.headers['X-testing'], 'value')


+ 3
- 1
ui/requirements.txt View File

@@ -1,7 +1,9 @@
urwid
-e git://github.com/jmgurney/pasn1@27ff594a2609c07205753ce24f74d8f45a7ea418#egg=pasn1
-e git+https://www.funkthat.com/gitea/jmg/pasn1.git@01d8efffd7bc3037dcb894ea44dbe959035948c6#egg=pasn1
coverage
mock
klein
cryptography
base58
# for kleintest
requests

+ 5
- 5
ui/server.py View File

@@ -9,7 +9,6 @@
# Going Async from Flask to Twisted Klein: https://crossbario.com/blog/Going-Asynchronous-from-Flask-to-Twisted-Klein/
# Klein POST docs: https://klein.readthedocs.io/en/latest/examples/handlingpost.html

from contextlib import nested
from klein import Klein
from kleintest import *
from twisted.trial import unittest
@@ -216,9 +215,9 @@ class _TestCases(_BaseServerTest):
persona = Persona()
persona.generate_key()

with nested(mock.patch('server.MEDAServer.addpubkey'),
with mock.patch('server.MEDAServer.addpubkey') as addpub, \
mock.patch('sys.argv', [ 'progname', '-a',
persona.get_pubkey() ])) as (addpub, argv):
persona.get_pubkey() ]) as argv:
main()

addpub.assert_called_with(persona.get_pubkey())
@@ -254,10 +253,11 @@ class _TestPostConfig(_BaseServerTest):

def test_hashlookup(self):
# that the hash of a file
h = hashlib.sha256(open('fixtures/testfiles/test.txt').read()).hexdigest()
h = hashlib.sha256(open('fixtures/testfiles/test.txt', 'rb').read()).hexdigest()

# when looked up
r = self.requests.get('/lookup/%s' % h)
print('hl')
r = self.requests.get(('/lookup/%s' % h).encode('ascii'))

# returns a 404
self.assertEqual(r.status_code, 404)


Loading…
Cancel
Save