Add 'ui/medashare/btv/' from commit '85f5c9a03e3773b6868ae83a075c9df8fc70c4d4'

git-subtree-dir: ui/medashare/btv
git-subtree-mainline: 4a28327b7d
git-subtree-split: 85f5c9a03e

ui/medashare/btv/__init__.py

@@ -0,0 +1,250 @@
+
+from . import bencode
+from functools import reduce
+from hashlib import sha1
+import importlib.resources
+import itertools
+import os
+import pathlib
+import shutil
+import sys
+import tempfile
+import unittest
+
+_encoding = 'utf-8'
+
+__all__ = [ 'validate' ]
+
+class Storage:
+	'''A class to help read pieces of a torrent.
+	'''
+
+	def __init__(self, rootpath, files, piecelen):
+		'''
+		rootpath - path to the dir of torrent files are in
+		files - the files dictionary from the torrent info key
+		piecelen - piece length from the torren info key
+		'''
+
+		self._rootpath = pathlib.Path(rootpath)
+		self._files = files
+		self._piecelen = piecelen
+
+		self._buildindex()
+
+	def _filepaths(self):
+		'''Iterates over all the files in the torrent.
+
+		Each item is a tuple of:
+			array of file path components (undecoded)
+			a pathlib.PurePath for the file
+			a pathlib.Path for file on disk
+		'''
+
+		for curfile in self._files:
+			fname = pathlib.PurePath(
+			    *(x.decode(_encoding) for x in
+			    curfile['path']))
+			curfilepath = self._rootpath / fname
+
+			yield curfile, fname, curfilepath
+
+	def allfiles(self):
+		'''Iterator that returns each on disk path name for
+		each file.'''
+
+		for x, y, curfilepath in self._filepaths():
+			yield curfilepath
+
+	def _buildindex(self):
+		'''Internal function to build the needed indexes for
+		pieces and files.'''
+
+		self._pieceindex = []
+		self._fileindex = {}
+		files = self._filepaths()
+		left = 0
+		curfile = None
+
+		while True:
+			if curfile is None or curfileoff == curfile['length']:
+				# next file
+				try:
+					curfile, fname, curfilepath = next(files)
+				except StopIteration:
+					break
+				curfileoff = 0
+
+			if left == 0:
+				current = []
+				self._fileindex.setdefault(fname,
+				    []).append(len(self._pieceindex))
+				self._pieceindex.append(current)
+				left = self._piecelen
+
+			sz = min(curfile['length'] - curfileoff, left)
+
+			current.append(dict(file=curfilepath, fname=fname,
+			    offset=curfileoff, size=sz))
+
+			curfileoff += sz
+			left -= sz
+
+	def filepieces(self):
+		'''Iterator that returns a pair, first item is the subpath
+		to a file (that is relative to the torrent dir), and the
+		pieces that cover the file.'''
+
+		return self._fileindex.items()
+
+	def filesforpiece(self, idx):
+		'''Return a list of files that are covered by piece idx.'''
+
+		for x in self._pieceindex[idx]:
+			yield x['file']
+
+	def apply_piece(self, idx, fun):
+		'''Read the parts of piece idx, and call fun w/ each part.
+
+		This is to hash the parts, e.g.
+		hash = sha1()
+		stor.apply_piece(num, hash.update)
+
+		hash now contains the digest for the piece.'''
+
+		for i in self._pieceindex[idx]:
+			with open(i['file'], 'rb') as fp:
+				fp.seek(i['offset'])
+				fun(fp.read(i['size']))
+
+def validate(torrent, basedir):
+	'''Take a decode torrent file, where it was stored in basedir,
+	verify the torrent.  Returns a pair of set, the first is all the
+	files that are valid, the second are all the invalid files.'''
+
+	info = torrent['info']
+
+	basedir = pathlib.Path(basedir)
+
+	torrentdir = basedir / info['name'].decode(_encoding)
+
+	stor = Storage(torrentdir, info['files'], info['piece length'])
+
+	pieces = info['pieces']
+	piecescnt = len(pieces) // 20
+	valid = [ None ] * piecescnt
+	for num, i in enumerate(pieces[x:x+20] for x in range(0, len(pieces),
+	    20)):
+		hash = sha1()
+
+		stor.apply_piece(num, hash.update)
+
+		if hash.digest() == i:
+			valid[num] = True
+		else:
+			valid[num] = False
+
+	# if any piece of a file is bad, it's bad
+	allfiles = set(stor.allfiles())
+
+	badfiles = { torrentdir / x for x, y in stor.filepieces() if
+	    not all(valid[i] for i in y) }
+
+	return allfiles - badfiles, badfiles
+
+class _TestCases(unittest.TestCase):
+	dirname = 'somedir'
+
+	# file contents for somedir.torrent
+	origfiledata = {
+		'filea.txt':	b'foo\n',
+		'fileb.txt':	b'bar\n',
+		'filec.txt':	b'bleha\n',
+		'filed.txt':	b'somehow\n',
+		'filee.txt':	b'nowab\n',
+		'filef/filef.txt':	b'\n',
+	}
+
+	# some munging to make some files bad
+	badfiles = {
+		'filea.txt': b'',
+		'filec.txt': b'\x00\x00\x00\x00a\n',
+		'filee.txt': b'no',
+	}
+
+	def setUp(self):
+		d = pathlib.Path(tempfile.mkdtemp()).resolve()
+
+		tor = importlib.resources.files(__name__)
+		tor = tor / 'fixtures' / 'somedir.torrent'
+		with tor.open('rb') as fp:
+			self.torrent = bencode.bdecode(fp.read())
+
+		self.basetempdir = d
+
+		self.oldcwd = os.getcwd()
+
+		os.chdir(d)
+
+	def tearDown(self):
+		shutil.rmtree(self.basetempdir)
+
+		os.chdir(self.oldcwd)
+
+	@staticmethod
+	def make_files(dname, fdict):
+		dname = pathlib.Path(dname)
+		for k, v in fdict.items():
+			k = dname / pathlib.PurePosixPath(k)
+			k.parent.mkdir(parents=True, exist_ok=True)
+			with open(k, 'wb') as fp:
+				fp.write(v)
+
+	def test_completeverif(self):
+		sd = self.basetempdir / self.dirname
+		sd.mkdir()
+
+		self.make_files(sd, self.origfiledata)
+
+		validate(self.torrent, self.basetempdir)
+
+		# that utf-8 encoded names work
+
+		sd = self.basetempdir / 'thai'
+		sd.mkdir()
+
+		self.make_files(sd, { 'thai - สวัสดี.txt': b'hello\n'
+		})
+
+		tor = importlib.resources.files(__name__)
+		tor = tor / 'fixtures' / 'thai.torrent'
+		with tor.open('rb') as fp:
+			torrent = bencode.bdecode(fp.read())
+
+		validate(torrent, self.basetempdir)
+
+	def test_verification(self):
+		# Testing for "missing" files
+		# piece size 2 (aka 4 bytes)
+		# empty file of 4 bytes		'foo\n'
+		# complete file of 4 bytes	'bar\n'
+		# partial missing file, 6 bytes, last two correct	'bleha\n'
+		# complete file of 8 bytes (multiple pieces)		'somehow\n'
+		# partial missing file, starting w/ 2 bytes, length 6	'nowab\n'
+		# complete file (length 1)				'\n'
+
+		missingfiles = self.origfiledata.copy()
+
+		missingfiles.update(self.badfiles)
+
+		sd = self.basetempdir / self.dirname
+		sd.mkdir()
+
+		self.make_files(sd, missingfiles)
+
+		val, inval = validate(self.torrent, self.basetempdir)
+
+		self.assertEqual(set(val), { sd / x for x in
+		    missingfiles.keys() if x not in self.badfiles })
+		self.assertEqual(set(inval), { sd / x for x in
+		    self.badfiles.keys() })

ui/medashare/btv/bencode.py

@@ -0,0 +1,341 @@
+# Written by Petru Paler, Uoti Urpala, Ross Cohen and John Hoffman
+# see LICENSE.txt for license information
+
+# LICENSE.txt:
+# Unless otherwise noted, all files are released under the MIT
+# license, exceptions contain licensing information in them.
+# 
+# Copyright (C) 2001-2002 Bram Cohen
+# 
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation files
+# (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge,
+# publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+# 
+# The Software is provided "AS IS", without warranty of any kind,
+# express or implied, including but not limited to the warranties of
+# merchantability,  fitness for a particular purpose and
+# noninfringement. In no event shall the  authors or copyright holders
+# be liable for any claim, damages or other liability, whether in an
+# action of contract, tort or otherwise, arising from, out of or in
+# connection with the Software or the use or other dealings in the
+# Software.
+
+
+try:
+    from types import BooleanType
+except ImportError:
+    BooleanType = None
+try:
+    from types import UnicodeType
+except ImportError:
+    UnicodeType = None
+from io import StringIO
+
+def decode_int(x, f):
+    f += 1
+    newf = x.index(b'e', f)
+    n = int(x[f:newf])
+    if x[f] == b'-'[0]:
+        if x[f + 1] == b'0'[0]:
+            raise ValueError
+    elif x[f] == b'0'[0] and newf != f+1:
+        raise ValueError
+    return (n, newf+1)
+
+def decode_string(x, f):
+    colon = x.index(b':', f)
+    n = int(x[f:colon])
+    if x[f] == b'0'[0] and colon != f+1:
+        raise ValueError
+    colon += 1
+    return (x[colon:colon+n], colon+n)
+
+def decode_unicode(x, f):
+    s, f = decode_string(x, f+1)
+    return (s.decode('UTF-8'),f)
+
+def decode_list(x, f):
+    r, f = [], f+1
+    while x[f] != b'e'[0]:
+        v, f = decode_func[x[f]](x, f)
+        r.append(v)
+    return (r, f + 1)
+
+def decode_dict(x, f):
+    r, f = {}, f+1
+    lastkey = ''
+    while x[f] != b'e'[0]:
+        k, f = decode_string(x, f)
+        k = k.decode('us-ascii')
+        if lastkey >= k:
+            raise ValueError
+        lastkey = k
+        r[k], f = decode_func[x[f]](x, f)
+    return (r, f + 1)
+
+decode_func = {}
+decode_func[b'l'[0]] = decode_list
+decode_func[b'd'[0]] = decode_dict
+decode_func[b'i'[0]] = decode_int
+decode_func[b'0'[0]] = decode_string
+decode_func[b'1'[0]] = decode_string
+decode_func[b'2'[0]] = decode_string
+decode_func[b'3'[0]] = decode_string
+decode_func[b'4'[0]] = decode_string
+decode_func[b'5'[0]] = decode_string
+decode_func[b'6'[0]] = decode_string
+decode_func[b'7'[0]] = decode_string
+decode_func[b'8'[0]] = decode_string
+decode_func[b'9'[0]] = decode_string
+#decode_func['u'[0]] = decode_unicode
+
+def bdecode(x, sloppy = 0):
+    try:
+        r, l = decode_func[x[0]](x, 0)
+#    except (IndexError, KeyError):
+    except (IndexError, KeyError, ValueError):
+        raise ValueError("bad bencoded data")
+    if not sloppy and l != len(x):
+        raise ValueError("bad bencoded data")
+    return r
+
+def test_bdecode():
+    try:
+        bdecode(b'0:0:')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode(b'ie')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode(b'i341foo382e')
+        assert 0
+    except ValueError:
+        pass
+    assert bdecode(b'i4e') == 4
+    assert bdecode(b'i0e') == 0
+    assert bdecode(b'i123456789e') == 123456789
+    assert bdecode(b'i-10e') == -10
+    try:
+        bdecode(b'i-0e')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode(b'i123')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode(b'')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode('')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode(b'i6easd')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode(b'35208734823ljdahflajhdf')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode(b'2:abfdjslhfld')
+        assert 0
+    except ValueError:
+        pass
+    assert bdecode(b'0:') == b''
+    assert bdecode(b'3:abc') == b'abc'
+    assert bdecode(b'10:1234567890') == b'1234567890'
+    try:
+        bdecode(b'02:xy')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode(b'l')
+        assert 0
+    except ValueError:
+        pass
+    assert bdecode(b'le') == []
+    try:
+        bdecode(b'leanfdldjfh')
+        assert 0
+    except ValueError:
+        pass
+    assert bdecode(b'l0:0:0:e') == [b'', b'', b'']
+    try:
+        bdecode(b'relwjhrlewjh')
+        assert 0
+    except ValueError:
+        pass
+    assert bdecode(b'li1ei2ei3ee') == [1, 2, 3]
+    assert bdecode(b'l3:asd2:xye') == [b'asd', b'xy']
+    assert bdecode(b'll5:Alice3:Bobeli2ei3eee') == [[b'Alice', b'Bob'], [2, 3]]
+    try:
+        bdecode(b'd')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode(b'defoobar')
+        assert 0
+    except ValueError:
+        pass
+    assert bdecode(b'de') == {}
+    assert bdecode(b'd3:agei25e4:eyes4:bluee') == {'age': 25, 'eyes': b'blue'}
+    assert bdecode(b'd8:spam.mp3d6:author5:Alice6:lengthi100000eee') == {'spam.mp3': {'author': b'Alice', 'length': 100000}}
+    try:
+        bdecode(b'd3:fooe')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode(b'di1e0:e')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode(b'd1:b0:1:a0:e')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode(b'd1:a0:1:a0:e')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode(b'i03e')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode(b'l01:ae')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode(b'9999:x')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode(b'l0:')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode(b'd0:0:')
+        assert 0
+    except ValueError:
+        pass
+    try:
+        bdecode(b'd0:')
+        assert 0
+    except ValueError:
+        pass
+
+bencached_marker = []
+
+class Bencached:
+    def __init__(self, s):
+        self.marker = bencached_marker
+        self.bencoded = s
+
+BencachedType = type(Bencached(b'')) # insufficient, but good as a filter
+
+def encode_bencached(x,r):
+    assert x.marker == bencached_marker
+    r.append(x.bencoded)
+
+def encode_int(x,r):
+    r.append(b'i%de' % x)
+
+def encode_bool(x,r):
+    encode_int(int(x),r)
+
+def encode_bytes(x,r):
+    r.extend((b'%d:' % len(x),x))
+
+def encode_string(x,r):
+    #r.append('u')
+    encode_bytes(x.encode('UTF-8'),r)
+
+def encode_list(x,r):
+        r.append(b'l')
+        for e in x:
+            encode_func[type(e)](e, r)
+        r.append(b'e')
+
+def encode_dict(x,r):
+    r.append(b'd')
+    for k,v in sorted(x.items()):
+        r.extend((b'%d:' % len(k),k.encode('UTF-8')))
+        encode_func[type(v)](v, r)
+    r.append(b'e')
+
+encode_func = {}
+encode_func[BencachedType] = encode_bencached
+encode_func[int] = encode_int
+encode_func[str] = encode_string
+encode_func[list] = encode_list
+encode_func[tuple] = encode_list
+encode_func[type({})] = encode_dict
+if BooleanType:
+    encode_func[BooleanType] = encode_bool
+if UnicodeType:
+    encode_func[UnicodeType] = encode_unicode
+
+def bencode(x):
+    r = []
+    try:
+        encode_func[type(x)](x, r)
+    except:
+        raise ValueError("could not encode type %s (value: %s)" % (type(x), x))
+    return b''.join(r)
+
+def test_bencode():
+    assert bencode(4) == b'i4e'
+    assert bencode(0) == b'i0e'
+    assert bencode(-10) == b'i-10e'
+    assert bencode(12345678901234567890) == b'i12345678901234567890e'
+    assert bencode('') == b'0:'
+    assert bencode('abc') == b'3:abc'
+    assert bencode('1234567890') == b'10:1234567890'
+    assert bencode([]) == b'le'
+    assert bencode([1, 2, 3]) == b'li1ei2ei3ee'
+    assert bencode([['Alice', 'Bob'], [2, 3]]) == b'll5:Alice3:Bobeli2ei3eee'
+    assert bencode({}) == b'de'
+    assert bencode({'age': 25, 'eyes': 'blue'}) == b'd3:agei25e4:eyes4:bluee'
+    assert bencode({'spam.mp3': {'author': 'Alice', 'length': 100000}}) == b'd8:spam.mp3d6:author5:Alice6:lengthi100000eee'
+    try:
+        bencode({1: 'foo'})
+        assert 0
+    except (ValueError, AssertionError):
+        pass
+
+
+try:
+    import psyco
+    psyco.bind(bdecode)
+    psyco.bind(bencode)
+except ImportError:
+    pass

ui/medashare/btv/fixtures/somedir.torrent

@@ -0,0 +1,4 @@
+d13:creation datei1661282749e4:infod5:filesld6:lengthi4e4:pathl9:filea.txteed6:lengthi4e4:pathl9:fileb.txteed6:lengthi6e4:pathl9:filec.txteed6:lengthi8e4:pathl9:filed.txteed6:lengthi6e4:pathl9:filee.txteed6:lengthi1e4:pathl5:filef9:filef.txteee4:name7:somedir12:piece lengthi4e6:pieces160:ñÒÒù$醬†ý÷³l”¼ß2¾ìâBí;ÿÌß'ºóN×-•7´/é ëå�Qä)ÞævŒçTÑUÕãBñY)¶-÷/…:�
+bBÜëPÿØ–€õs
Óé+Gï3
+‡Ñ)d3Õp%ê*™†Äi?gÕ&S2·˜²
+ú_.7î[2#e­È;ç“In ý‹FÍŸ2å’üe5:nodeslee