Browse Source

use threads to read/hash multiple datablocks at a time..

as we might have a lot of work to submit, BUT it might fail early,
don't submit too much work early on, just for it to fail, so we
limit how much work we submit..
main
John-Mark Gurney 2 years ago
parent
commit
7212192801
1 changed files with 26 additions and 9 deletions
  1. +26
    -9
      ui/medashare/btv/__init__.py

+ 26
- 9
ui/medashare/btv/__init__.py View File

@@ -1,5 +1,6 @@


from . import bencode from . import bencode
import concurrent
import fnmatch import fnmatch
from functools import reduce from functools import reduce
from hashlib import sha1, sha512 from hashlib import sha1, sha512
@@ -247,7 +248,7 @@ def validate(torrent, basedir, with_file_hashes=None):


file_hashes = dict() file_hashes = dict()


def apply_fun(data, fname, offset):
def apply_fun(hash, data, fname, offset):
if with_file_hashes is not None: if with_file_hashes is not None:
try: try:
hio = file_hashes[fname] hio = file_hashes[fname]
@@ -262,16 +263,32 @@ def validate(torrent, basedir, with_file_hashes=None):
pieces = info['pieces'] pieces = info['pieces']
piecescnt = len(pieces) // 20 piecescnt = len(pieces) // 20
valid = [ None ] * piecescnt valid = [ None ] * piecescnt
for num, i in enumerate(pieces[x:x+20] for x in range(0, len(pieces),
20)):
hash = sha1()
with concurrent.futures.ThreadPoolExecutor(max_workers=os.cpu_count()) as executor:
futures = {}


stor.apply_piece(num, apply_fun)
pending_work = [ x for x in enumerate(pieces[x:x+20] for x in range(0, len(pieces), 20)) ]


if hash.digest() == i:
valid[num] = True
else:
valid[num] = False
def submit_work(num, i):
hash = sha1()

fut = executor.submit(stor.apply_piece, num, lambda *args: apply_fun(hash, *args))

futures[fut] = num, hash, i

for i in range(min(len(pending_work), os.cpu_count() + 1)):
submit_work(*pending_work.pop(0))

for future in concurrent.futures.as_completed(futures):
if pending_work:
submit_work(*pending_work.pop(0))

future.result()
num, hash, i = futures[future]

if hash.digest() == i:
valid[num] = True
else:
valid[num] = False


if files is None: if files is None:
filesizes = { pathlib.PurePosixPath(info['name'].decode( filesizes = { pathlib.PurePosixPath(info['name'].decode(


Loading…
Cancel
Save