Browse Source

use threads to read/hash multiple datablocks at a time..

as we might have a lot of work to submit, BUT it might fail early,
don't submit too much work early on, just for it to fail, so we
limit how much work we submit..
main
John-Mark Gurney 1 year ago
parent
commit
7212192801
1 changed files with 26 additions and 9 deletions
  1. +26
    -9
      ui/medashare/btv/__init__.py

+ 26
- 9
ui/medashare/btv/__init__.py View File

@@ -1,5 +1,6 @@

from . import bencode
import concurrent
import fnmatch
from functools import reduce
from hashlib import sha1, sha512
@@ -247,7 +248,7 @@ def validate(torrent, basedir, with_file_hashes=None):

file_hashes = dict()

def apply_fun(data, fname, offset):
def apply_fun(hash, data, fname, offset):
if with_file_hashes is not None:
try:
hio = file_hashes[fname]
@@ -262,16 +263,32 @@ def validate(torrent, basedir, with_file_hashes=None):
pieces = info['pieces']
piecescnt = len(pieces) // 20
valid = [ None ] * piecescnt
for num, i in enumerate(pieces[x:x+20] for x in range(0, len(pieces),
20)):
hash = sha1()
with concurrent.futures.ThreadPoolExecutor(max_workers=os.cpu_count()) as executor:
futures = {}

stor.apply_piece(num, apply_fun)
pending_work = [ x for x in enumerate(pieces[x:x+20] for x in range(0, len(pieces), 20)) ]

if hash.digest() == i:
valid[num] = True
else:
valid[num] = False
def submit_work(num, i):
hash = sha1()

fut = executor.submit(stor.apply_piece, num, lambda *args: apply_fun(hash, *args))

futures[fut] = num, hash, i

for i in range(min(len(pending_work), os.cpu_count() + 1)):
submit_work(*pending_work.pop(0))

for future in concurrent.futures.as_completed(futures):
if pending_work:
submit_work(*pending_work.pop(0))

future.result()
num, hash, i = futures[future]

if hash.digest() == i:
valid[num] = True
else:
valid[num] = False

if files is None:
filesizes = { pathlib.PurePosixPath(info['name'].decode(


Loading…
Cancel
Save