#!/usr/bin/env python # Copyright 2006-2008 John-Mark Gurney __version__ = '$Change$' # $Id$ import itertools import os.path import sets import time import iterzipfile zipfile = iterzipfile import itertarfile tarfile = itertarfile try: import iterrarfile rarfile = iterrarfile except ImportError: class rarfile: pass rarfile = rarfile() rarfile.is_rarfile = lambda x: False import FileDIDL from DIDLLite import StorageFolder, Item, VideoItem, AudioItem, TextItem, ImageItem, Resource from FSStorage import FSObject, registerklassfun from twisted.python import log, threadable from twisted.spread import pb from twisted.web import http from twisted.web import server from twisted.web import resource def inserthierdict(d, name, obj, sep): if not name: return if sep is not None: i = name.find(sep) if sep is None or i == -1: d[name] = obj return dname = name[:i] rname = name[i + 1:] # remaining path components try: inserthierdict(d[dname], rname, obj, sep) except KeyError: d[dname] = {} inserthierdict(d[dname], rname, obj, sep) def buildNameHier(names, objs, sep): ret = {} for n, o in itertools.izip(names, objs): #Skip directories in a TarFile or RarFile if hasattr(o, 'isdir') and o.isdir(): continue inserthierdict(ret, n, o, sep) return ret def zipinfocmp(za, zb): for i in [ 'filename', 'date_time', 'file_size', 'CRC' ]: r = cmp(getattr(za, i), getattr(zb, i)) if r: return r return 0 def zipinfohash(za): r = 0 for i in [ 'filename', 'date_time', 'file_size', 'CRC' ]: r ^= getattr(za, i).__hash__ return r class ZIWrap(object): __slots__ = [ '_zi' ] def __init__(self, zi): self._zi = zi __hash__ = zipinfohash __cmp__ = zipinfocmp def __getattr__(self, n): return getattr(self._zi, n) def __setattr__(self, n, k): if n == '_zi': object.__setattr__(self, n, k) return return setattr(self._zi, n, k) def __delattr__(sefl, n): return delattr(self._zi, n) class ZipFileTransfer(pb.Viewable): def __init__(self, zf, name, request): self.zf = zf self.size = zf.getinfo(name).file_size self.iter = zf.readiter(name) self.request = request self.written = 0 request.registerProducer(self, 0) def resumeProducing(self): if not self.request: return # get data and write to request. try: data = self.iter.next() if data: self.written += len(data) # this .write will spin the reactor, calling # .doWrite and then .resumeProducing again, so # be prepared for a re-entrant call self.request.write(data) except StopIteration: if self.request: self.request.unregisterProducer() self.request.finish() self.request = None def pauseProducing(self): pass def stopProducing(self): # close zipfile self.request = None # Remotely relay producer interface. def view_resumeProducing(self, issuer): self.resumeProducing() def view_pauseProducing(self, issuer): self.pauseProducing() def view_stopProducing(self, issuer): self.stopProducing() synchronized = ['resumeProducing', 'stopProducing'] threadable.synchronize(ZipFileTransfer) class ZipResource(resource.Resource): # processors = {} isLeaf = True def __init__(self, zf, name, mt): resource.Resource.__init__(self) self.zf = zf self.zi = zf.getinfo(name) self.name = name self.mt = mt def getFileSize(self): return self.zi.file_size def render(self, request): request.setHeader('content-type', self.mt) # We could possibly send the deflate data directly! if None and self.encoding: request.setHeader('content-encoding', self.encoding) if request.setLastModified(time.mktime(list(self.zi.date_time) + [ 0, 0, -1])) is http.CACHED: return '' request.setHeader('content-length', str(self.getFileSize())) if request.method == 'HEAD': return '' # return data ZipFileTransfer(self.zf, self.name, request) # and make sure the connection doesn't get closed return server.NOT_DONE_YET class ZipItem: '''Basic zip stuff initalization''' def __init__(self, *args, **kwargs): self.zo = kwargs['zo'] del kwargs['zo'] self.zf = kwargs['zf'] del kwargs['zf'] self.name = kwargs['name'] del kwargs['name'] def checkUpdate(self): # XXX - is this the correct order? self.doUpdate() self.zo.checkUpdate() class ZipFile(ZipItem, Item): def __init__(self, *args, **kwargs): self.mimetype = kwargs['mimetype'] del kwargs['mimetype'] ZipItem.__init__(self, *args, **kwargs) del kwargs['zf'] del kwargs['zo'] del kwargs['name'] self.zi = self.zf.getinfo(self.name) kwargs['content'] = ZipResource(self.zf, self.name, self.mimetype) Item.__init__(self, *args, **kwargs) self.url = '%s/%s' % (self.cd.urlbase, self.id) self.res = Resource(self.url, 'http-get:*:%s:*' % self.mimetype) self.res.size = self.zi.file_size class ZipChildDir(ZipItem, StorageFolder): '''This is to represent a child dir of the zip file.''' def __init__(self, *args, **kwargs): self.hier = kwargs['hier'] self.sep = kwargs['sep'] del kwargs['hier'], kwargs['sep'] ZipItem.__init__(self, *args, **kwargs) del kwargs['zf'], kwargs['zo'], kwargs['name'] StorageFolder.__init__(self, *args, **kwargs) def genChildren(self): return self.hier def genCurrent(self): return ((x.id, x.name.rsplit(self.sep, 1)[1]) for x in self) def createObject(self, i, arg): pathname = self.sep.join((self.name, i)) kwargs = { 'zf': self.zf, 'zo': self, 'name': pathname } if isinstance(self.hier[i], dict): # must be a dir klass = ZipChildDir kwargs['sep'] = self.sep kwargs['hier'] = self.hier[i] else: klass, mt = FileDIDL.buildClassMT(ZipFile, i) kwargs['mimetype'] = mt return klass, i, (), kwargs def __repr__(self): return '' % len(self) def tryTar(path): # Try to see if it's a tar file if path[-2:] == 'gz': comp = tarfile.TAR_GZIPPED elif path[-3:] == 'bz2': comp = tarfile.TAR_BZ2 else: comp = tarfile.TAR_PLAIN return tarfile.TarFileCompat(path, compression=comp) def canHandle(path): if zipfile.is_zipfile(path): return True if rarfile.is_rarfile(path): return True #tar is cheaper on __init__ than zipfile return tryTar(path) def genZipFile(path): if zipfile.is_zipfile(path): return zipfile.ZipFile(path) if rarfile.is_rarfile(path): return rarfile.RarFile(path) try: return tryTar(path) except: #import traceback #traceback.print_exc(file=log.logfile) raise class ZipObject(FSObject, StorageFolder): seps = [ '/', '\\' ] def __init__(self, *args, **kwargs): '''If a zip argument is passed it, use that as the zip archive.''' path = kwargs.pop('path') StorageFolder.__init__(self, *args, **kwargs) FSObject.__init__(self, path) def genChildren(self): # open the zipfile as necessary. # XXX - this might leave too many zip files around self.zip = genZipFile(self.FSpath) nl = self.zip.namelist() cnt = 0 cursep = None for i in self.seps: newsum = sum([ j.count(i) for j in nl ]) if newsum > cnt: cursep = i cnt = newsum self.sep = cursep hier = buildNameHier(nl, [ ZIWrap(x) for x in self.zip.infolist() ], cursep) return hier def genCurrent(self): return ((x.id, x.name) for x in self) def createObject(self, i, arg): kwargs = { 'zf': self.zip, 'zo': self, 'name': i } if isinstance(arg, dict): # must be a dir klass = ZipChildDir kwargs['sep'] = self.sep kwargs['hier'] = arg else: klass, mt = FileDIDL.buildClassMT(ZipFile, i) kwargs['mimetype'] = mt return klass, i, (), kwargs def detectzipfile(path, fobj): try: canHandle(path) except: return None, None return ZipObject, { 'path': path } registerklassfun(detectzipfile)