From bc52915c6425ba6471df421a18e8238d633c0572 Mon Sep 17 00:00:00 2001 From: John-Mark Gurney Date: Tue, 4 Jul 2006 22:08:33 -0800 Subject: [PATCH] make zip files w/ subdirs work.. also support tar files, and ones that are bzip2'd... add a repr... [git-p4: depot-paths = "//depot/": change = 817] --- ZipStorage.py | 101 +++++++++++++++++++++++++++++++++++++++++------- itertarfile.py | 33 ++++++++++++++++ iterzipfile.pyc | Bin 0 -> 2554 bytes 3 files changed, 121 insertions(+), 13 deletions(-) create mode 100644 itertarfile.py create mode 100644 iterzipfile.pyc diff --git a/ZipStorage.py b/ZipStorage.py index c68a6a0..a17768a 100644 --- a/ZipStorage.py +++ b/ZipStorage.py @@ -10,6 +10,8 @@ import sets import time import iterzipfile zipfile = iterzipfile +import itertarfile +tarfile = itertarfile import FileDIDL from DIDLLite import StorageFolder, Item, VideoItem, AudioItem, TextItem, ImageItem, Resource @@ -129,8 +131,8 @@ class ZipResource(resource.Resource): # and make sure the connection doesn't get closed return server.NOT_DONE_YET -class ZipItem(Item): - '''An item in the zip file''' +class ZipItem: + '''Basic zip stuff initalization''' def __init__(self, *args, **kwargs): self.zo = kwargs['zo'] @@ -139,18 +141,22 @@ class ZipItem(Item): del kwargs['zf'] self.name = kwargs['name'] del kwargs['name'] - self.zi = self.zf.getinfo(self.name) + + def checkUpdate(self): + self.doUpdate() + return self.zo.checkUpdate() + +class ZipFile(ZipItem, Item): + def __init__(self, *args, **kwargs): self.mimetype = kwargs['mimetype'] del kwargs['mimetype'] + ZipItem.__init__(self, *args, **kwargs) + self.zi = self.zf.getinfo(self.name) kwargs['content'] = ZipResource(self.zf, self.name, self.mimetype) Item.__init__(self, *args, **kwargs) self.url = '%s/%s' % (self.cd.urlbase, self.id) - def checkUpdate(self): - self.doUpdate() - return self.zo.checkUpdate() - def doUpdate(self): self.res = Resource(self.url, 'http-get:*:%s:*' % self.mimetype) self.res.size = self.zi.file_size @@ -159,8 +165,71 @@ class ZipItem(Item): class ZipChildDir(ZipItem, StorageFolder): '''This is to represent a child dir of the zip file.''' - def __init__(self): - raise NotImplementedError + def __init__(self, *args, **kwargs): + self.hier = kwargs['hier'] + del kwargs['hier'] + ZipItem.__init__(self, *args, **kwargs) + del kwargs['zf'], kwargs['zo'], kwargs['name'] + StorageFolder.__init__(self, *args, **kwargs) + + # mapping from path to objectID + self.pathObjmap = {} + + def doUpdate(self): + children = sets.Set(self.hier.keys()) + for i in self.pathObjmap.keys(): + if i not in children: + # delete + self.cd.delItem(self.pathObjmap[i]) + del self.pathObjmap[i] + + for i in children: + if i in self.pathObjmap: + continue + + # new object + pathname = os.path.join(self.name, i) + if isinstance(self.hier[i], dict): + # must be a dir + self.pathObjmap[i] = self.cd.addItem(self.id, + ZipChildDir, i, zf = self.zf, zo = self, + name = pathname, hier = self.hier[i]) + else: + klass, mt = FileDIDL.buildClassMT(ZipFile, i) + if klass is None: + continue + self.pathObjmap[i] = self.cd.addItem(self.id, + klass, i, zf = self.zf, zo = self, + name = pathname, mimetype = mt) + + # sort our children + self.sort(lambda x, y: cmp(x.title, y.title)) + + def __repr__(self): + return '' % len(self.pathObjmap) + +def genZipFile(path): + try: + return zipfile.ZipFile(path) + except: + import traceback + traceback.print_exc(file=log.logfile) + pass + + log.msg('trying tar now:', `path`) + try: + # Try to see if it's a tar file + if path[-2:] == 'gz': + comp = tarfile.TAR_GZIPPED + elif path[-3:] == 'bz2': + comp = tarfile.TAR_BZ2 + else: + comp = tarfile.TAR_PLAIN + return tarfile.TarFileCompat(path, compression=comp) + except: + import traceback + traceback.print_exc(file=log.logfile) + raise class ZipObject(FSObject, StorageFolder): def __init__(self, *args, **kwargs): @@ -176,7 +245,7 @@ class ZipObject(FSObject, StorageFolder): def doUpdate(self): # open the zipfile as necessary. - self.zip = zipfile.ZipFile(self.FSpath) + self.zip = genZipFile(self.FSpath) hier = buildNameHier(self.zip.namelist(), self.zip.infolist()) children = sets.Set(hier.keys()) @@ -194,9 +263,12 @@ class ZipObject(FSObject, StorageFolder): if isinstance(hier[i], dict): # must be a dir self.pathObjmap[i] = self.cd.addItem(self.id, - ZipChildDir, i, self, i) + ZipChildDir, i, zf = self.zip, zo = self, + name = i, hier = hier[i]) else: - klass, mt = FileDIDL.buildClassMT(ZipItem, i) + klass, mt = FileDIDL.buildClassMT(ZipFile, i) + if klass is None: + continue self.pathObjmap[i] = self.cd.addItem(self.id, klass, i, zf = self.zip, zo = self, name = i, mimetype = mt) @@ -204,9 +276,12 @@ class ZipObject(FSObject, StorageFolder): # sort our children self.sort(lambda x, y: cmp(x.title, y.title)) + def __repr__(self): + return '' % len(self.path) + def detectzipfile(path, fobj): try: - z = zipfile.ZipFile(fobj) + genZipFile(path) except: return None, None diff --git a/itertarfile.py b/itertarfile.py new file mode 100644 index 0000000..906c97c --- /dev/null +++ b/itertarfile.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python + +import tarfile +from tarfile import * + +TAR_PLAIN = tarfile.TAR_PLAIN +TAR_GZIPPED = tarfile.TAR_GZIPPED +TAR_BZ2 = 'bz2' + +__all__ = tarfile.__all__ + +class TarFileCompat(tarfile.TarFileCompat): + def __init__(self, file, mode="r", compression=TAR_PLAIN): + if compression != TAR_BZ2: + tarfile.TarFileCompat.__init__(self, file, mode, compression) + return + + self.tarfile = TarFile.bz2open(file, mode) + if mode[0:1] == "r": + members = self.tarfile.getmembers() + for i in xrange(len(members)): + m = members[i] + m.filename = m.name + m.file_size = m.size + m.date_time = time.gmtime(m.mtime)[:6] + + def readiter(self, name, blksize=16384): + f = self.tarfile.extractfile(self.tarfile.getmember(name)) + while True: + data = f.read(blksize) + if data == '': + break + yield data diff --git a/iterzipfile.pyc b/iterzipfile.pyc new file mode 100644 index 0000000000000000000000000000000000000000..28e5008d58ffb9f6f89068078cd6ba714db0917f GIT binary patch literal 2554 zcmZ`)&2k$>5boKP^=rirPGSNiY)le1R3%j;ha5PNA176zC_~xCCc3EB+EG06YInUe zBC$)lBs>8xz?};Z!F%uu6udzAx>vFkD#)JRnV$anx_hSQtAEG z>`RcC=!mFOP84;Na6IaG6e~LNX!oQoET1|)?WdQiqAN6LQr0GgpI%~q>XG_Udt|EA zsZ#9G&#+me*e6}5K5SN)^M>eor_R=PLI$xFDu%)2iyl)C!^e1xVK+e>(HPJaz42(7 zJvzYh=(R_w!onw0A*M?#2Iuhb2EvLaipGj|#1&iIvQ^N&NpzaotC$W))&&PT$Z}mo z&ZIAbEDZ`BB>@YCb%@v+$YEq7=L*Zv`nhf5vlk^J{iGg-q-@5aV=lYH~(#CqEGku*XZ@wG52ZdDT`|m27rQ{uG-%=JStz zJi@RSK#)JKDE?6rRM(^Jolx>&=mAlWr)>-d!xro|)H#(irM^dln$(&z@6R&NBL2K0 zl|_DqGtVVxk*kR@UzR?NQK>qOTVxs(FL6y3FPh|9H1H^UUapr#q|L0nhAN+<*C;j= zCUtuCyBe1!mT5qgU4sg&u97=XV^0*JUZ)RX92oYFel}aSOM7na|cOkZtWoMJjhJI5=_I24Q4_D6p=; z9lRCvvm!{Np}r$sp%5C`Poass8HcxUMUO|OfZi~Rb+A!v1kl)sHtapfJakSEa~HUb zuW}r8o_!xgMeor3qyu*tx!@%`OL*@j=p~uev4I-dFF`i-?Ory_VcQz%1ye`vPzT}A zfXe_+6LY}6M~OCnzkT~w5O(Y(7hjkx5DOKTjORNx*YN20|cv6IawPrrS*3pX4!=U5u(C}l{(lL>>2^$ccqFpy>% z_e5eVOSSmu6+MamX_Q!9Hqfk}jO?LZfj-+rHHJFwy2vEbP)mz|$Vro9DHeB&QJIFh zPmw5%aqq-)A_fjj8rhyPvJLx3+Fj@eUcSgwj9vimHrQ$c}BIb4vgcNy~@ZPjk+gnj9 zYE7-Gk5pCpw5o1mURQ0uiuDRc$!+1giggoO=e&lh`B%ITR0X`YTJzTNz39s;B!P6h zoPlmvGS%&tcMHo!Fmu@di=7iNeGB>Jlaovmsy9 z@&xl~lyk+>-ENd504Av}pJbU&y@t#e#!QeSI6E0<2Yb;m&!_6|m@P5Rqkdnpxb9zb VS>>f~DE_M{pL4ESUc+lt{{_!K9dZBw literal 0 HcmV?d00001