make zip files w/ subdirs work.. also support tar files, and ones

that are bzip2'd...

add a repr...

[git-p4: depot-paths = "//depot/": change = 817]

ZipStorage.py

@@ -10,6 +10,8 @@ import sets
 import time
 import iterzipfile
 zipfile = iterzipfile
+import itertarfile
+tarfile = itertarfile
 
 import FileDIDL
 from DIDLLite import StorageFolder, Item, VideoItem, AudioItem, TextItem, ImageItem, Resource
@@ -129,8 +131,8 @@ class ZipResource(resource.Resource):
 		# and make sure the connection doesn't get closed
 		return server.NOT_DONE_YET
 
-class ZipItem(Item):
-	'''An item in the zip file'''
+class ZipItem:
+	'''Basic zip stuff initalization'''
 
 	def __init__(self, *args, **kwargs):
 		self.zo = kwargs['zo']
@@ -139,18 +141,22 @@ class ZipItem(Item):
 		del kwargs['zf']
 		self.name = kwargs['name']
 		del kwargs['name']
-		self.zi = self.zf.getinfo(self.name)
+
+	def checkUpdate(self):
+		self.doUpdate()
+		return self.zo.checkUpdate()
+
+class ZipFile(ZipItem, Item):
+	def __init__(self, *args, **kwargs):
 		self.mimetype = kwargs['mimetype']
 		del kwargs['mimetype']
+		ZipItem.__init__(self, *args, **kwargs)
+		self.zi = self.zf.getinfo(self.name)
 		kwargs['content'] = ZipResource(self.zf, self.name,
 		    self.mimetype)
 		Item.__init__(self, *args, **kwargs)
 		self.url = '%s/%s' % (self.cd.urlbase, self.id)
 
-	def checkUpdate(self):
-		self.doUpdate()
-		return self.zo.checkUpdate()
-
 	def doUpdate(self):
 		self.res = Resource(self.url, 'http-get:*:%s:*' % self.mimetype)
 		self.res.size = self.zi.file_size
@@ -159,8 +165,71 @@ class ZipItem(Item):
 class ZipChildDir(ZipItem, StorageFolder):
 	'''This is to represent a child dir of the zip file.'''
 
-	def __init__(self):
-		raise NotImplementedError
+	def __init__(self, *args, **kwargs):
+		self.hier = kwargs['hier']
+		del kwargs['hier']
+		ZipItem.__init__(self, *args, **kwargs)
+		del kwargs['zf'], kwargs['zo'], kwargs['name']
+		StorageFolder.__init__(self, *args, **kwargs)
+
+		# mapping from path to objectID
+		self.pathObjmap = {}
+
+	def doUpdate(self):
+		children = sets.Set(self.hier.keys())
+		for i in self.pathObjmap.keys():
+			if i not in children:
+				# delete
+				self.cd.delItem(self.pathObjmap[i])
+				del self.pathObjmap[i]
+
+		for i in children:
+			if i in self.pathObjmap:
+				continue
+
+			# new object
+			pathname = os.path.join(self.name, i)
+			if isinstance(self.hier[i], dict):
+				# must be a dir
+				self.pathObjmap[i] = self.cd.addItem(self.id,
+				    ZipChildDir, i, zf = self.zf, zo = self,
+				    name = pathname, hier = self.hier[i])
+			else:
+				klass, mt = FileDIDL.buildClassMT(ZipFile, i)
+				if klass is None:
+					continue
+				self.pathObjmap[i] = self.cd.addItem(self.id,
+				    klass, i, zf = self.zf, zo = self,
+				    name = pathname, mimetype = mt)
+
+		# sort our children
+		self.sort(lambda x, y: cmp(x.title, y.title))
+
+	def __repr__(self):
+		return '<ZipChildDir: len: %d>' % len(self.pathObjmap)
+
+def genZipFile(path):
+	try:
+		return zipfile.ZipFile(path)
+	except:
+		import traceback
+		traceback.print_exc(file=log.logfile)
+		pass
+
+	log.msg('trying tar now:', `path`)
+	try:
+		# Try to see if it's a tar file
+		if path[-2:] == 'gz':
+			comp = tarfile.TAR_GZIPPED
+		elif path[-3:] == 'bz2':
+			comp = tarfile.TAR_BZ2
+		else:
+			comp = tarfile.TAR_PLAIN
+		return tarfile.TarFileCompat(path, compression=comp)
+	except:
+		import traceback
+		traceback.print_exc(file=log.logfile)
+		raise
 
 class ZipObject(FSObject, StorageFolder):
 	def __init__(self, *args, **kwargs):
@@ -176,7 +245,7 @@ class ZipObject(FSObject, StorageFolder):
 
 	def doUpdate(self):
 		# open the zipfile as necessary.
-		self.zip = zipfile.ZipFile(self.FSpath)
+		self.zip = genZipFile(self.FSpath)
 		hier = buildNameHier(self.zip.namelist(), self.zip.infolist())
 
 		children = sets.Set(hier.keys())
@@ -194,9 +263,12 @@ class ZipObject(FSObject, StorageFolder):
 			if isinstance(hier[i], dict):
 				# must be a dir
 				self.pathObjmap[i] = self.cd.addItem(self.id,
-				    ZipChildDir, i, self, i)
+				    ZipChildDir, i, zf = self.zip, zo = self,
+				    name = i, hier = hier[i])
 			else:
-				klass, mt = FileDIDL.buildClassMT(ZipItem, i)
+				klass, mt = FileDIDL.buildClassMT(ZipFile, i)
+				if klass is None:
+					continue
 				self.pathObjmap[i] = self.cd.addItem(self.id,
 				    klass, i, zf = self.zip, zo = self,
 				    name = i, mimetype = mt)
@@ -204,9 +276,12 @@ class ZipObject(FSObject, StorageFolder):
 		# sort our children
 		self.sort(lambda x, y: cmp(x.title, y.title))
 
+	def __repr__(self):
+		return '<ZipObject: path: %s>' % len(self.path)
+
 def detectzipfile(path, fobj):
 	try:
-		z = zipfile.ZipFile(fobj)
+		genZipFile(path)
 	except:
 		return None, None
 

itertarfile.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+
+import tarfile
+from tarfile import *
+
+TAR_PLAIN = tarfile.TAR_PLAIN
+TAR_GZIPPED = tarfile.TAR_GZIPPED
+TAR_BZ2 = 'bz2'
+
+__all__ = tarfile.__all__
+
+class TarFileCompat(tarfile.TarFileCompat):
+	def __init__(self, file, mode="r", compression=TAR_PLAIN):
+		if compression != TAR_BZ2:
+			tarfile.TarFileCompat.__init__(self, file, mode, compression)
+			return
+
+		self.tarfile = TarFile.bz2open(file, mode)
+		if mode[0:1] == "r":
+			members = self.tarfile.getmembers()
+			for i in xrange(len(members)):
+				m = members[i]
+				m.filename = m.name
+				m.file_size = m.size
+				m.date_time = time.gmtime(m.mtime)[:6]
+
+	def readiter(self, name, blksize=16384):
+		f = self.tarfile.extractfile(self.tarfile.getmember(name))
+		while True:
+			data = f.read(blksize)
+			if data == '':
+				break
+			yield data