add support for looking at the logs, getting specific entries, and

getting ranges

RainEagle/parse.py

@@ -1,9 +1,16 @@
 from StringIO import StringIO
+from bisect import bisect
 
 import collections
+import glob
 import itertools
+import os.path
+import shutil
+import tempfile
 import unittest
 
+# meterts is the timestamp of the read from the meter.  readts is the
+# timestamp of the local machine fetching the data
 class MeterRead(collections.namedtuple('MeterRead', [ 'meterts', 'readts', 'status', 'load', 'maxload', 'loadunit', 'sink', 'source', 'ssunit' ])):
 	pass
 
@@ -45,6 +52,83 @@ def _pread(fp, off, sz):
 # be a discontinuity, where it skips forward or backward an hour, and this
 # is when the new timezone takes effect.
 
+class LogDir(object):
+	def __init__(self, basename):
+		'''Pass in the base name for the logs.  The pattern is
+		<basename>.<number>.log.   Files w/ an extension of .idx
+		are indexes for those files and can be safely removed.'''
+
+		self._basename = basename
+
+		self.verifycreateindexes()
+
+	def verifycreateindexes(self):
+		self._files = {}
+		for i in glob.iglob(self._basename + '.*.log'):
+			with open(i) as fp:
+				self._files[i] = ParseLog.generateIndex(fp)
+
+		self._poses = [ (y[0], fname, y[1]) for fname, x in self._files.iteritems() for y in x['index'] ]
+		self._poses.sort()
+
+	def __getitem__(self, rng):
+		if not isinstance(rng, slice):
+			return self.get(rng)
+
+		if rng.stop < rng.start:
+			return []
+
+		ret = []
+		start, end = rng.start, rng.stop
+
+		pos = bisect(self._poses, (start, '')) - 1
+		while True:
+			posinfo = self._poses[pos]
+
+			with open(posinfo[1]) as fp:
+				for i in ParseLog(fp).iterfp(posinfo[2]):
+					#print `start, end, i`
+					if i.meterts < start:
+						# not to the start yet
+						continue
+					elif i.meterts >= end:
+						# past the end
+						break
+					else:
+						ret.append(i)
+
+			if i.meterts >= end:
+				break
+
+			pos += 1
+
+		return ret
+
+	def get(self, utc):
+		pos = bisect(self._poses, (utc, '')) - 1
+
+		posinfo = self._poses[pos]
+
+		previ = None
+		with open(posinfo[1]) as fp:
+			for i in ParseLog(fp).iterfp(posinfo[2]):
+				if utc == i.meterts:
+					break
+				elif utc < i.meterts:
+					break
+
+				previ = i
+
+		# decide which is closer, previ or i
+		if previ is None:
+			return i
+
+		middle = previ.meterts + (i.meterts - previ.meterts) / 2
+		if utc >= middle:
+			return i
+
+		return previ
+
 class ParseLog(object):
 	def __init__(self, fp):
 		self._fp = fp
@@ -66,12 +150,15 @@ class ParseLog(object):
 			raise ValueError('unknown type: %s' % repr(data[0]))
 
 	def __iter__(self):
+		return self.iterfp()
+
+	def iterfp(self, pos=0):
+		'''Iterate the lines in the file starting as pos.'''
 		# this can be suspended/resumed between yields, so
-		# keep track of pointer.
+		# keep track of fp possition.
 
 		fp = self._fp
 
-		pos = 0
 		tz = None
 		done = False
 		while not done:
@@ -194,6 +281,18 @@ class MiscTests(unittest.TestCase):
 		self.assertEqual(sio.tell(), pos)
 
 class Tests(unittest.TestCase):
+	def setUp(self):
+		d = os.path.realpath(tempfile.mkdtemp())
+		self.basetempdir = d
+		self.tempdir = os.path.join(d, 'subdir')
+
+		shutil.copytree(os.path.join('fixtures'), self.tempdir)
+
+	def tearDown(self):
+		shutil.rmtree(self.basetempdir)
+		self.tempdir = None
+		self.basetempdir = None
+
 	oldlines = '''l	1571848576	Connected	1.1260	9.155000	W	65375.946	0.000	Wh
 z	GMT+7	1571848569	1571873769	-0700
 l	1571848585	Connected	1.0890	9.155000	kW	15.946	0.000	kWh
@@ -203,7 +302,7 @@ l	1571848593	Connected	1.0500	9.155000	kW	15.946	0.000	kWh
 		zi = ZoneInfo(tz='GMT+8', lcl='1577132476', utc='1577161276', offset='-0800')
 
 		self.assertEqual(zi.getOffset(), -8 * 60 * 60)
-		
+
 		zi = ZoneInfo(tz='GMT+8', lcl='1577132476', utc='1577161276', offset='-0700')
 
 		self.assertEqual(zi.getOffset(), -7 * 60 * 60)
@@ -298,6 +397,49 @@ m	1577161298.96	1577132488	Connected	0.1450	1.992000	kW	90.404	1.660	kWh
 		self.assertEqual([ x.meterts for x in lines ], [ 1577161272, 1577161280, 1577161288 ])
 		self.assertEqual([ x.load for x in lines ], [ 0.2580, 0.3410, 0.1450 ])
 
+	def test_logdir(self):
+		ld = LogDir(os.path.join(self.tempdir, 'data'))
+
+		mr = ld.get(1577952703)
+		self.assertEqual(mr.load, .3596)
+
+		mr = ld.get(1577953233)
+		self.assertEqual(mr.load, .4799)
+
+		mr = ld.get(1577953252.71)
+		self.assertEqual(mr.load, .2738)
+		self.assertEqual(ld[mr.meterts].load, .2738)
+
+		# data.1.log
+		# XXX - fix, include first line of file
+		#mr = ld[1577952662]
+		#self.assertEqual(mr.load, .3967)
+
+		mr = ld.get(1577954523)
+		#print `mr`
+		self.assertEqual(mr.load, .1539)
+
+		rng = ld[1577953704:1577953541]
+		self.assertEqual(rng, [])
+
+		rng = ld[1577953541:1577953704]
+		tzline = 'z       GMT+8   1577924671      1577953471      -0800'
+		tz = ParseLog.parseline(tzline)
+
+		# XXX - not including the first line of the file
+		#m       1577953674.37   1577924872      Connected       0.2909  1.483000        kW      1.000   1.000   kWh
+		reslines = '''m       1577953544.45   1577924742      Connected       0.4826  1.483000        kW      1.000   1.000   kWh
+m       1577953553.54   1577924752      Connected       0.3117  1.483000        kW      1.000   1.000   kWh
+m       1577953562.83   1577924762      Connected       0.4346  1.483000        kW      1.000   1.000   kWh
+m       1577953683.88   1577924882      Connected       0.3879  1.483000        kW      1.000   1.000   kWh
+m       1577953693.96   1577924892      Connected       0.1687  1.483000        kW      1.000   1.000   kWh
+m       1577953704.06   1577924902      Connected       0.3323  1.483000        kW      1.000   1.000   kWh'''
+
+		rngres = [ ParseLog.parseline(x, tz) for x in reslines.split('\n') ]
+		self.assertEqual(rng, rngres)
+
+		# XXX - test out of range items as well
+
 	def test_close(self):
 		# test to make sure the file object is closed
 		pass