From 5dedc3d970399f31d91604514b003a18b3b41787 Mon Sep 17 00:00:00 2001 From: John-Mark Gurney Date: Fri, 3 Jan 2020 13:07:43 -0800 Subject: [PATCH] add support for looking at the logs, getting specific entries, and getting ranges --- RainEagle/parse.py | 148 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 145 insertions(+), 3 deletions(-) diff --git a/RainEagle/parse.py b/RainEagle/parse.py index 975afb0..b35137f 100644 --- a/RainEagle/parse.py +++ b/RainEagle/parse.py @@ -1,9 +1,16 @@ from StringIO import StringIO +from bisect import bisect import collections +import glob import itertools +import os.path +import shutil +import tempfile import unittest +# meterts is the timestamp of the read from the meter. readts is the +# timestamp of the local machine fetching the data class MeterRead(collections.namedtuple('MeterRead', [ 'meterts', 'readts', 'status', 'load', 'maxload', 'loadunit', 'sink', 'source', 'ssunit' ])): pass @@ -45,6 +52,83 @@ def _pread(fp, off, sz): # be a discontinuity, where it skips forward or backward an hour, and this # is when the new timezone takes effect. +class LogDir(object): + def __init__(self, basename): + '''Pass in the base name for the logs. The pattern is + ..log. Files w/ an extension of .idx + are indexes for those files and can be safely removed.''' + + self._basename = basename + + self.verifycreateindexes() + + def verifycreateindexes(self): + self._files = {} + for i in glob.iglob(self._basename + '.*.log'): + with open(i) as fp: + self._files[i] = ParseLog.generateIndex(fp) + + self._poses = [ (y[0], fname, y[1]) for fname, x in self._files.iteritems() for y in x['index'] ] + self._poses.sort() + + def __getitem__(self, rng): + if not isinstance(rng, slice): + return self.get(rng) + + if rng.stop < rng.start: + return [] + + ret = [] + start, end = rng.start, rng.stop + + pos = bisect(self._poses, (start, '')) - 1 + while True: + posinfo = self._poses[pos] + + with open(posinfo[1]) as fp: + for i in ParseLog(fp).iterfp(posinfo[2]): + #print `start, end, i` + if i.meterts < start: + # not to the start yet + continue + elif i.meterts >= end: + # past the end + break + else: + ret.append(i) + + if i.meterts >= end: + break + + pos += 1 + + return ret + + def get(self, utc): + pos = bisect(self._poses, (utc, '')) - 1 + + posinfo = self._poses[pos] + + previ = None + with open(posinfo[1]) as fp: + for i in ParseLog(fp).iterfp(posinfo[2]): + if utc == i.meterts: + break + elif utc < i.meterts: + break + + previ = i + + # decide which is closer, previ or i + if previ is None: + return i + + middle = previ.meterts + (i.meterts - previ.meterts) / 2 + if utc >= middle: + return i + + return previ + class ParseLog(object): def __init__(self, fp): self._fp = fp @@ -66,12 +150,15 @@ class ParseLog(object): raise ValueError('unknown type: %s' % repr(data[0])) def __iter__(self): + return self.iterfp() + + def iterfp(self, pos=0): + '''Iterate the lines in the file starting as pos.''' # this can be suspended/resumed between yields, so - # keep track of pointer. + # keep track of fp possition. fp = self._fp - pos = 0 tz = None done = False while not done: @@ -194,6 +281,18 @@ class MiscTests(unittest.TestCase): self.assertEqual(sio.tell(), pos) class Tests(unittest.TestCase): + def setUp(self): + d = os.path.realpath(tempfile.mkdtemp()) + self.basetempdir = d + self.tempdir = os.path.join(d, 'subdir') + + shutil.copytree(os.path.join('fixtures'), self.tempdir) + + def tearDown(self): + shutil.rmtree(self.basetempdir) + self.tempdir = None + self.basetempdir = None + oldlines = '''l 1571848576 Connected 1.1260 9.155000 W 65375.946 0.000 Wh z GMT+7 1571848569 1571873769 -0700 l 1571848585 Connected 1.0890 9.155000 kW 15.946 0.000 kWh @@ -203,7 +302,7 @@ l 1571848593 Connected 1.0500 9.155000 kW 15.946 0.000 kWh zi = ZoneInfo(tz='GMT+8', lcl='1577132476', utc='1577161276', offset='-0800') self.assertEqual(zi.getOffset(), -8 * 60 * 60) - + zi = ZoneInfo(tz='GMT+8', lcl='1577132476', utc='1577161276', offset='-0700') self.assertEqual(zi.getOffset(), -7 * 60 * 60) @@ -298,6 +397,49 @@ m 1577161298.96 1577132488 Connected 0.1450 1.992000 kW 90.404 1.660 kWh self.assertEqual([ x.meterts for x in lines ], [ 1577161272, 1577161280, 1577161288 ]) self.assertEqual([ x.load for x in lines ], [ 0.2580, 0.3410, 0.1450 ]) + def test_logdir(self): + ld = LogDir(os.path.join(self.tempdir, 'data')) + + mr = ld.get(1577952703) + self.assertEqual(mr.load, .3596) + + mr = ld.get(1577953233) + self.assertEqual(mr.load, .4799) + + mr = ld.get(1577953252.71) + self.assertEqual(mr.load, .2738) + self.assertEqual(ld[mr.meterts].load, .2738) + + # data.1.log + # XXX - fix, include first line of file + #mr = ld[1577952662] + #self.assertEqual(mr.load, .3967) + + mr = ld.get(1577954523) + #print `mr` + self.assertEqual(mr.load, .1539) + + rng = ld[1577953704:1577953541] + self.assertEqual(rng, []) + + rng = ld[1577953541:1577953704] + tzline = 'z GMT+8 1577924671 1577953471 -0800' + tz = ParseLog.parseline(tzline) + + # XXX - not including the first line of the file + #m 1577953674.37 1577924872 Connected 0.2909 1.483000 kW 1.000 1.000 kWh + reslines = '''m 1577953544.45 1577924742 Connected 0.4826 1.483000 kW 1.000 1.000 kWh +m 1577953553.54 1577924752 Connected 0.3117 1.483000 kW 1.000 1.000 kWh +m 1577953562.83 1577924762 Connected 0.4346 1.483000 kW 1.000 1.000 kWh +m 1577953683.88 1577924882 Connected 0.3879 1.483000 kW 1.000 1.000 kWh +m 1577953693.96 1577924892 Connected 0.1687 1.483000 kW 1.000 1.000 kWh +m 1577953704.06 1577924902 Connected 0.3323 1.483000 kW 1.000 1.000 kWh''' + + rngres = [ ParseLog.parseline(x, tz) for x in reslines.split('\n') ] + self.assertEqual(rng, rngres) + + # XXX - test out of range items as well + def test_close(self): # test to make sure the file object is closed pass