diff --git a/bitelab/__init__.py b/bitelab/__init__.py index f1369dd..80dfc31 100644 --- a/bitelab/__init__.py +++ b/bitelab/__init__.py @@ -26,10 +26,10 @@ # SUCH DAMAGE. # -from typing import Optional, Union, Dict, Any from dataclasses import dataclass from functools import lru_cache, wraps from io import StringIO +from typing import Optional, Union, Dict, Any from fastapi import APIRouter, Body, Depends, FastAPI, HTTPException from fastapi import Path, Request @@ -53,6 +53,7 @@ from .data import * from .abstract import * from .snmp import * from .mocks import * +from .iso8601 import parse_date import asyncio import contextlib @@ -72,6 +73,8 @@ import unittest import urllib import websockets +epsilon = sys.float_info.epsilon + # fix up parse_socket_addr for hypercorn from hypercorn.utils import parse_socket_addr from hypercorn.asyncio import tcp_server @@ -83,6 +86,47 @@ def new_parse_socket_addr(domain, addr): tcp_server.parse_socket_addr = new_parse_socket_addr +def looptoutc(looptime): + '''The argument looptime, which is a time stamp relative to the + current event loop's clock, to UTC. + + It does this by calculating the current offset, and applying that + offset. This will deal with any time drift issues as it is + expected that the loop's clock does not stay in sync w/ UTC, but + it does mean that large differences from the current time are less + accurate. That is if the returned value - current UTC is large, + then the accuracy of the time is not very high. + + Modern clocks are pretty accurate, but modern crystals do have an + error that will accumulate over time. + + This is only tested to nanosecond precision, as floating point does + not allow higher precision (and even if it did, as there is no way + to get the offset between the two in a single call, it will likely + introduce a larger offset than nanoseconds). + ''' + + loop = asyncio.get_running_loop() + curlooptime = loop.time() + utctime = time.time() + + off = looptime - curlooptime + + return utctime + off + +def utctoloop(utctime): + '''For documentation, see looptoutc. This is the inverse, but + all the warnings in there apply here as well. + ''' + + loop = asyncio.get_running_loop() + looptime = loop.time() + curutctime = time.time() + + off = utctime - curutctime + + return looptime + off + async def log_event(tag, board=None, user=None, extra={}): info = extra.copy() info['event'] = tag @@ -103,8 +147,101 @@ async def log_event(tag, board=None, user=None, extra={}): logging.info(json.dumps(info)) -class TimeOut(DefROAttribute): - defattername = 'timeout' +class TimeOut(Attribute): + ''' + Implement a TimeOut functionality. The argument val (first and + only) to __init__ is a number of seconds for the timeout to + last. This will start the time ticking on activation. If it + is not deactivated before the timer expires, it will deactivate + the board itself. + + Not that this uses the asyncio loop timescale and NOT UTC. This + means that over large durations, the clock will drift. This means + that over time, the "expired" time will change. + + While the board is not activated, it will display the timeout in + seconds. When the board is activated, the getvalue will return + the time the board will be deactivated. + ''' + + defattrname = 'timeout' + + def __init__(self, val): + self._value = val + self._brd = None + + # proteted by brd.lock + self._cb = None + self._task = None + self._exp = None + + async def getvalue(self): + if self._exp is None: + return self._value + + t = looptoutc(self._exp) + + return time.strftime('%Y-%m-%dT%H:%M:%S', time.gmtime(t)) + \ + '.%03dZ' % (int((t * 1000) % 1000),) + + async def setvalue(self, v): + if self._exp is None: + raise RuntimeError('cannot set when not activate') + + loop = asyncio.get_running_loop() + + t = parse_date(v).timestamp() + loopvalue = utctoloop(t) + + async with self._brd.lock: + if loopvalue > self._exp: + raise ValueError('value in the future') + + # I really don't know how test this + if self._task is not None: + raise ValueError('should never happen') + + await self.deactivate(self._brd) + self._cb = loop.call_at(loopvalue, self.timeout_callback) + self._exp = self._cb.when() + + async def activate(self, brd): + assert brd.lock.locked() + + loop = asyncio.get_running_loop() + self._brd = brd + self._cb = loop.call_later(self._value, self.timeout_callback) + self._exp = self._cb.when() + self._task = None + + async def deactivate(self, brd): + assert brd.lock.locked() + + if self._cb is not None: + self._cb.cancel() + self._cb = None + + if self._task is not None: + self._task.cancel() + + # awaiting on a canceled task blocks, spin the + # loop and make sure it was cancelled + await asyncio.sleep(0) + assert self._task.cancelled() + self._task = None + + self._exp = None + + @_tbprinter + async def timeout_coro(self): + print('tc1') + async with self._brd.lock: + print('tc2') + await self._brd.release() + print('tc3') + + def timeout_callback(self): + self._task = asyncio.create_task(self.timeout_coro()) class EtherIface(DefROAttribute): defattrname = 'eiface' @@ -147,7 +284,13 @@ class BoardImpl: self.attrmap = {} self.lock = asyncio.Lock() for i in options: - self.attrmap[i.defattrname] = i + cls, kwargs = i + opt = cls(**kwargs) + if opt.defattrname in self.attrmap: + raise ValueError( + 'attribute name %s duplicated' % + repr(opt.defattrname)) + self.attrmap[opt.defattrname] = opt self.attrcache = {} @@ -177,13 +320,13 @@ class BoardImpl: async def activate(self): assert self.lock.locked() and self.reserved - for i in self.options: + for i in self.attrmap.values(): await i.activate(self) async def deactivate(self): assert self.lock.locked() and self.reserved - for i in self.options: + for i in self.attrmap.values(): await i.deactivate(self) def add_info(self, d): @@ -230,7 +373,7 @@ class BoardManager(object): classes = conf['classes'] brds = conf['boards'] - makeopt = lambda x: cls._option_map[x['cls']](**{ k: v for k, v in x.items() if k != 'cls' }) + makeopt = lambda x: (cls._option_map[x['cls']], { k: v for k, v in x.items() if k != 'cls' }) for i in brds: opt = i['options'] opt[:] = [ makeopt(x) for x in opt ] @@ -1232,8 +1375,9 @@ class TestBiteLab(TestCommon): class TestBoardImpl(unittest.IsolatedAsyncioTestCase): async def test_activate(self): # that a board impl - opt = create_autospec(Attribute) - brd = BoardImpl('foo', 'bar', [ opt ]) + opttup = create_autospec, dict(spec=Attribute) + brd = BoardImpl('foo', 'bar', [ opttup ]) + (opt,) = tuple(brd.attrmap.values()) async with brd.lock: await brd.reserve() @@ -1243,8 +1387,9 @@ class TestBoardImpl(unittest.IsolatedAsyncioTestCase): async def test_deactivate(self): # that a board impl - opt = create_autospec(Attribute) - brd = BoardImpl('foo', 'bar', [ opt ]) + opttup = create_autospec, dict(spec=Attribute) + brd = BoardImpl('foo', 'bar', [ opttup ]) + (opt,) = tuple(brd.attrmap.values()) async with brd.lock: await brd.reserve() @@ -1299,7 +1444,11 @@ class TestAttrs(unittest.IsolatedAsyncioTestCase): @patch('asyncio.create_subprocess_exec') async def test_serialconsole(self, cse): data = 'somepath' - sc = SerialConsole(data) + + sctup = (SerialConsole, dict(val=data)) + + brd = BoardImpl('foo', 'bar', [ sctup ]) + sc = brd.attrmap['console'] self.assertEqual(sc.defattrname, 'console') @@ -1310,7 +1459,6 @@ class TestAttrs(unittest.IsolatedAsyncioTestCase): devfspath = 'eifd' - brd = BoardImpl('foo', 'bar', [ sc ]) brd.add_info(dict(devfspath=devfspath)) wrap_subprocess_exec(cse, retcode=0) @@ -1337,7 +1485,11 @@ class TestAttrs(unittest.IsolatedAsyncioTestCase): async def test_etheriface(self, cse): eiface = 'aneiface' - ei = EtherIface(eiface) + eitup = EtherIface, dict(val=eiface) + + brd = BoardImpl('foo', 'bar', [ eitup ]) + + ei = brd.attrmap['eiface'] self.assertEqual(ei.defattrname, 'eiface') @@ -1346,8 +1498,6 @@ class TestAttrs(unittest.IsolatedAsyncioTestCase): with self.assertRaises(TypeError): await ei.setvalue('randomdata') - brd = BoardImpl('foo', 'bar', [ ei ]) - wrap_subprocess_exec(cse, retcode=0) await ei.activate(brd) @@ -1360,9 +1510,172 @@ class TestAttrs(unittest.IsolatedAsyncioTestCase): with self.assertRaises(RuntimeError): await ei.activate(brd) + async def test_multipleattrs(self): + attrs = [ (Power, dict()) ] * 2 + + # That multiple attributes w/ same name raises ValueError + with self.assertRaises(ValueError): + BoardImpl('foo', 'bar', attrs) + + # Enough of this code depends upon the event loop using the + # code in BaseEventLoop wrt scheduling that this is not a + # terrible test. If this fails, it is likely the selector + # doesn't use a sane event loop. + @patch('asyncio.BaseEventLoop.time') + @patch('time.time') + async def test_looptoutc(self, ttime, beltime): + loop = asyncio.get_running_loop() + + utctime = 19239 + belsrctime = 892934 + ttime.return_value = utctime + beltime.return_value = belsrctime + + # that when given the current loop time, that + # it returns the current utc time + self.assertEqual(looptoutc(belsrctime), utctime) + + # then when an offset is applied + import random + offset = random.random() * 1000000 + offset = .000999 * 100000 + + # the utc has the same offset + # it'd be nice if this was exact, but it's not because + # floating point. 9 places gets us nanosecond precision + self.assertAlmostEqual(looptoutc(belsrctime + offset), utctime + offset, places=9) + + # make sure w/ the new code, it round trips + sometime = 238974.34 + self.assertAlmostEqual(utctoloop(looptoutc(sometime)), sometime) + self.assertAlmostEqual(looptoutc(utctoloop(sometime)), sometime) + + @timeout(2) + @patch('asyncio.BaseEventLoop.time') + @patch('time.time') + async def test_timeout_vals(self, ttime, belt): + # that a TimeOut with args + totup = TimeOut, dict(val=10) + + # passed to a board w/ the totup + brd = BoardImpl('foo', 'bar', [ totup ]) + + to = brd.attrmap['timeout'] + + with self.assertRaises(RuntimeError): + # that setting the value when not activate errors + await to.setvalue(234987) + + # that an update will populate the attrs. + await brd.update() + + # and that the board attrs will be present + # and contain the current timeout + self.assertEqual(brd.attrs, dict(timeout=10)) + + # that a given loop time + looptime = 100.384 + belt.return_value = 100.384 + + # and a given UTC time (hu Dec 10 14:06:35 UTC 2020) + utctime = 1607609195.28 + ttime.return_value = utctime + + # that when reserved/activated + async with brd.lock: + await brd.reserve() + await brd.activate() + + await brd.update() + + # That it returns timeout seconds in the future. + self.assertEqual(brd.attrs, dict(timeout='2020-12-10T14:06:45.280Z')) + + with self.assertRaises(ValueError): + # that setting it to a value farther into + # the future fails + await to.setvalue('2020-12-10T14:06:55.280Z') + + with self.assertRaises(ValueError): + # that passing a non-Z ending (not UTC) date fails + await to.setvalue('2020-12-10T14:06:55.28') + + # that setting it to a time slightly earlier + await to.setvalue('2020-12-10T14:06:44.280Z') + + await brd.update() + + # That it returns that time + self.assertEqual(brd.attrs, dict(timeout='2020-12-10T14:06:44.280Z')) + + @timeout(2) async def test_timeout(self): - # that a TimeOut can be created - to = TimeOut(.1) + # that a TimeOut with args + totup = TimeOut, dict(val=.01) + + # passed to a board w/ the totup + brd = BoardImpl('foo', 'bar', [ totup ]) + + to = brd.attrmap['timeout'] + + # that when reserved/activated + async with brd.lock: + await brd.reserve() + await brd.activate() + + evt = asyncio.Event() + loop = asyncio.get_running_loop() + loop.call_at(to._exp + epsilon, evt.set) + await evt.wait() + + # that the board is no longer reserved + self.assertFalse(brd.reserved) + + # that when reserved/activated/deactivated/released + async with brd.lock: + await brd.reserve() + await brd.activate() + exp = to._exp + await brd.deactivate() + await brd.release() + + # that the expiration is no longer there + self.assertIsNone(to._exp) + print('z') + + # and the timeout passes + evt = asyncio.Event() + loop = asyncio.get_running_loop() + loop.call_at(exp + epsilon, evt.set) + await evt.wait() + + print('a') + # that when reserved/activated + async with brd.lock: + await brd.reserve() + await brd.activate() + + print('b') + # but the board is locked for some reason + await brd.lock.acquire() + + print('c') + # and the callback is called + await asyncio.sleep(.02) + + print('d') + # that the task has been scheduled + self.assertIsNotNone(to._task) + + print('e') + # that it can be deactivated + await brd.deactivate() + + print('f') + # and when the board lock is released + brd.lock.release() + + print('g') + # that the board was not released + self.assertTrue(brd.reserved) - # that a board w/ the to - brd = BoardImpl('foo', 'bar', [ to ]) diff --git a/bitelab/iso8601.py b/bitelab/iso8601.py new file mode 100644 index 0000000..566beec --- /dev/null +++ b/bitelab/iso8601.py @@ -0,0 +1,240 @@ +# From: +# https://github.com/micktwomey/pyiso8601 +# +# +# This file is licensed: +# Copyright (c) 2007 - 2015 Michael Twomey +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +"""ISO 8601 date time string parsing + +Basic usage: +>>> import iso8601 +>>> iso8601.parse_date("2007-01-25T12:00:00Z") +datetime.datetime(2007, 1, 25, 12, 0, tzinfo=) +>>> + +""" + +import datetime +from decimal import Decimal +import sys +import re + +__all__ = ["parse_date", "ParseError", "UTC", + "FixedOffset"] + +if sys.version_info >= (3, 0, 0): + _basestring = str +else: + _basestring = basestring + + +# Adapted from http://delete.me.uk/2005/03/iso8601.html +ISO8601_REGEX = re.compile( + r""" + (?P[0-9]{4}) + ( + ( + (-(?P[0-9]{1,2})) + | + (?P[0-9]{2}) + (?!$) # Don't allow YYYYMM + ) + ( + ( + (-(?P[0-9]{1,2})) + | + (?P[0-9]{2}) + ) + ( + ( + (?P[ T]) + (?P[0-9]{2}) + (:{0,1}(?P[0-9]{2})){0,1} + ( + :{0,1}(?P[0-9]{1,2}) + ([.,](?P[0-9]+)){0,1} + ){0,1} + (?P + Z + | + ( + (?P[-+]) + (?P[0-9]{2}) + :{0,1} + (?P[0-9]{2}){0,1} + ) + ){0,1} + ){0,1} + ) + ){0,1} # YYYY-MM + ){0,1} # YYYY only + $ + """, + re.VERBOSE +) + +class ParseError(Exception): + """Raised when there is a problem parsing a date string""" + +if sys.version_info >= (3, 2, 0): + UTC = datetime.timezone.utc + def FixedOffset(offset_hours, offset_minutes, name): + return datetime.timezone( + datetime.timedelta( + hours=offset_hours, minutes=offset_minutes), + name) +else: + # Yoinked from python docs + ZERO = datetime.timedelta(0) + class Utc(datetime.tzinfo): + """UTC Timezone + + """ + def utcoffset(self, dt): + return ZERO + + def tzname(self, dt): + return "UTC" + + def dst(self, dt): + return ZERO + + def __repr__(self): + return "" + + UTC = Utc() + + class FixedOffset(datetime.tzinfo): + """Fixed offset in hours and minutes from UTC + + """ + def __init__(self, offset_hours, offset_minutes, name): + self.__offset_hours = offset_hours # Keep for later __getinitargs__ + self.__offset_minutes = offset_minutes # Keep for later __getinitargs__ + self.__offset = datetime.timedelta( + hours=offset_hours, minutes=offset_minutes) + self.__name = name + + def __eq__(self, other): + if isinstance(other, FixedOffset): + return ( + (other.__offset == self.__offset) + and + (other.__name == self.__name) + ) + return NotImplemented + + def __getinitargs__(self): + return (self.__offset_hours, self.__offset_minutes, self.__name) + + def utcoffset(self, dt): + return self.__offset + + def tzname(self, dt): + return self.__name + + def dst(self, dt): + return ZERO + + def __repr__(self): + return "" % (self.__name, self.__offset) + + +def to_int(d, key, default_to_zero=False, default=None, required=True): + """Pull a value from the dict and convert to int + + :param default_to_zero: If the value is None or empty, treat it as zero + :param default: If the value is missing in the dict use this default + + """ + value = d.get(key) or default + if (value in ["", None]) and default_to_zero: + return 0 + if value is None: + if required: + raise ParseError("Unable to read %s from %s" % (key, d)) + else: + return int(value) + +def parse_timezone(matches, default_timezone=UTC): + """Parses ISO 8601 time zone specs into tzinfo offsets + + """ + + if matches["timezone"] == "Z": + return UTC + # This isn't strictly correct, but it's common to encounter dates without + # timezones so I'll assume the default (which defaults to UTC). + # Addresses issue 4. + if matches["timezone"] is None: + return default_timezone + sign = matches["tz_sign"] + hours = to_int(matches, "tz_hour") + minutes = to_int(matches, "tz_minute", default_to_zero=True) + description = "%s%02d:%02d" % (sign, hours, minutes) + if sign == "-": + hours = -hours + minutes = -minutes + return FixedOffset(hours, minutes, description) + +def parse_date(datestring, default_timezone=UTC): + """Parses ISO 8601 dates into datetime objects + + The timezone is parsed from the date string. However it is quite common to + have dates without a timezone (not strictly correct). In this case the + default timezone specified in default_timezone is used. This is UTC by + default. + + :param datestring: The date to parse as a string + :param default_timezone: A datetime tzinfo instance to use when no timezone + is specified in the datestring. If this is set to + None then a naive datetime object is returned. + :returns: A datetime.datetime instance + :raises: ParseError when there is a problem parsing the date or + constructing the datetime instance. + + """ + if not isinstance(datestring, _basestring): + raise ParseError("Expecting a string %r" % datestring) + m = ISO8601_REGEX.match(datestring) + if not m: + raise ParseError("Unable to parse date string %r" % datestring) + groups = m.groupdict() + + tz = parse_timezone(groups, default_timezone=default_timezone) + + groups["second_fraction"] = int(Decimal("0.%s" % (groups["second_fraction"] or 0)) * Decimal("1000000.0")) + + try: + return datetime.datetime( + year=to_int(groups, "year"), + month=to_int(groups, "month", default=to_int(groups, "monthdash", required=False, default=1)), + day=to_int(groups, "day", default=to_int(groups, "daydash", required=False, default=1)), + hour=to_int(groups, "hour", default_to_zero=True), + minute=to_int(groups, "minute", default_to_zero=True), + second=to_int(groups, "second", default_to_zero=True), + microsecond=groups["second_fraction"], + tzinfo=tz, + ) + except Exception as e: + raise ParseError(e)