add a script that generates fake data.. definitely needs work...

Also, get basic data being loaded, and get the fetching to work
somewhat ok.  Needed to deal w/ the fact that HighStocks does stupid
requests data...

Issues still present:
	not timezone aware
	caps between days in the data
	not all navigator series are bars

.gitignore

@@ -3,6 +3,7 @@ solardash.egg-info
 p
 *.pyc
 .coverage
+fakedata.js
 root/js/solardash.file.js
 root/js/solardash.https.js
 root/js/solardash.http.js

Makefile

@@ -14,10 +14,21 @@ FILES=$(PROJNAME)/__init__.py
 
 JSFILES = root/js/solardash.file.js root/js/solardash.https.js root/js/solardash.http.js
 
-THIRDPARTYJS = root/js/jquery.js root/js/highstock.js
+THIRDPARTYJS = \
+	root/js/jquery.js \
+	root/js/highstock.js \
+
+#	root/js/moment.min.js \
+#	root/js/moment-timezone-with-data.min.js
 
 root/js/jquery.js:
-	wget -O $@ "https://code.jquery.com/jquery-3.4.1.min.js"
+	wget -O $@ "https://code.jquery.com/jquery-3.4.1.min.js" || (rm "$@"; false)
+
+root/js/moment.min.js:
+	wget -O $@ "https://momentjs.com/downloads/moment.min.js" || (rm "$@"; false)
+
+root/js/moment-timezone-with-data.min.js:
+	wget -O $@ "https://momentjs.com/downloads/moment-timezone-with-data-1970-2030.js" || (rm "$@"; false)
 
 root/js/highstock.js: Makefile
 	wget -O - "https://code.highcharts.com/stock/8.0.0/highstock.js" | grep -v '^//# sourceMappingURL=' > $@ || (rm "$@"; false)
@@ -36,8 +47,15 @@ run: $(JSFILES)
 
 $(JSFILES): $(THIRDPARTYJS) $(JSBASE)
 
+root/js/solardash.file.js: fakedata.js
+
+fakedata.js: fakedata.py
+	python $< > $@ || (rm $@; false)
+
 .jspp.js:
-	cat $< $(THIRDPARTYJS) $(JSBASE) > $@ || (rm "$@"; false)
+	# bsdmake uses $>, gmake uses $^
+	(echo '// DO NOT EDIT FILE!!!!  THIS IS AUTOMATICALLY GENERATED!!!'; cat $^) > $@ || (rm "$@"; false)
+	#cat $< $(THIRDPARTYJS) $(JSBASE) > $@ || (rm $@; false)
 
 keepupdate:
 	find . -name '*.js' -o -name '*.jspp' | entr make all

NOTES.md

@@ -52,6 +52,14 @@ Maybe heatmap for solar panels
 Use removePoint and addPoint: https://web.archive.org/web/20200109083308/https://api.highcharts.com/class-reference/Highcharts.Series.html
 to add/remove the end null point and add points as they dynamically arrive
 
+Sample Home power Consumption dataset:
+https://archive.ics.uci.edu/ml/datasets/individual+household+electric+power+consumption
+
+List of Solar resources:
+https://energydemo.github.io/SolarDatasets/
+
+https://pvoutput.org
+
 WebSocket Definition
 --------------------
 
@@ -73,6 +81,13 @@ Messages from the websocket:
 	ov	overview of data, contains gird, production and consumption data.
 		following is JSON object
 
+	win	The graph has set a window size.  The two arguments are start and end.  If this is the
+		first time, NaN will be sent for both, which means to send all data.
+
+	windata	Data in response to a window command.  The format is an object w/ the keys, production,
+		grid and consumption.  Each will be a list of data points.  Each point is a pair of
+		timestamp in miliseconds and the value for the point.
+
 One the first connection, the following messages/data will be sent:
 	c, ng, p, ov
 

README.md

@@ -2,3 +2,13 @@ Solar Dashboard
 ===============
 
 This is a solar dashboard for displaying information about a solar install.
+
+Structure
+---------
+
+Frontend
+
+The front end is located in root.  The meat of the logic is in
+`js/solardash.base.js`.  The testing infrastructure is in
+`js/solardash.file.js` as if you are launching the page from a file
+url, there is no way to make a url to access the backend.

cmds.txt

@@ -0,0 +1 @@
+ls fakedata.py | entr sh -c 'python fakedata.py > fake.txt; echo plot \"fake.txt\" | gnuplot'

fakedata.py

@@ -0,0 +1,133 @@
+import arrow
+import random
+import pprint
+import json
+
+from datetime import datetime, timedelta
+
+rand = random.Random('a seed')
+
+tz = 'US/Pacific'
+
+startdate = arrow.Arrow(2019, 11, 1, tzinfo=tz)
+enddate = arrow.Arrow(2019, 12, 10, tzinfo=tz)
+
+meanwhprod = 20000
+sigwhprod = 2000
+
+def drange(s, e, interval):
+	cmpfun = lambda s, e, ts, te: te < e
+	if e < s:
+		interval = -interval
+		cmpfun = lambda s, e, ts, te: te > e
+	ts = s.clone()
+	te = ts + interval
+	#print('dr:', repr((s, e, ts, te, cmpfun(s, e, ts, te), interval)))
+	while cmpfun(s, e, ts, te):
+		yield ts + (te - ts)
+		ts, te = te, te + interval
+
+# idea:
+# first hour linear ramp up (25% in first half, 75% in second half)
+# middle 5 hours near constant generation
+# first/tailing linear is equiv of an hour total, so total power / 6
+
+# approx 7 hours time
+def makestartend(t):
+	s = t.replace(hour=9).shift(minutes=rand.gauss(30, 10))
+	e = t.replace(hour=16).shift(minutes=rand.gauss(30, 10))
+
+	return (s, e)
+
+def normdist(small, big, amount, minsize):
+	'''Distribute most twoards the big side, total distribute amount
+	over the entire range, [small, big].
+	'''
+
+	ret = []
+
+	timediff = abs(big - small)
+	if timediff < minsize:
+		scaledamount = amount * (timedelta(hours=1) / timediff)
+		midpnt = small + (big - small) / 2
+		#print('ndf:', repr((small, big, midpnt, amount, scaledamount)))
+		return [ (midpnt, scaledamount) ]
+
+	#print('nd:', repr((small, big, amount)))
+	dist = big - small
+	halfpoint = small + (dist / 9 * 5)
+	ret.extend(normdist(small, halfpoint, amount / 2, minsize))
+	ret.extend(normdist(halfpoint, big, amount / 2, minsize))
+
+	#print('ndr:')
+	#pprint.pprint(ret)
+	return ret
+
+def linramp(start, end, wtarget, minsize):
+	mid = start + (end - start) / 2
+	timediff = abs(end - start)
+	ret = []
+
+	ndates = abs((end - start) / minsize)
+	#print('lr', ndates)
+	for x, i in enumerate(drange(start, end, minsize)):
+		#print(repr((x, i)))
+		yield (i, (x + 1) / ndates * wtarget)
+
+def distribute(s, e, prod, minsize):
+	onehour = timedelta(hours=1)
+	totaltime = e - s
+	mid = s + totaltime / 2
+	startrampend = s + onehour
+	endrampstart = e - onehour
+
+	# prod == wh
+	wtarget = prod / ((totaltime + onehour).seconds / 60 / 60)
+
+	ret = []
+
+	#print('d:', repr((s, e)))
+	ret.extend(linramp(s, startrampend, wtarget, minsize))
+
+	for i in drange(startrampend, endrampstart, minsize):
+		ret.append((i, wtarget))
+
+	ret.extend(linramp(e, endrampstart, wtarget, minsize))
+
+	ret.sort()
+	#pprint.pprint(ret)
+
+	return ret
+
+#print('start')
+
+points = []
+index = []
+
+def serializearrowasmili(obj):
+	if not isinstance(obj, arrow.Arrow):
+		raise TypeError
+
+	return int(obj.float_timestamp*1000)
+
+for i in arrow.Arrow.range('day', startdate, enddate):
+	whprod = rand.gauss(meanwhprod, sigwhprod)
+	s, e = makestartend(i)
+	noon = i.replace(hour=12)
+
+	index.append((noon, whprod))
+	#print(repr(i), whprod)
+	dist = distribute(s, e, whprod, timedelta(seconds=20))
+
+	# print timestamps as miliseconds
+	if False:
+		dist = ((int(a.float_timestamp*1000), b) for a, b in dist)
+
+	# print space delimited time, else json
+	if False:
+		print('\n'.join('%s %s' % (a, b) for a, b in dist))
+	else:
+		#print(json.dumps(tuple(dist), indent=2))
+		points.extend(dist)
+
+print('fakedata =', json.dumps(dict(production=points, index=index), default=serializearrowasmili))

root/js/solardash.base.js

@@ -1,3 +1,9 @@
+//Highcharts.setOptions({
+//    time: {
+//        timezone: 'America/Los_Angeles'
+//    }
+//});
+
 socket.onopen = function() {
 	// connected
 }
@@ -20,6 +26,19 @@ function netgridcolor(v) {
 		return "#ff0000";
 }
 
+function afterSetExtremes(e) {
+	// hack to deal w/ HighStocks being stupid
+	if (Date.now() - solarchart.sd_lastwindata < 500) {
+		// Make sure we trigger again, and that the user doesn't
+		// have to wait too long
+		solarchart.sd_lastwindata = Date.now() - 500;
+		return;
+	}
+	solarchart.showLoading('Loading data from server...');
+	console.log(e)
+	socket.send('win ' + Math.round(e.min).toString() + ' ' + Math.round(e.max).toString());
+}
+
 socket.onmessage = function(m) {
 	var msg = m.data.split(" ");
 	if (msg[0] in msgNumbs) {
@@ -40,6 +59,15 @@ socket.onmessage = function(m) {
 		solarchart.navigator.series[gridIdx].setData(data.grid, false);
 		solarchart.navigator.series[prodIdx].setData(data.production, false);
 		solarchart.redraw();
+	} else if (msg[0] == 'windata') {
+		var data = JSON.parse(msg[1]);
+		console.log(data);
+		solarchart.sd_lastwindata = Date.now();
+		solarchart.series[consumIdx].setData(data.consumption, false);
+		//solarchart.navigator.series[gridIdx].setData(data.grid, false);
+		//solarchart.navigator.series[prodIdx].setData(data.production, false);
+		solarchart.redraw();
+		solarchart.hideLoading()
 	}
 }
 
@@ -59,16 +87,19 @@ var solarchart = Highcharts.stockChart('solarchart', {
 		adaptToUpdatedData: false, // XXX - keep?
 		series: [
 			{
+				type: 'bar',
 				name: 'consumptionNav',
 				showInNavigator: true,
 				data: [],
 			},
 			{
+				type: 'bar',
 				name: 'gridNav',
 				showInNavigator: true,
 				data: [],
 			},
 			{
+				type: 'bar',
 				name: 'productionNav',
 				showInNavigator: true,
 				data: [],
@@ -76,6 +107,9 @@ var solarchart = Highcharts.stockChart('solarchart', {
 		]
 	},
 	xAxis: {
+		events: {
+			afterSetExtremes: afterSetExtremes,
+		},
 		type: "datetime",
 		title: {
 			enabled: false
@@ -83,12 +117,12 @@ var solarchart = Highcharts.stockChart('solarchart', {
 	},
 	yAxis: {
 		title: {
-			text: 'kW'
+			text: 'W'
 		}
 	},
 	tooltip: {
 		split: true,
-		valueSuffix: ' kW'
+		valueSuffix: ' W'
 	},
 	plotOptions: {
 		area: {
@@ -104,7 +138,10 @@ var solarchart = Highcharts.stockChart('solarchart', {
 	series: [
 		{
 			name: 'grid',
-			data: fakeData,
+			type: 'area',
+			gapSize: 1,
+			//gapUnit: 'value',
+			data: [ ],
 		},
 	]
 });

root/js/solardash.file.jspp

@@ -7,7 +7,45 @@ function WebSocketTest(actions) {
 	this.processNextItem()
 }
 
-// WebSocketTest.prototype.send = function ()
+Array.prototype.bisect = function (val, lo, hi) {
+	var mid;
+	if (lo == null)
+		lo = 0;
+	if (hi == null)
+		hi = this.length;
+
+	while (lo < hi) {
+		mid = Math.floor((lo + hi) / 2);
+		if (this[mid] < val)
+			lo = mid + 1;
+		else
+			hi = mid;
+	}
+	return lo;
+};
+
+WebSocketTest.prototype.send = function (s) {
+	console.log("ws send: " + s);
+	var msg = s.split(" ");
+
+	if (msg[0] == 'win') {
+		if (msg[1] == 'NaN')
+			msg[1] = -Infinity;
+		if (msg[2] == 'NaN')
+			msg[2] = Infinity;
+		var loidx, hiidx;
+		loidx = fakedata.production.bisect([msg[1], 0]);
+		hiidx = fakedata.production.bisect([msg[2], 0]);
+		var subar = fakedata.production.slice(loidx, hiidx + 1);
+		var data = {
+			"production": subar,
+			"consumption": subar,
+			"grid": subar,
+		}
+		this.makercv('windata ' + JSON.stringify(data));
+	}
+}
+
 // WebSocketTest.prototype.close = function ()
 
 // Internal
@@ -26,6 +64,9 @@ WebSocketTest.prototype.processNextItem = function () {
 
 	return this;
 }
+WebSocketTest.prototype.makercv = function (m) {
+	this.onmessage(new MessageEvent('websockettest', { data: m }));
+}
 
 fakeData = [
 	[ 1578544199000, 0.3934 ],
@@ -59,26 +100,21 @@ fakeOverviewData = {
 	production: fakeConData,
 }
 
+function getoverviewdata() {
+	return {
+		grid: fakedata.index,
+		consumption: fakedata.index,
+		production: fakedata.index,
+	}
+}
 
 // Setup the socket that will be used
 var socket = new WebSocketTest([
 	[ 10, function(a) { a.onopen(new Object()) } ],
-	[ 10, function(a) { a.onmessage(new MessageEvent('websockettest', {
-		  data : 'o ' + JSON.stringify(fakeOverviewData)
-		})) } ],
-	[ 10, function(a) { a.onmessage(new MessageEvent('websockettest', {
-		  data : 'p .123'
-		})) } ],
-	[ 10, function(a) { a.onmessage(new MessageEvent('websockettest', {
-		  data : 'c .302'
-		})) } ],
-	[ 10, function(a) { a.onmessage(new MessageEvent('websockettest', {
-		  data : 'ng .758'
-		})) } ],
-	[ 2000, function(a) { a.onmessage(new MessageEvent('websockettest', {
-		  data : 'p .234'
-		})) } ],
-	[ 10, function(a) { a.onmessage(new MessageEvent('websockettest', {
-		  data : 'ng -.584'
-		})) } ],
+	[ 10, function(a) { a.makercv('o ' + JSON.stringify(getoverviewdata())) } ],
+	[ 10, function(a) { a.makercv('p .123') } ],
+	[ 10, function(a) { a.makercv('c .302') } ],
+	[ 10, function(a) { a.makercv('ng .758') } ],
+	[ 2000, function(a) { a.makercv('p .234') } ],
+	[ 10, function(a) { a.makercv('ng -.584') } ],
 ]);