add first cut of converting twitter threads into blog posts...

`twitfetch.py` fetches the threads defined in `tweets.txt`, and
then the tmux.html has an example (will likely be made a macro
in the future)...

Makefile

@@ -14,6 +14,10 @@ vnu.jar_$(VNU_RELEASE).zip:
 vnu.jar: vnu.jar_$(VNU_RELEASE).zip
 	unzip -p $< dist/vnu.jar > $@ || (rm "$@"; false)
 
+.PHONY:
+test:
+	ls fixtures/* encthenet_plugins.py | entr python -m unittest encthenet_plugins
+
 .PHONY:
 validate: vnu.jar
 	hyde gen

NOTES.md

@@ -91,6 +91,34 @@ immediate children.  Add a `>` to restrict it.
 Hack to make a CSS only clickable be exposed in order to handle the nav bar:
 https://stackoverflow.com/questions/6019845/show-hide-div-on-click-with-css
 
+List of filters: `jinja2/filters.py` (`FILTERS`) and `hyde/ext/templates/jinja.py` (`self.env.filters`)
+
+Notes on Twitter
+================
+
+A Python module `twauth.py` stores credentials.  It should contain
+a memember `twit_argskw` that is a tuple of (args, kwargs).  These will be used
+as arguments to the Twitter instance.  Here is a sample:
+
+```
+from twitter import OAuth
+
+tok = 'thetoken'
+tokkey = 'tokenkey'
+
+apikey = 'apikeyfromrunningthis'
+apisec = 'apisecretkey'
+
+twit_argskw = (), dict(auth=OAuth(tok, tokkey, apikey, apisec))
+
+__all__ = [ 'twit_argskw' ]
+
+if __name__ == '__main__':
+	from twitter import oauth_dance
+
+	oauth_dance('pytwscr', tok, tokkey)
+```
+
 TODO
 ====
 

content/drafts/index.html

@@ -4,3 +4,5 @@ description: Blog posts in progress
 extends: drafts.j2
 listable: false
 ---
+
+Empty

content/drafts/tmux.html

@@ -0,0 +1,20 @@
+---
+title: tmux beginners guide
+description: Quick simple beginners guide to using tmux
+created: !!timestamp '2021-01-01'
+extends: drafts.j2
+listable: false
+time: 12:00 PM
+tags:
+ - tweets
+---
+
+<p>Originally Posted: {{ site.content.resource_from_relative_path('twitter/1556349580932833280.yaml').meta.created_at }}</p>
+{% for twdata in get_tweet_thread(1556349580932833280, 1556353959723356160)  %}
+{{ twdata.meta | gettweettext }}
+{% for media in twdata.meta.extended_entities.media %}
+<p>
+<img src="/media/twitter/{{ (media.media_url | make_path).name }}" alt="{{ media.ext_alt_text | attr_escape }}">
+</p>
+{% endfor %}
+{% endfor %}

content/twitter/1556349580932833280.yaml

@@ -0,0 +1,3 @@
+---
+{"created_at": "Sun Aug 07 18:40:30 +0000 2022", "id": 1556349580932833280, "id_str": "1556349580932833280", "full_text": "@0xKruzr First, here's the .tmux.conf I use (alt text to copy/paste).\n\nThis remaps the control key from ctrl-b (^B) to ctrl-a which is more natural for me.\n\nNext is split windows, to have two windows side by side: ^A %, to have top/bottom: ^A \". https://t.co/3jPtg5Z41z", "truncated": false, "display_text_range": [9, 245], "entities": {"hashtags": [], "symbols": [], "user_mentions": [{"screen_name": "0xKruzr", "name": "\ud83c\udd79 (same handle at IG, Masto coming soon)", "id": 5424492, "id_str": "5424492", "indices": [0, 8]}], "urls": [], "media": [{"id": 1556348891120877569, "id_str": "1556348891120877569", "indices": [246, 269], "media_url": "http://pbs.twimg.com/media/FZlDEhHVQAEj5eA.png", "media_url_https": "https://pbs.twimg.com/media/FZlDEhHVQAEj5eA.png", "url": "https://t.co/3jPtg5Z41z", "display_url": "pic.twitter.com/3jPtg5Z41z", "expanded_url": "https://twitter.com/encthenet/status/1556349580932833280/photo/1", "type": "photo", "sizes": {"large": {"w": 714, "h": 712, "resize": "fit"}, "small": {"w": 680, "h": 678, "resize": "fit"}, "thumb": {"w": 150, "h": 150, "resize": "crop"}, "medium": {"w": 714, "h": 712, "resize": "fit"}}}]}, "extended_entities": {"media": [{"id": 1556348891120877569, "id_str": "1556348891120877569", "indices": [246, 269], "media_url": "http://pbs.twimg.com/media/FZlDEhHVQAEj5eA.png", "media_url_https": "https://pbs.twimg.com/media/FZlDEhHVQAEj5eA.png", "url": "https://t.co/3jPtg5Z41z", "display_url": "pic.twitter.com/3jPtg5Z41z", "expanded_url": "https://twitter.com/encthenet/status/1556349580932833280/photo/1", "type": "photo", "sizes": {"large": {"w": 714, "h": 712, "resize": "fit"}, "small": {"w": 680, "h": 678, "resize": "fit"}, "thumb": {"w": 150, "h": 150, "resize": "crop"}, "medium": {"w": 714, "h": 712, "resize": "fit"}}, "ext_alt_text": "unbind C-b\nset -g prefix C-a\nbind C-a send-prefix\n\nbind ^L refresh-client\nset-window-option -g mode-keys vi\n\nbind < resize-pane -L 4\nbind > resize-pane -R 4\nbind - resize-pane -D 4\nbind + resize-pane -U 4\n\nset -g mode-mouse off\nset -g mouse-resize-pane off\nset -g mouse-select-pane off\nset -g mouse-select-window off\n\nbind k select-pane -U\nbind j select-pane -D\nbind h select-pane -L\nbind l select-pane -R\n"}]}, "source": "<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>", "in_reply_to_status_id": 1556120685713522690, "in_reply_to_status_id_str": "1556120685713522690", "in_reply_to_user_id": 5424492, "in_reply_to_user_id_str": "5424492", "in_reply_to_screen_name": "0xKruzr", "user": {"id": 1972539134, "id_str": "1972539134", "name": "John-Mark Gurney \ud83d\uddd1\ufe0f\ud83d\udd25", "screen_name": "encthenet", "location": "Oakland, CA", "description": "he/they FreeBSD developer. consultant. documentation! crypto means cryptography. open dms.  tech is inherently political. Do your part, encrypt the Net!", "url": null, "entities": {"description": {"urls": []}}, "protected": false, "followers_count": 1870, "friends_count": 1956, "listed_count": 136, "created_at": "Sat Oct 19 18:59:42 +0000 2013", "favourites_count": 231707, "utc_offset": null, "time_zone": null, "geo_enabled": true, "verified": false, "statuses_count": 130080, "lang": null, "contributors_enabled": false, "is_translator": false, "is_translation_enabled": false, "profile_background_color": "352726", "profile_background_image_url": "http://abs.twimg.com/images/themes/theme5/bg.gif", "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme5/bg.gif", "profile_background_tile": false, "profile_image_url": "http://pbs.twimg.com/profile_images/622190067939827712/txSkhH63_normal.jpg", "profile_image_url_https": "https://pbs.twimg.com/profile_images/622190067939827712/txSkhH63_normal.jpg", "profile_banner_url": "https://pbs.twimg.com/profile_banners/1972539134/1591081516", "profile_image_extensions_alt_text": null, "profile_banner_extensions_alt_text": null, "profile_link_color": "D02B55", "profile_sidebar_border_color": "829D5E", "profile_sidebar_fill_color": "99CC33", "profile_text_color": "3E4415", "profile_use_background_image": true, "has_extended_profile": false, "default_profile": false, "default_profile_image": false, "following": false, "follow_request_sent": false, "notifications": false, "translator_type": "none", "withheld_in_countries": []}, "geo": null, "coordinates": null, "place": null, "contributors": null, "is_quote_status": false, "retweet_count": 1, "favorite_count": 3, "favorited": false, "retweeted": false, "possibly_sensitive": false, "possibly_sensitive_appealable": false, "lang": "en"}
+---

content/twitter/1556350064749998081.yaml

@@ -0,0 +1,3 @@
+---
+{"created_at": "Sun Aug 07 18:42:25 +0000 2022", "id": 1556350064749998081, "id_str": "1556350064749998081", "full_text": "@0xKruzr To move between windows, ^A then one of h (up) j (left) k (right) l (ell down), aka the vi movement keys, or you can use arrow keys.\n\nTo resize the current window, ^A &lt; or &gt; or + or -.\n\nTo swap two windows (to move them around), ^A { or ^A }.", "truncated": false, "display_text_range": [9, 257], "entities": {"hashtags": [], "symbols": [], "user_mentions": [{"screen_name": "0xKruzr", "name": "\ud83c\udd79 Is Thankful for the Cyrus SASL Project \ud83e\udd83", "id": 5424492, "id_str": "5424492", "indices": [0, 8]}], "urls": []}, "source": "<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>", "in_reply_to_status_id": 1556349580932833280, "in_reply_to_status_id_str": "1556349580932833280", "in_reply_to_user_id": 1972539134, "in_reply_to_user_id_str": "1972539134", "in_reply_to_screen_name": "encthenet", "user": {"id": 1972539134, "id_str": "1972539134", "name": "John-Mark Gurney \ud83d\uddd1\ufe0f\ud83d\udd25", "screen_name": "encthenet", "location": "Oakland, CA", "description": "he/they FreeBSD developer. consultant. documentation! crypto means cryptography. open dms.  tech is inherently political. Do your part, encrypt the Net!", "url": null, "entities": {"description": {"urls": []}}, "protected": false, "followers_count": 1870, "friends_count": 1956, "listed_count": 136, "created_at": "Sat Oct 19 18:59:42 +0000 2013", "favourites_count": 231707, "utc_offset": null, "time_zone": null, "geo_enabled": true, "verified": false, "statuses_count": 130080, "lang": null, "contributors_enabled": false, "is_translator": false, "is_translation_enabled": false, "profile_background_color": "352726", "profile_background_image_url": "http://abs.twimg.com/images/themes/theme5/bg.gif", "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme5/bg.gif", "profile_background_tile": false, "profile_image_url": "http://pbs.twimg.com/profile_images/622190067939827712/txSkhH63_normal.jpg", "profile_image_url_https": "https://pbs.twimg.com/profile_images/622190067939827712/txSkhH63_normal.jpg", "profile_banner_url": "https://pbs.twimg.com/profile_banners/1972539134/1591081516", "profile_image_extensions_alt_text": null, "profile_banner_extensions_alt_text": null, "profile_link_color": "D02B55", "profile_sidebar_border_color": "829D5E", "profile_sidebar_fill_color": "99CC33", "profile_text_color": "3E4415", "profile_use_background_image": true, "has_extended_profile": false, "default_profile": false, "default_profile_image": false, "following": false, "follow_request_sent": false, "notifications": false, "translator_type": "none", "withheld_in_countries": []}, "geo": null, "coordinates": null, "place": null, "contributors": null, "is_quote_status": false, "retweet_count": 0, "favorite_count": 1, "favorited": false, "retweeted": false, "lang": "en"}
+---

content/twitter/1556350651805798401.yaml

@@ -0,0 +1,3 @@
+---
+{"created_at": "Sun Aug 07 18:44:45 +0000 2022", "id": 1556350651805798401, "id_str": "1556350651805798401", "full_text": "@0xKruzr To copy between windows, ^A [.  Then move the cursor to where you want to start, hit space to start selection, move to the end, hit enter.  Now that you have your selection, to paste it, ^A ].\n\nTo create another workspace in the same session ^A c.  To switch workspaces, ^A num.", "truncated": false, "display_text_range": [9, 287], "entities": {"hashtags": [], "symbols": [], "user_mentions": [{"screen_name": "0xKruzr", "name": "\ud83c\udd79 Is Thankful for the Cyrus SASL Project \ud83e\udd83", "id": 5424492, "id_str": "5424492", "indices": [0, 8]}], "urls": []}, "source": "<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>", "in_reply_to_status_id": 1556350064749998081, "in_reply_to_status_id_str": "1556350064749998081", "in_reply_to_user_id": 1972539134, "in_reply_to_user_id_str": "1972539134", "in_reply_to_screen_name": "encthenet", "user": {"id": 1972539134, "id_str": "1972539134", "name": "John-Mark Gurney \ud83d\uddd1\ufe0f\ud83d\udd25", "screen_name": "encthenet", "location": "Oakland, CA", "description": "he/they FreeBSD developer. consultant. documentation! crypto means cryptography. open dms.  tech is inherently political. Do your part, encrypt the Net!", "url": null, "entities": {"description": {"urls": []}}, "protected": false, "followers_count": 1870, "friends_count": 1956, "listed_count": 136, "created_at": "Sat Oct 19 18:59:42 +0000 2013", "favourites_count": 231707, "utc_offset": null, "time_zone": null, "geo_enabled": true, "verified": false, "statuses_count": 130080, "lang": null, "contributors_enabled": false, "is_translator": false, "is_translation_enabled": false, "profile_background_color": "352726", "profile_background_image_url": "http://abs.twimg.com/images/themes/theme5/bg.gif", "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme5/bg.gif", "profile_background_tile": false, "profile_image_url": "http://pbs.twimg.com/profile_images/622190067939827712/txSkhH63_normal.jpg", "profile_image_url_https": "https://pbs.twimg.com/profile_images/622190067939827712/txSkhH63_normal.jpg", "profile_banner_url": "https://pbs.twimg.com/profile_banners/1972539134/1591081516", "profile_image_extensions_alt_text": null, "profile_banner_extensions_alt_text": null, "profile_link_color": "D02B55", "profile_sidebar_border_color": "829D5E", "profile_sidebar_fill_color": "99CC33", "profile_text_color": "3E4415", "profile_use_background_image": true, "has_extended_profile": false, "default_profile": false, "default_profile_image": false, "following": false, "follow_request_sent": false, "notifications": false, "translator_type": "none", "withheld_in_countries": []}, "geo": null, "coordinates": null, "place": null, "contributors": null, "is_quote_status": false, "retweet_count": 0, "favorite_count": 1, "favorited": false, "retweeted": false, "lang": "en"}
+---

content/twitter/1556350779216130048.yaml

@@ -0,0 +1,3 @@
+---
+{"created_at": "Sun Aug 07 18:45:16 +0000 2022", "id": 1556350779216130048, "id_str": "1556350779216130048", "full_text": "@0xKruzr Note that my window/workspace terminology doesn't line up w/ tmux's, but makes more sense to me.\n\nfin.", "truncated": false, "display_text_range": [9, 111], "entities": {"hashtags": [], "symbols": [], "user_mentions": [{"screen_name": "0xKruzr", "name": "\ud83c\udd79 (same handle at IG, Masto coming soon)", "id": 5424492, "id_str": "5424492", "indices": [0, 8]}], "urls": []}, "source": "<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>", "in_reply_to_status_id": 1556350651805798401, "in_reply_to_status_id_str": "1556350651805798401", "in_reply_to_user_id": 1972539134, "in_reply_to_user_id_str": "1972539134", "in_reply_to_screen_name": "encthenet", "user": {"id": 1972539134, "id_str": "1972539134", "name": "John-Mark Gurney \ud83d\uddd1\ufe0f\ud83d\udd25", "screen_name": "encthenet", "location": "Oakland, CA", "description": "he/they FreeBSD developer. consultant. documentation! crypto means cryptography. open dms.  tech is inherently political. Do your part, encrypt the Net!", "url": null, "entities": {"description": {"urls": []}}, "protected": false, "followers_count": 1870, "friends_count": 1956, "listed_count": 136, "created_at": "Sat Oct 19 18:59:42 +0000 2013", "favourites_count": 231707, "utc_offset": null, "time_zone": null, "geo_enabled": true, "verified": false, "statuses_count": 130080, "lang": null, "contributors_enabled": false, "is_translator": false, "is_translation_enabled": false, "profile_background_color": "352726", "profile_background_image_url": "http://abs.twimg.com/images/themes/theme5/bg.gif", "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme5/bg.gif", "profile_background_tile": false, "profile_image_url": "http://pbs.twimg.com/profile_images/622190067939827712/txSkhH63_normal.jpg", "profile_image_url_https": "https://pbs.twimg.com/profile_images/622190067939827712/txSkhH63_normal.jpg", "profile_banner_url": "https://pbs.twimg.com/profile_banners/1972539134/1591081516", "profile_image_extensions_alt_text": null, "profile_banner_extensions_alt_text": null, "profile_link_color": "D02B55", "profile_sidebar_border_color": "829D5E", "profile_sidebar_fill_color": "99CC33", "profile_text_color": "3E4415", "profile_use_background_image": true, "has_extended_profile": false, "default_profile": false, "default_profile_image": false, "following": false, "follow_request_sent": false, "notifications": false, "translator_type": "none", "withheld_in_countries": []}, "geo": null, "coordinates": null, "place": null, "contributors": null, "is_quote_status": false, "retweet_count": 0, "favorite_count": 1, "favorited": false, "retweeted": false, "lang": "en"}
+---

content/twitter/1556353959723356160.yaml

@@ -0,0 +1,3 @@
+---
+{"created_at": "Sun Aug 07 18:57:54 +0000 2022", "id": 1556353959723356160, "id_str": "1556353959723356160", "full_text": "@0xKruzr Oh, another thing.  You have have multiple terminals connected to the same session (tmux attach).  This is great for pair programming, so you both don't have to crowd around a terminal and swap keyboards, each can be on their own laptop, or now w/ WFH, be far away.", "truncated": false, "display_text_range": [9, 274], "entities": {"hashtags": [], "symbols": [], "user_mentions": [{"screen_name": "0xKruzr", "name": "\ud83c\udd79 (same handle at IG, Masto coming soon)", "id": 5424492, "id_str": "5424492", "indices": [0, 8]}], "urls": []}, "source": "<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>", "in_reply_to_status_id": 1556350779216130048, "in_reply_to_status_id_str": "1556350779216130048", "in_reply_to_user_id": 1972539134, "in_reply_to_user_id_str": "1972539134", "in_reply_to_screen_name": "encthenet", "user": {"id": 1972539134, "id_str": "1972539134", "name": "John-Mark Gurney \ud83d\uddd1\ufe0f\ud83d\udd25", "screen_name": "encthenet", "location": "Oakland, CA", "description": "he/they FreeBSD developer. consultant. documentation! crypto means cryptography. open dms.  tech is inherently political. Do your part, encrypt the Net!", "url": null, "entities": {"description": {"urls": []}}, "protected": false, "followers_count": 1870, "friends_count": 1956, "listed_count": 136, "created_at": "Sat Oct 19 18:59:42 +0000 2013", "favourites_count": 231707, "utc_offset": null, "time_zone": null, "geo_enabled": true, "verified": false, "statuses_count": 130080, "lang": null, "contributors_enabled": false, "is_translator": false, "is_translation_enabled": false, "profile_background_color": "352726", "profile_background_image_url": "http://abs.twimg.com/images/themes/theme5/bg.gif", "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme5/bg.gif", "profile_background_tile": false, "profile_image_url": "http://pbs.twimg.com/profile_images/622190067939827712/txSkhH63_normal.jpg", "profile_image_url_https": "https://pbs.twimg.com/profile_images/622190067939827712/txSkhH63_normal.jpg", "profile_banner_url": "https://pbs.twimg.com/profile_banners/1972539134/1591081516", "profile_image_extensions_alt_text": null, "profile_banner_extensions_alt_text": null, "profile_link_color": "D02B55", "profile_sidebar_border_color": "829D5E", "profile_sidebar_fill_color": "99CC33", "profile_text_color": "3E4415", "profile_use_background_image": true, "has_extended_profile": false, "default_profile": false, "default_profile_image": false, "following": false, "follow_request_sent": false, "notifications": false, "translator_type": "none", "withheld_in_countries": []}, "geo": null, "coordinates": null, "place": null, "contributors": null, "is_quote_status": false, "retweet_count": 0, "favorite_count": 1, "favorited": false, "retweeted": false, "lang": "en"}
+---

encthenet_plugins.py

@@ -1,9 +1,134 @@
-from jinja2 import contextfilter, environmentfilter
+from jinja2 import contextfilter, contextfunction, environmentfilter
 from jinja2.ext import Extension
 
+import functools
+import json
+import pathlib
+import re
+import unittest
+
+import hyde.model
+
 from io import StringIO
 from lxml import etree
 
+#from commando.util import getLoggerWithNullHandler
+#from logging import DEBUG
+
+#logger = getLoggerWithNullHandler('hyde.server')
+#logger.setLevel(DEBUG)
+
+def tbwrapper(f):
+	@functools.wraps(f)
+	def wrapper(*args, **kwargs):
+		try:
+			return f(*args, **kwargs)
+		except:
+			import traceback
+			traceback.print_exc()
+			raise
+
+	return wrapper
+
+@contextfunction
+@tbwrapper
+def do_get_tweet_thread(context, start, stop):
+	#print('gtt:', repr(start), repr(stop))
+
+	nodes = []
+	nextnode = stop
+
+	while True:
+		#print('proc:', nextnode)
+		curnode = context['site'].content.node_from_relative_path(
+		    'twitter').get_resource('%d.yaml' % nextnode)
+		#print('cn:', repr(curnode))
+		nodes.append(curnode)
+
+		if nextnode == start:
+			break
+
+		nextnode = curnode.meta.in_reply_to_status_id
+
+	return iter(reversed(nodes))
+
+@contextfilter
+def do_attr_escape(context, value):
+	#print('ae:', repr(value))
+
+	# jinja2 requires <, >, and new line to be escaped, which isn't
+	# required per html spec:
+	# https://html.spec.whatwg.org/multipage/syntax.html#attributes-2
+
+	return value.replace('&', '&amp;').replace('"', '&quot;').replace('<', '&lt;').replace('>', '&gt;').replace('\n', ' ')
+
+@contextfilter
+def make_path(context, value):
+	#print('mp:', repr(value))
+
+	return pathlib.PurePosixPath(value)
+
+@contextfilter
+def do_gettweettext(context, value):
+	if isinstance(value, str):
+		raise TypeError('got string expected tweet object')
+
+	#print('rd:', type(value), repr(value), type(value.entities))
+
+	if hasattr(value, 'extended_entities'):
+		mediaindices = [ x.indices for x in value.extended_entities.media ]
+	else:
+		mediaindices = [ ]
+
+	mentionsindices = [ x.indices for x in value.entities.user_mentions ]
+
+	delindices = mediaindices + mentionsindices
+
+	text = value.full_text
+
+	for start, stop in reversed(sorted(delindices)):
+		text = text[:start] + text[stop:]
+
+	text = text.strip()
+
+	return ''.join(('<p>%s</p>' % x for x in text.split('\n') if x))
+
+class TweetExtension(Extension):
+	"""
+	A wrapper around the rellinktoabs filter for syntactic sugar.
+	"""
+	tags = { 'gettweettext' }
+
+	def __init__(self, env):
+		super(TweetExtension, self).__init__(env)
+
+		env.filters['gettweettext'] = do_gettweettext
+		env.filters['make_path'] = make_path
+		env.filters['attr_escape'] = do_attr_escape
+		env.globals['get_tweet_thread'] = do_get_tweet_thread
+
+	def parse(self, parser):
+		"""
+		Parses the statements and defers to the callback
+		for rellinktoabs processing.
+		"""
+
+		lineno = next(parser.stream).lineno
+		body = parser.parse_statements(['name:endcleantweet'], drop_needle=True)
+
+		return nodes.CallBlock(
+		    self.call_method('_render_cleantweet'),
+		    [], [], body).set_lineno(lineno)
+
+	def _render_cleantweet(self, caller=None):
+		"""
+		Calls the cleantweet filter to transform the output.
+		"""
+		if not caller:
+			return ''
+		output = caller().strip()
+		return do_gettweettext(self.environment, output)
+
 @contextfilter
 def rellinktoabs(context, value):
 	env = context.environment
@@ -82,3 +207,36 @@ class RelLinktoAbs(Extension):
 			return ''
 		output = caller().strip()
 		return rellinktoabs(self.environment, output)
+
+class Tests(unittest.TestCase):
+	_fnameparser = re.compile('funcs_(.*)_[^_]+\\.json')
+
+	def test_fixtures(self):
+		for i in sorted(pathlib.Path('fixtures').glob(
+		    'funcs_*_*.json')):
+			with self.subTest(file=i):
+				mat = self._fnameparser.match(i.name)
+				self.assertIsNotNone(mat)
+
+				with i.open() as fp:
+					obj = json.load(fp)
+
+				if obj['do_expando']:
+					obj['args'][1] = hyde.model.Expando(
+					    obj['args'][1])
+
+				fun = globals()[mat.group(1)]
+
+				self.assertEqual(fun(*obj['args']), obj['res'])
+
+	def test_errors(self):
+		# make sure we get a more useful error message
+		with self.assertRaises(TypeError):
+			do_gettweettext(None, 'foo')
+
+	def test_basepath(self):
+		v = 'https://foo/bar/baz'
+		self.assertEqual(make_path(None, v), pathlib.PurePosixPath(v))
+
+	def test_attr_escape(self):
+		self.assertEqual(do_attr_escape(None, 'some & < > \n\n ""'), 'some &amp; &lt; &gt;    &quot;&quot;')

fixtures/funcs_do_gettweettext_1.json

@@ -0,0 +1,5 @@
+{
+	"do_expando": true,
+	"args": [ null, {"created_at": "Sun Aug 07 18:40:30 +0000 2022", "id": 1556349580932833280, "id_str": "1556349580932833280", "full_text": "@0xKruzr First, here's the .tmux.conf I use (alt text to copy/paste).\n\nThis remaps the control key from ctrl-b (^B) to ctrl-a which is more natural for me.\n\nNext is split windows, to have two windows side by side: ^A %, to have top/bottom: ^A \". https://t.co/3jPtg5Z41z", "truncated": false, "display_text_range": [9, 245], "entities": {"hashtags": [], "symbols": [], "user_mentions": [{"screen_name": "0xKruzr", "name": "\ud83c\udd79 Is Thankful for the Cyrus SASL Project \ud83e\udd83", "id": 5424492, "id_str": "5424492", "indices": [0, 8]}], "urls": [], "media": [{"id": 1556348891120877569, "id_str": "1556348891120877569", "indices": [246, 269], "media_url": "http://pbs.twimg.com/media/FZlDEhHVQAEj5eA.png", "media_url_https": "https://pbs.twimg.com/media/FZlDEhHVQAEj5eA.png", "url": "https://t.co/3jPtg5Z41z", "display_url": "pic.twitter.com/3jPtg5Z41z", "expanded_url": "https://twitter.com/encthenet/status/1556349580932833280/photo/1", "type": "photo", "sizes": {"large": {"w": 714, "h": 712, "resize": "fit"}, "small": {"w": 680, "h": 678, "resize": "fit"}, "thumb": {"w": 150, "h": 150, "resize": "crop"}, "medium": {"w": 714, "h": 712, "resize": "fit"}}}]}, "extended_entities": {"media": [{"id": 1556348891120877569, "id_str": "1556348891120877569", "indices": [246, 269], "media_url": "http://pbs.twimg.com/media/FZlDEhHVQAEj5eA.png", "media_url_https": "https://pbs.twimg.com/media/FZlDEhHVQAEj5eA.png", "url": "https://t.co/3jPtg5Z41z", "display_url": "pic.twitter.com/3jPtg5Z41z", "expanded_url": "https://twitter.com/encthenet/status/1556349580932833280/photo/1", "type": "photo", "sizes": {"large": {"w": 714, "h": 712, "resize": "fit"}, "small": {"w": 680, "h": 678, "resize": "fit"}, "thumb": {"w": 150, "h": 150, "resize": "crop"}, "medium": {"w": 714, "h": 712, "resize": "fit"}}, "ext_alt_text": "unbind C-b\nset -g prefix C-a\nbind C-a send-prefix\n\nbind ^L refresh-client\nset-window-option -g mode-keys vi\n\nbind < resize-pane -L 4\nbind > resize-pane -R 4\nbind - resize-pane -D 4\nbind + resize-pane -U 4\n\nset -g mode-mouse off\nset -g mouse-resize-pane off\nset -g mouse-select-pane off\nset -g mouse-select-window off\n\nbind k select-pane -U\nbind j select-pane -D\nbind h select-pane -L\nbind l select-pane -R\n"}]}, "source": "<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>", "in_reply_to_status_id": 1556120685713522690, "in_reply_to_status_id_str": "1556120685713522690", "in_reply_to_user_id": 5424492, "in_reply_to_user_id_str": "5424492", "in_reply_to_screen_name": "0xKruzr", "user": {"id": 1972539134, "id_str": "1972539134", "name": "John-Mark Gurney \ud83d\uddd1\ufe0f\ud83d\udd25", "screen_name": "encthenet", "location": "Oakland, CA", "description": "he/they FreeBSD developer. consultant. documentation! crypto means cryptography. open dms.  tech is inherently political. Do your part, encrypt the Net!", "url": null, "entities": {"description": {"urls": []}}, "protected": false, "followers_count": 1867, "friends_count": 1959, "listed_count": 136, "created_at": "Sat Oct 19 18:59:42 +0000 2013", "favourites_count": 231737, "utc_offset": null, "time_zone": null, "geo_enabled": true, "verified": false, "statuses_count": 130160, "lang": null, "contributors_enabled": false, "is_translator": false, "is_translation_enabled": false, "profile_background_color": "352726", "profile_background_image_url": "http://abs.twimg.com/images/themes/theme5/bg.gif", "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme5/bg.gif", "profile_background_tile": false, "profile_image_url": "http://pbs.twimg.com/profile_images/622190067939827712/txSkhH63_normal.jpg", "profile_image_url_https": "https://pbs.twimg.com/profile_images/622190067939827712/txSkhH63_normal.jpg", "profile_banner_url": "https://pbs.twimg.com/profile_banners/1972539134/1591081516", "profile_image_extensions_alt_text": null, "profile_banner_extensions_alt_text": null, "profile_link_color": "D02B55", "profile_sidebar_border_color": "829D5E", "profile_sidebar_fill_color": "99CC33", "profile_text_color": "3E4415", "profile_use_background_image": true, "has_extended_profile": false, "default_profile": false, "default_profile_image": false, "following": false, "follow_request_sent": false, "notifications": false, "translator_type": "none", "withheld_in_countries": []}, "geo": null, "coordinates": null, "place": null, "contributors": null, "is_quote_status": false, "retweet_count": 1, "favorite_count": 3, "favorited": false, "retweeted": false, "possibly_sensitive": false, "possibly_sensitive_appealable": false, "lang": "en"} ],
+	"res": "<p>First, here's the .tmux.conf I use (alt text to copy/paste).</p><p>This remaps the control key from ctrl-b (^B) to ctrl-a which is more natural for me.</p><p>Next is split windows, to have two windows side by side: ^A %, to have top/bottom: ^A \".</p>"
+}

fixtures/funcs_do_gettweettext_2.json

@@ -0,0 +1,5 @@
+{
+	"do_expando": true,
+	"args": [ null, {"created_at": "Sun Aug 07 18:42:25 +0000 2022", "id": 1556350064749998081, "id_str": "1556350064749998081", "full_text": "@0xKruzr To move between windows, ^A then one of h (up) j (left) k (right) l (ell down), aka the vi movement keys, or you can use arrow keys.\n\nTo resize the current window, ^A &lt; or &gt; or + or -.\n\nTo swap two windows (to move them around), ^A { or ^A }.", "truncated": false, "display_text_range": [9, 257], "entities": {"hashtags": [], "symbols": [], "user_mentions": [{"screen_name": "0xKruzr", "name": "\ud83c\udd79 Is Thankful for the Cyrus SASL Project \ud83e\udd83", "id": 5424492, "id_str": "5424492", "indices": [0, 8]}], "urls": []}, "source": "<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>", "in_reply_to_status_id": 1556349580932833280, "in_reply_to_status_id_str": "1556349580932833280", "in_reply_to_user_id": 1972539134, "in_reply_to_user_id_str": "1972539134", "in_reply_to_screen_name": "encthenet", "user": {"id": 1972539134, "id_str": "1972539134", "name": "John-Mark Gurney \ud83d\uddd1\ufe0f\ud83d\udd25", "screen_name": "encthenet", "location": "Oakland, CA", "description": "he/they FreeBSD developer. consultant. documentation! crypto means cryptography. open dms.  tech is inherently political. Do your part, encrypt the Net!", "url": null, "entities": {"description": {"urls": []}}, "protected": false, "followers_count": 1867, "friends_count": 1959, "listed_count": 136, "created_at": "Sat Oct 19 18:59:42 +0000 2013", "favourites_count": 231737, "utc_offset": null, "time_zone": null, "geo_enabled": true, "verified": false, "statuses_count": 130160, "lang": null, "contributors_enabled": false, "is_translator": false, "is_translation_enabled": false, "profile_background_color": "352726", "profile_background_image_url": "http://abs.twimg.com/images/themes/theme5/bg.gif", "profile_background_image_url_https": "https://abs.twimg.com/images/themes/theme5/bg.gif", "profile_background_tile": false, "profile_image_url": "http://pbs.twimg.com/profile_images/622190067939827712/txSkhH63_normal.jpg", "profile_image_url_https": "https://pbs.twimg.com/profile_images/622190067939827712/txSkhH63_normal.jpg", "profile_banner_url": "https://pbs.twimg.com/profile_banners/1972539134/1591081516", "profile_image_extensions_alt_text": null, "profile_banner_extensions_alt_text": null, "profile_link_color": "D02B55", "profile_sidebar_border_color": "829D5E", "profile_sidebar_fill_color": "99CC33", "profile_text_color": "3E4415", "profile_use_background_image": true, "has_extended_profile": false, "default_profile": false, "default_profile_image": false, "following": false, "follow_request_sent": false, "notifications": false, "translator_type": "none", "withheld_in_countries": []}, "geo": null, "coordinates": null, "place": null, "contributors": null, "is_quote_status": false, "retweet_count": 0, "favorite_count": 1, "favorited": false, "retweeted": false, "lang": "en"} ],
+	"res": "<p>To move between windows, ^A then one of h (up) j (left) k (right) l (ell down), aka the vi movement keys, or you can use arrow keys.</p><p>To resize the current window, ^A &lt; or &gt; or + or -.</p><p>To swap two windows (to move them around), ^A { or ^A }.</p>"
+}

layout/drafts.j2

@@ -19,6 +19,7 @@
 
 <div class="clear"></div>
 <section>
+<h1 id="{{ slug(resource) }}-title"><a class="no-tufte-underline" href="{{ itemurl(resource) }}">{{ resource.meta.title }}</a></h1>
 {% filter markdown|typogrify|rellinktoabs -%}
 {% block post -%}{%- endblock %}
 {%- endfilter %}

site.yaml

@@ -1,5 +1,6 @@
 mode: development
 media_root: media # Relative path from content folder.
+twitter_root: twitter # Relative path from content folder.
 media_url: /media # URL where the media files are served from.
 base_url: / # The base url for autogenerated links.
 plugins:
@@ -18,6 +19,7 @@ context:
 jinja2:
     extensions:
         - encthenet_plugins.RelLinktoAbs
+        - encthenet_plugins.TweetExtension
 markdown:
     extensions:
         - toc

tweets.txt

@@ -0,0 +1 @@
+my_thread	1556349580932833280	1556353959723356160

twitfetch.py

@@ -0,0 +1,96 @@
+from twitter import *
+
+import pprint
+
+import json
+import pathlib
+import urllib
+
+import twauth
+
+forcerefetch = False
+
+base_dir = pathlib.Path('content') / 'twitter'
+imgbase_dir = pathlib.Path('content') / 'media' / 'twitter'
+
+# Info for getting alt text.
+# Requires a project, which requires a phone number, which it rejects my home phone
+# https://developer.twitter.com/en/docs/twitter-api/data-dictionary/object-model/media
+
+def write_media(media):
+	mediaurl = media['media_url_https']
+
+	parts = urllib.parse.urlparse(mediaurl)
+	fname = pathlib.PurePosixPath(parts.path).name
+	imgpath = imgbase_dir / fname
+
+	if not forcerefetch and imgpath.exists():
+		print('media exists', fname)
+		return
+
+	resp = urllib.request.urlopen(mediaurl)
+	try:
+		if resp.getcode() == 200:
+			with open(imgpath, 'wb') as fp:
+				d = None
+				while d != b'':
+					d = resp.read1()
+					fp.write(d)
+	finally:
+		resp.close()
+
+def write_tweet(tweet):
+	twpath = base_dir / (tweet['id_str'] + '.yaml')
+
+	if not forcerefetch and twpath.exists():
+		print('exists', tweet['id_str'])
+		return
+
+	#pprint.pprint(repr(tweet))
+
+	try:
+		# Has media to d/l
+		for i in tweet['entities']['media']:
+			write_media(i)
+	except KeyError as e:
+		pass
+
+	with open(twpath, 'w') as fp:
+		print('write:', tweet['id_str'])
+
+		# wrap in --- to mark as metadata for hyde/jinja2
+		print('---', file=fp)
+		json.dump(tweet, fp)
+		# dump doesn't terminate w/ nl
+		print('', file=fp)
+		print('---', file=fp)
+
+def act_my_thread(tw, start_tw_id, stop_tw_id):
+	tw_id = int(stop_tw_id)
+
+	while True:
+		# https://alexwlchan.net/2022/11/tweet-alt-text/
+		base_tw = tw.statuses.show(_id=tw_id, tweet_mode='extended', include_ext_alt_text=True)
+
+		write_tweet(base_tw)
+
+		if start_tw_id == tw_id:
+			break
+
+		tw_id = base_tw['in_reply_to_status_id']
+
+if __name__ == '__main__':
+	args, kwargs = twauth.twit_argskw
+	tw = Twitter(*args, **kwargs)
+
+	# https://developer.twitter.com/en/docs/twitter-api/v1/tweets/post-and-engage/api-reference/get-statuses-show-id
+	#print(repr(tw.statuses.show(_id=1556349580932833280)))
+
+	with open('tweets.txt') as fp:
+		for i in fp:
+			action, tw_id, *extra = i.split()
+			tw_id = int(tw_id)
+
+			locals()['act_%s' % action](tw, tw_id, *extra)
+
+	print('done')