inital sphinx gallery

docs/conf.py

@@ -37,6 +37,7 @@ extensions = [
     'sphinx.ext.napoleon',
     'sphinx.ext.coverage',
     'recommonmark',
+    'sphinx_gallery.gen_gallery'
 ]
 
 # Add any paths that contain templates here, relative to this directory.

docs/examples

@@ -0,0 +1 @@
+../examples/

docs/index.rst

@@ -24,6 +24,7 @@ Welcome to Lark's documentation!
    how_to_use
    how_to_develop
    recipes
+   auto_examples/index
 
 
 .. toctree::

examples/README.md

@@ -1,34 +0,0 @@
-# Examples for Lark
-
-#### How to run the examples
-
-After cloning the repo, open the terminal into the root directory of the project, and run the following:
-
-```bash
-[lark]$ python -m examples.<name_of_example>
-```
-
-For example, the following will parse all the Python files in the standard library of your local installation:
-
-```bash
-[lark]$ python -m examples.python_parser
-```
-
-### Beginners
-
-- [calc.py](calc.py) - A simple example of a REPL calculator
-- [json\_parser.py](json_parser.py) - A simple JSON parser (comes with a tutorial, see docs)
-- [indented\_tree.py](indented\_tree.py) - A demonstration of parsing indentation ("whitespace significant" language)
-- [fruitflies.py](fruitflies.py) - A demonstration of ambiguity
-- [turtle\_dsl.py](turtle_dsl.py) - Implements a LOGO-like toy language for Python's turtle, with interpreter.
-- [lark\_grammar.py](lark_grammar.py) + [lark.lark](lark.lark) - A reference implementation of the Lark grammar (using LALR(1) + standard lexer)
-
-### Advanced
-
-- [error\_reporting\_lalr.py](error_reporting_lalr.py) - A demonstration of example-driven error reporting with the LALR parser
-- [python\_parser.py](python_parser.py) - A fully-working Python 2 & 3 parser (but not production ready yet!)
-- [python\_bytecode.py](python_bytecode.py) - A toy example showing how to compile Python directly to bytecode
-- [conf\_lalr.py](conf_lalr.py) - Demonstrates the power of LALR's contextual lexer on a toy configuration language
-- [conf\_earley.py](conf_earley.py) - Demonstrates the power of Earley's dynamic lexer on a toy configuration language
-- [custom\_lexer.py](custom_lexer.py) - Demonstrates using a custom lexer to parse a non-textual stream of data
-- [reconstruct\_json.py](reconstruct_json.py) - Demonstrates the experimental text-reconstruction feature

examples/README.rst

@@ -0,0 +1,66 @@
+Examples for Lark
+=================
+
+How to run the examples
+^^^^^^^^^^^^^^^^^^^^^^^
+
+After cloning the repo, open the terminal into the root directory of the
+project, and run the following:
+
+.. code:: bash
+
+   [lark]$ python -m examples.<name_of_example>
+
+For example, the following will parse all the Python files in the
+standard library of your local installation:
+
+.. code:: bash
+
+   [lark]$ python -m examples.python_parser
+
+Beginners
+~~~~~~~~~
+
+-  `calc.py`_ - A simple example of a REPL calculator
+-  `json_parser.py`_ - A simple JSON parser (comes with a tutorial, see
+   docs)
+-  `indented_tree.py`_ - A demonstration of parsing indentation
+   (“whitespace significant” language)
+-  `fruitflies.py`_ - A demonstration of ambiguity
+-  `turtle_dsl.py`_ - Implements a LOGO-like toy language for Python’s
+   turtle, with interpreter.
+-  `lark_grammar.py`_ + `lark.lark`_ - A reference implementation of the
+   Lark grammar (using LALR(1) + standard lexer)
+
+Advanced
+~~~~~~~~
+
+-  `error_reporting_lalr.py`_ - A demonstration of example-driven error
+   reporting with the LALR parser
+-  `python_parser.py`_ - A fully-working Python 2 & 3 parser (but not
+   production ready yet!)
+-  `python_bytecode.py`_ - A toy example showing how to compile Python
+   directly to bytecode
+-  `conf_lalr.py`_ - Demonstrates the power of LALR’s contextual lexer
+   on a toy configuration language
+-  `conf_earley.py`_ - Demonstrates the power of Earley’s dynamic lexer
+   on a toy configuration language
+-  `custom_lexer.py`_ - Demonstrates using a custom lexer to parse a
+   non-textual stream of data
+-  `reconstruct_json.py`_ - Demonstrates the experimental
+   text-reconstruction feature
+
+.. _calc.py: calc.py
+.. _json_parser.py: json_parser.py
+.. _indented_tree.py: indented_tree.py
+.. _fruitflies.py: fruitflies.py
+.. _turtle_dsl.py: turtle_dsl.py
+.. _lark_grammar.py: lark_grammar.py
+.. _lark.lark: lark.lark
+.. _error_reporting_lalr.py: error_reporting_lalr.py
+.. _python_parser.py: python_parser.py
+.. _python_bytecode.py: python_bytecode.py
+.. _conf_lalr.py: conf_lalr.py
+.. _conf_earley.py: conf_earley.py
+.. _custom_lexer.py: custom_lexer.py
+.. _reconstruct_json.py: reconstruct_json.py

examples/calc.py

@@ -1,7 +1,11 @@
-#
-# This example shows how to write a basic calculator with variables.
-#
+"""
+Basic calculator
+================
+
+A simple example of a REPL calculator
 
+This example shows how to write a basic calculator with variables.
+"""
 from lark import Lark, Transformer, v_args
 
 

examples/conf_earley.py

@@ -1,17 +1,19 @@
-#
-# This example demonstrates parsing using the dynamic-lexer earley frontend
-#
-# Using a lexer for configuration files is tricky, because values don't
-# have to be surrounded by delimiters. Using a standard lexer for this just won't work.
-#
-# In this example we use a dynamic lexer and let the Earley parser resolve the ambiguity.
-#
-# Another approach is to use the contextual lexer with LALR. It is less powerful than Earley,
-# but it can handle some ambiguity when lexing and it's much faster.
-# See examples/conf_lalr.py for an example of that approach.
-#
+"""
+Earley’s dynamic lexer
+======================
+
+Demonstrates the power of Earley’s dynamic lexer on a toy configuration language
+
+Using a lexer for configuration files is tricky, because values don't
+have to be surrounded by delimiters. Using a standard lexer for this just won't work.
 
+In this example we use a dynamic lexer and let the Earley parser resolve the ambiguity.
 
+Another approach is to use the contextual lexer with LALR. It is less powerful than Earley,
+but it can handle some ambiguity when lexing and it's much faster.
+See examples/conf_lalr.py for an example of that approach.
+
+"""
 from lark import Lark
 
 parser = Lark(r"""

examples/conf_lalr.py

@@ -1,18 +1,20 @@
-#
-# This example demonstrates the power of the contextual lexer, by parsing a config file.
-#
-# The tokens NAME and VALUE match the same input. A standard lexer would arbitrarily
-# choose one over the other, which would lead to a (confusing) parse error.
-# However, due to the unambiguous structure of the grammar, Lark's LALR(1) algorithm knows
-# which one of them to expect at each point during the parse.
-# The lexer then only matches the tokens that the parser expects.
-# The result is a correct parse, something that is impossible with a regular lexer.
-#
-# Another approach is to discard a lexer altogether and use the Earley algorithm.
-# It will handle more cases than the contextual lexer, but at the cost of performance.
-# See examples/conf_earley.py for an example of that approach.
-#
+"""
+LALR’s contextual lexer
+=======================
+
+Demonstrates the power of LALR’s contextual lexer on a toy configuration language.
 
+The tokens NAME and VALUE match the same input. A standard lexer would arbitrarily
+choose one over the other, which would lead to a (confusing) parse error.
+However, due to the unambiguous structure of the grammar, Lark's LALR(1) algorithm knows
+which one of them to expect at each point during the parse.
+The lexer then only matches the tokens that the parser expects.
+The result is a correct parse, something that is impossible with a regular lexer.
+
+Another approach is to discard a lexer altogether and use the Earley algorithm.
+It will handle more cases than the contextual lexer, but at the cost of performance.
+See examples/conf_earley.py for an example of that approach.
+"""
 from lark import Lark
 
 parser = Lark(r"""

examples/custom_lexer.py

@@ -1,13 +1,14 @@
-#
-# This example demonstrates using Lark with a custom lexer.
-#
-# You can use a custom lexer to tokenize text when the lexers offered by Lark
-# are too slow, or not flexible enough.
-#
-# You can also use it (as shown in this example) to tokenize streams of objects.
-#
+"""
+Custom lexer
+============
 
+Demonstrates using a custom lexer to parse a non-textual stream of data
 
+You can use a custom lexer to tokenize text when the lexers offered by Lark
+are too slow, or not flexible enough.
+
+You can also use it (as shown in this example) to tokenize streams of objects.
+"""
 from lark import Lark, Transformer, v_args
 from lark.lexer import Lexer, Token
 

examples/error_puppet.py

@@ -1,11 +1,14 @@
-#
-# This example demonstrates error handling using a parsing puppet in LALR
-#
-# When the parser encounters an UnexpectedToken exception, it creates a
-# parsing puppet with the current parse-state, and lets you control how
-# to proceed step-by-step. When you've achieved the correct parse-state,
-# you can resume the run by returning True.
-#
+"""
+Error handling with parsing puppet
+==================================
+
+This example demonstrates error handling using a parsing puppet in LALR
+
+When the parser encounters an UnexpectedToken exception, it creates a
+parsing puppet with the current parse-state, and lets you control how
+to proceed step-by-step. When you've achieved the correct parse-state,
+you can resume the run by returning True.
+"""
 
 from lark import UnexpectedToken, Token
 

examples/error_reporting_lalr.py

@@ -1,7 +1,10 @@
-#
-# This demonstrates example-driven error reporting with the LALR parser
-#
+"""
+Example Driver Error Reporting
+==============================
 
+A demonstration of example-driven error reporting with the LALR parser
+
+"""
 from lark import Lark, UnexpectedInput
 
 from .json_parser import json_grammar   # Using the grammar from the json_parser example

examples/fruitflies.py

@@ -1,7 +1,12 @@
-#
-# This example shows how to use get explicit ambiguity from Lark's Earley parser.
-#
+"""
+Handling Ambiguity
+==================
 
+A demonstration of ambiguity
+
+This example shows how to use get explicit ambiguity from Lark's Earley parser.
+
+"""
 import sys
 from lark import Lark, tree
 

examples/indented_tree.py

@@ -1,13 +1,16 @@
-#
-# This example demonstrates usage of the Indenter class.
-#
-# Since indentation is context-sensitive, a postlex stage is introduced to
-# manufacture INDENT/DEDENT tokens.
-#
-# It is crucial for the indenter that the NL_type matches
-# the spaces (and tabs) after the newline.
-#
+"""
+Parsing Indentation
+===================
+
+A demonstration of parsing indentation (“whitespace significant” language)
+and the usage of the Indenter class.
 
+Since indentation is context-sensitive, a postlex stage is introduced to
+manufacture INDENT/DEDENT tokens.
+
+It is crucial for the indenter that the NL_type matches
+the spaces (and tabs) after the newline.
+"""
 from lark import Lark
 from lark.indenter import Indenter
 

examples/json_parser.py

@@ -1,10 +1,12 @@
-#
-# This example shows how to write a basic JSON parser
-#
-# The code is short and clear, and outperforms every other parser (that's written in Python).
-# For an explanation, check out the JSON parser tutorial at /docs/json_tutorial.md
-#
+"""
+Simple JSON Parser
+==================
+
+A simple JSON parser (comes with a tutorial, see docs)
 
+The code is short and clear, and outperforms every other parser (that's written in Python).
+For an explanation, check out the JSON parser tutorial at /docs/json_tutorial.md
+"""
 import sys
 
 from lark import Lark, Transformer, v_args

examples/lark_grammar.py

@@ -1,3 +1,9 @@
+"""
+Lark Grammar
+============
+
+A reference implementation of the Lark grammar (using LALR(1) + standard lexer)
+"""
 from lark import Lark
 
 parser = Lark(open('examples/lark.lark'), parser="lalr")

examples/python_bytecode.py

@@ -1,12 +1,16 @@
-#
-# This is a toy example that compiles Python directly to bytecode, without generating an AST.
-# It currently only works for very very simple Python code.
-#
-# It requires the 'bytecode' library. You can get it using
-#
-#     $ pip install bytecode
-#
+"""
+Compile Python to Bytecode
+==========================
+ 
+A toy example that compiles Python directly to bytecode, without generating an AST.
+It currently only works for very very simple Python code.
 
+It requires the 'bytecode' library. You can get it using
+::
+
+    $ pip install bytecode
+
+"""
 from lark import Lark, Transformer, v_args
 from lark.indenter import Indenter
 

examples/python_parser.py

@@ -1,7 +1,11 @@
-#
-# This example demonstrates usage of the included Python grammars
-#
+"""
+Real Python Parser
+==================
 
+A fully-working Python 2 & 3 parser (but not production ready yet!)
+
+This example demonstrates usage of the included Python grammars
+"""
 import sys
 import os, os.path
 from io import open

examples/qscintilla_json.py

@@ -1,10 +1,14 @@
-#
-# This example shows how to write a syntax-highlighted editor with Qt and Lark
-#
-# Requirements:
-#
-#   PyQt5==5.10.1
-#   QScintilla==2.10.4
+"""
+Syntax Highlighting
+===================
+
+This example shows how to write a syntax-highlighted editor with Qt and Lark
+
+Requirements:
+
+  PyQt5==5.10.1
+  QScintilla==2.10.4
+"""
 
 import sys
 import textwrap

examples/reconstruct_json.py

@@ -1,9 +1,13 @@
-#
-# This example demonstrates an experimental feature: Text reconstruction
-# The Reconstructor takes a parse tree (already filtered from punctuation, of course),
-# and reconstructs it into correct text, that can be parsed correctly.
-# It can be useful for creating "hooks" to alter data before handing it to other parsers. You can also use it to generate samples from scratch.
-#
+"""
+Reconstruct a JSON
+==================
+
+Demonstrates the experimental text-reconstruction feature
+
+The Reconstructor takes a parse tree (already filtered from punctuation, of course),
+and reconstructs it into correct text, that can be parsed correctly.
+It can be useful for creating "hooks" to alter data before handing it to other parsers. You can also use it to generate samples from scratch.
+"""
 
 import json
 

examples/templates.py

@@ -1,7 +1,10 @@
-#
-# This example shows how to use Lark's templates to achieve cleaner grammars
-#
+"""
+Templates
+=========
+
+This example shows how to use Lark's templates to achieve cleaner grammars
 
+""""
 from lark import Lark
 
 grammar = r"""

examples/turtle_dsl.py

@@ -1,4 +1,9 @@
-# This example implements a LOGO-like toy language for Python's turtle, with interpreter.
+"""
+Turtle DSL
+==========
+
+Implements a LOGO-like toy language for Python’s turtle, with interpreter.
+"""
 
 try:
     input = raw_input   # For Python2 compatibility