From e6ad86f373a12edb9a4b941d2ea80cb81f2384e2 Mon Sep 17 00:00:00 2001 From: Michael Milton Date: Mon, 3 Aug 2020 20:30:24 +1000 Subject: [PATCH 1/3] For Nearley: add argparse, extras install, allow es6 compiling --- lark/tools/nearley.py | 30 ++++++++++++++++++------------ setup.py | 3 ++- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/lark/tools/nearley.py b/lark/tools/nearley.py index 0b04fb5..c3df234 100644 --- a/lark/tools/nearley.py +++ b/lark/tools/nearley.py @@ -3,6 +3,7 @@ import os.path import sys import codecs +import argparse from lark import Lark, InlineTransformer @@ -137,7 +138,7 @@ def _nearley_to_lark(g, builtin_path, n2l, js_code, folder_path, includes): return rule_defs -def create_code_for_nearley_grammar(g, start, builtin_path, folder_path): +def create_code_for_nearley_grammar(g, start, builtin_path, folder_path, es6=False): import js2py emit_code = [] @@ -160,7 +161,10 @@ def create_code_for_nearley_grammar(g, start, builtin_path, folder_path): for alias, code in n2l.alias_js_code.items(): js_code.append('%s = (%s);' % (alias, code)) - emit(js2py.translate_js('\n'.join(js_code))) + if es6: + emit(js2py.translate_js6('\n'.join(js_code))) + else: + emit(js2py.translate_js('\n'.join(js_code))) emit('class TransformNearley(Transformer):') for alias in n2l.alias_js_code: emit(" %s = var.get('%s').to_python()" % (alias, alias)) @@ -173,18 +177,20 @@ def create_code_for_nearley_grammar(g, start, builtin_path, folder_path): return ''.join(emit_code) -def main(fn, start, nearley_lib): +def main(fn, start, nearley_lib, es6=False): with codecs.open(fn, encoding='utf8') as f: grammar = f.read() - return create_code_for_nearley_grammar(grammar, start, os.path.join(nearley_lib, 'builtin'), os.path.abspath(os.path.dirname(fn))) + return create_code_for_nearley_grammar(grammar, start, os.path.join(nearley_lib, 'builtin'), os.path.abspath(os.path.dirname(fn)), es6=es6) +def get_parser(): + parser = argparse.ArgumentParser('Reads Nearley grammar (with js functions) outputs an equivalent lark parser.') + parser.add_argument('nearley_grammar', help='Path to the file containing the nearley grammar') + parser.add_argument('start_rule', help='Rule within the nearley grammar to make the base rule') + parser.add_argument('nearley_lib', help='') + parser.add_argument('--es6', help='Enable experimental ES6 support', action='store_true') + return parser if __name__ == '__main__': - if len(sys.argv) < 4: - print("Reads Nearley grammar (with js functions) outputs an equivalent lark parser.") - print("Usage: %s " % sys.argv[0]) - sys.exit(1) - - fn, start, nearley_lib = sys.argv[1:] - - print(main(fn, start, nearley_lib)) + parser = get_parser() + args = parser.parse_args() + print(main(fn=args.nearley_grammar, start=args.start_rule, nearley_lib=args.nearley_lib, es6=args.es6)) diff --git a/setup.py b/setup.py index 5e7bda3..382943e 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,8 @@ setup( install_requires = [], extras_require = { - "regex": ["regex"] + "regex": ["regex"], + "nearley": ["js2py"] }, package_data = {'': ['*.md', '*.lark'], 'lark-stubs': ['*.pyi']}, From 193dcf032d549a48cc57a7bc39181be73cccae18 Mon Sep 17 00:00:00 2001 From: Michael Milton Date: Mon, 3 Aug 2020 20:35:45 +1000 Subject: [PATCH 2/3] Update readme --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 464f409..fb00841 100644 --- a/README.md +++ b/README.md @@ -163,6 +163,11 @@ Using Lark? Send me a message and I'll add your project! Lark comes with a tool to convert grammars from [Nearley](https://github.com/Hardmath123/nearley), a popular Earley library for Javascript. It uses [Js2Py](https://github.com/PiotrDabkowski/Js2Py) to convert and run the Javascript postprocessing code segments. +First, ensure you have Lark installed with the `nearley` component included: +```bash +pip install lark-parser[nearley] +``` + Here's an example: ```bash git clone https://github.com/Hardmath123/nearley @@ -177,6 +182,12 @@ You can use the output as a regular python module: 0.38981434460254655 ``` +The Nearley converter also supports an experimental converter for newer JavaScript (ES6+), using the `--es6` flag: + +```bash +git clone https://github.com/Hardmath123/nearley +python -m lark.tools.nearley nearley/examples/calculator/arithmetic.ne main nearley --es6 > ncalc.py +``` ## License From b0feed8c12838f6e0078b119e2eee97a1fafa66c Mon Sep 17 00:00:00 2001 From: Erez Sh Date: Wed, 5 Aug 2020 16:05:31 +0300 Subject: [PATCH 3/3] A few adjustments to Nearley PR (mostly docs) --- README.md | 32 +---------------------------- docs/features.md | 2 +- docs/index.md | 1 + docs/nearley.md | 47 +++++++++++++++++++++++++++++++++++++++++++ lark/tools/nearley.py | 6 +++--- 5 files changed, 53 insertions(+), 35 deletions(-) create mode 100644 docs/nearley.md diff --git a/README.md b/README.md index fb00841..aa19ab0 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ See more [examples here](https://github.com/lark-parser/lark/tree/master/example - **Python 2 & 3** compatible - Automatic line & column tracking - Standard library of terminals (strings, numbers, names, etc.) - - Import grammars from Nearley.js + - Import grammars from Nearley.js ([read more](/docs/nearley.md)) - Extensive test suite [![codecov](https://codecov.io/gh/erezsh/lark/branch/master/graph/badge.svg)](https://codecov.io/gh/erezsh/lark) - MyPy support using type stubs - And much more! @@ -159,36 +159,6 @@ Check out the [JSON tutorial](/docs/json_tutorial.md#conclusion) for more detail Using Lark? Send me a message and I'll add your project! -### How to use Nearley grammars in Lark - -Lark comes with a tool to convert grammars from [Nearley](https://github.com/Hardmath123/nearley), a popular Earley library for Javascript. It uses [Js2Py](https://github.com/PiotrDabkowski/Js2Py) to convert and run the Javascript postprocessing code segments. - -First, ensure you have Lark installed with the `nearley` component included: -```bash -pip install lark-parser[nearley] -``` - -Here's an example: -```bash -git clone https://github.com/Hardmath123/nearley -python -m lark.tools.nearley nearley/examples/calculator/arithmetic.ne main nearley > ncalc.py -``` - -You can use the output as a regular python module: - -```python ->>> import ncalc ->>> ncalc.parse('sin(pi/4) ^ e') -0.38981434460254655 -``` - -The Nearley converter also supports an experimental converter for newer JavaScript (ES6+), using the `--es6` flag: - -```bash -git clone https://github.com/Hardmath123/nearley -python -m lark.tools.nearley nearley/examples/calculator/arithmetic.ne main nearley --es6 > ncalc.py -``` - ## License Lark uses the [MIT license](LICENSE). diff --git a/docs/features.md b/docs/features.md index d8a4340..9346989 100644 --- a/docs/features.md +++ b/docs/features.md @@ -21,7 +21,7 @@ # Extra features - Import rules and tokens from other Lark grammars, for code reuse and modularity. - - Import grammars from Nearley.js + - Import grammars from Nearley.js ([read more](/docs/nearley.md)) - CYK parser ### Experimental features diff --git a/docs/index.md b/docs/index.md index 20257b5..1310be2 100644 --- a/docs/index.md +++ b/docs/index.md @@ -49,6 +49,7 @@ $ pip install lark-parser * [Visitors & Transformers](visitors.md) * [Classes](classes.md) * [Cheatsheet (PDF)](lark_cheatsheet.pdf) + * [Importing grammars from Nearley](nearley.md) * Discussion * [Gitter](https://gitter.im/lark-parser/Lobby) * [Forum (Google Groups)](https://groups.google.com/forum/#!forum/lark-parser) diff --git a/docs/nearley.md b/docs/nearley.md new file mode 100644 index 0000000..4ab8595 --- /dev/null +++ b/docs/nearley.md @@ -0,0 +1,47 @@ +# Importing grammars from Nearley + +Lark comes with a tool to convert grammars from [Nearley](https://github.com/Hardmath123/nearley), a popular Earley library for Javascript. It uses [Js2Py](https://github.com/PiotrDabkowski/Js2Py) to convert and run the Javascript postprocessing code segments. + +## Requirements + +1. Install Lark with the `nearley` component: +```bash +pip install lark-parser[nearley] +``` + +2. Acquire a copy of the nearley codebase. This can be done using: +```bash +git clone https://github.com/Hardmath123/nearley +``` + +## Usage + +Here's an example of how to import nearley's calculator example into Lark: + +```bash +git clone https://github.com/Hardmath123/nearley +python -m lark.tools.nearley nearley/examples/calculator/arithmetic.ne main nearley > ncalc.py +``` + +You can use the output as a regular python module: + +```python +>>> import ncalc +>>> ncalc.parse('sin(pi/4) ^ e') +0.38981434460254655 +``` + +The Nearley converter also supports an experimental converter for newer JavaScript (ES6+), using the `--es6` flag: + +```bash +git clone https://github.com/Hardmath123/nearley +python -m lark.tools.nearley nearley/examples/calculator/arithmetic.ne main nearley --es6 > ncalc.py +``` + +## Notes + +- Lark currently cannot import templates from Nearley + +- Lark currently cannot export grammars to Nearley + +These might get added in the future, if enough users ask for them. \ No newline at end of file diff --git a/lark/tools/nearley.py b/lark/tools/nearley.py index c3df234..0237fcd 100644 --- a/lark/tools/nearley.py +++ b/lark/tools/nearley.py @@ -1,4 +1,4 @@ -"Converts between Lark and Nearley grammars. Work in progress!" +"Converts Nearley grammars to Lark" import os.path import sys @@ -182,7 +182,7 @@ def main(fn, start, nearley_lib, es6=False): grammar = f.read() return create_code_for_nearley_grammar(grammar, start, os.path.join(nearley_lib, 'builtin'), os.path.abspath(os.path.dirname(fn)), es6=es6) -def get_parser(): +def get_arg_parser(): parser = argparse.ArgumentParser('Reads Nearley grammar (with js functions) outputs an equivalent lark parser.') parser.add_argument('nearley_grammar', help='Path to the file containing the nearley grammar') parser.add_argument('start_rule', help='Rule within the nearley grammar to make the base rule') @@ -191,6 +191,6 @@ def get_parser(): return parser if __name__ == '__main__': - parser = get_parser() + parser = get_arg_parser() args = parser.parse_args() print(main(fn=args.nearley_grammar, start=args.start_rule, nearley_lib=args.nearley_lib, es6=args.es6))