From bc3923aed85fc33b788790d0f61b66cb7298471a Mon Sep 17 00:00:00 2001
From: Erez Sh <erezshin@gmail.com>
Date: Sun, 25 Oct 2020 11:15:03 +0200
Subject: [PATCH] Added more terminals to grammars/ folder

---
 examples/advanced/python3.lark | 18 +++++-------------
 lark/grammars/common.lark      | 11 ++++++++++-
 lark/grammars/python.lark      | 19 +++++++++++++++++++
 lark/lark.py                   |  4 ++--
 4 files changed, 36 insertions(+), 16 deletions(-)
 create mode 100644 lark/grammars/python.lark

diff --git a/examples/advanced/python3.lark b/examples/advanced/python3.lark
index 78c9875..9be6b43 100644
--- a/examples/advanced/python3.lark
+++ b/examples/advanced/python3.lark
@@ -163,22 +163,14 @@ yield_arg: "from" test | testlist
 
 number: DEC_NUMBER | HEX_NUMBER | BIN_NUMBER | OCT_NUMBER | FLOAT_NUMBER | IMAG_NUMBER
 string: STRING | LONG_STRING
-// Tokens
-
-NAME: /[a-zA-Z_]\w*/
-COMMENT: /#[^\n]*/
-_NEWLINE: ( /\r?\n[\t ]*/ | COMMENT )+
 
+// Import terminals from standard library (grammars/python.lark)
+%import python (NAME, COMMENT, STRING, LONG_STRING)
+%import python (DEC_NUMBER, HEX_NUMBER, OCT_NUMBER, BIN_NUMBER, FLOAT_NUMBER, IMAG_NUMBER)
 
-STRING : /[ubf]?r?("(?!"").*?(?<!\\)(\\\\)*?"|'(?!'').*?(?<!\\)(\\\\)*?')/i
-LONG_STRING: /[ubf]?r?(""".*?(?<!\\)(\\\\)*?"""|'''.*?(?<!\\)(\\\\)*?''')/is
+// Other terminals
 
-DEC_NUMBER: /0|[1-9]\d*/i
-HEX_NUMBER.2: /0x[\da-f]*/i
-OCT_NUMBER.2: /0o[0-7]*/i
-BIN_NUMBER.2 : /0b[0-1]*/i
-FLOAT_NUMBER.2: /((\d+\.\d*|\.\d+)(e[-+]?\d+)?|\d+(e[-+]?\d+))/i
-IMAG_NUMBER.2: /\d+j/i | FLOAT_NUMBER "j"i
+_NEWLINE: ( /\r?\n[\t ]*/ | COMMENT )+
 
 %ignore /[\t \f]+/  // WS
 %ignore /\\[\t \f]*\r?\n/   // LINE_CONT
diff --git a/lark/grammars/common.lark b/lark/grammars/common.lark
index a675ca4..1158026 100644
--- a/lark/grammars/common.lark
+++ b/lark/grammars/common.lark
@@ -1,3 +1,6 @@
+// Basic terminals for common use
+
+
 //
 // Numbers
 //
@@ -21,7 +24,7 @@ SIGNED_NUMBER: ["+"|"-"] NUMBER
 // Strings
 //
 _STRING_INNER: /.*?/
-_STRING_ESC_INNER: _STRING_INNER /(?<!\\)(\\\\)*?/ 
+_STRING_ESC_INNER: _STRING_INNER /(?<!\\)(\\\\)*?/
 
 ESCAPED_STRING : "\"" _STRING_ESC_INNER "\""
 
@@ -48,3 +51,9 @@ CR : /\r/
 LF : /\n/
 NEWLINE: (CR? LF)+
 
+
+// Comments
+SH_COMMENT: /#[^\n]*/
+CPP_COMMENT: /\/\/[^\n]*/
+C_COMMENT: "/*" /.*?/s "*/"
+SQL_COMMENT: /--[^\n]*/
diff --git a/lark/grammars/python.lark b/lark/grammars/python.lark
new file mode 100644
index 0000000..ba2b852
--- /dev/null
+++ b/lark/grammars/python.lark
@@ -0,0 +1,19 @@
+// Python terminals
+
+NAME: /[a-zA-Z_]\w*/
+COMMENT: /#[^\n]*/
+
+STRING : /[ubf]?r?("(?!"").*?(?<!\\)(\\\\)*?"|'(?!'').*?(?<!\\)(\\\\)*?')/i
+LONG_STRING: /[ubf]?r?(""".*?(?<!\\)(\\\\)*?"""|'''.*?(?<!\\)(\\\\)*?''')/is
+
+DEC_NUMBER: /0|[1-9]\d*/i
+HEX_NUMBER.2: /0x[\da-f]*/i
+OCT_NUMBER.2: /0o[0-7]*/i
+BIN_NUMBER.2 : /0b[0-1]*/i
+FLOAT_NUMBER.2: /((\d+\.\d*|\.\d+)(e[-+]?\d+)?|\d+(e[-+]?\d+))/i
+IMAG_NUMBER.2: /\d+j/i | FLOAT_NUMBER "j"i
+
+
+// Comma-separated list (with an optional trailing comma)
+cs_list{item}: item ("," item)* ","?
+_cs_list{item}: item ("," item)* ","?
\ No newline at end of file
diff --git a/lark/lark.py b/lark/lark.py
index 642e408..b8a1231 100644
--- a/lark/lark.py
+++ b/lark/lark.py
@@ -294,8 +294,8 @@ class Lark(Serialize):
         if self.options.ambiguity not in _VALID_AMBIGUITY_OPTIONS:
             raise ValueError("invalid ambiguity option: %r. Must be one of %r" % (self.options.ambiguity, _VALID_AMBIGUITY_OPTIONS))
 
-        # Parse the grammar file and compose the grammars (TODO)
-        self.grammar = load_grammar(grammar, self.source, self.options.import_paths, self.options.keep_all_tokens)
+        # Parse the grammar file and compose the grammars
+        self.grammar = load_grammar(grammar, self.source_path, self.options.import_paths, self.options.keep_all_tokens)
 
         if self.options.postlex is not None:
             terminals_to_keep = set(self.options.postlex.always_accept)