diff --git a/.github/ISSUE_TEMPLATE/other.md b/.github/ISSUE_TEMPLATE/other.md index 1c8fcda2..a8f765fc 100644 --- a/.github/ISSUE_TEMPLATE/other.md +++ b/.github/ISSUE_TEMPLATE/other.md @@ -6,5 +6,3 @@ labels: '' assignees: '' --- - - diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 21450d6f..9624cd9c 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -1,7 +1,7 @@ name: Python type check on: [push, pull_request] jobs: - build: + type: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -9,3 +9,11 @@ jobs: submodules: recursive - name: Lint with mypy run: pipx run tox -e type + + pre-commit: + name: Format + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v3 + - uses: pre-commit/action@v2.0.3 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f15e487f..5d06f090 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -6,14 +6,14 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11-dev", "pypy-3.7"] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "pypy-3.7"] steps: - uses: actions/checkout@v3 with: submodules: recursive - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/.gitignore b/.gitignore index b30399ed..26275e4b 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,4 @@ tags /dist /build docs/_build -docs/examples \ No newline at end of file +docs/examples diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..2852d8ed --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,32 @@ +# To use: +# +# pre-commit run -a +# +# Or: +# +# pre-commit install # (runs every time you commit in git) +# +# To update this file: +# +# pre-commit autoupdate +# +# See https://github.com/pre-commit/pre-commit + +repos: +# Standard hooks +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: "v4.2.0" + hooks: + - id: check-added-large-files + - id: check-case-conflict + - id: check-merge-conflict + - id: check-symlinks + - id: check-toml + - id: check-yaml + - id: debug-statements + - id: end-of-file-fixer + exclude: '(^tests/.*\.lark|\.svg)$' + - id: mixed-line-ending + - id: requirements-txt-fixer + - id: trailing-whitespace + exclude: '(^tests/.*\.lark|\.svg)$' diff --git a/CHANGELOG.md b/CHANGELOG.md index 22f4b8c1..7e94cf99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,4 +10,4 @@ v1.0 - `use_accepts` in `UnexpectedInput.match_examples()` is now True by default -- `v_args(meta=True)` now gives meta as the first argument. i.e. `(meta, children)` \ No newline at end of file +- `v_args(meta=True)` now gives meta as the first argument. i.e. `(meta, children)` diff --git a/LICENSE b/LICENSE index efcb9665..aaf210b1 100644 --- a/LICENSE +++ b/LICENSE @@ -16,4 +16,3 @@ FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - diff --git a/README.md b/README.md index 23184f7a..4468243b 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ Lark is great at handling ambiguity. Here is the result of parsing the phrase "f - Grammar composition - Import terminals and rules from other grammars - Standard library of terminals (strings, numbers, names, etc.) - Import grammars from Nearley.js ([read more](/docs/tools.md#importing-grammars-from-nearleyjs)) - - Extensive test suite [![codecov](https://codecov.io/gh/lark-parser/lark/branch/master/graph/badge.svg?token=lPxgVhCVPK)](https://codecov.io/gh/lark-parser/lark) + - Extensive test suite [![codecov](https://codecov.io/gh/lark-parser/lark/branch/master/graph/badge.svg?token=lPxgVhCVPK)](https://codecov.io/gh/lark-parser/lark) - Type annotations (MyPy support) - And much more! @@ -193,4 +193,3 @@ Questions about code are best asked on [gitter](https://gitter.im/lark-parser/Lo For anything else, I can be reached by email at erezshin at gmail com. -- [Erez](https://github.com/erezsh) - diff --git a/docs/Makefile b/docs/Makefile index 58127b42..ff51a731 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -17,4 +17,4 @@ help: # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/_static/sppf/sppf.html b/docs/_static/sppf/sppf.html index c9c3d218..c2fd532a 100644 --- a/docs/_static/sppf/sppf.html +++ b/docs/_static/sppf/sppf.html @@ -209,4 +209,4 @@

Transformation to an abstract - \ No newline at end of file + diff --git a/docs/classes.rst b/docs/classes.rst index 6e88fae4..965cbccd 100644 --- a/docs/classes.rst +++ b/docs/classes.rst @@ -81,7 +81,7 @@ InteractiveParser ast_utils --------- -For an example of using ``ast_utils``, see `/examples/advanced/create_ast.py`_ +For an example of using ``ast_utils``, see `/examples/advanced/create_ast.py`_ .. autoclass:: lark.ast_utils.Ast @@ -89,4 +89,4 @@ For an example of using ``ast_utils``, see `/examples/advanced/create_ast.py`_ .. autofunction:: lark.ast_utils.create_transformer -.. _/examples/advanced/create_ast.py: examples/advanced/create_ast.html \ No newline at end of file +.. _/examples/advanced/create_ast.py: examples/advanced/create_ast.html diff --git a/docs/conf.py b/docs/conf.py index c3d89fe4..0cf22a53 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -182,4 +182,4 @@ sphinx_gallery_conf = { 'examples_dirs': ['../examples'], 'gallery_dirs': ['examples'], -} \ No newline at end of file +} diff --git a/docs/features.md b/docs/features.md index 5baa16eb..121089e6 100644 --- a/docs/features.md +++ b/docs/features.md @@ -29,4 +29,3 @@ - Visualize your parse trees as dot or png files ([see_example](https://github.com/lark-parser/lark/blob/master/examples/fruitflies.py)) - Automatic reconstruction of input from parse-tree (see examples) - Use Lark grammars in Julia and Javascript. - diff --git a/docs/how_to_develop.md b/docs/how_to_develop.md index 311ad313..cf95fed0 100644 --- a/docs/how_to_develop.md +++ b/docs/how_to_develop.md @@ -63,5 +63,5 @@ pytest tests Another way to run the tests is using setup.py: ```bash -python setup.py test +python setup.py test ``` diff --git a/docs/ide/app/app.py b/docs/ide/app/app.py index 146aee98..69745462 100644 --- a/docs/ide/app/app.py +++ b/docs/ide/app/app.py @@ -80,4 +80,3 @@ def start(): html5.Body().appendChild( App() ) - diff --git a/docs/ide/app/examples.py b/docs/ide/app/examples.py index af9c38c4..afe003a7 100644 --- a/docs/ide/app/examples.py +++ b/docs/ide/app/examples.py @@ -147,4 +147,4 @@ "favoriteFruit": "strawberry" } ]""") -} \ No newline at end of file +} diff --git a/docs/ide/app/files.json b/docs/ide/app/files.json index b2308999..98dfdb44 100644 --- a/docs/ide/app/files.json +++ b/docs/ide/app/files.json @@ -6,4 +6,4 @@ "ext.py", "ignite.py", "utils.py" -] \ No newline at end of file +] diff --git a/docs/ide/app/html5.py b/docs/ide/app/html5.py index b62a821b..64d201f2 100644 --- a/docs/ide/app/html5.py +++ b/docs/ide/app/html5.py @@ -2,5 +2,3 @@ from .core import * from . import ext, utils, ignite - - diff --git a/docs/philosophy.md b/docs/philosophy.md index ebdbd4fc..d2baa39a 100644 --- a/docs/philosophy.md +++ b/docs/philosophy.md @@ -62,4 +62,3 @@ In short, "Premature optimization is the root of all evil." - Automatically resolve terminal collisions whenever possible - Automatically keep track of line & column numbers - diff --git a/docs/recipes.md b/docs/recipes.md index 1aadd047..336be23e 100644 --- a/docs/recipes.md +++ b/docs/recipes.md @@ -171,4 +171,4 @@ try: print( t.transform(tree)) except VisitError as e: raise e.orig_exc -``` \ No newline at end of file +``` diff --git a/docs/requirements.txt b/docs/requirements.txt index 5509c25c..92ccc7f2 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,6 +1,6 @@ # https://docs.readthedocs.io/en/stable/guides/specifying-dependencies.html#specifying-a-requirements-file +pillow +recommonmark sphinx-gallery sphinx_markdown_tables -recommonmark sphinx_rtd_theme -pillow \ No newline at end of file diff --git a/docs/tools.md b/docs/tools.md index ee9d2cfc..bf445d52 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -68,4 +68,3 @@ python -m lark.tools.nearley nearley/examples/calculator/arithmetic.ne main near - Lark currently cannot export grammars to Nearley These might get added in the future, if enough users ask for them. - diff --git a/docs/visitors.rst b/docs/visitors.rst index 11d1b643..323c1837 100644 --- a/docs/visitors.rst +++ b/docs/visitors.rst @@ -119,4 +119,4 @@ Discard VisitError ---------- -.. autoclass:: lark.exceptions.VisitError \ No newline at end of file +.. autoclass:: lark.exceptions.VisitError diff --git a/examples/advanced/error_handling.py b/examples/advanced/error_handling.py index 1fb9be3f..aeaa21b4 100644 --- a/examples/advanced/error_handling.py +++ b/examples/advanced/error_handling.py @@ -34,4 +34,3 @@ def main(): print(res) # prints [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0] main() - diff --git a/examples/advanced/error_reporting_earley.py b/examples/advanced/error_reporting_earley.py index f0bcc20d..3d47a53e 100644 --- a/examples/advanced/error_reporting_earley.py +++ b/examples/advanced/error_reporting_earley.py @@ -75,5 +75,3 @@ def test(): if __name__ == '__main__': test() - - diff --git a/examples/advanced/error_reporting_lalr.py b/examples/advanced/error_reporting_lalr.py index c2cb2393..722a71fd 100644 --- a/examples/advanced/error_reporting_lalr.py +++ b/examples/advanced/error_reporting_lalr.py @@ -75,5 +75,3 @@ def test(): if __name__ == '__main__': test() - - diff --git a/examples/advanced/py3to2.py b/examples/advanced/py3to2.py index e08967c8..3c474c28 100644 --- a/examples/advanced/py3to2.py +++ b/examples/advanced/py3to2.py @@ -45,12 +45,12 @@ def parse_code(s): # # 2. Define translations using templates (each template code is parsed to a template tree) -# +# pytemplate = TemplateConf(parse=parse_template) translations_3to2 = { - 'yield from $a': + 'yield from $a': 'for _tmp in $a: yield _tmp', 'raise $e from $x': @@ -63,7 +63,7 @@ def parse_code(s): # # 3. Translate and reconstruct Python 3 code into valid Python 2 code -# +# python_reconstruct = PythonReconstructor(parser) @@ -91,4 +91,4 @@ def test(): print(translate_py3to2(_TEST_CODE)) if __name__ == '__main__': - test() \ No newline at end of file + test() diff --git a/examples/advanced/python2.lark b/examples/advanced/python2.lark index 6fbae459..fd49e4dd 100644 --- a/examples/advanced/python2.lark +++ b/examples/advanced/python2.lark @@ -165,4 +165,3 @@ IMAG_NUMBER: (_INT | FLOAT) ("j"|"J") %ignore /\\[\t \f]*\r?\n/ // LINE_CONT %ignore COMMENT %declare _INDENT _DEDENT - diff --git a/examples/advanced/python_parser.py b/examples/advanced/python_parser.py index 776f6e12..78b91ca0 100644 --- a/examples/advanced/python_parser.py +++ b/examples/advanced/python_parser.py @@ -76,4 +76,4 @@ def test_earley_equals_lalr(): if __name__ == '__main__': test_python_lib() # test_earley_equals_lalr() - # python_parser3.parse(_read(sys.argv[1]) + '\n') \ No newline at end of file + # python_parser3.parse(_read(sys.argv[1]) + '\n') diff --git a/examples/advanced/reconstruct_python.py b/examples/advanced/reconstruct_python.py index a318485f..95fb4798 100644 --- a/examples/advanced/reconstruct_python.py +++ b/examples/advanced/reconstruct_python.py @@ -83,4 +83,4 @@ def test(): if __name__ == '__main__': - test() \ No newline at end of file + test() diff --git a/examples/advanced/templates.py b/examples/advanced/templates.py index ac59b7a9..6d4b6056 100644 --- a/examples/advanced/templates.py +++ b/examples/advanced/templates.py @@ -26,4 +26,4 @@ parser = Lark(grammar) print(parser.parse('[1, "a", 2]')) -print(parser.parse('{"a": 2, "b": 6}')) \ No newline at end of file +print(parser.parse('{"a": 2, "b": 6}')) diff --git a/examples/composition/README.md b/examples/composition/README.md index 259a66a5..1c32e59a 100644 --- a/examples/composition/README.md +++ b/examples/composition/README.md @@ -7,4 +7,4 @@ file format that allows both CSV and JSON to co-exist. We show how, by using namespaces, Lark grammars and their transformers can be fully reused - they don't need to care if their grammar is used directly, or being imported, or who is doing the importing. -See [``main.py``](main.py) for more details. \ No newline at end of file +See [``main.py``](main.py) for more details. diff --git a/examples/composition/main.py b/examples/composition/main.py index c6f150ff..4c803f5a 100644 --- a/examples/composition/main.py +++ b/examples/composition/main.py @@ -48,4 +48,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/examples/composition/storage.lark b/examples/composition/storage.lark index 09bb0ae7..fc730277 100644 --- a/examples/composition/storage.lark +++ b/examples/composition/storage.lark @@ -6,4 +6,3 @@ start: (csv__start | json__start _NL?)+ %import .json.start -> json__start %import .csv._NL -> _NL - diff --git a/examples/grammars/README.md b/examples/grammars/README.md index cdd3b75b..71b3232d 100644 --- a/examples/grammars/README.md +++ b/examples/grammars/README.md @@ -2,4 +2,4 @@ This directory is a collection of lark grammars, taken from real world projects. -- [Verilog](verilog.lark) - Taken from https://github.com/circuitgraph/circuitgraph/blob/master/circuitgraph/parsing/verilog.lark \ No newline at end of file +- [Verilog](verilog.lark) - Taken from https://github.com/circuitgraph/circuitgraph/blob/master/circuitgraph/parsing/verilog.lark diff --git a/examples/grammars/verilog.lark b/examples/grammars/verilog.lark index 0120fe71..0b0a4c35 100644 --- a/examples/grammars/verilog.lark +++ b/examples/grammars/verilog.lark @@ -3,9 +3,9 @@ // 1. Source Text start: description* - + ?description: module - + module: "module" name_of_module list_of_ports? ";" module_item* "endmodule" ?name_of_module: IDENTIFIER @@ -13,7 +13,7 @@ module: "module" name_of_module list_of_ports? ";" module_item* "endmodule" list_of_ports: "(" port ("," port)* ")" ?port: IDENTIFIER - + ?module_item: input_declaration | output_declaration | net_declaration diff --git a/examples/indented_tree.py b/examples/indented_tree.py index 6cdaf374..5ac928c7 100644 --- a/examples/indented_tree.py +++ b/examples/indented_tree.py @@ -52,4 +52,3 @@ def test(): if __name__ == '__main__': test() - diff --git a/examples/standalone/README.md b/examples/standalone/README.md index 6ec80355..2f4541d8 100644 --- a/examples/standalone/README.md +++ b/examples/standalone/README.md @@ -17,4 +17,3 @@ Then run using: ```bash python json_parser_main.py ``` - diff --git a/examples/standalone/json_parser_main.py b/examples/standalone/json_parser_main.py index 3d9b5a6d..0afabde6 100644 --- a/examples/standalone/json_parser_main.py +++ b/examples/standalone/json_parser_main.py @@ -34,4 +34,3 @@ def string(self, s): if __name__ == '__main__': with open(sys.argv[1]) as f: print(parser.parse(f.read())) - diff --git a/examples/tests/no_newline_at_end.lark b/examples/tests/no_newline_at_end.lark index 8ab64d7a..9beaff7f 100644 --- a/examples/tests/no_newline_at_end.lark +++ b/examples/tests/no_newline_at_end.lark @@ -1 +1 @@ -start: "a" \ No newline at end of file +start: "a" diff --git a/lark/__init__.py b/lark/__init__.py index 1819d8d6..d8d0c88a 100644 --- a/lark/__init__.py +++ b/lark/__init__.py @@ -1,9 +1,38 @@ -from .utils import logger -from .tree import Tree, ParseTree -from .visitors import Transformer, Visitor, v_args, Discard, Transformer_NonRecursive -from .exceptions import (ParseError, LexError, GrammarError, UnexpectedToken, - UnexpectedInput, UnexpectedCharacters, UnexpectedEOF, LarkError) -from .lexer import Token +from .exceptions import ( + GrammarError, + LarkError, + LexError, + ParseError, + UnexpectedCharacters, + UnexpectedEOF, + UnexpectedInput, + UnexpectedToken, +) from .lark import Lark +from .lexer import Token +from .tree import ParseTree, Tree +from .utils import logger +from .visitors import Discard, Transformer, Transformer_NonRecursive, Visitor, v_args + +__version__: str = "1.1.4" -__version__: str = "1.1.3" +__all__ = ( + "GrammarError", + "LarkError", + "LexError", + "ParseError", + "UnexpectedCharacters", + "UnexpectedEOF", + "UnexpectedInput", + "UnexpectedToken", + "Lark", + "Token", + "ParseTree", + "Tree", + "logger", + "Discard", + "Transformer", + "Transformer_NonRecursive", + "Visitor", + "v_args", +) diff --git a/lark/__pyinstaller/__init__.py b/lark/__pyinstaller/__init__.py index fa02fc92..9da62a33 100644 --- a/lark/__pyinstaller/__init__.py +++ b/lark/__pyinstaller/__init__.py @@ -3,4 +3,4 @@ import os def get_hook_dirs(): - return [os.path.dirname(__file__)] \ No newline at end of file + return [os.path.dirname(__file__)] diff --git a/lark/ast_utils.py b/lark/ast_utils.py index faa17d0f..0ceee98f 100644 --- a/lark/ast_utils.py +++ b/lark/ast_utils.py @@ -44,7 +44,7 @@ def create_transformer(ast_module: types.ModuleType, Parameters: ast_module: A Python module containing all the subclasses of ``ast_utils.Ast`` transformer (Optional[Transformer]): An initial transformer. Its attributes may be overwritten. - decorator_factory (Callable): An optional callable accepting two booleans, inline, and meta, + decorator_factory (Callable): An optional callable accepting two booleans, inline, and meta, and returning a decorator for the methods of ``transformer``. (default: ``v_args``). """ t = transformer or Transformer() diff --git a/lark/exceptions.py b/lark/exceptions.py index dafe0b37..35b986af 100644 --- a/lark/exceptions.py +++ b/lark/exceptions.py @@ -168,7 +168,7 @@ def __str__(self): class UnexpectedCharacters(LexError, UnexpectedInput): - """An exception that is raised by the lexer, when it cannot match the next + """An exception that is raised by the lexer, when it cannot match the next string of characters to any of its terminals. """ diff --git a/lark/grammars/python.lark b/lark/grammars/python.lark index 8286c856..ab4a26a4 100644 --- a/lark/grammars/python.lark +++ b/lark/grammars/python.lark @@ -50,14 +50,14 @@ lambda_kwparams: "**" name ","? ?stmt: simple_stmt | compound_stmt ?simple_stmt: small_stmt (";" small_stmt)* [";"] _NEWLINE ?small_stmt: (expr_stmt | assign_stmt | del_stmt | pass_stmt | flow_stmt | import_stmt | global_stmt | nonlocal_stmt | assert_stmt) -expr_stmt: testlist_star_expr +expr_stmt: testlist_star_expr assign_stmt: annassign | augassign | assign annassign: testlist_star_expr ":" test ["=" test] assign: testlist_star_expr ("=" (yield_expr|testlist_star_expr))+ augassign: testlist_star_expr augassign_op (yield_expr|testlist) !augassign_op: "+=" | "-=" | "*=" | "@=" | "/=" | "%=" | "&=" | "|=" | "^=" | "<<=" | ">>=" | "**=" | "//=" -?testlist_star_expr: test_or_star_expr +?testlist_star_expr: test_or_star_expr | test_or_star_expr ("," test_or_star_expr)+ ","? -> tuple | test_or_star_expr "," -> tuple @@ -95,13 +95,13 @@ for_stmt: "for" exprlist "in" testlist ":" suite ["else" ":" suite] try_stmt: "try" ":" suite except_clauses ["else" ":" suite] [finally] | "try" ":" suite finally -> try_finally finally: "finally" ":" suite -except_clauses: except_clause+ +except_clauses: except_clause+ except_clause: "except" [test ["as" name]] ":" suite // NB compile.c makes sure that the default except clause is last with_stmt: "with" with_items ":" suite -with_items: with_item ("," with_item)* +with_items: with_item ("," with_item)* with_item: test ["as" name] match_stmt: "match" test ":" _NEWLINE _INDENT case+ _DEDENT @@ -204,7 +204,7 @@ AWAIT: "await" | "{" _set_exprlist "}" -> set | "{" comprehension{test} "}" -> set_comprehension | name -> var - | number + | number | string_concat | "(" test ")" | "..." -> ellipsis @@ -217,7 +217,7 @@ AWAIT: "await" _testlist_comp: test | _tuple_inner _tuple_inner: test_or_star_expr (("," test_or_star_expr)+ [","] | ",") - + ?test_or_star_expr: test | star_expr @@ -253,7 +253,7 @@ kwargs: "**" test ("," argvalue)* comprehension{comp_result}: comp_result comp_fors [comp_if] -comp_fors: comp_for+ +comp_fors: comp_for+ comp_for: [ASYNC] "for" exprlist "in" or_test ASYNC: "async" ?comp_if: "if" test_nocond diff --git a/lark/lark.py b/lark/lark.py index 9f80ef06..c93e9e19 100644 --- a/lark/lark.py +++ b/lark/lark.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod import getpass -import sys, os, pickle, hashlib +import sys, os, pickle import tempfile import types import re @@ -17,7 +17,7 @@ else: from typing_extensions import Literal from .parser_frontends import ParsingFrontend - + from .exceptions import ConfigurationError, assert_config, UnexpectedInput from .utils import Serialize, SerializeMemoizer, FS, isascii, logger from .load_grammar import load_grammar, FromPackageLoader, Grammar, verify_used_files, PackageResource, md5_digest diff --git a/lark/lexer.py b/lark/lexer.py index 6f5fb67d..5e6d6d40 100644 --- a/lark/lexer.py +++ b/lark/lexer.py @@ -5,9 +5,10 @@ from contextlib import suppress from typing import ( TypeVar, Type, List, Dict, Iterator, Collection, Callable, Optional, FrozenSet, Any, - Pattern as REPattern, ClassVar, TYPE_CHECKING + Pattern as REPattern, ClassVar, TYPE_CHECKING, overload ) from types import ModuleType +import warnings if TYPE_CHECKING: from .common import LexerConf @@ -147,18 +148,61 @@ class Token(str): """ __slots__ = ('type', 'start_pos', 'value', 'line', 'column', 'end_line', 'end_column', 'end_pos') + __match_args__ = ('type', 'value') + type: str - start_pos: int + start_pos: Optional[int] value: Any - line: int - column: int - end_line: int - end_column: int - end_pos: int + line: Optional[int] + column: Optional[int] + end_line: Optional[int] + end_column: Optional[int] + end_pos: Optional[int] + + + @overload + def __new__( + cls, + type: str, + value: Any, + start_pos: Optional[int]=None, + line: Optional[int]=None, + column: Optional[int]=None, + end_line: Optional[int]=None, + end_column: Optional[int]=None, + end_pos: Optional[int]=None + ) -> 'Token': + ... + + @overload + def __new__( + cls, + type_: str, + value: Any, + start_pos: Optional[int]=None, + line: Optional[int]=None, + column: Optional[int]=None, + end_line: Optional[int]=None, + end_column: Optional[int]=None, + end_pos: Optional[int]=None + ) -> 'Token': ... + + def __new__(cls, *args, **kwargs): + if "type_" in kwargs: + warnings.warn("`type_` is deprecated use `type` instead", DeprecationWarning) + + if "type" in kwargs: + raise TypeError("Error: using both 'type' and the deprecated 'type_' as arguments.") + kwargs["type"] = kwargs.pop("type_") + + return cls._future_new(*args, **kwargs) + - def __new__(cls, type_, value, start_pos=None, line=None, column=None, end_line=None, end_column=None, end_pos=None): + @classmethod + def _future_new(cls, type, value, start_pos=None, line=None, column=None, end_line=None, end_column=None, end_pos=None): inst = super(Token, cls).__new__(cls, value) - inst.type = type_ + + inst.type = type inst.start_pos = start_pos inst.value = value inst.line = line @@ -168,9 +212,27 @@ def __new__(cls, type_, value, start_pos=None, line=None, column=None, end_line= inst.end_pos = end_pos return inst + @overload + def update(self, type: Optional[str]=None, value: Optional[Any]=None) -> 'Token': + ... + + @overload def update(self, type_: Optional[str]=None, value: Optional[Any]=None) -> 'Token': + ... + + def update(self, *args, **kwargs): + if "type_" in kwargs: + warnings.warn("`type_` is deprecated use `type` instead", DeprecationWarning) + + if "type" in kwargs: + raise TypeError("Error: using both 'type' and the deprecated 'type_' as arguments.") + kwargs["type"] = kwargs.pop("type_") + + return self._future_update(*args, **kwargs) + + def _future_update(self, type: Optional[str]=None, value: Optional[Any]=None) -> 'Token': return Token.new_borrow_pos( - type_ if type_ is not None else self.type, + type if type is not None else self.type, value if value is not None else self.value, self ) diff --git a/lark/parse_tree_builder.py b/lark/parse_tree_builder.py index 888cc736..a6003a92 100644 --- a/lark/parse_tree_builder.py +++ b/lark/parse_tree_builder.py @@ -73,6 +73,8 @@ def _pp_get_meta(self, children): return c.meta elif isinstance(c, Token): return c + elif hasattr(c, '__lark_meta__'): + return c.__lark_meta__() def make_propagate_positions(option): if callable(option): diff --git a/lark/parsers/earley.py b/lark/parsers/earley.py index 044f7b05..2a047b03 100644 --- a/lark/parsers/earley.py +++ b/lark/parsers/earley.py @@ -170,7 +170,7 @@ def is_quasi_complete(item): return True # def create_leo_transitives(origin, start): - # ... # removed at commit 4c1cfb2faf24e8f8bff7112627a00b94d261b420 + # ... # removed at commit 4c1cfb2faf24e8f8bff7112627a00b94d261b420 def scan(i, token, to_scan): """The core Earley Scanner. diff --git a/lark/parsers/earley_common.py b/lark/parsers/earley_common.py index 844ff108..46e242b4 100644 --- a/lark/parsers/earley_common.py +++ b/lark/parsers/earley_common.py @@ -39,4 +39,4 @@ def __repr__(self): # class TransitiveItem(Item): -# ... # removed at commit 4c1cfb2faf24e8f8bff7112627a00b94d261b420 \ No newline at end of file +# ... # removed at commit 4c1cfb2faf24e8f8bff7112627a00b94d261b420 diff --git a/lark/parsers/lalr_interactive_parser.py b/lark/parsers/lalr_interactive_parser.py index c9658daf..0013ddf3 100644 --- a/lark/parsers/lalr_interactive_parser.py +++ b/lark/parsers/lalr_interactive_parser.py @@ -37,20 +37,20 @@ def iter_parse(self) -> Iterator[Token]: Returns an iterator of the tokens it encounters. - When the parse is over, the resulting tree can be found in ``InteractiveParser.result``. + When the parse is over, the resulting tree can be found in ``InteractiveParser.result``. """ for token in self.lexer_thread.lex(self.parser_state): yield token self.result = self.feed_token(token) - + def exhaust_lexer(self) -> List[Token]: """Try to feed the rest of the lexer state into the interactive parser. - + Note that this modifies the instance in place and does not feed an '$END' Token """ return list(self.iter_parse()) - + def feed_eof(self, last_token=None): """Feed a '$END' Token. Borrows from 'last_token' if given.""" eof = Token.new_borrow_pos('$END', '', last_token) if last_token is not None else self.lexer_thread._Token('$END', '', 0, 1, 1) @@ -146,4 +146,3 @@ def as_mutable(self): """Convert to an ``InteractiveParser``.""" p = copy(self) return InteractiveParser(p.parser, p.parser_state, p.lexer_thread) - diff --git a/lark/parsers/lalr_parser.py b/lark/parsers/lalr_parser.py index 48dac7b7..c89c49df 100644 --- a/lark/parsers/lalr_parser.py +++ b/lark/parsers/lalr_parser.py @@ -197,4 +197,3 @@ def parse_from_state(self, state): raise ###} - diff --git a/lark/tools/__init__.py b/lark/tools/__init__.py index 6b0bd6ab..391f991f 100644 --- a/lark/tools/__init__.py +++ b/lark/tools/__init__.py @@ -53,7 +53,7 @@ def showwarning_as_comment(message, category, filename, lineno, file=None, line= if file is None: file = sys.stderr if file is None: - return + return try: file.write(text) except OSError: diff --git a/lark/tools/standalone.py b/lark/tools/standalone.py index 87e90315..3ae2cdb6 100644 --- a/lark/tools/standalone.py +++ b/lark/tools/standalone.py @@ -29,7 +29,7 @@ from types import ModuleType from typing import ( TypeVar, Generic, Type, Tuple, List, Dict, Iterator, Collection, Callable, Optional, FrozenSet, Any, - Union, Iterable, IO, TYPE_CHECKING, + Union, Iterable, IO, TYPE_CHECKING, overload, Pattern as REPattern, ClassVar, Set, Mapping ) ###} diff --git a/lark/visitors.py b/lark/visitors.py index 44e4caa6..932fbee1 100644 --- a/lark/visitors.py +++ b/lark/visitors.py @@ -72,7 +72,7 @@ def __class_getitem__(cls, _): class Transformer(_Decoratable, ABC, Generic[_Leaf_T, _Return_T]): - """Transformers work bottom-up (or depth-first), starting with visiting the leaves and working + """Transformers work bottom-up (or depth-first), starting with visiting the leaves and working their way up until ending at the root of the tree. For each node visited, the transformer will call the appropriate method (callbacks), according to the @@ -83,7 +83,7 @@ class Transformer(_Decoratable, ABC, Generic[_Leaf_T, _Return_T]): If the transformer cannot find a method with the right name, it will instead call ``__default__``, which by default creates a copy of the node. - + To discard a node, return Discard (``lark.visitors.Discard``). ``Transformer`` can do anything ``Visitor`` can do, but because it reconstructs the tree, diff --git a/setup.py b/setup.py index a6b5bef0..6c4a3676 100644 --- a/setup.py +++ b/setup.py @@ -69,4 +69,3 @@ ] }, ) - diff --git a/test-requirements.txt b/test-requirements.txt index d304ee8f..43014967 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,2 +1,2 @@ Js2Py==0.68 -regex \ No newline at end of file +regex diff --git a/tests/__main__.py b/tests/__main__.py index bbf99865..c5298a77 100644 --- a/tests/__main__.py +++ b/tests/__main__.py @@ -2,6 +2,7 @@ import unittest import logging +import sys from lark import logger from .test_trees import TestTrees @@ -26,6 +27,9 @@ from .test_parser import * # We define __all__ to list which TestSuites to run +if sys.version_info >= (3, 10): + from .test_pattern_matching import TestPatternMatching + logger.setLevel(logging.INFO) if __name__ == '__main__': diff --git a/tests/test_cache.py b/tests/test_cache.py index 99ec1339..fd653bff 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -71,12 +71,12 @@ def append_zero(t): class TestCache(TestCase): g = '''start: "a"''' - + def setUp(self): self.fs = lark_module.FS self.mock_fs = MockFS() lark_module.FS = self.mock_fs - + def tearDown(self): self.mock_fs.files = {} lark_module.FS = self.fs @@ -88,7 +88,7 @@ def test_simple(self): assert fn in self.mock_fs.files parser = Lark(self.g, parser='lalr', cache=fn) assert parser.parse('a') == Tree('start', []) - + def test_automatic_naming(self): assert len(self.mock_fs.files) == 0 Lark(self.g, parser='lalr', cache=True) @@ -102,7 +102,7 @@ def test_automatic_naming(self): parser = Lark(self.g, parser='lalr', cache=True) assert parser.parse('a') == Tree('start', []) - + def test_custom_lexer(self): parser = Lark(self.g, parser='lalr', lexer=CustomLexer, cache=True) @@ -115,7 +115,7 @@ def test_options(self): Lark(self.g, parser="lalr", debug=True, cache=True) parser = Lark(self.g, parser="lalr", debug=True, cache=True) assert parser.options.options['debug'] - + def test_inline(self): # Test inline transformer (tree-less) & lexer_callbacks # Note: the Transformer should not be saved to the file, @@ -136,7 +136,7 @@ def test_inline(self): res1 = parser.parse(text) res2 = InlineTestT().transform(Lark(g, parser="lalr", cache=True, lexer_callbacks={'NUM': append_zero}).parse(text)) assert res0 == res1 == res2 == expected - + def test_imports(self): g = """ %import .grammars.ab (startab, expr) @@ -167,7 +167,7 @@ def test_recursive_pattern(self): # should only have the dummy log self.assertCountEqual(cm.output, ["ERROR:lark:dummy message"]) - + if __name__ == '__main__': diff --git a/tests/test_grammar.py b/tests/test_grammar.py index e388cb75..624b0799 100644 --- a/tests/test_grammar.py +++ b/tests/test_grammar.py @@ -302,6 +302,3 @@ def test_line_breaks(self): if __name__ == '__main__': main() - - - diff --git a/tests/test_lexer.py b/tests/test_lexer.py index 411ef942..0996c897 100644 --- a/tests/test_lexer.py +++ b/tests/test_lexer.py @@ -20,4 +20,4 @@ def test_basic(self): if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/tests/test_parser.py b/tests/test_parser.py index 94427ca6..b48ea039 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -193,11 +193,11 @@ def test_visit_tokens2(self): class T(base): def add(self, children): return sum(children if isinstance(children, list) else children.children) - + def NUM(self, token): return int(token) - - + + parser = Lark(g, parser='lalr', transformer=T()) result = parser.parse(text) self.assertEqual(result, expected) @@ -385,7 +385,7 @@ def test_lexer_token_limit(self): tokens = {'A%d'%i:'"%d"'%i for i in range(300)} g = """start: %s %s""" % (' '.join(tokens), '\n'.join("%s: %s"%x for x in tokens.items())) - + p = Lark(g, parser='lalr') @@ -2478,7 +2478,7 @@ def test_unicode_word(self): NAME: /[\w]+/ """, regex=True) self.assertEqual(g.parse('வணக்கம்'), 'வணக்கம்') - + @unittest.skipIf(not regex, "regex not installed") def test_regex_width_fallback(self): g = r""" @@ -2488,7 +2488,7 @@ def test_regex_width_fallback(self): self.assertRaises((GrammarError, LexError, re.error), _Lark, g) p = _Lark(g, regex=True) self.assertEqual(p.parse("123abc"), Tree('start', ['123', 'abc'])) - + g = r""" start: NAME NAME? NAME: /(?(?=\d)\d+|\w*)/ @@ -2503,7 +2503,7 @@ def test_parser_interactive_parser(self): A: "a" B: "b" ''') - + ip = g.parse_interactive() self.assertRaises(UnexpectedToken, ip.feed_eof) @@ -2526,7 +2526,7 @@ def test_parser_interactive_parser(self): res = ip.feed_eof(ip.lexer_thread.state.last_token) self.assertEqual(res, Tree('start', ['a', 'b'])) self.assertRaises(UnexpectedToken ,ip.feed_eof) - + self.assertRaises(UnexpectedToken, ip_copy.feed_token, Token('A', 'a')) ip_copy.feed_token(Token('B', 'b')) res = ip_copy.feed_eof() diff --git a/tests/test_pattern_matching.py b/tests/test_pattern_matching.py new file mode 100644 index 00000000..b86bd543 --- /dev/null +++ b/tests/test_pattern_matching.py @@ -0,0 +1,52 @@ +from unittest import TestCase, main + +from lark import Token + + +class TestPatternMatching(TestCase): + token = Token('A', 'a') + + def setUp(self): + pass + + def test_matches_with_string(self): + match self.token: + case 'a': + pass + case _: + assert False + + def test_matches_with_str_positional_arg(self): + match self.token: + case str('a'): + pass + case _: + assert False + + def test_matches_with_token_positional_arg(self): + match self.token: + case Token('a'): + assert False + case Token('A'): + pass + case _: + assert False + + def test_matches_with_token_kwarg_type(self): + match self.token: + case Token(type='A'): + pass + case _: + assert False + + def test_matches_with_bad_token_type(self): + match self.token: + case Token(type='B'): + assert False + case _: + pass + + + +if __name__ == '__main__': + main() diff --git a/tests/test_python_grammar.py b/tests/test_python_grammar.py index 837369ea..345452b7 100644 --- a/tests/test_python_grammar.py +++ b/tests/test_python_grammar.py @@ -190,7 +190,7 @@ case str() | bytes(): print("Something string-like") case _: - print("Something else") + print("Something else") """), # guards diff --git a/tox.ini b/tox.ini index ef895658..0b1426dc 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py36, py37, py38, py39, py310, pypy3, type +envlist = lint, type, py36, py37, py38, py39, py310, py311, pypy3 skip_missing_interpreters = true [testenv] @@ -28,3 +28,12 @@ deps = rich commands = mypy + +[testenv:lint] +description = run linters on code base +skip_install = true +recreate = false +deps = + pre-commit +commands = + pre-commit run --all-files --show-diff-on-failure pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy