Skip to content

Commit e352670

Browse files
ncwncw
committed
parser: fix "single" mode and return correct EOF error
1 parent 5775015 commit e352670

File tree

8 files changed

+6200
-6153
lines changed

8 files changed

+6200
-6153
lines changed

‎notes.txt

Lines changed: 0 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,6 @@ Things to do before release
1818
===========================
1919

2020
* Line numbers
21-
* compile single
22-
* interactive interpreter
2321
* Subclass builtins
2422
* pygen
2523

@@ -29,36 +27,12 @@ FIXME recursive types for __repr__ in list, dict, tuple
2927
>>> L
3028
[[...]]
3129

32-
interactive interpreter
33-
calls compile(..."single) until doesn't get "SyntaxError: unexpected EOF while parsing" ?
34-
see pythonrun.c
35-
PyRun_InteractiveOneObject(FILE *fp, PyObject *filename, PyCompilerFlags *flags)
36-
...
37-
mod = PyParser_ASTFromFileObject(fp, filename, enc,
38-
Py_single_input, ps1, ps2,
39-
flags, &errcode, arena);
40-
Py_XDECREF(v);
41-
Py_XDECREF(w);
42-
Py_XDECREF(oenc);
43-
if (mod == NULL) {
44-
PyArena_Free(arena);
45-
if (errcode == E_EOF) {
46-
PyErr_Clear();
47-
return E_EOF;
48-
}
49-
PyErr_Print();
50-
return -1;
51-
}
52-
53-
5430
Limitations & Missing parts
5531
===========================
5632
* string keys only in dictionaries
5733
* intermediate fix - stop it panicing!
5834
* line numbers missing in SyntaxErrors
5935
* \N{...} escapes not implemented
60-
* Interactive interpreter does single lines only
61-
* compile(..., "single") not working
6236
* lots of builtins still to implement
6337
* FIXME eq && ne should throw an error for a type which doesn' have eq implemented
6438
* repr/str

‎parser/grammar.y

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -259,18 +259,20 @@ inputs:
259259
}
260260

261261
single_input:
262-
NEWLINE
262+
/* NEWLINE
263263
{
264-
// panic("FIXME no coverage")
264+
// This is in the python grammar, but the interpreter
265+
// just gives "unexpected EOF while parsing" when you
266+
// give it a \n
265267
$$ = &ast.Interactive{ModBase: ast.ModBase{Pos: $<pos>$}}
266268
}
267-
| simple_stmt
269+
|*/ simple_stmt
268270
{
269271
$$ = &ast.Interactive{ModBase: ast.ModBase{Pos: $<pos>$}, Body: $1}
270272
}
271273
| compound_stmt NEWLINE
272274
{
273-
// panic("FIXME no coverage")
275+
// NB: compound_stmt in single_input is followed by extra NEWLINE!
274276
$$ = &ast.Interactive{ModBase: ast.ModBase{Pos: $<pos>$}, Body: []ast.Stmt{$1}}
275277
}
276278

‎parser/grammar_data_test.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,9 @@ var grammarTestData = []struct {
2929
{"b'abc' b'''123'''", "eval", "Expression(body=Bytes(s=b'abc123'))", nil, ""},
3030
{"1234", "eval", "Expression(body=Num(n=1234))", nil, ""},
3131
{"01234", "eval", "", py.SyntaxError, "illegal decimal with leading zero"},
32-
{"1234d", "eval", "", py.SyntaxError, "invalid syntax"},
32+
{"1234d", "eval", "", py.SyntaxError, "unexpected EOF while parsing"},
3333
{"1234d", "exec", "", py.SyntaxError, "invalid syntax"},
34+
{"1234d", "single", "", py.SyntaxError, "unexpected EOF while parsing"},
3435
{"0x1234", "eval", "Expression(body=Num(n=4660))", nil, ""},
3536
{"12.34", "eval", "Expression(body=Num(n=12.34))", nil, ""},
3637
{"1,", "eval", "Expression(body=Tuple(elts=[Num(n=1)], ctx=Load()))", nil, ""},
@@ -311,5 +312,9 @@ var grammarTestData = []struct {
311312
{"@dec(a,b,c=d,*args,**kwargs)\ndef fn():\n pass\n", "exec", "Module(body=[FunctionDef(name='fn', args=arguments(args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[Call(func=Name(id='dec', ctx=Load()), args=[Name(id='a', ctx=Load()), Name(id='b', ctx=Load())], keywords=[keyword(arg='c', value=Name(id='d', ctx=Load()))], starargs=Name(id='args', ctx=Load()), kwargs=Name(id='kwargs', ctx=Load()))], returns=None)])", nil, ""},
312313
{"@dec1\n@dec2()\n@dec3(a)\n@dec4(a,b)\ndef fn():\n pass\n", "exec", "Module(body=[FunctionDef(name='fn', args=arguments(args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[Pass()], decorator_list=[Name(id='dec1', ctx=Load()), Call(func=Name(id='dec2', ctx=Load()), args=[], keywords=[], starargs=None, kwargs=None), Call(func=Name(id='dec3', ctx=Load()), args=[Name(id='a', ctx=Load())], keywords=[], starargs=None, kwargs=None), Call(func=Name(id='dec4', ctx=Load()), args=[Name(id='a', ctx=Load()), Name(id='b', ctx=Load())], keywords=[], starargs=None, kwargs=None)], returns=None)])", nil, ""},
313314
{"@dec1\n@dec2()\n@dec3(a)\n@dec4(a,b)\nclass A(B):\n pass\n", "exec", "Module(body=[ClassDef(name='A', bases=[Name(id='B', ctx=Load())], keywords=[], starargs=None, kwargs=None, body=[Pass()], decorator_list=[Name(id='dec1', ctx=Load()), Call(func=Name(id='dec2', ctx=Load()), args=[], keywords=[], starargs=None, kwargs=None), Call(func=Name(id='dec3', ctx=Load()), args=[Name(id='a', ctx=Load())], keywords=[], starargs=None, kwargs=None), Call(func=Name(id='dec4', ctx=Load()), args=[Name(id='a', ctx=Load()), Name(id='b', ctx=Load())], keywords=[], starargs=None, kwargs=None)])])", nil, ""},
315+
{"", "single", "", py.SyntaxError, "unexpected EOF while parsing"},
316+
{"\n", "single", "", py.SyntaxError, "unexpected EOF while parsing"},
314317
{"pass\n", "single", "Interactive(body=[Pass()])", nil, ""},
318+
{"if True:\n pass\n\n", "single", "Interactive(body=[If(test=NameConstant(value=True), body=[Pass()], orelse=[])])", nil, ""},
319+
{"while True:\n pass\nelse:\n return\n", "single", "Interactive(body=[While(test=NameConstant(value=True), body=[Pass()], orelse=[Return(value=None)])])", nil, ""},
315320
}

‎parser/lexer.go

Lines changed: 65 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ type yyLex struct {
4747
parenthesis int // number of open ( )
4848
brace int // number of open { }
4949
mod ast.Mod // output
50-
startToken int // initial token to output
50+
tokens []int // buffered tokens to output
5151
}
5252

5353
// Create a new lexer
@@ -64,19 +64,41 @@ func NewLex(r io.Reader, mode string) (*yyLex, error) {
6464
}
6565
switch mode {
6666
case "exec":
67-
x.startToken = FILE_INPUT
67+
x.queue(FILE_INPUT)
6868
x.exec = true
6969
case "eval":
70-
x.startToken = EVAL_INPUT
70+
x.queue(EVAL_INPUT)
7171
case "single":
72-
x.startToken = SINGLE_INPUT
72+
x.queue(SINGLE_INPUT)
7373
x.interactive = true
7474
default:
7575
return nil, py.ExceptionNewf(py.ValueError, "compile mode must be 'exec', 'eval' or 'single'")
7676
}
7777
return x, nil
7878
}
7979

80+
// queue tokens for later return
81+
func (x *yyLex) queue(tokens ...int) {
82+
x.tokens = append(x.tokens, tokens...)
83+
}
84+
85+
// Return whether the token queue is empty
86+
func (x *yyLex) queueEmpty() bool {
87+
return len(x.tokens) == 0
88+
}
89+
90+
// dequeue a token for return
91+
//
92+
// panic if no token available
93+
func (x *yyLex) dequeue() int {
94+
if x.queueEmpty() {
95+
panic("token queue empty")
96+
}
97+
token := x.tokens[0]
98+
x.tokens = x.tokens[1:]
99+
return token
100+
}
101+
80102
// Refill line
81103
func (x *yyLex) refill() {
82104
var err error
@@ -328,6 +350,14 @@ func (lts LexTokens) String() string {
328350
return buf.String()
329351
}
330352

353+
// Queue any remaining DEDENTS
354+
func (x *yyLex) queueDedents() {
355+
for i := len(x.indentStack) - 1; i >= 1; i-- {
356+
x.queue(DEDENT)
357+
}
358+
x.indentStack = x.indentStack[:1]
359+
}
360+
331361
// The parser calls this method to get each new token. This
332362
// implementation returns operators and NUM.
333363
func (x *yyLex) Lex(yylval *yySymType) (ret int) {
@@ -340,11 +370,9 @@ func (x *yyLex) Lex(yylval *yySymType) (ret int) {
340370
}()
341371
}
342372

343-
// Return initial token
344-
if x.startToken != eof {
345-
token := x.startToken
346-
x.startToken = eof
347-
return token
373+
// Return queued tokens if there are any
374+
if !x.queueEmpty() {
375+
return x.dequeue()
348376
}
349377

350378
// FIXME keep x.pos up to date
@@ -358,10 +386,13 @@ func (x *yyLex) Lex(yylval *yySymType) (ret int) {
358386
x.state++
359387
if x.line == "" && x.eof {
360388
x.state = checkEof
361-
// an empty line while reading interactive input should return a NEWLINE
362-
// Don't output NEWLINE if brackets are open
389+
// During interactive input of statements an entirely blank logical
390+
// line (i.e. one containing not even whitespace or a comment)
391+
// terminates a multi-line statement.
363392
if x.interactive && !x.openBrackets() {
364-
return NEWLINE
393+
x.queueDedents()
394+
x.queue(NEWLINE)
395+
return x.dequeue()
365396
}
366397
continue
367398
}
@@ -380,32 +411,33 @@ func (x *yyLex) Lex(yylval *yySymType) (ret int) {
380411
}
381412
x.state++
382413
case checkIndent:
414+
x.state++
383415
// Don't output INDENT or DEDENT if brackets are open
384416
if x.openBrackets() {
385-
x.state++
386417
continue
387418
}
388419
// See if indent has changed and issue INDENT / DEDENT
389420
indent := countIndent(x.currentIndent)
390-
indentStackTop := x.indentStack[len(x.indentStack)-1]
391-
switch {
392-
case indent > indentStackTop:
421+
i := len(x.indentStack) - 1
422+
indentStackTop := x.indentStack[i]
423+
if indent == indentStackTop {
424+
continue
425+
} else if indent > indentStackTop {
393426
x.indentStack = append(x.indentStack, indent)
394-
x.state++
395427
return INDENT
396-
case indent < indentStackTop:
397-
for i := len(x.indentStack) - 1; i >= 0; i-- {
428+
} else {
429+
for ; i >= 0; i-- {
398430
if x.indentStack[i] == indent {
399431
goto foundIndent
400432
}
433+
x.queue(DEDENT)
401434
}
402435
x.SyntaxError("Inconsistent indent")
403436
return eof
404437
foundIndent:
405-
x.indentStack = x.indentStack[:len(x.indentStack)-1]
406-
return DEDENT
438+
x.indentStack = x.indentStack[:i+1]
439+
return x.dequeue()
407440
}
408-
x.state++
409441
case parseTokens:
410442
// Skip white space
411443
x.line = strings.TrimLeft(x.line, " \t")
@@ -493,18 +525,16 @@ func (x *yyLex) Lex(yylval *yySymType) (ret int) {
493525
return eof
494526
case checkEof:
495527
if x.eof {
496-
// Return any remaining DEDENTS
497-
if len(x.indentStack) > 1 {
498-
x.indentStack = x.indentStack[:len(x.indentStack)-1]
499-
x.state = checkEof
500-
return DEDENT
501-
}
528+
x.queueDedents()
502529
// then return ENDMARKER
503530
x.state = isEof
504-
if x.interactive {
531+
if !x.interactive {
532+
x.queue(ENDMARKER)
533+
}
534+
if x.queueEmpty() {
505535
continue
506536
}
507-
return ENDMARKER
537+
return x.dequeue()
508538
}
509539
x.state = readString
510540
case isEof:
@@ -857,7 +887,11 @@ func (x *yyLex) SyntaxErrorf(format string, a ...interface{}) {
857887
func (x *yyLex) ErrorReturn() error {
858888
if x.error {
859889
if x.errorString == "" {
860-
x.errorString = "invalid syntax"
890+
if x.eof && !x.exec {
891+
x.errorString = "unexpected EOF while parsing"
892+
} else {
893+
x.errorString = "invalid syntax"
894+
}
861895
}
862896
return py.ExceptionNewf(py.SyntaxError, "%s", x.errorString)
863897
}

‎parser/lexer_test.go

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -361,6 +361,56 @@ func TestLex(t *testing.T) {
361361
{"$asdasd", "invalid syntax", "eval", LexTokens{
362362
{EVAL_INPUT, nil},
363363
}},
364+
{"if True:\n pass\n\n", "", "single", LexTokens{
365+
{SINGLE_INPUT, nil},
366+
{IF, nil},
367+
{TRUE, nil},
368+
{':', nil},
369+
{NEWLINE, nil},
370+
{INDENT, nil},
371+
{PASS, nil},
372+
{NEWLINE, nil},
373+
{DEDENT, nil},
374+
{NEWLINE, nil},
375+
}},
376+
{"while True:\n pass\nelse:\n return\n", "", "single", LexTokens{
377+
{SINGLE_INPUT, nil},
378+
{WHILE, nil},
379+
{TRUE, nil},
380+
{':', nil},
381+
{NEWLINE, nil},
382+
{INDENT, nil},
383+
{PASS, nil},
384+
{NEWLINE, nil},
385+
{DEDENT, nil},
386+
{ELSE, nil},
387+
{':', nil},
388+
{NEWLINE, nil},
389+
{INDENT, nil},
390+
{RETURN, nil},
391+
{NEWLINE, nil},
392+
{DEDENT, nil},
393+
{NEWLINE, nil},
394+
}},
395+
{"while True:\n pass\nelse:\n return\n", "", "exec", LexTokens{
396+
{FILE_INPUT, nil},
397+
{WHILE, nil},
398+
{TRUE, nil},
399+
{':', nil},
400+
{NEWLINE, nil},
401+
{INDENT, nil},
402+
{PASS, nil},
403+
{NEWLINE, nil},
404+
{DEDENT, nil},
405+
{ELSE, nil},
406+
{':', nil},
407+
{NEWLINE, nil},
408+
{INDENT, nil},
409+
{RETURN, nil},
410+
{NEWLINE, nil},
411+
{DEDENT, nil},
412+
{ENDMARKER, nil},
413+
}},
364414
} {
365415
lts, err := LexString(test.in, test.mode)
366416
errString := ""

‎parser/make_grammar_test.py

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,9 @@
2525
("b'abc' b'''123'''", "eval"),
2626
("1234", "eval"),
2727
("01234", "eval", SyntaxError, "illegal decimal with leading zero"),
28-
("1234d", "eval", SyntaxError, "invalid syntax"),
28+
("1234d", "eval", SyntaxError),
2929
("1234d", "exec", SyntaxError),
30+
("1234d", "single", SyntaxError),
3031
("0x1234", "eval"),
3132
("12.34", "eval"),
3233
("1,", "eval"),
@@ -461,10 +462,11 @@ class A(B):
461462
""", "exec"),
462463

463464
# single input
464-
#("\n", "single"),
465+
("", "single", SyntaxError),
466+
("\n", "single", SyntaxError),
465467
("pass\n", "single"),
466-
# FIXME ("if True:\n pass\n\n", "single"),
467-
468+
("if True:\n pass\n\n", "single"),
469+
("while True:\n pass\nelse:\n return\n", "single"),
468470
]
469471

470472
def dump(source, mode):

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy