Skip to content

Commit 323ea3b

Browse files
Support incomplete parsing (#5764)
* continue accepting REPL input for multiline strings * Match cpython behavior for all multi-line statements (execute when complete) * Emit _IncompleteInputError when compiling with incomplete flag * Refine when _IncompleteInputError is emitted * Support multiline strings emitting _IncompleteInputError * lint * Undo accidental change to PyTabError * match -> if let * Fix test_baseexception and test_codeop * fix spelling * fix exception name * Skip pickle test of _IncompleteInputError * Use py3.15's codeop implementation * Update Lib/test/test_baseexception.py --------- Co-authored-by: Jeong, YunWon <69878+youknowone@users.noreply.github.com>
1 parent e27d031 commit 323ea3b

File tree

10 files changed

+231
-38
lines changed

10 files changed

+231
-38
lines changed

Lib/codeop.py

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -65,14 +65,10 @@ def _maybe_compile(compiler, source, filename, symbol):
6565
try:
6666
compiler(source + "\n", filename, symbol)
6767
return None
68+
except _IncompleteInputError as e:
69+
return None
6870
except SyntaxError as e:
69-
# XXX: RustPython; support multiline definitions in REPL
70-
# See also: https://github.com/RustPython/RustPython/pull/5743
71-
strerr = str(e)
72-
if source.endswith(":") and "expected an indented block" in strerr:
73-
return None
74-
elif "incomplete input" in str(e):
75-
return None
71+
pass
7672
# fallthrough
7773

7874
return compiler(source, filename, symbol, incomplete_input=False)

Lib/test/test_baseexception.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,8 @@ def test_inheritance(self):
8383
exc_set = set(e for e in exc_set if not e.startswith('_'))
8484
# RUSTPYTHON specific
8585
exc_set.discard("JitError")
86+
# TODO: RUSTPYTHON; this will be officially introduced in Python 3.15
87+
exc_set.discard("IncompleteInputError")
8688
self.assertEqual(len(exc_set), 0, "%s not accounted for" % exc_set)
8789

8890
interface_tests = ("length", "args", "str", "repr")

Lib/test/test_pickle.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -664,6 +664,9 @@ def test_exceptions(self):
664664
BaseExceptionGroup,
665665
ExceptionGroup):
666666
continue
667+
# TODO: RUSTPYTHON: fix name mapping for _IncompleteInputError
668+
if exc is _IncompleteInputError:
669+
continue
667670
if exc is not OSError and issubclass(exc, OSError):
668671
self.assertEqual(reverse_mapping('builtins', name),
669672
('exceptions', 'OSError'))

compiler/src/lib.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ pub enum CompileErrorType {
2525
pub struct ParseError {
2626
#[source]
2727
pub error: parser::ParseErrorType,
28+
pub raw_location: ruff_text_size::TextRange,
2829
pub location: SourceLocation,
2930
pub source_path: String,
3031
}
@@ -48,6 +49,7 @@ impl CompileError {
4849
let location = source_code.source_location(error.location.start());
4950
Self::Parse(ParseError {
5051
error: error.error,
52+
raw_location: error.location,
5153
location,
5254
source_path: source_code.path.to_owned(),
5355
})

src/shell.rs

Lines changed: 75 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
mod helper;
22

33
use rustpython_compiler::{
4-
CompileError, ParseError, parser::LexicalErrorType, parser::ParseErrorType,
4+
CompileError, ParseError, parser::FStringErrorType, parser::LexicalErrorType,
5+
parser::ParseErrorType,
56
};
67
use rustpython_vm::{
78
AsObject, PyResult, VirtualMachine,
@@ -14,19 +15,26 @@ use rustpython_vm::{
1415
enum ShellExecResult {
1516
Ok,
1617
PyErr(PyBaseExceptionRef),
17-
Continue,
18+
ContinueBlock,
19+
ContinueLine,
1820
}
1921

2022
fn shell_exec(
2123
vm: &VirtualMachine,
2224
source: &str,
2325
scope: Scope,
2426
empty_line_given: bool,
25-
continuing: bool,
27+
continuing_block: bool,
2628
) -> ShellExecResult {
29+
// compiling expects only UNIX style line endings, and will replace windows line endings
30+
// internally. Since we might need to analyze the source to determine if an error could be
31+
// resolved by future input, we need the location from the error to match the source code that
32+
// was actually compiled.
33+
#[cfg(windows)]
34+
let source = &source.replace("\r\n", "\n");
2735
match vm.compile(source, compiler::Mode::Single, "<stdin>".to_owned()) {
2836
Ok(code) => {
29-
if empty_line_given || !continuing {
37+
if empty_line_given || !continuing_block {
3038
// We want to execute the full code
3139
match vm.run_code_obj(code, scope) {
3240
Ok(_val) => ShellExecResult::Ok,
@@ -40,8 +48,32 @@ fn shell_exec(
4048
Err(CompileError::Parse(ParseError {
4149
error: ParseErrorType::Lexical(LexicalErrorType::Eof),
4250
..
43-
})) => ShellExecResult::Continue,
51+
})) => ShellExecResult::ContinueLine,
52+
Err(CompileError::Parse(ParseError {
53+
error:
54+
ParseErrorType::Lexical(LexicalErrorType::FStringError(
55+
FStringErrorType::UnterminatedTripleQuotedString,
56+
)),
57+
..
58+
})) => ShellExecResult::ContinueLine,
4459
Err(err) => {
60+
// Check if the error is from an unclosed triple quoted string (which should always
61+
// continue)
62+
if let CompileError::Parse(ParseError {
63+
error: ParseErrorType::Lexical(LexicalErrorType::UnclosedStringError),
64+
raw_location,
65+
..
66+
}) = err
67+
{
68+
let loc = raw_location.start().to_usize();
69+
let mut iter = source.chars();
70+
if let Some(quote) = iter.nth(loc) {
71+
if iter.next() == Some(quote) && iter.next() == Some(quote) {
72+
return ShellExecResult::ContinueLine;
73+
}
74+
}
75+
};
76+
4577
// bad_error == true if we are handling an error that should be thrown even if we are continuing
4678
// if its an indentation error, set to true if we are continuing and the error is on column 0,
4779
// since indentations errors on columns other than 0 should be ignored.
@@ -50,10 +82,12 @@ fn shell_exec(
5082
let bad_error = match err {
5183
CompileError::Parse(ref p) => {
5284
match &p.error {
53-
ParseErrorType::Lexical(LexicalErrorType::IndentationError) => continuing, // && p.location.is_some()
85+
ParseErrorType::Lexical(LexicalErrorType::IndentationError) => {
86+
continuing_block
87+
} // && p.location.is_some()
5488
ParseErrorType::OtherError(msg) => {
5589
if msg.starts_with("Expected an indented block") {
56-
continuing
90+
continuing_block
5791
} else {
5892
true
5993
}
@@ -68,7 +102,7 @@ fn shell_exec(
68102
if empty_line_given || bad_error {
69103
ShellExecResult::PyErr(vm.new_syntax_error(&err, Some(source)))
70104
} else {
71-
ShellExecResult::Continue
105+
ShellExecResult::ContinueBlock
72106
}
73107
}
74108
}
@@ -93,10 +127,19 @@ pub fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> {
93127
println!("No previous history.");
94128
}
95129

96-
let mut continuing = false;
130+
// We might either be waiting to know if a block is complete, or waiting to know if a multiline
131+
// statement is complete. In the former case, we need to ensure that we read one extra new line
132+
// to know that the block is complete. In the latter, we can execute as soon as the statement is
133+
// valid.
134+
let mut continuing_block = false;
135+
let mut continuing_line = false;
97136

98137
loop {
99-
let prompt_name = if continuing { "ps2" } else { "ps1" };
138+
let prompt_name = if continuing_block || continuing_line {
139+
"ps2"
140+
} else {
141+
"ps1"
142+
};
100143
let prompt = vm
101144
.sys_module
102145
.get_attr(prompt_name, vm)
@@ -105,6 +148,8 @@ pub fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> {
105148
Ok(ref s) => s.as_str(),
106149
Err(_) => "",
107150
};
151+
152+
continuing_line = false;
108153
let result = match repl.readline(prompt) {
109154
ReadlineResult::Line(line) => {
110155
debug!("You entered {:?}", line);
@@ -120,39 +165,44 @@ pub fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> {
120165
}
121166
full_input.push('\n');
122167

123-
match shell_exec(vm, &full_input, scope.clone(), empty_line_given, continuing) {
168+
match shell_exec(
169+
vm,
170+
&full_input,
171+
scope.clone(),
172+
empty_line_given,
173+
continuing_block,
174+
) {
124175
ShellExecResult::Ok => {
125-
if continuing {
176+
if continuing_block {
126177
if empty_line_given {
127-
// We should be exiting continue mode
128-
continuing = false;
178+
// We should exit continue mode since the block successfully executed
179+
continuing_block = false;
129180
full_input.clear();
130-
Ok(())
131-
} else {
132-
// We should stay in continue mode
133-
continuing = true;
134-
Ok(())
135181
}
136182
} else {
137183
// We aren't in continue mode so proceed normally
138-
continuing = false;
139184
full_input.clear();
140-
Ok(())
141185
}
186+
Ok(())
187+
}
188+
// Continue, but don't change the mode
189+
ShellExecResult::ContinueLine => {
190+
continuing_line = true;
191+
Ok(())
142192
}
143-
ShellExecResult::Continue => {
144-
continuing = true;
193+
ShellExecResult::ContinueBlock => {
194+
continuing_block = true;
145195
Ok(())
146196
}
147197
ShellExecResult::PyErr(err) => {
148-
continuing = false;
198+
continuing_block = false;
149199
full_input.clear();
150200
Err(err)
151201
}
152202
}
153203
}
154204
ReadlineResult::Interrupt => {
155-
continuing = false;
205+
continuing_block = false;
156206
full_input.clear();
157207
let keyboard_interrupt =
158208
vm.new_exception_empty(vm.ctx.exceptions.keyboard_interrupt.to_owned());

vm/src/compiler.rs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,3 +49,10 @@ impl crate::convert::ToPyException for (CompileError, Option<&str>) {
4949
vm.new_syntax_error(&self.0, self.1)
5050
}
5151
}
52+
53+
#[cfg(any(feature = "parser", feature = "codegen"))]
54+
impl crate::convert::ToPyException for (CompileError, Option<&str>, bool) {
55+
fn to_pyexception(&self, vm: &crate::VirtualMachine) -> crate::builtins::PyBaseExceptionRef {
56+
vm.new_syntax_error_maybe_incomplete(&self.0, self.1, self.2)
57+
}
58+
}

vm/src/exceptions.rs

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -495,6 +495,7 @@ pub struct ExceptionZoo {
495495
pub not_implemented_error: &'static Py<PyType>,
496496
pub recursion_error: &'static Py<PyType>,
497497
pub syntax_error: &'static Py<PyType>,
498+
pub incomplete_input_error: &'static Py<PyType>,
498499
pub indentation_error: &'static Py<PyType>,
499500
pub tab_error: &'static Py<PyType>,
500501
pub system_error: &'static Py<PyType>,
@@ -743,6 +744,7 @@ impl ExceptionZoo {
743744
let recursion_error = PyRecursionError::init_builtin_type();
744745

745746
let syntax_error = PySyntaxError::init_builtin_type();
747+
let incomplete_input_error = PyIncompleteInputError::init_builtin_type();
746748
let indentation_error = PyIndentationError::init_builtin_type();
747749
let tab_error = PyTabError::init_builtin_type();
748750

@@ -817,6 +819,7 @@ impl ExceptionZoo {
817819
not_implemented_error,
818820
recursion_error,
819821
syntax_error,
822+
incomplete_input_error,
820823
indentation_error,
821824
tab_error,
822825
system_error,
@@ -965,6 +968,7 @@ impl ExceptionZoo {
965968
"end_offset" => ctx.none(),
966969
"text" => ctx.none(),
967970
});
971+
extend_exception!(PyIncompleteInputError, ctx, excs.incomplete_input_error);
968972
extend_exception!(PyIndentationError, ctx, excs.indentation_error);
969973
extend_exception!(PyTabError, ctx, excs.tab_error);
970974

@@ -1623,6 +1627,28 @@ pub(super) mod types {
16231627
}
16241628
}
16251629

1630+
#[pyexception(
1631+
name = "_IncompleteInputError",
1632+
base = "PySyntaxError",
1633+
ctx = "incomplete_input_error"
1634+
)]
1635+
#[derive(Debug)]
1636+
pub struct PyIncompleteInputError {}
1637+
1638+
#[pyexception]
1639+
impl PyIncompleteInputError {
1640+
#[pyslot]
1641+
#[pymethod(name = "__init__")]
1642+
pub(crate) fn slot_init(
1643+
zelf: PyObjectRef,
1644+
_args: FuncArgs,
1645+
vm: &VirtualMachine,
1646+
) -> PyResult<()> {
1647+
zelf.set_attr("name", vm.ctx.new_str("SyntaxError"), vm)?;
1648+
Ok(())
1649+
}
1650+
}
1651+
16261652
#[pyexception(name, base = "PySyntaxError", ctx = "indentation_error", impl)]
16271653
#[derive(Debug)]
16281654
pub struct PyIndentationError {}

vm/src/stdlib/ast.rs

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -245,6 +245,7 @@ pub(crate) fn parse(
245245
let top = parser::parse(source, mode.into())
246246
.map_err(|parse_error| ParseError {
247247
error: parse_error.error,
248+
raw_location: parse_error.location,
248249
location: text_range_to_source_range(&source_code, parse_error.location)
249250
.start
250251
.to_source_location(),
@@ -295,8 +296,8 @@ pub const PY_COMPILE_FLAG_AST_ONLY: i32 = 0x0400;
295296
// The following flags match the values from Include/cpython/compile.h
296297
// Caveat emptor: These flags are undocumented on purpose and depending
297298
// on their effect outside the standard library is **unsupported**.
298-
const PY_CF_DONT_IMPLY_DEDENT: i32 = 0x200;
299-
const PY_CF_ALLOW_INCOMPLETE_INPUT: i32 = 0x4000;
299+
pub const PY_CF_DONT_IMPLY_DEDENT: i32 = 0x200;
300+
pub const PY_CF_ALLOW_INCOMPLETE_INPUT: i32 = 0x4000;
300301

301302
// __future__ flags - sync with Lib/__future__.py
302303
// TODO: These flags aren't being used in rust code

vm/src/stdlib/builtins.rs

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,8 @@ mod builtins {
186186
return Err(vm.new_value_error("compile() unrecognized flags".to_owned()));
187187
}
188188

189+
let allow_incomplete = !(flags & ast::PY_CF_ALLOW_INCOMPLETE_INPUT).is_zero();
190+
189191
if (flags & ast::PY_COMPILE_FLAG_AST_ONLY).is_zero() {
190192
#[cfg(not(feature = "compiler"))]
191193
{
@@ -207,14 +209,17 @@ mod builtins {
207209
args.filename.to_string_lossy().into_owned(),
208210
opts,
209211
)
210-
.map_err(|err| (err, Some(source)).to_pyexception(vm))?;
212+
.map_err(|err| {
213+
(err, Some(source), allow_incomplete).to_pyexception(vm)
214+
})?;
211215
Ok(code.into())
212216
}
213217
} else {
214218
let mode = mode_str
215219
.parse::<parser::Mode>()
216220
.map_err(|err| vm.new_value_error(err.to_string()))?;
217-
ast::parse(vm, source, mode).map_err(|e| (e, Some(source)).to_pyexception(vm))
221+
ast::parse(vm, source, mode)
222+
.map_err(|e| (e, Some(source), allow_incomplete).to_pyexception(vm))
218223
}
219224
}
220225
}
@@ -1056,6 +1061,7 @@ pub fn init_module(vm: &VirtualMachine, module: &Py<PyModule>) {
10561061
"NotImplementedError" => ctx.exceptions.not_implemented_error.to_owned(),
10571062
"RecursionError" => ctx.exceptions.recursion_error.to_owned(),
10581063
"SyntaxError" => ctx.exceptions.syntax_error.to_owned(),
1064+
"_IncompleteInputError" => ctx.exceptions.incomplete_input_error.to_owned(),
10591065
"IndentationError" => ctx.exceptions.indentation_error.to_owned(),
10601066
"TabError" => ctx.exceptions.tab_error.to_owned(),
10611067
"SystemError" => ctx.exceptions.system_error.to_owned(),

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy