WIP: mpremote: Support bytecode raw paste for 'mpremote run module.mpy'

projectgus · projectgus · commit a933f6997d0e · 2022-06-09T13:08:50.000+10:00
- Adds a new raw paste command 'B' for 'raw paste bytecode' ('A' is
'paste source')

- Adds an escaping mechanism (Ctrl-F &lt;char + 8&gt;) during raw paste so
bytes less than 8 can be sent without triggering Ctrl-C or Ctrl-D
handlers. The two-byte escape sequence still counts as one byte in
the paste window.

- Adds relevant support to mpremote.py
diff --git a/shared/runtime/pyexec.c b/shared/runtime/pyexec.c
@@ -35,6 +35,7 @@
 #include "py/gc.h"
 #include "py/frozenmod.h"
 #include "py/mphal.h"
+#include "py/persistentcode.h" // TODO guard for this?
 #if MICROPY_HW_ENABLE_USB
 #include "irq.h"
 #include "usb.h"
@@ -58,6 +59,8 @@ STATIC bool repl_display_debugging_info = 0;
 #define EXEC_FLAG_SOURCE_IS_FILENAME    (1 << 5)
 #define EXEC_FLAG_SOURCE_IS_READER      (1 << 6)
 
+#define NUM_ESCAPED 8 // This value has to match the value in tools/pyboard.py
+
 // parses, compiles and executes the code in the lexer
 // frees the lexer before returning
 // EXEC_FLAG_PRINT_EOF prints 2 EOF chars: 1 after normal output, 1 after exception output
@@ -82,12 +85,20 @@ STATIC int parse_compile_execute(const void *source, mp_parse_input_kind_t input
         mp_obj_t module_fun;
         #if MICROPY_MODULE_FROZEN_MPY
         if (exec_flags & EXEC_FLAG_SOURCE_IS_RAW_CODE) {
-            // source is a raw_code object, create the function
-            const mp_frozen_module_t *frozen = source;
             mp_module_context_t *ctx = m_new_obj(mp_module_context_t);
             ctx->module.globals = mp_globals_get();
-            ctx->constants = frozen->constants;
-            module_fun = mp_make_function_from_raw_code(frozen->rc, ctx, NULL);
+            if ((exec_flags & EXEC_FLAG_SOURCE_IS_READER) == 0) {
+                // source is a raw_code object, create the module function from it
+                const mp_frozen_module_t *frozen = source;
+                ctx->constants = frozen->constants;
+                module_fun = mp_make_function_from_raw_code(frozen->rc, ctx, NULL);
+            } else {
+                #if 1 // TODO conditions for compilation
+                // source is a reader that will give us raw code (mpy file equivalent)
+                mp_compiled_module_t cm = mp_raw_code_load((mp_reader_t *)source, ctx);
+                module_fun = mp_make_function_from_raw_code(cm.rc, ctx, NULL);
+                #endif
+            }
         } else
         #endif
         {
@@ -220,6 +231,10 @@ STATIC mp_uint_t mp_reader_stdin_readbyte(void *data) {
         } else {
             return MP_READER_EOF;
         }
+    } else if (c == CHAR_CTRL_F) {
+        // escape sequence, next character is escaped by adding NUM_ESCAPED to it
+        int e = mp_hal_stdin_rx_chr();
+        c = e - NUM_ESCAPED;
     }
 
     if (--reader->window_remain == 0) {
@@ -261,7 +276,7 @@ STATIC void mp_reader_new_stdin(mp_reader_t *reader, mp_reader_stdin_t *reader_s
 }
 
 STATIC int do_reader_stdin(int c) {
-    if (c != 'A') {
+    if (c != 'A' && c != 'B') {
         // Unsupported command.
         mp_hal_stdout_tx_strn("R\x00", 2);
         return 0;
@@ -270,10 +285,15 @@ STATIC int do_reader_stdin(int c) {
     // Indicate reception of command.
     mp_hal_stdout_tx_strn("R\x01", 2);
 
+    // Entering raw paste mode
+    // c == 'A' input is source, c == 'B' input is bytecode
     mp_reader_t reader;
     mp_reader_stdin_t reader_stdin;
     mp_reader_new_stdin(&reader, &reader_stdin, MICROPY_REPL_STDIN_BUFFER_MAX);
     int exec_flags = EXEC_FLAG_PRINT_EOF | EXEC_FLAG_SOURCE_IS_READER;
+    if (c == 'B') {
+        exec_flags |= EXEC_FLAG_SOURCE_IS_RAW_CODE;
+    }
     return parse_compile_execute(&reader, MP_PARSE_FILE_INPUT, exec_flags);
 }
 
diff --git a/tools/mpremote/mpremote/main.py b/tools/mpremote/mpremote/main.py
@@ -352,8 +352,10 @@ def do_repl_main_loop(pyb, console_in, console_out_write, *, code_to_inject, fil
                 pyb.enter_raw_repl(soft_reset=False)
                 with open(file_to_inject, "rb") as f:
                     pyfile = f.read()
+
+                is_bytecode = pyfile[0] == ord("M") and file_to_inject.endswith(".mpy")
                 try:
-                    pyb.exec_raw_no_follow(pyfile)
+                    pyb.exec_raw_no_follow(pyfile, is_bytecode)
                 except pyboard.PyboardError as er:
                     console_out_write(b"Error:\r\n")
                     console_out_write(er)
@@ -430,10 +432,10 @@ def console_out_write(b):
             capture_file.close()
 
 
-def execbuffer(pyb, buf, follow):
+def execbuffer(pyb, buf, follow, is_bytecode=False):
     ret_val = 0
     try:
-        pyb.exec_raw_no_follow(buf)
+        pyb.exec_raw_no_follow(buf, is_bytecode)
         if follow:
             ret, ret_err = pyb.follow(timeout=None, data_consumer=pyboard.stdout_write_bytes)
             if ret_err:
@@ -546,6 +548,7 @@ def main():
             elif cmd == "umount":
                 pyb.umount_local()
             elif cmd in ("exec", "eval", "run"):
+                is_bytecode = False
                 follow = True
                 if args[0] == "--no-follow":
                     args.pop(0)
@@ -554,15 +557,16 @@ def main():
                     buf = args.pop(0)
                 elif cmd == "eval":
                     buf = "print(" + args.pop(0) + ")"
-                else:
+                else:  # run
                     filename = args.pop(0)
                     try:
                         with open(filename, "rb") as f:
                             buf = f.read()
+                            is_bytecode = buf[0] == ord("M") and filename.endswith(".mpy")
                     except OSError:
                         print(f"{_PROG}: could not read file '{filename}'")
                         return 1
-                ret = execbuffer(pyb, buf, follow)
+                ret = execbuffer(pyb, buf, follow, is_bytecode)
                 if ret:
                     return ret
             elif cmd == "fs":
diff --git a/tools/pyboard.py b/tools/pyboard.py
@@ -70,6 +70,7 @@
 import sys
 import time
 import os
+import re
 import ast
 
 try:
@@ -399,10 +400,22 @@ def raw_paste_write(self, command_bytes):
                     raise PyboardError("unexpected read during raw paste: {}".format(data))
             # Send out as much data as possible that fits within the allowed window.
             b = command_bytes[i : min(i + window_remain, len(command_bytes))]
-            self.serial.write(b)
+
             window_remain -= len(b)
             i += len(b)
 
+            # escape any characters that need to be escaped. Note this doesn't
+            # count towards the window size, as unescaping happens before filling
+            # the window buffer in the device
+            NUM_ESCAPED = 8  # this values has to match value in pyexec.c
+            b = re.sub(
+                rb"[" + bytes(range(NUM_ESCAPED)) + rb"]",
+                lambda c: bytes((0x06, c.group()[0] + NUM_ESCAPED)),
+                b,
+            )
+
+            self.serial.write(b)
+
         # Indicate end of data.
         self.serial.write(b"\x04")
 
@@ -411,7 +424,7 @@ def raw_paste_write(self, command_bytes):
         if not data.endswith(b"\x04"):
             raise PyboardError("could not complete raw paste: {}".format(data))
 
-    def exec_raw_no_follow(self, command):
+    def exec_raw_no_follow(self, command, is_bytecode=False):
         if isinstance(command, bytes):
             command_bytes = command
         else:
@@ -424,20 +437,25 @@ def exec_raw_no_follow(self, command):
 
         if self.use_raw_paste:
             # Try to enter raw-paste mode.
-            self.serial.write(b"\x05A\x01")
+            raw_paste_cmd = b"\x05A\x01" if not is_bytecode else b"\x05B\x01"
+            self.serial.write(raw_paste_cmd)
             data = self.serial.read(2)
             if data == b"R\x00":
                 # Device understood raw-paste command but doesn't support it.
                 pass
             elif data == b"R\x01":
                 # Device supports raw-paste mode, write out the command using this mode.
                 return self.raw_paste_write(command_bytes)
-            else:
+            elif not is_bytecode:
                 # Device doesn't support raw-paste, fall back to normal raw REPL.
                 data = self.read_until(1, b"w REPL; CTRL-B to exit\r\n>")
                 if not data.endswith(b"w REPL; CTRL-B to exit\r\n>"):
                     print(data)
                     raise PyboardError("could not enter raw repl")
+            else:
+                raise RuntimeError(
+                    "Sending bytecode without raw paste is not yet implemented"
+                )  # TODO
             # Don't try to use raw-paste mode again for this connection.
             self.use_raw_paste = False
 
