tests/run-tests.py: Change --target/--device options to --test-instance.

dpgeorge · dpgeorge · commit 8978102f3595 · 2024-11-04T12:47:47.000+11:00
Previously to this commit, running the test suite on a bare-metal board
required specifying the target (really platform) and device, eg:

    $ ./run-tests.py --target pyboard --device /dev/ttyACM1

That's quite a lot to type, and you also need to know what the target
platform is, when a lot of the time you either don't care or it doesn't
matter.

This commit makes it easier to run the tests by replacing both of these
options with a single `--test-instance` (`-t` for short) option.  That
option specifies the executable/port/device to test.  Then the target
platform is automatically detected.

The `--test-instance` can be passed:
- "unix" (the default) to use the unix version of MicroPython
- "webassembly" to test the webassembly port
- anything else is considered a port/device to pass to Pyboard

There are also some shortcuts to specify a port/device, following
`mpremote`:
- a&lt;n&gt; is short for /dev/ttyACM&lt;n&gt;
- u&lt;n&gt; is short for /dev/ttyUSB&lt;n&gt;
- c&lt;n&gt; is short for COM&lt;n&gt;

For example:

    $ ./run-tests.py -t a1

Note that the default test instance is "unix" and so this commit does not
change the standard way to run tests on the unix port, by just doing
`./run-tests.py`.

As part of this change, the platform (and it's native architecture if it
supports importing native .mpy files) is show at the start of the test run.

Signed-off-by: Damien George &lt;damien@micropython.org&gt;
diff --git a/docs/develop/gettingstarted.rst b/docs/develop/gettingstarted.rst
@@ -278,7 +278,7 @@ To run a selection of tests on a board/device connected over USB use:
 .. code-block:: bash
 
    $ cd tests
-   $ ./run-tests.py --target minimal --device /dev/ttyACM0
+   $ ./run-tests.py -t /dev/ttyACM0
 
 See also :ref:`writingtests`.
 
diff --git a/docs/develop/writingtests.rst b/docs/develop/writingtests.rst
@@ -60,7 +60,7 @@ Then to run on a board:
 
 .. code-block:: bash
 
-   $ ./run-tests.py --target minimal --device /dev/ttyACM0
+   $ ./run-tests.py -t /dev/ttyACM0
 
 And to run only a certain set of tests (eg a directory):
 
diff --git a/ports/cc3200/tools/smoke.py b/ports/cc3200/tools/smoke.py
@@ -6,7 +6,7 @@
 """
 Execute it like this:
 
-python3 run-tests.py --target wipy --device 192.168.1.1 ../cc3200/tools/smoke.py
+python3 run-tests.py -t 192.168.1.1 ../cc3200/tools/smoke.py
 """
 
 pin_map = [23, 24, 11, 12, 13, 14, 15, 16, 17, 22, 28, 10, 9, 8, 7, 6, 30, 31, 3, 0, 4, 5]
diff --git a/ports/qemu/Makefile b/ports/qemu/Makefile
@@ -166,7 +166,7 @@ run: $(BUILD)/firmware.elf
 .PHONY: test
 test: $(BUILD)/firmware.elf
 	$(eval DIRNAME=ports/$(notdir $(CURDIR)))
-	cd $(TOP)/tests && ./run-tests.py --target qemu --device execpty:"$(QEMU_SYSTEM) $(QEMU_ARGS) -serial pty -kernel ../$(DIRNAME)/$<" $(RUN_TESTS_ARGS) $(RUN_TESTS_EXTRA)
+	cd $(TOP)/tests && ./run-tests.py -t execpty:"$(QEMU_SYSTEM) $(QEMU_ARGS) -serial pty -kernel ../$(DIRNAME)/$<" $(RUN_TESTS_ARGS) $(RUN_TESTS_EXTRA)
 
 $(BUILD)/firmware.elf: $(LDSCRIPT) $(OBJ)
 	$(Q)$(CC) $(LDFLAGS) -o $@ $(OBJ) $(LIBS)
diff --git a/ports/qemu/README.md b/ports/qemu/README.md
@@ -95,7 +95,7 @@ Or manually by first starting the emulation with `make run` and then running the
 tests against the serial device, for example:
 
     $ cd ../../tests
-    $ ./run-tests.py --target qemu --device /dev/pts/1
+    $ ./run-tests.py -t /dev/pts/1
 
 Extra make options
 ------------------
diff --git a/ports/webassembly/Makefile b/ports/webassembly/Makefile
@@ -148,10 +148,10 @@ repl: $(BUILD)/micropython.mjs
 min: $(BUILD)/micropython.min.mjs
 
 test: $(BUILD)/micropython.mjs $(TOP)/tests/run-tests.py
-	cd $(TOP)/tests && MICROPY_MICROPYTHON_MJS=../ports/webassembly/$< ./run-tests.py --target webassembly
+	cd $(TOP)/tests && MICROPY_MICROPYTHON_MJS=../ports/webassembly/$< ./run-tests.py -t webassembly
 
 test_min: $(BUILD)/micropython.min.mjs $(TOP)/tests/run-tests.py
-	cd $(TOP)/tests && MICROPY_MICROPYTHON_MJS=../ports/webassembly/$< ./run-tests.py --target webassembly
+	cd $(TOP)/tests && MICROPY_MICROPYTHON_MJS=../ports/webassembly/$< ./run-tests.py -t webassembly
 
 ################################################################################
 # Remaining make rules.
diff --git a/tests/README.md b/tests/README.md
@@ -1,7 +1,19 @@
 # MicroPython Test Suite
 
-This directory contains tests for various functionality areas of MicroPython.
-To run all stable tests, run "run-tests.py" script in this directory.
+This directory contains tests for most parts of MicroPython.
+
+To run all stable tests, run the "run-tests.py" script in this directory.  By default
+that will run the test suite against the unix port of MicroPython.
+
+To run the test suite against a bare-metal target (a board running MicroPython firmware)
+use the `-t` option to specify the serial port.  This will automatically detect the
+target platform and run the appropriate set of tests for that platform.  For example:
+
+    $ ./run-tests.py -t /dev/ttyACM0
+
+That will run tests on the `/dev/ttyACM0` serial port.  You can also use shortcut
+device names like `a<n>` for `/dev/ttyACM<n>` and `c<n>` for `COM<n>`.  Use
+`./run-tests.py --help` to see all of the device possibilites, and other options.
 
 Tests of capabilities not supported on all platforms should be written
 to check for the capability being present. If it is not, the test
diff --git a/tests/feature_check/target_info.py b/tests/feature_check/target_info.py
@@ -4,6 +4,7 @@
 
 import sys
 
+platform = getattr(sys, "platform", "minimal")
 sys_mpy = getattr(sys.implementation, "_mpy", 0)
 arch = [
     None,
@@ -19,4 +20,4 @@
     "xtensawin",
     "rv32imc",
 ][sys_mpy >> 10]
-print(arch)
+print(platform, arch)
diff --git a/tests/run-tests.py b/tests/run-tests.py
@@ -89,9 +89,12 @@ def open(self, path, mode):
 __import__('__injected_test')
 """
 
+# Platforms associated with the unix port, values of `sys.platform`.
+PC_PLATFORMS = ("darwin", "linux", "win32")
+
 # Tests to skip on specific targets.
 # These are tests that are difficult to detect that they should not be run on the given target.
-target_tests_to_skip = {
+platform_tests_to_skip = {
     "esp8266": (
         "micropython/viper_args.py",  # too large
         "micropython/viper_binop_arith.py",  # too large
@@ -161,7 +164,7 @@ def open(self, path, mode):
         "micropython/extreme_exc.py",
         "micropython/heapalloc_exc_compressed_emg_exc.py",
     ),
-    "wipy": (
+    "WiPy": (
         "misc/print_exception.py",  # requires error reporting full
     ),
     "zephyr": (
@@ -197,6 +200,53 @@ def convert_regex_escapes(line):
     return bytes("".join(cs), "utf8")
 
 
+def get_test_instance(test_instance, baudrate, user, password):
+    if test_instance.startswith("port:"):
+        _, port = test_instance.split(":", 1)
+    elif test_instance == "unix":
+        return None
+    elif test_instance == "webassembly":
+        return PyboardNodeRunner()
+    elif test_instance.startswith("a") and test_instance[1:].isdigit():
+        port = "/dev/ttyACM" + test_instance[1:]
+    elif test_instance.startswith("u") and test_instance[1:].isdigit():
+        port = "/dev/ttyUSB" + test_instance[1:]
+    elif test_instance.startswith("c") and test_instance[1:].isdigit():
+        port = "COM" + test_instance[1:]
+    else:
+        # Assume it's a device path.
+        port = test_instance
+
+    global pyboard
+    sys.path.append(base_path("../tools"))
+    import pyboard
+
+    pyb = pyboard.Pyboard(port, baudrate, user, password)
+    pyboard.Pyboard.run_script_on_remote_target = run_script_on_remote_target
+    pyb.enter_raw_repl()
+    return pyb
+
+
+def detect_test_platform(pyb, args):
+    # Run a script to detect various bits of information about the target test instance.
+    output = run_feature_check(pyb, args, "target_info.py")
+    if output.endswith(b"CRASH"):
+        raise ValueError("cannot detect platform: {}".format(output))
+    platform, arch = str(output, "ascii").strip().split()
+    if arch == "None":
+        arch = None
+
+    args.platform = platform
+    args.arch = arch
+    if arch and not args.mpy_cross_flags:
+        args.mpy_cross_flags = "-march=" + arch
+
+    print("platform={}".format(platform), end="")
+    if arch:
+        print(" arch={}".format(arch), end="")
+    print()
+
+
 def prepare_script_for_target(args, *, script_filename=None, script_text=None, force_plain=False):
     if force_plain or (not args.via_mpy and args.emit == "bytecode"):
         if script_filename is not None:
@@ -706,18 +756,18 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1):
         skip_tests.add("extmod/ssl_poll.py")
 
     # Skip thread mutation tests on targets that don't have the GIL.
-    if args.target in ("rp2", "unix"):
+    if args.platform in PC_PLATFORMS + ("rp2",):
         for t in tests:
             if t.startswith("thread/mutate_"):
                 skip_tests.add(t)
 
     # Some tests shouldn't be run on pyboard
-    if args.target != "unix":
+    if args.platform not in PC_PLATFORMS:
         skip_tests.add("basics/exception_chain.py")  # warning is not printed
         skip_tests.add("micropython/meminfo.py")  # output is very different to PC output
 
-    # Skip target-specific tests.
-    skip_tests.update(target_tests_to_skip.get(args.target, ()))
+    # Skip platform-specific tests.
+    skip_tests.update(platform_tests_to_skip.get(args.platform, ()))
 
     # Some tests are known to fail on 64-bit machines
     if pyb is None and platform.architecture()[0] == "64bit":
@@ -932,17 +982,38 @@ def main():
         formatter_class=argparse.RawDescriptionHelpFormatter,
         description="""Run and manage tests for MicroPython.
 
+By default the tests are run against the unix port of MicroPython. To run it
+against something else, use the -t option.  See below for details.
+
 Tests are discovered by scanning test directories for .py files or using the
 specified test files. If test files nor directories are specified, the script
 expects to be ran in the tests directory (where this file is located) and the
 builtin tests suitable for the target platform are ran.
+
 When running tests, run-tests.py compares the MicroPython output of the test with the output
 produced by running the test through CPython unless a <test>.exp file is found, in which
 case it is used as comparison.
+
 If a test fails, run-tests.py produces a pair of <test>.out and <test>.exp files in the result
 directory with the MicroPython output and the expectations, respectively.
 """,
         epilog="""\
+The -t option accepts the following for the test instance:
+- unix - use the unix port of MicroPython, specified by the MICROPY_MICROPYTHON
+  environment variable (which defaults to the standard variant of either the unix
+  or windows ports, depending on the host platform)
+- webassembly - use the webassembly port of MicroPython, specified by the
+  MICROPY_MICROPYTHON_MJS environment variable (which defaults to the standard
+  variant of the webassembly port)
+- port:<device> - connect to and use the given serial port device
+- a<n> - connect to and use /dev/ttyACM<n>
+- u<n> - connect to and use /dev/ttyUSB<n>
+- c<n> - connect to and use COM<n>
+- exec:<command> - execute a command and attach to its stdin/stdout
+- execpty:<command> - execute a command and attach to the printed /dev/pts/<n> device
+- <a>.<b>.<c>.<d> - connect to the given IPv4 address
+- anything else specifies a serial port
+
 Options -i and -e can be multiple and processed in the order given. Regex
 "search" (vs "match") operation is used. An action (include/exclude) of
 the last matching regex is used:
@@ -951,11 +1022,8 @@ def main():
   run-tests.py -e async -i async_foo - include all, exclude async, yet still include async_foo
 """,
     )
-    cmd_parser.add_argument("--target", default="unix", help="the target platform")
     cmd_parser.add_argument(
-        "--device",
-        default="/dev/ttyACM0",
-        help="the serial device or the IP address of the pyboard",
+        "-t", "--test-instance", default="unix", help="the MicroPython instance to test"
     )
     cmd_parser.add_argument(
         "-b", "--baudrate", default=115200, help="the baud rate of the serial device"
@@ -1039,43 +1107,11 @@ def main():
 
         sys.exit(0)
 
-    LOCAL_TARGETS = (
-        "unix",
-        "webassembly",
-    )
-    EXTERNAL_TARGETS = (
-        "pyboard",
-        "wipy",
-        "esp8266",
-        "esp32",
-        "minimal",
-        "nrf",
-        "qemu",
-        "renesas-ra",
-        "rp2",
-        "zephyr",
-    )
-    if args.target in LOCAL_TARGETS:
-        pyb = None
-        if args.target == "webassembly":
-            pyb = PyboardNodeRunner()
-    elif args.target in EXTERNAL_TARGETS:
-        global pyboard
-        sys.path.append(base_path("../tools"))
-        import pyboard
-
-        pyb = pyboard.Pyboard(args.device, args.baudrate, args.user, args.password)
-        pyboard.Pyboard.run_script_on_remote_target = run_script_on_remote_target
-        pyb.enter_raw_repl()
-    else:
-        raise ValueError("target must be one of %s" % ", ".join(LOCAL_TARGETS + EXTERNAL_TARGETS))
+    # Get the test instance to run on.
+    pyb = get_test_instance(args.test_instance, args.baudrate, args.user, args.password)
 
-    # Automatically detect the native architecture for mpy-cross if not given.
-    if not args.mpy_cross_flags:
-        output = run_feature_check(pyb, args, "target_info.py")
-        arch = str(output, "ascii").strip()
-        if arch != "None":
-            args.mpy_cross_flags = "-march=" + arch
+    # Automatically detect the platform.
+    detect_test_platform(pyb, args)
 
     if args.run_failures and (any(args.files) or args.test_dirs is not None):
         raise ValueError(
@@ -1091,7 +1127,7 @@ def main():
             tests = []
     elif len(args.files) == 0:
         test_extensions = ("*.py",)
-        if args.target == "webassembly":
+        if args.platform == "webassembly":
             test_extensions += ("*.js", "*.mjs")
 
         if args.test_dirs is None:
@@ -1101,23 +1137,23 @@ def main():
                 "misc",
                 "extmod",
             )
-            if args.target == "pyboard":
+            if args.platform == "pyboard":
                 # run pyboard tests
                 test_dirs += ("float", "stress", "inlineasm", "ports/stm32")
-            elif args.target in ("renesas-ra"):
+            elif args.platform == "renesas-ra":
                 test_dirs += ("float", "inlineasm", "ports/renesas-ra")
-            elif args.target == "rp2":
+            elif args.platform == "rp2":
                 test_dirs += ("float", "stress", "thread", "ports/rp2")
                 if "arm" in args.mpy_cross_flags:
                     test_dirs += ("inlineasm",)
-            elif args.target == "esp32":
+            elif args.platform == "esp32":
                 test_dirs += ("float", "stress", "thread")
-            elif args.target in ("esp8266", "minimal", "nrf"):
+            elif args.platform in ("esp8266", "minimal", "nrf"):
                 test_dirs += ("float",)
-            elif args.target == "wipy":
+            elif args.platform == "WiPy":
                 # run WiPy tests
                 test_dirs += ("ports/cc3200",)
-            elif args.target == "unix":
+            elif args.platform in PC_PLATFORMS:
                 # run PC tests
                 test_dirs += (
                     "float",
@@ -1128,13 +1164,13 @@ def main():
                     "cmdline",
                     "ports/unix",
                 )
-            elif args.target == "qemu":
+            elif args.platform == "qemu":
                 test_dirs += (
                     "float",
                     "inlineasm",
                     "ports/qemu",
                 )
-            elif args.target == "webassembly":
+            elif args.platform == "webassembly":
                 test_dirs += ("float", "ports/webassembly")
         else:
             # run tests from these directories
diff --git a/tools/ci.sh b/tools/ci.sh
@@ -785,5 +785,5 @@ function ci_zephyr_run_tests {
     docker exec zephyr-ci west build -p auto -b qemu_cortex_m3 -- -DCONF_FILE=prj_minimal.conf
     # Issues with zephyr tests:
     # - inf_nan_arith fails pow(-1, nan) test
-    (cd tests && ./run-tests.py --target minimal --device execpty:"qemu-system-arm -cpu cortex-m3 -machine lm3s6965evb -nographic -monitor null -serial pty -kernel ../ports/zephyr/build/zephyr/zephyr.elf" -d basics float --exclude inf_nan_arith)
+    (cd tests && ./run-tests.py -t execpty:"qemu-system-arm -cpu cortex-m3 -machine lm3s6965evb -nographic -monitor null -serial pty -kernel ../ports/zephyr/build/zephyr/zephyr.elf" -d basics float --exclude inf_nan_arith)
 }

Original file line number	Diff line number	Diff line change
`@@ -785,5 +785,5 @@ function ci_zephyr_run_tests {`
`785`	`785`	`docker exec zephyr-ci west build -p auto -b qemu_cortex_m3 -- -DCONF_FILE=prj_minimal.conf`
`786`	`786`	`# Issues with zephyr tests:`
`787`	`787`	`# - inf_nan_arith fails pow(-1, nan) test`
`788`		`- (cd tests && ./run-tests.py --target minimal --device execpty:"qemu-system-arm -cpu cortex-m3 -machine lm3s6965evb -nographic -monitor null -serial pty -kernel ../ports/zephyr/build/zephyr/zephyr.elf" -d basics float --exclude inf_nan_arith)`
	`788`	`+ (cd tests && ./run-tests.py -t execpty:"qemu-system-arm -cpu cortex-m3 -machine lm3s6965evb -nographic -monitor null -serial pty -kernel ../ports/zephyr/build/zephyr/zephyr.elf" -d basics float --exclude inf_nan_arith)`
`789`	`789`	`}`