tools/tinytest-codegen, qemu-arm: Externalise tests list.

agatti · agatti · commit edadfd54c6c8 · 2023-11-02T15:17:00.000+01:00
Remove port-specific test directories and excluded tests from
tinytest-codegen, and let it read said information from an external
file.  This way tinytest-codegen is not limited to always generate tests
for the `qemu-arm` target.

This allows having port-specific test directory and excluded tests for
more than one QEMU bare-metal target.

The `qemu-arm` port Makefile was modified to work with the generator
changes and a tests profile file was added to said port.

Signed-off-by: Alessandro Gatti &lt;a.gatti@frob.it&gt;
diff --git a/ports/qemu-arm/Makefile.test b/ports/qemu-arm/Makefile.test
@@ -13,10 +13,12 @@ CFLAGS += -DTEST
 
 .PHONY: $(BUILD)/genhdr/tests.h
 
+TESTS_PROFILE = $(dir $(abspath $(firstword $(MAKEFILE_LIST))))/tests_profile.txt
+
 $(BUILD)/test_main.o: $(BUILD)/genhdr/tests.h
 $(BUILD)/genhdr/tests.h:
 	(cd $(TOP)/tests; ./run-tests.py --target=qemu-arm --write-exp)
-	$(Q)echo "Generating $@";(cd $(TOP)/tests; ../tools/tinytest-codegen.py $(addprefix --exclude ,$(TESTS_EXCLUDE))) > $@
+	$(Q)echo "Generating $@";(cd $(TOP)/tests; ../tools/tinytest-codegen.py --profile $(TESTS_PROFILE) $(addprefix --exclude ,$(TESTS_EXCLUDE))) > $@
 
 $(BUILD)/lib/tinytest/tinytest.o: CFLAGS += -DNO_FORKING
 
diff --git a/ports/qemu-arm/tests_profile.txt b/ports/qemu-arm/tests_profile.txt
@@ -0,0 +1,26 @@
+# Test profile for `qemu-arm`
+
+[test_directories]
+
+inlineasm
+micropython
+qemu-arm
+
+[excluded_tests]
+
+# inline asm FP tests (require Cortex-M4)
+inlineasm/asmfpaddsub.py
+inlineasm/asmfpcmp.py
+inlineasm/asmfpldrstr.py
+inlineasm/asmfpmuldiv.py
+inlineasm/asmfpsqrt.py
+
+# different filename in output
+micropython/emg_exc.py
+micropython/heapalloc_traceback.py
+
+# don't have emergency exception buffer
+micropython/heapalloc_exc_compressed_emg_exc.py
+
+# pattern matching in .exp
+micropython/meminfo.py
diff --git a/tools/tinytest-codegen.py b/tools/tinytest-codegen.py
@@ -32,6 +32,35 @@ def script_to_map(test_file):
     return r
 
 
+def load_profile(profile_file):
+    in_test_dirs = False
+    in_exclusion_list = False
+
+    test_dirs = set()
+    exclusion_list = set()
+
+    for line in profile_file:
+        # Allow quoted comment markers if that's absolutely needed.
+        if line := sub(r"\s*(?<!\\)#.*$", "", line.strip()):
+            if line == "[test_directories]":
+                in_test_dirs = True
+                in_exclusion_list = False
+                continue
+            if line == "[excluded_tests]":
+                in_test_dirs = False
+                in_exclusion_list = True
+                continue
+
+            if in_test_dirs:
+                test_dirs.add(line)
+            elif in_exclusion_list:
+                exclusion_list.add(line)
+            else:
+                pass
+
+    return test_dirs, exclusion_list
+
+
 test_function = (
     "void {name}(void* data) {{\n"
     "  static const char pystr[] = {script};\n"
@@ -50,17 +79,15 @@ def script_to_map(test_file):
 testgroup_member = '  {{ "{name}", {name}_tests }},'
 
 ## XXX: may be we could have `--without <groups>` argument...
-# currently these tests are selected because they pass on qemu-arm
-test_dirs = (
+
+test_dirs = set((
     "basics",
-    "micropython",
-    "misc",
     "extmod",
     "float",
-    "inlineasm",
-    "qemu-arm",
-)  # 'import', 'io',)
-exclude_tests = (
+    "misc",
+))
+
+exclude_tests = set((
     # pattern matching in .exp
     "basics/bytes_compare3.py",
     "extmod/ticks_diff.py",
@@ -80,19 +107,6 @@ def script_to_map(test_file):
     "float/float2int_doubleprec_intbig.py",
     "float/float_format_ints_doubleprec.py",
     "float/float_parse_doubleprec.py",
-    # inline asm FP tests (require Cortex-M4)
-    "inlineasm/asmfpaddsub.py",
-    "inlineasm/asmfpcmp.py",
-    "inlineasm/asmfpldrstr.py",
-    "inlineasm/asmfpmuldiv.py",
-    "inlineasm/asmfpsqrt.py",
-    # different filename in output
-    "micropython/emg_exc.py",
-    "micropython/heapalloc_traceback.py",
-    # don't have emergency exception buffer
-    "micropython/heapalloc_exc_compressed_emg_exc.py",
-    # pattern matching in .exp
-    "micropython/meminfo.py",
     # needs sys stdfiles
     "misc/print_exception.py",
     # settrace .exp files are too large
@@ -102,7 +116,7 @@ def script_to_map(test_file):
     # don't have f-string
     "basics/string_fstring.py",
     "basics/string_fstring_debug.py",
-)
+))
 
 output = []
 tests = []
@@ -112,11 +126,20 @@ def script_to_map(test_file):
 )
 argparser.add_argument("--stdin", action="store_true", help="read list of tests from stdin")
 argparser.add_argument("--exclude", action="append", help="exclude test by name")
+argparser.add_argument(
+    "--profile",
+    type=argparse.FileType('rt', encoding='utf-8'),
+    help="optional profile file providing test directories and exclusion list"
+)
 args = argparser.parse_args()
 
 if not args.stdin:
+    if args.profile:
+        directories, excluded = load_profile(args.profile)
+        test_dirs = test_dirs.union(directories)
+        exclude_tests = exclude_tests.union(excluded)
     if args.exclude:
-        exclude_tests += tuple(args.exclude)
+        exclude_tests = exclude_tests.union(args.exclude)
     for group in test_dirs:
         tests += [test for test in glob("{}/*.py".format(group)) if test not in exclude_tests]
 else:
