This allows the test runner to execute test_multiprocessing and test_asyncio in parallel.

zitterbewegung · zitterbewegung · commit 557e6b74fae8 · 2023-04-27T14:00:52.000-06:00
By sharding the individual tests in test_multiprocessing and test_asyncio we are able to reduce considerably the time it takes to run the whole test suite.
diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py
@@ -19,10 +19,10 @@
 
 class TestResult:
     def __init__(
-        self,
-        name: str,
-        duration_sec: float = 0.0,
-        xml_data: list[str] | None = None,
+            self,
+            name: str,
+            duration_sec: float = 0.0,
+            xml_data: list[str] | None = None,
     ) -> None:
         self.name = name
         self.duration_sec = duration_sec
@@ -39,12 +39,12 @@ def __str__(self) -> str:
 
 class Failed(TestResult):
     def __init__(
-        self,
-        name: str,
-        duration_sec: float = 0.0,
-        xml_data: list[str] | None = None,
-        errors: list[tuple[str, str]] | None = None,
-        failures: list[tuple[str, str]] | None = None,
+            self,
+            name: str,
+            duration_sec: float = 0.0,
+            xml_data: list[str] | None = None,
+            errors: list[tuple[str, str]] | None = None,
+            failures: list[tuple[str, str]] | None = None,
     ) -> None:
         super().__init__(name, duration_sec=duration_sec, xml_data=xml_data)
         self.errors = errors
@@ -128,21 +128,30 @@ def __str__(self) -> str:
 # small set of tests to determine if we have a basically functioning interpreter
 # (i.e. if any of these fail, then anything else is likely to follow)
 STDTESTS = [
-    'test_grammar',
-    'test_opcodes',
-    'test_dict',
-    'test_builtin',
-    'test_exceptions',
-    'test_types',
-    'test_unittest',
-    'test_doctest',
-    'test_doctest2',
-    'test_support'
+        'test_grammar',
+        'test_opcodes',
+        'test_dict',
+        'test_builtin',
+        'test_exceptions',
+        'test_types',
+        'test_unittest',
+        'test_doctest',
+        'test_doctest2',
+        'test_support'
 ]
 
 # set of tests that we don't want to be executed when using regrtest
 NOTTESTS = set()
 
+#If these test directories are encountered recurse into them and treat each
+# test_ .py or dir as a separate test module. This can increase parallelism.
+# Beware this can't generally be done for any directory with sub-tests as the
+# __init__.py may do things which alter what tests are to be run.
+
+SPLITTESTDIRS = {
+        "test_asyncio",
+        "test_compiler",
+}
 
 # Storage of uncollectable objects
 FOUND_GARBAGE = []
@@ -158,16 +167,24 @@ def findtestdir(path=None):
     return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir
 
 
-def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS):
+def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS, splittestdirs=SPLITTESTDIRS, base_mod=""):
     """Return a list of all applicable test modules."""
     testdir = findtestdir(testdir)
     names = os.listdir(testdir)
     tests = []
     others = set(stdtests) | nottests
     for name in names:
         mod, ext = os.path.splitext(name)
-        if mod[:5] == "test_" and ext in (".py", "") and mod not in others:
-            tests.append(mod)
+        if mod[:5] == "test_" and mod not in others:
+            if mod in splittestdirs:
+                subdir = os.path.join(testdir, mod)
+                if len(base_mod):
+                    mod = f"{base_mod}.{mod}"
+                else:
+                    mod = f"test.{mod}"
+                tests.extend(findtests(subdir, [], nottests, splittestdirs, mod))
+            elif ext in (".py", ""):
+                tests.append(f"{base_mod}.{mod}" if len(base_mod) else mod)
     return stdtests + sorted(tests)
 
 
@@ -186,7 +203,7 @@ def _runtest(ns: Namespace, test_name: str) -> TestResult:
     output_on_failure = ns.verbose3
 
     use_timeout = (
-        ns.timeout is not None and threading_helper.can_start_thread
+            ns.timeout is not None and threading_helper.can_start_thread
     )
     if use_timeout:
         faulthandler.dump_traceback_later(ns.timeout, exit=True)
@@ -217,7 +234,7 @@ def _runtest(ns: Namespace, test_name: str) -> TestResult:
                 print_warning.orig_stderr = stream
 
                 result = _runtest_inner(ns, test_name,
-                                        display_failure=False)
+                                                                display_failure=False)
                 if not isinstance(result, Passed):
                     output = stream.getvalue()
             finally:
@@ -233,13 +250,13 @@ def _runtest(ns: Namespace, test_name: str) -> TestResult:
             support.verbose = ns.verbose
 
             result = _runtest_inner(ns, test_name,
-                                    display_failure=not ns.verbose)
+                                                            display_failure=not ns.verbose)
 
         if xml_list:
             import xml.etree.ElementTree as ET
             result.xml_data = [
-                ET.tostring(x).decode('us-ascii')
-                for x in xml_list
+                    ET.tostring(x).decode('us-ascii')
+                    for x in xml_list
             ]
 
         result.duration_sec = time.perf_counter() - start_time
@@ -267,7 +284,7 @@ def runtest(ns: Namespace, test_name: str) -> TestResult:
         if not ns.pgo:
             msg = traceback.format_exc()
             print(f"test {test_name} crashed -- {msg}",
-                  file=sys.stderr, flush=True)
+                      file=sys.stderr, flush=True)
         return Failed(test_name)
 
 
@@ -328,7 +345,7 @@ def _runtest_inner2(ns: Namespace, test_name: str) -> bool:
     if gc.garbage:
         support.environment_altered = True
         print_warning(f"{test_name} created {len(gc.garbage)} "
-                      f"uncollectable object(s).")
+                                  f"uncollectable object(s).")
 
         # move the uncollectable objects somewhere,
         # so we don't see them again
@@ -341,7 +358,7 @@ def _runtest_inner2(ns: Namespace, test_name: str) -> bool:
 
 
 def _runtest_inner(
-    ns: Namespace, test_name: str, display_failure: bool = True
+        ns: Namespace, test_name: str, display_failure: bool = True
 ) -> TestResult:
     # Detect environment changes, handle exceptions.
 
@@ -387,7 +404,7 @@ def _runtest_inner(
         if not ns.pgo:
             msg = traceback.format_exc()
             print(f"test {test_name} crashed -- {msg}",
-                  file=sys.stderr, flush=True)
+                      file=sys.stderr, flush=True)
         return UncaughtException(test_name)
 
     if refleak:
@@ -415,7 +432,7 @@ def cleanup_test_droppings(test_name: str, verbose: int) -> None:
             kind, nuker = "file", os.unlink
         else:
             raise RuntimeError(f"os.path says {name!r} exists but is neither "
-                               f"directory nor file")
+                                               f"directory nor file")
 
         if verbose:
             print_warning(f"{test_name} left behind {kind} {name!r}")
@@ -428,4 +445,4 @@ def cleanup_test_droppings(test_name: str, verbose: int) -> None:
             nuker(name)
         except Exception as exc:
             print_warning(f"{test_name} left behind {kind} {name!r} "
-                          f"and it couldn't be removed: {exc}")
+                                      f"and it couldn't be removed: {exc}")
