diff --git a/Doc/library/test.rst b/Doc/library/test.rst index f3b5383658b5ac..e26e9d8575a7ac 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -513,6 +513,8 @@ The :mod:`!test.support` module defines the following functions: ``True`` if called by a function whose ``__name__`` is ``'__main__'``. Used when tests are executed by :mod:`test.regrtest`. + If called at the top level, sets label "requires\_\ *resource*" on the module. + .. function:: sortdict(dict) @@ -529,6 +531,29 @@ The :mod:`!test.support` module defines the following functions: rather than looking directly in the path directories. +.. function:: mark(label, value=True, /) + + Add a label to a test. Use ``@mark('label')`` as a decorator of a test + method or class. + + The optional *value* (``True`` by default) is matched on the command line + by ``--label label=value``, whereas ``--label label`` matches any value. + + Many :mod:`test.support` decorators like :func:`requires_resource`, + :func:`~test.support.cpython_only` or :func:`bigmemtest` add labels + automatically. + + +.. function:: mark_module(label, value=True, /, *, globals=None) + + Add a label to every test of a module. Call it at the module level:: + + test.support.mark_module('pickletest') + + The module is the caller, unless its :func:`globals` dict is passed as the + *globals* argument. + + .. function:: get_pagesize() Get size of a page in bytes. @@ -773,26 +798,31 @@ The :mod:`!test.support` module defines the following functions: .. decorator:: requires_zlib Decorator for skipping tests if :mod:`zlib` doesn't exist. + Adds label "requires_zlib". .. decorator:: requires_gzip Decorator for skipping tests if :mod:`gzip` doesn't exist. + Adds label "requires_gzip". .. decorator:: requires_bz2 Decorator for skipping tests if :mod:`bz2` doesn't exist. + Adds label "requires_bz2". .. decorator:: requires_lzma Decorator for skipping tests if :mod:`lzma` doesn't exist. + Adds label "requires_lzma". .. decorator:: requires_resource(resource) Decorator for skipping tests if *resource* is not available. + Adds label "requires\_\ *resource*". .. decorator:: requires_docstrings @@ -809,12 +839,17 @@ The :mod:`!test.support` module defines the following functions: .. decorator:: cpython_only Decorator for tests only applicable to CPython. + Adds label "impl_detail_cpython". .. decorator:: impl_detail(msg=None, **guards) Decorator for invoking :func:`check_impl_detail` on *guards*. If that returns ``False``, then uses *msg* as the reason for skipping the test. + For every keyword argument *name* adds a label + "impl_detail\_\ *name*" if its value is true or + "impl_detail_no\_\ *name*" otherwise. + .. decorator:: thread_unsafe(reason=None) @@ -849,10 +884,13 @@ The :mod:`!test.support` module defines the following functions: method may be less than the requested value. If *dry_run* is ``False``, it means the test doesn't support dummy runs when ``-M`` is not specified. + Adds label "bigmemtest". + .. decorator:: bigaddrspacetest Decorator for tests that fill the address space. + Adds label "bigaddrspacetest". .. function:: linked_to_musl() @@ -1746,6 +1784,8 @@ The :mod:`!test.support.import_helper` module provides support for import tests. optional for others, set *required_on* to an iterable of platform prefixes which will be compared against :data:`sys.platform`. + If called at the top level, sets label "requires\_\ *name*" on the module. + .. versionadded:: 3.1 diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 64c035307e6654..ab0e863a0dc526 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -180,6 +180,7 @@ def __init__(self, **kwargs) -> None: self.header = False self.failfast = False self.match_tests: TestFilter = [] + self.match_labels: TestFilter = [] self.pgo = False self.pgo_extended = False self.tsan = False @@ -299,6 +300,14 @@ def _create_parser(): group.add_argument('-i', '--ignore', metavar='PAT', dest='match_tests', action=FilterAction, const=False, help='ignore test cases and methods with glob pattern `PAT`') + group.add_argument('--label', metavar='NAME[=VALUE]', + dest='match_labels', action=FilterAction, const=True, + help='match test cases and methods with label `NAME` ' + '(optionally only if its value is `VALUE`)') + group.add_argument('--no-label', metavar='NAME[=VALUE]', + dest='match_labels', action=FilterAction, const=False, + help='ignore test cases and methods with label `NAME` ' + '(optionally only if its value is `VALUE`)') group.add_argument('--matchfile', metavar='FILENAME', dest='match_tests', action=FromFileFilterAction, const=True, diff --git a/Lib/test/libregrtest/filter.py b/Lib/test/libregrtest/filter.py index 41372e427ffd03..a190a97fd9e3ae 100644 --- a/Lib/test/libregrtest/filter.py +++ b/Lib/test/libregrtest/filter.py @@ -1,21 +1,61 @@ import itertools import operator import re +import sys # By default, don't filter tests _test_matchers = () _test_patterns = () +_match_labels = () + +# Sentinel returned by _get_label() when the test has no such label. +_no_label = object() def match_test(test): # Function used by support.run_unittest() and regrtest --list-cases + return match_test_id(test) and match_test_label(test) + +def match_test_id(test): result = False for matcher, result in reversed(_test_matchers): if matcher(test.id()): return result return not result +def match_test_label(test): + result = False + for name, value, result in reversed(_match_labels): + actual = _get_label(test, name) + if actual is _no_label: + continue + # value is None for a plain "--label name" (match any value). + if value is None or value == str(actual): + return result + return not result + +def _get_label(test, label): + attrname = f'_label_{label}' + value = getattr(test, attrname, _no_label) + if value is not _no_label: + return value + testMethod = getattr(test, test._testMethodName) + while testMethod is not None: + value = getattr(testMethod, attrname, _no_label) + if value is not _no_label: + return value + testMethod = getattr(testMethod, '__wrapped__', None) + try: + module = sys.modules[test.__class__.__module__] + except KeyError: + pass + else: + value = getattr(module, attrname, _no_label) + if value is not _no_label: + return value + return _no_label + def _is_full_match_test(pattern): # If a pattern contains at least one dot, it's considered @@ -32,8 +72,8 @@ def get_match_tests(): return _test_patterns -def set_match_tests(patterns): - global _test_matchers, _test_patterns +def set_match_tests(patterns=None, match_labels=None): + global _test_matchers, _test_patterns, _match_labels if not patterns: _test_matchers = () @@ -48,6 +88,17 @@ def set_match_tests(patterns): ] _test_patterns = patterns + if not match_labels: + _match_labels = () + else: + # "name" matches a label with any value, "name=value" matches only + # the specified value. + _match_labels = tuple( + (name, value if sep else None, result) + for label, result in match_labels + for name, sep, value in [label.partition('=')] + ) + def _compile_match_function(patterns): patterns = list(patterns) diff --git a/Lib/test/libregrtest/findtests.py b/Lib/test/libregrtest/findtests.py index 6c0e50846a466b..3614949e266e0d 100644 --- a/Lib/test/libregrtest/findtests.py +++ b/Lib/test/libregrtest/findtests.py @@ -91,9 +91,10 @@ def _list_cases(suite: unittest.TestSuite) -> None: def list_cases(tests: TestTuple, *, match_tests: TestFilter | None = None, + match_labels: TestFilter | None = None, test_dir: StrPath | None = None) -> None: support.verbose = False - set_match_tests(match_tests) + set_match_tests(match_tests, match_labels) skipped = [] for test_name in tests: diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 8773e9df73263b..67155b4e695708 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -85,6 +85,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): # Select tests self.match_tests: TestFilter = ns.match_tests + self.match_labels: TestFilter = ns.match_labels self.exclude: bool = ns.exclude self.fromfile: StrPath | None = ns.fromfile self.starting_test: TestName | None = ns.start @@ -501,6 +502,7 @@ def create_run_tests(self, tests: TestTuple) -> RunTests: fail_fast=self.fail_fast, fail_env_changed=self.fail_env_changed, match_tests=self.match_tests, + match_labels=self.match_labels, match_tests_dict=None, rerun=False, forever=self.forever, @@ -788,6 +790,7 @@ def main(self, tests: TestList | None = None) -> NoReturn: elif self.want_list_cases: list_cases(selected, match_tests=self.match_tests, + match_labels=self.match_labels, test_dir=self.test_dir) else: exitcode = self.run_tests(selected, tests) diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index 0a9edce1085be5..d689c28a664607 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -81,6 +81,7 @@ class RunTests: fail_fast: bool fail_env_changed: bool match_tests: TestFilter + match_labels: TestFilter match_tests_dict: FilterDict | None rerun: bool forever: bool diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index d62194acd9c29e..3c6ca753d06a9d 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -110,7 +110,7 @@ def setup_tests(runtests: RunTests) -> None: support.PGO = runtests.pgo support.PGO_EXTENDED = runtests.pgo_extended - set_match_tests(runtests.match_tests) + set_match_tests(runtests.match_tests, runtests.match_labels) if runtests.use_junit: support.junit_xml_list = [] diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 74d3794289bf69..2b293d8f21b267 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -338,6 +338,9 @@ def get_resource_value(resource): def requires(resource, msg=None): """Raise ResourceDenied if the specified resource is not available.""" + f = sys._getframe(1) + if f.f_globals is f.f_locals: + mark_module(f'requires_{resource}', globals=f.f_globals) if not is_resource_enabled(resource): if msg is None: msg = "Use of the %r resource not enabled" % resource @@ -551,35 +554,35 @@ def requires_zlib(reason='requires zlib'): import zlib except ImportError: zlib = None - return unittest.skipUnless(zlib, reason) + return skipUnless(zlib, reason, label='requires_zlib') def requires_gzip(reason='requires gzip'): try: import gzip except ImportError: gzip = None - return unittest.skipUnless(gzip, reason) + return skipUnless(gzip, reason, label='requires_gzip') def requires_bz2(reason='requires bz2'): try: import bz2 except ImportError: bz2 = None - return unittest.skipUnless(bz2, reason) + return skipUnless(bz2, reason, label='requires_bz2') def requires_lzma(reason='requires lzma'): try: import lzma except ImportError: lzma = None - return unittest.skipUnless(lzma, reason) + return skipUnless(lzma, reason, label='requires_lzma') def requires_zstd(reason='requires zstd'): try: from compression import zstd except ImportError: zstd = None - return unittest.skipUnless(zstd, reason) + return skipUnless(zstd, reason, label='requires_zstd') def has_no_debug_ranges(): try: @@ -594,7 +597,7 @@ def requires_debug_ranges(reason='requires co_positions / debug_ranges'): except unittest.SkipTest as e: skip = True reason = e.args[0] if e.args else reason - return unittest.skipIf(skip, reason) + return skipIf(skip, reason, label='requires_debug_ranges') MS_WINDOWS = (sys.platform == 'win32') @@ -643,7 +646,8 @@ def skip_wasi_stack_overflow(): ) def requires_fork(): - return unittest.skipUnless(has_fork_support, "requires working os.fork()") + return skipUnless(has_fork_support, "requires working os.fork()", + label='requires_fork') has_subprocess_support = not ( # WASM and Apple mobile platforms do not support subprocesses. @@ -659,7 +663,8 @@ def requires_fork(): def requires_subprocess(): """Used for subprocess, os.spawn calls, fd inheritance""" - return unittest.skipUnless(has_subprocess_support, "requires subprocess support") + return skipUnless(has_subprocess_support, "requires subprocess support", + label='requires_subprocess') # Emscripten's socket emulation and WASI sockets have limitations. has_socket_support = not ( @@ -667,17 +672,21 @@ def requires_subprocess(): or is_wasi ) -def requires_working_socket(*, module=False): +def requires_working_socket(*, module=False, globals=None): """Skip tests or modules that require working sockets Can be used as a function/class decorator or to skip an entire module. """ + label = 'requires_socket' msg = "requires socket support" - if module: + if module or globals is not None: + if globals is None: + globals = sys._getframe(1).f_globals + mark_module(label, globals=globals) if not has_socket_support: raise unittest.SkipTest(msg) else: - return unittest.skipUnless(has_socket_support, msg) + return skipUnless(has_socket_support, msg, label=label) @functools.cache @@ -1293,6 +1302,7 @@ def bigmemtest(size, memuse, dry_run=True): test doesn't support dummy runs when -M is not specified. """ def decorator(f): + @mark('bigmemtest') @functools.wraps(f) def wrapper(self): size = wrapper.size @@ -1346,6 +1356,7 @@ def internal(*args, **kwargs): def bigaddrspacetest(f): """Decorator for tests that fill the address space.""" + @mark('bigaddrspacetest') @functools.wraps(f) def wrapper(self): if max_memuse < MAX_Py_ssize_t: @@ -1366,13 +1377,47 @@ def wrapper(self): def _id(obj): return obj +def mark(label, value=True, /): + """Add a label to a test method or class. Use it as a decorator. + + The optional value (``True`` by default) can be matched on the command + line with ``--label name=value``. + """ + def decorator(test): + setattr(test, f'_label_{label}', value) + return test + return decorator + +def mark_module(label, value=True, /, *, globals=None): + """Add a label to every test of a module. + + The module is the caller, unless its globals() dict is passed as the + globals argument. + """ + if globals is None: + globals = sys._getframe(1).f_globals + globals[f'_label_{label}'] = value + +def combine(*decorators): + def decorator(test): + for deco in reversed(decorators): + test = deco(test) + return test + return decorator + +def skipUnless(condition, reason, *, label): + return combine(unittest.skipUnless(condition, reason), mark(label)) + +def skipIf(condition, reason, *, label): + return combine(unittest.skipIf(condition, reason), mark(label)) + def requires_resource(resource): + label = 'requires_' + resource if resource == 'gui' and not _is_gui_available(): - return unittest.skip(_is_gui_available.reason) - if is_resource_enabled(resource): - return _id - else: - return unittest.skip("resource {0!r} is not enabled".format(resource)) + return skipUnless(False, _is_gui_available.reason, label=label) + return skipUnless(is_resource_enabled(resource), + f"resource {resource!r} is not enabled", + label=label) def cpython_only(test): """ @@ -1381,8 +1426,16 @@ def cpython_only(test): return impl_detail(cpython=True)(test) def impl_detail(msg=None, **guards): + guards, _ = _parse_guards(guards) + decorators = [] + for name in reversed(guards): + if guards[name]: + label = f'impl_detail_{name}' + else: + label = f'impl_detail_no_{name}' + decorators.append(mark(label)) if check_impl_detail(**guards): - return _id + return combine(*decorators) if msg is None: guardnames, default = _parse_guards(guards) if default: @@ -1391,7 +1444,7 @@ def impl_detail(msg=None, **guards): msg = "implementation detail specific to {0}" guardnames = sorted(guardnames.keys()) msg = msg.format(' or '.join(guardnames)) - return unittest.skip(msg) + return combine(unittest.skip(msg), *decorators) def _parse_guards(guards): # Returns a tuple ({platform_name: run_me}, default_value) diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index e8a3d176ad6943..f5c81e6e89f8b5 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -88,6 +88,10 @@ def import_module(name, deprecated=False, *, required_on=()): compared against sys.platform. """ with _ignore_deprecated_imports(deprecated): + f = sys._getframe(1) + if f.f_globals is f.f_locals: + from test.support import mark_module + mark_module(f'requires_{name}', globals=f.f_globals) try: return importlib.import_module(name) except ImportError as msg: diff --git a/Lib/test/support/socket_helper.py b/Lib/test/support/socket_helper.py index 66e5379d2c6e25..34451896024cce 100644 --- a/Lib/test/support/socket_helper.py +++ b/Lib/test/support/socket_helper.py @@ -147,6 +147,7 @@ def _is_ipv6_enabled(): _bind_nix_socket_error = None def skip_unless_bind_unix_socket(test): """Decorator for tests requiring a functional bind() for unix sockets.""" + test = support.mark('requires_unix_sockets')(test) if not hasattr(socket, 'AF_UNIX'): return unittest.skip('No UNIX Sockets')(test) if sys.platform == 'cygwin': diff --git a/Lib/test/support/threading_helper.py b/Lib/test/support/threading_helper.py index cf87233f0e2e93..3cef70ffeebb2d 100644 --- a/Lib/test/support/threading_helper.py +++ b/Lib/test/support/threading_helper.py @@ -237,17 +237,21 @@ def _can_start_thread() -> bool: can_start_thread = _can_start_thread() -def requires_working_threading(*, module=False): +def requires_working_threading(*, module=False, globals=None): """Skip tests or modules that require working threading. Can be used as a function/class decorator or to skip an entire module. """ + label = 'requires_threading' msg = "requires threading support" - if module: + if module or globals is not None: + if globals is None: + globals = sys._getframe(1).f_globals + support.mark_module(label, globals=globals) if not can_start_thread: raise unittest.SkipTest(msg) else: - return unittest.skipUnless(can_start_thread, msg) + return support.skipUnless(can_start_thread, msg, label=label) def run_concurrently(worker_func, nthreads=None, args=(), kwargs={}): diff --git a/Misc/NEWS.d/next/Tests/2023-09-03-12-53-53.gh-issue-108828.zoWIyX.rst b/Misc/NEWS.d/next/Tests/2023-09-03-12-53-53.gh-issue-108828.zoWIyX.rst new file mode 100644 index 00000000000000..eac54c262a9cbb --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-03-12-53-53.gh-issue-108828.zoWIyX.rst @@ -0,0 +1,9 @@ +Add support of labels in tests. The ``@test.support.mark('label')`` +decorator adds a label to method or class. ``test.support.mark('label', +globals=globals())`` adds a label to the whole module. Many +:mod:`test.support` decorators like :func:`~test.support.requires_resource`, +:func:`~test.support.cpython_only` or :func:`~test.support.bigmemtest` add +labels automatically. A label can have a value (``True`` by default). Tests +which have or have not the specified label can be filtered by options +``--label`` and ``--no-label``; ``--label name=value`` matches a specific +value, while ``--label name`` matches any value.