diff --git a/docs/source/command_line.rst b/docs/source/command_line.rst index b371062e8bff..cbd72a217d29 100644 --- a/docs/source/command_line.rst +++ b/docs/source/command_line.rst @@ -1037,19 +1037,22 @@ Parallel type-checking ********************** By default, mypy checks all modules in the same Python process. This can be slow -for large code bases. Mypy offers experimental parallel type-checking mode using +for large code bases. Mypy offers parallel type-checking mode using multiple worker processes. In parallel mode, modules that do not depend om each other are type-checked in parallel. :ref:`Incremental cache ` is used to manage most of the shared state. Parallel type-checking also requires :option:`--local-partial-types `, which is enabled by default starting from mypy 2.0. -.. option:: -n NUMBER, --num-workers NUMBER +.. option:: -n VALUE, --num-workers VALUE - Use ``NUMBER`` parallel worker processes (in addition to the coordinator - process) to perform type-checking. Specifying ``--num-workers 0`` (default) - disables parallel checking. Automatic detection of the optimal number - of workers is not supported yet. + Use the specified amount of parallel worker processes (in addition to the + coordinator process) to perform type-checking. Specifying ``--num-workers 0`` + (default) disables parallel checking. Specifying ``--num-workers auto`` + selects the number based on the CPU resources available to mypy. + + Automatic selection uses at most 8 workers because each worker adds roughly + 10% memory overhead. This cap does not apply when an explicit value is specified. This setting will override the ``MYPY_NUM_WORKERS`` environment variable if it is set. diff --git a/docs/source/config_file.rst b/docs/source/config_file.rst index adb6c9f21a72..61ec382795f2 100644 --- a/docs/source/config_file.rst +++ b/docs/source/config_file.rst @@ -1013,13 +1013,14 @@ These options may only be set in the global section (``[mypy]``). .. confval:: num_workers - :type: integer + :type: integer or string :default: 0 - Use specific number of parallel worker processes for type-checking, see - :ref:`parallel type-checking ` for more details. - This setting will be overridden by the ``MYPY_NUM_WORKERS`` environment - variable. + Use a specific number of parallel worker processes for type-checking or + the value ``auto`` to select the number based on the CPUs available to mypy. + See :ref:`parallel type-checking ` for more details. + The ``MYPY_NUM_WORKERS`` environment variable accepts the same values and + overrides this setting. Advanced options diff --git a/mypy/config_parser.py b/mypy/config_parser.py index 97fa01b8dd21..14e85d8dcef2 100644 --- a/mypy/config_parser.py +++ b/mypy/config_parser.py @@ -19,6 +19,7 @@ from mypy import defaults from mypy.options import PER_MODULE_OPTIONS, Options +from mypy.util import get_available_threads _CONFIG_VALUE_TYPES: TypeAlias = ( str | bool | int | float | dict[str, str] | list[str] | tuple[int, int] @@ -58,6 +59,19 @@ def parse_version(v: str | float) -> tuple[int, int]: return major, minor +def parse_num_workers(v: str | int) -> int: + value = v.strip() if isinstance(v, str) else v + if value == "auto": + return min(defaults.MAX_AUTO_WORKERS, get_available_threads()) + + try: + return int(value) + except (TypeError, ValueError) as err: + raise argparse.ArgumentTypeError( + f"Invalid number of workers '{v}' (expected an integer or 'auto')" + ) from err + + def try_split(v: str | Sequence[str] | object, split_regex: str = ",") -> list[str]: """Split and trim a str or sequence (eg: list) of str into a list of str. If an element of the input is not str, a type error will be raised.""" @@ -207,6 +221,7 @@ def split_commas(value: str) -> list[str]: "exclude": lambda s: [s.strip()], "packages": try_split, "modules": try_split, + "num_workers": parse_num_workers, } # Reuse the ini_config_types and overwrite the diff diff --git a/mypy/defaults.py b/mypy/defaults.py index cb95763b8191..d3bbe6f91dd3 100644 --- a/mypy/defaults.py +++ b/mypy/defaults.py @@ -48,6 +48,10 @@ RECURSION_LIMIT: Final = 2**14 +# Cap the automatic selection of workers since each worker adds roughly 10% of memory overhead. +# Users can specify an explicit --num-workers value which can exceed this limit. +MAX_AUTO_WORKERS: Final = 8 + # It looks like Windows & riscv64 are both slow with processes, causing test # flakiness even with our generous timeouts, so we set them higher. slow_fs = sys.platform == "win32" or platform.machine() == "riscv64" diff --git a/mypy/main.py b/mypy/main.py index 1beb7aaf5be4..c83ac180817b 100644 --- a/mypy/main.py +++ b/mypy/main.py @@ -23,7 +23,12 @@ sys.exit(2) from mypy import build, defaults, state, util -from mypy.config_parser import parse_config_file, parse_version, validate_package_allow_list +from mypy.config_parser import ( + parse_config_file, + parse_num_workers, + parse_version, + validate_package_allow_list, +) from mypy.defaults import RECURSION_LIMIT from mypy.error_formatter import OUTPUT_CHOICES from mypy.errors import CompileError @@ -100,6 +105,14 @@ def main( if options.num_workers: # Supporting both parsers would be really tricky, so just support the new one. options.native_parser = True + if not options.incremental and os.path.isdir(options.cache_dir): + print( + "Warning: disabling incremental mode may severely reduce performance", file=stdout + ) + print( + f"If this is intentional, delete '{options.cache_dir}' to suppress this warning", + file=stdout, + ) if options.num_workers < 0: fail("error: Number of workers cannot be negative", stderr, options) if options.cache_dir == os.devnull: @@ -1185,13 +1198,14 @@ def add_invertible_flag( # This undocumented feature exports limited line-level dependency information. internals_group.add_argument("--export-ref-info", action="store_true", help=argparse.SUPPRESS) - # Experimental parallel type-checking support. + # Parallel type-checking support. internals_group.add_argument( "-n", "--num-workers", - type=int, + type=parse_num_workers, + metavar="VALUE", default=0, - help="Number of separate mypy worker processes (experimental)", + help="Number of separate mypy worker processes, or 'auto'", ) report_group = parser.add_argument_group( @@ -1477,9 +1491,9 @@ def set_strict_flags() -> None: environ_num_workers = os.getenv("MYPY_NUM_WORKERS", "") if environ_num_workers.strip(): try: - options.num_workers = int(environ_num_workers) - except ValueError: - parser.error(f"MYPY_NUM_WORKERS must be an integer, got {environ_num_workers!r}") + options.num_workers = parse_num_workers(environ_num_workers) + except argparse.ArgumentTypeError as error: + parser.error(f"MYPY_NUM_WORKERS: {error}") # Parse command line for real, using a split namespace. special_opts = argparse.Namespace() diff --git a/mypy/test/testargs.py b/mypy/test/testargs.py index 767b36fbedd7..e233d5a42a1a 100644 --- a/mypy/test/testargs.py +++ b/mypy/test/testargs.py @@ -8,15 +8,139 @@ from __future__ import annotations import argparse +import contextlib +import os import sys +import tempfile +from collections.abc import Iterator +from io import StringIO +from pathlib import Path from typing import Any, cast +from unittest import mock -from mypy.main import infer_python_executable, process_options +from mypy import util +from mypy.config_parser import parse_num_workers +from mypy.main import infer_python_executable, main, process_options from mypy.options import Options from mypy.test.helpers import Suite +@contextlib.contextmanager +def _available_threads(value: int) -> Iterator[None]: + previous = util._AVAILABLE_THREADS + util._AVAILABLE_THREADS = value + try: + yield + finally: + util._AVAILABLE_THREADS = previous + + class ArgSuite(Suite): + def test_parse_num_workers(self) -> None: + with _available_threads(4): + assert parse_num_workers("auto") == 4 + with _available_threads(32): + assert parse_num_workers("auto") == 8 + + assert parse_num_workers(12) == 12 + assert parse_num_workers("12") == 12 + + assert parse_num_workers("0") == 0 + assert parse_num_workers("1") == 1 + + assert parse_num_workers("-1") == -1 + with self.assertRaises(argparse.ArgumentTypeError): + parse_num_workers("automatic") + + def test_num_workers_auto_from_command_line_and_environment(self) -> None: + with _available_threads(6): + with mock.patch.dict(os.environ, {"MYPY_NUM_WORKERS": ""}): + _, cli_options = process_options( + ["--config-file=", "--num-workers=auto"], require_targets=False + ) + with mock.patch.dict(os.environ, {"MYPY_NUM_WORKERS": "auto"}): + _, env_options = process_options(["--config-file="], require_targets=False) + + assert cli_options.num_workers == 6 + assert env_options.num_workers == 6 + + def test_invalid_num_workers_environment(self) -> None: + stderr = StringIO() + with mock.patch.dict(os.environ, {"MYPY_NUM_WORKERS": "automatic"}): + with self.assertRaises(SystemExit) as context: + process_options( + ["--config-file="], require_targets=False, stdout=StringIO(), stderr=stderr + ) + + assert context.exception.code == 2 + assert "MYPY_NUM_WORKERS: Invalid number of workers 'automatic'" in stderr.getvalue() + + def test_num_workers_auto_from_config(self) -> None: + configs = ( + ("mypy.ini", "[mypy]\nnum_workers = auto\n"), + ("pyproject.toml", '[tool.mypy]\nnum_workers = "auto"\n'), + ) + with tempfile.TemporaryDirectory() as temp_dir: + with _available_threads(6): + for filename, contents in configs: + config = Path(temp_dir) / filename + config.write_text(contents, encoding="utf-8") + with mock.patch.dict(os.environ, {"MYPY_NUM_WORKERS": ""}): + _, options = process_options( + ["--config-file", str(config)], require_targets=False + ) + assert options.num_workers == 6, filename + + def test_num_workers_precedence(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + config = Path(temp_dir) / "mypy.ini" + config.write_text("[mypy]\nnum_workers = 2\n", encoding="utf-8") + + with _available_threads(4): + with mock.patch.dict(os.environ, {"MYPY_NUM_WORKERS": ""}): + _, config_options = process_options( + ["--config-file", str(config)], require_targets=False + ) + + with mock.patch.dict(os.environ, {"MYPY_NUM_WORKERS": "7"}): + _, env_options = process_options( + ["--config-file", str(config)], require_targets=False + ) + + _, cli_options = process_options( + ["--config-file", str(config), "--num-workers=auto"], require_targets=False + ) + + assert config_options.num_workers == 2 + assert env_options.num_workers == 7 + assert cli_options.num_workers == 4 + + def test_parallel_mode_warns_when_incremental_disabled(self) -> None: + with tempfile.TemporaryDirectory() as cache_dir: + stdout = StringIO() + stderr = StringIO() + main( + args=[ + "--config-file=", + "--num-workers=1", + "--no-incremental", + "--no-site-packages", + "--no-error-summary", + f"--cache-dir={cache_dir}", + "-c", + "pass", + ], + stdout=stdout, + stderr=stderr, + clean_exit=True, + ) + + assert stdout.getvalue() == ( + "Warning: disabling incremental mode may severely reduce performance\n" + f"If this is intentional, delete '{cache_dir}' to suppress this warning\n" + ) + assert stderr.getvalue() == "" + def test_coherence(self) -> None: options = Options() _, parsed_options = process_options([], require_targets=False)