From 9518c9bee58d08fd2de3c3f798a522fe0184fe90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jouni=20K=2E=20Sepp=C3=A4nen?= Date: Sat, 5 Jul 2025 10:56:42 +0300 Subject: [PATCH] Run subprocess-starting tests without pytest-xdist I suspect that we are running out of process IDs, or the scheduler is not letting all subprocesses execute, leading to timeouts in tests that start subprocesses when other long-running tests are executing. This happens often with Python 3.14 tests for some reason. Add a new pytest marker for the tests that start a subprocess at top level, and run these tests in a separate pass without pytest-xdist parallelization. --- .github/workflows/tests.yml | 15 ++++++++++++--- azure-pipelines.yml | 18 ++++++++++++++---- lib/matplotlib/testing/conftest.py | 1 + lib/matplotlib/tests/test_backend_inline.py | 1 + lib/matplotlib/tests/test_backend_nbagg.py | 1 + lib/matplotlib/tests/test_backend_webagg.py | 1 + .../tests/test_backends_interactive.py | 3 +++ lib/matplotlib/tests/test_basic.py | 3 +++ lib/matplotlib/tests/test_determinism.py | 2 ++ lib/matplotlib/tests/test_font_manager.py | 1 + lib/matplotlib/tests/test_matplotlib.py | 2 ++ lib/matplotlib/tests/test_preprocess_data.py | 1 + lib/matplotlib/tests/test_pyplot.py | 1 + lib/matplotlib/tests/test_rcparams.py | 3 +++ lib/matplotlib/tests/test_sphinxext.py | 1 + lib/matplotlib/tests/test_texmanager.py | 1 + 16 files changed, 48 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 85ace93445b6..0f272bbcf531 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -337,9 +337,18 @@ jobs: if [[ "${{ matrix.python-version }}" == '3.13t' ]]; then export PYTHON_GIL=0 fi - pytest -rfEsXR -n auto \ - --maxfail=50 --timeout=300 --durations=25 \ - --cov-report=xml --cov=lib --log-level=DEBUG --color=yes + FLAGS=( + -rfEsXR + --maxfail=50 + --timeout=300 + --durations=25 + --cov-report=xml + --cov=lib + --log-level=DEBUG + --color=yes + ) + pytest "${FLAGS[@]}" -m 'not subprocess' -n auto + pytest "${FLAGS[@]}" -m subprocess --cov-append - name: Cleanup non-failed image files if: failure() diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d68a9d36f0d3..4ab1df028013 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -120,9 +120,18 @@ stages: echo "##vso[task.setvariable variable=VS_COVERAGE_TOOL]$TOOL" - PYTHONFAULTHANDLER=1 pytest -rfEsXR -n 2 \ - --maxfail=50 --timeout=300 --durations=25 \ - --junitxml=junit/test-results.xml --cov-report=xml --cov=lib + FLAGS=( + -rfEsXR + --maxfail=50 + --timeout=300 + --durations=25 + --cov-report=xml + --cov=lib + ) + PYTHONFAULTHANDLER=1 pytest "${FLAGS[@]}" -m 'not subprocess' -n 2 \ + --junitxml=junit/test-results-1.xml + PYTHONFAULTHANDLER=1 pytest "${FLAGS[@]}" -m subprocess \ + --junitxml=junit/test-results-2.xml --cov-append if [[ $VS_VER == 2022 ]]; then "$TOOL" shutdown $SESSION_ID @@ -153,7 +162,8 @@ stages: - task: PublishTestResults@2 inputs: - testResultsFiles: '**/test-results.xml' + mergeTestResults: true + testResultsFiles: '**/test-results*.xml' testRunTitle: 'Python $(python.version)' condition: succeededOrFailed() diff --git a/lib/matplotlib/testing/conftest.py b/lib/matplotlib/testing/conftest.py index 2961e7f02f3f..4efc10766be7 100644 --- a/lib/matplotlib/testing/conftest.py +++ b/lib/matplotlib/testing/conftest.py @@ -15,6 +15,7 @@ def pytest_configure(config): ("markers", "backend: Set alternate Matplotlib backend temporarily."), ("markers", "baseline_images: Compare output against references."), ("markers", "pytz: Tests that require pytz to be installed."), + ("markers", "subprocess: Tests that start a subprocess."), ("filterwarnings", "error"), ("filterwarnings", "ignore:.*The py23 module has been deprecated:DeprecationWarning"), diff --git a/lib/matplotlib/tests/test_backend_inline.py b/lib/matplotlib/tests/test_backend_inline.py index 997e1e7186b1..2cc1022b7939 100644 --- a/lib/matplotlib/tests/test_backend_inline.py +++ b/lib/matplotlib/tests/test_backend_inline.py @@ -10,6 +10,7 @@ pytest.importorskip('nbconvert') pytest.importorskip('ipykernel') pytest.importorskip('matplotlib_inline') +pytestmark = pytest.mark.subprocess def test_ipynb(): diff --git a/lib/matplotlib/tests/test_backend_nbagg.py b/lib/matplotlib/tests/test_backend_nbagg.py index ccf74df20aab..5b082f159c36 100644 --- a/lib/matplotlib/tests/test_backend_nbagg.py +++ b/lib/matplotlib/tests/test_backend_nbagg.py @@ -9,6 +9,7 @@ nbformat = pytest.importorskip('nbformat') pytest.importorskip('nbconvert') pytest.importorskip('ipykernel') +pytestmark = pytest.mark.subprocess # From https://blog.thedataincubator.com/2016/06/testing-jupyter-notebooks/ diff --git a/lib/matplotlib/tests/test_backend_webagg.py b/lib/matplotlib/tests/test_backend_webagg.py index 1d6769494ef9..e09d85ce2e26 100644 --- a/lib/matplotlib/tests/test_backend_webagg.py +++ b/lib/matplotlib/tests/test_backend_webagg.py @@ -6,6 +6,7 @@ from matplotlib.testing import subprocess_run_for_testing +@pytest.mark.subprocess @pytest.mark.parametrize("backend", ["webagg", "nbagg"]) def test_webagg_fallback(backend): pytest.importorskip("tornado") diff --git a/lib/matplotlib/tests/test_backends_interactive.py b/lib/matplotlib/tests/test_backends_interactive.py index 9f8522a9df4a..ceb3ee501279 100644 --- a/lib/matplotlib/tests/test_backends_interactive.py +++ b/lib/matplotlib/tests/test_backends_interactive.py @@ -22,6 +22,9 @@ from matplotlib.testing import subprocess_run_helper as _run_helper, is_ci_environment +pytestmark = pytest.mark.subprocess + + class _WaitForStringPopen(subprocess.Popen): """ A Popen that passes flags that allow triggering KeyboardInterrupt. diff --git a/lib/matplotlib/tests/test_basic.py b/lib/matplotlib/tests/test_basic.py index f6aa1e458555..10e94ee3be9d 100644 --- a/lib/matplotlib/tests/test_basic.py +++ b/lib/matplotlib/tests/test_basic.py @@ -5,6 +5,8 @@ from matplotlib.testing import subprocess_run_for_testing +import pytest + def test_simple(): assert 1 + 1 == 2 @@ -28,6 +30,7 @@ def test_override_builtins(): assert overridden <= ok_to_override +@pytest.mark.subprocess def test_lazy_imports(): source = textwrap.dedent(""" import sys diff --git a/lib/matplotlib/tests/test_determinism.py b/lib/matplotlib/tests/test_determinism.py index 2ecc40dbd3c0..dd1669894d98 100644 --- a/lib/matplotlib/tests/test_determinism.py +++ b/lib/matplotlib/tests/test_determinism.py @@ -21,6 +21,8 @@ from matplotlib.text import TextPath from matplotlib.transforms import IdentityTransform +pytestmark = pytest.mark.subprocess + def _save_figure(objects='mhip', fmt="pdf", usetex=False): mpl.use(fmt) diff --git a/lib/matplotlib/tests/test_font_manager.py b/lib/matplotlib/tests/test_font_manager.py index 24421b8e30b3..b49bc65e285d 100644 --- a/lib/matplotlib/tests/test_font_manager.py +++ b/lib/matplotlib/tests/test_font_manager.py @@ -289,6 +289,7 @@ def test_fontcache_thread_safe(): subprocess_run_helper(_test_threading, timeout=10) +@pytest.mark.subprocess def test_lockfilefailure(tmp_path): # The logic here: # 1. get a temp directory from pytest diff --git a/lib/matplotlib/tests/test_matplotlib.py b/lib/matplotlib/tests/test_matplotlib.py index d0a3f8c617e1..3a28ecb985a7 100644 --- a/lib/matplotlib/tests/test_matplotlib.py +++ b/lib/matplotlib/tests/test_matplotlib.py @@ -19,6 +19,7 @@ def test_parse_to_version_info(version_str, version_tuple): assert matplotlib._parse_to_version_info(version_str) == version_tuple +@pytest.mark.subprocess @pytest.mark.skipif(sys.platform == "win32", reason="chmod() doesn't work as is on Windows") @pytest.mark.skipif(sys.platform != "win32" and os.geteuid() == 0, @@ -37,6 +38,7 @@ def test_tmpconfigdir_warning(tmp_path): os.chmod(tmp_path, mode) +@pytest.mark.subprocess def test_importable_with_no_home(tmp_path): subprocess_run_for_testing( [sys.executable, "-c", diff --git a/lib/matplotlib/tests/test_preprocess_data.py b/lib/matplotlib/tests/test_preprocess_data.py index c983d78786e1..3825a0aecd09 100644 --- a/lib/matplotlib/tests/test_preprocess_data.py +++ b/lib/matplotlib/tests/test_preprocess_data.py @@ -245,6 +245,7 @@ def funcy(ax, x, y, z, t=None): funcy.__doc__) +@pytest.mark.subprocess def test_data_parameter_replacement(): """ Test that the docstring contains the correct *data* parameter stub diff --git a/lib/matplotlib/tests/test_pyplot.py b/lib/matplotlib/tests/test_pyplot.py index 55f7c33cb52e..336847f4b7ef 100644 --- a/lib/matplotlib/tests/test_pyplot.py +++ b/lib/matplotlib/tests/test_pyplot.py @@ -12,6 +12,7 @@ from matplotlib import pyplot as plt +@pytest.mark.subprocess def test_pyplot_up_to_date(tmp_path): pytest.importorskip("black") diff --git a/lib/matplotlib/tests/test_rcparams.py b/lib/matplotlib/tests/test_rcparams.py index 2235f98b720f..ee9a6476703a 100644 --- a/lib/matplotlib/tests/test_rcparams.py +++ b/lib/matplotlib/tests/test_rcparams.py @@ -528,6 +528,7 @@ def test_rcparams_reset_after_fail(): assert mpl.rcParams['text.usetex'] is False +@pytest.mark.subprocess @pytest.mark.skipif(sys.platform != "linux", reason="Linux only") def test_backend_fallback_headless_invalid_backend(tmp_path): env = {**os.environ, @@ -545,6 +546,7 @@ def test_backend_fallback_headless_invalid_backend(tmp_path): env=env, check=True, stderr=subprocess.DEVNULL) +@pytest.mark.subprocess @pytest.mark.skipif(sys.platform != "linux", reason="Linux only") def test_backend_fallback_headless_auto_backend(tmp_path): # specify a headless mpl environment, but request a graphical (tk) backend @@ -567,6 +569,7 @@ def test_backend_fallback_headless_auto_backend(tmp_path): assert backend.strip().lower() == "agg" +@pytest.mark.subprocess @pytest.mark.skipif( sys.platform == "linux" and not _c_internal_utils.xdisplay_is_valid(), reason="headless") diff --git a/lib/matplotlib/tests/test_sphinxext.py b/lib/matplotlib/tests/test_sphinxext.py index ede3166a2e1b..5b6e2e6b6e70 100644 --- a/lib/matplotlib/tests/test_sphinxext.py +++ b/lib/matplotlib/tests/test_sphinxext.py @@ -16,6 +16,7 @@ tinypages = Path(__file__).parent / 'data/tinypages' +@pytest.mark.subprocess def build_sphinx_html(source_dir, doctree_dir, html_dir, extra_args=None): # Build the pages with warnings turned into errors extra_args = [] if extra_args is None else extra_args diff --git a/lib/matplotlib/tests/test_texmanager.py b/lib/matplotlib/tests/test_texmanager.py index 64dcbf46456d..d7adda36e615 100644 --- a/lib/matplotlib/tests/test_texmanager.py +++ b/lib/matplotlib/tests/test_texmanager.py @@ -63,6 +63,7 @@ def test_unicode_characters(): fig.canvas.draw() +@pytest.mark.subprocess @needs_usetex def test_openin_any_paranoid(): completed = subprocess_run_for_testing(