From ec302927b68e9dc8356695e46506985713c51df8 Mon Sep 17 00:00:00 2001 From: Brennan Magee Date: Thu, 2 Jul 2026 14:40:00 +0100 Subject: [PATCH 1/2] Add failing parallel build test --- test/roots/test-parallel-build/conf.py | 2 ++ test/roots/test-parallel-build/index.rst | 11 ++++++++++ test/roots/test-parallel-build/page1.rst | 8 +++++++ test/roots/test-parallel-build/page2.rst | 8 +++++++ test/roots/test-parallel-build/page3.rst | 8 +++++++ test/roots/test-parallel-build/page4.rst | 8 +++++++ test/roots/test-parallel-build/page5.rst | 8 +++++++ test/roots/test-parallel-build/page6.rst | 8 +++++++ test/test_parallel_build.py | 28 ++++++++++++++++++++++++ 9 files changed, 89 insertions(+) create mode 100644 test/roots/test-parallel-build/conf.py create mode 100644 test/roots/test-parallel-build/index.rst create mode 100644 test/roots/test-parallel-build/page1.rst create mode 100644 test/roots/test-parallel-build/page2.rst create mode 100644 test/roots/test-parallel-build/page3.rst create mode 100644 test/roots/test-parallel-build/page4.rst create mode 100644 test/roots/test-parallel-build/page5.rst create mode 100644 test/roots/test-parallel-build/page6.rst create mode 100644 test/test_parallel_build.py diff --git a/test/roots/test-parallel-build/conf.py b/test/roots/test-parallel-build/conf.py new file mode 100644 index 0000000..5921a0f --- /dev/null +++ b/test/roots/test-parallel-build/conf.py @@ -0,0 +1,2 @@ +extensions = ['sphinxarg.ext'] +sphinxarg_build_commands_index = True diff --git a/test/roots/test-parallel-build/index.rst b/test/roots/test-parallel-build/index.rst new file mode 100644 index 0000000..2d7e706 --- /dev/null +++ b/test/roots/test-parallel-build/index.rst @@ -0,0 +1,11 @@ +Test Parallel Build +==================== + +.. toctree:: + + page1 + page2 + page3 + page4 + page5 + page6 diff --git a/test/roots/test-parallel-build/page1.rst b/test/roots/test-parallel-build/page1.rst new file mode 100644 index 0000000..6d37ddb --- /dev/null +++ b/test/roots/test-parallel-build/page1.rst @@ -0,0 +1,8 @@ +Page 1 +====== + +.. argparse:: + :filename: sample-directive-opts.py + :func: get_parser + :nosubcommands: + :prog: page1-command diff --git a/test/roots/test-parallel-build/page2.rst b/test/roots/test-parallel-build/page2.rst new file mode 100644 index 0000000..52b6a6f --- /dev/null +++ b/test/roots/test-parallel-build/page2.rst @@ -0,0 +1,8 @@ +Page 2 +====== + +.. argparse:: + :filename: sample-directive-opts.py + :func: get_parser + :nosubcommands: + :prog: page2-command diff --git a/test/roots/test-parallel-build/page3.rst b/test/roots/test-parallel-build/page3.rst new file mode 100644 index 0000000..07d3cc2 --- /dev/null +++ b/test/roots/test-parallel-build/page3.rst @@ -0,0 +1,8 @@ +Page 3 +====== + +.. argparse:: + :filename: sample-directive-opts.py + :func: get_parser + :nosubcommands: + :prog: page3-command diff --git a/test/roots/test-parallel-build/page4.rst b/test/roots/test-parallel-build/page4.rst new file mode 100644 index 0000000..8b564f3 --- /dev/null +++ b/test/roots/test-parallel-build/page4.rst @@ -0,0 +1,8 @@ +Page 4 +====== + +.. argparse:: + :filename: sample-directive-opts.py + :func: get_parser + :nosubcommands: + :prog: page4-command diff --git a/test/roots/test-parallel-build/page5.rst b/test/roots/test-parallel-build/page5.rst new file mode 100644 index 0000000..6c44602 --- /dev/null +++ b/test/roots/test-parallel-build/page5.rst @@ -0,0 +1,8 @@ +Page 5 +====== + +.. argparse:: + :filename: sample-directive-opts.py + :func: get_parser + :nosubcommands: + :prog: page5-command diff --git a/test/roots/test-parallel-build/page6.rst b/test/roots/test-parallel-build/page6.rst new file mode 100644 index 0000000..a3d329e --- /dev/null +++ b/test/roots/test-parallel-build/page6.rst @@ -0,0 +1,8 @@ +Page 6 +====== + +.. argparse:: + :filename: sample-directive-opts.py + :func: get_parser + :nosubcommands: + :prog: page6-command diff --git a/test/test_parallel_build.py b/test/test_parallel_build.py new file mode 100644 index 0000000..4630794 --- /dev/null +++ b/test/test_parallel_build.py @@ -0,0 +1,28 @@ +"""Regression test for parallel reads (see issue with merge_domaindata). + +Sphinx only performs a parallel *read* when there are more than 5 source +documents and ``parallel > 1`` (see ``Builder.read`` in sphinx/builders/__init__.py). +The test root therefore needs at least 6 documents with ``argparse`` +directives so that reading is actually split across worker processes and +``ArgParseDomain.merge_domaindata`` is exercised. +""" + +import pytest + +from sphinxarg.ext import ArgParseDomain + + +@pytest.mark.sphinx('dummy', testroot='parallel-build', parallel=2) +def test_parallel_read_merges_domain_data(app): + # Building must not raise NotImplementedError / KeyError from + # sphinx.util.parallel when merging results from worker processes. + app.build() + assert app._warning.getvalue() == '' + + domain = app.env.domains[ArgParseDomain.name] + full_commands = {entry[0] for entry in domain.get_objects()} + expected = {f'page{i}-command' for i in range(1, 7)} + assert full_commands == expected + + # Each command should only be merged in once, from the worker that read it. + assert len(list(domain.get_objects())) == len(expected) From d39479127dfa16eb6bae47c9868658885373984c Mon Sep 17 00:00:00 2001 From: Brennan Magee Date: Thu, 2 Jul 2026 14:45:46 +0100 Subject: [PATCH 2/2] Add missing clear_doc and merge_domaindata methods to the ArgParseDomain This enables parallel builds --- sphinxarg/ext.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/sphinxarg/ext.py b/sphinxarg/ext.py index f796b17..38476ab 100644 --- a/sphinxarg/ext.py +++ b/sphinxarg/ext.py @@ -910,6 +910,36 @@ class ArgParseDomain(Domain): def get_objects(self) -> Iterable[_ObjectDescriptionTuple]: yield from self.data['commands'] + def clear_doc(self, docname: str) -> None: + """Remove traces of a document in the domain-specific inventories.""" + self.data['commands'] = [ + entry for entry in self.data['commands'] if entry[3] != docname + ] + commands_by_group: dict[str, list[_ObjectDescriptionTuple]] + commands_by_group = self.data['commands-by-group'] + for group in list(commands_by_group): + entries = [e for e in commands_by_group[group] if e[3] != docname] + if entries: + commands_by_group[group] = entries + else: + del commands_by_group[group] + + def merge_domaindata(self, docnames: Iterable[str], otherdata: dict) -> None: + """Merge in data regarding *docnames* from a different domaindata + inventory (coming from a subprocess in parallel builds). + """ + docnames = set(docnames) + # No need to check for duplicates: each docname is only ever merged + # once, from the single worker process that read it. + for entry in otherdata['commands']: + if entry[3] in docnames: + self.data['commands'].append(entry) + for group, entries in otherdata['commands-by-group'].items(): + merged = self.data['commands-by-group'].setdefault(group, []) + for entry in entries: + if entry[3] in docnames: + merged.append(entry) + def resolve_xref( self, env: BuildEnvironment,