|
7 | 7 | """ |
8 | 8 | import dataclasses |
9 | 9 | import itertools |
10 | | -from collections.abc import Sequence, Set |
| 10 | +from collections.abc import Callable, Sequence, Set |
11 | 11 |
|
12 | 12 | import pytest |
13 | 13 |
|
|
16 | 16 | from nameparser._pipeline import STAGES |
17 | 17 | from nameparser._pipeline._assign import assign |
18 | 18 | from nameparser._pipeline._classify import classify |
| 19 | +from nameparser._pipeline import _group |
19 | 20 | from nameparser._pipeline._group import group |
20 | 21 | from nameparser._pipeline._pieces import ( |
21 | 22 | _anchors, _numeral_behind_the_initial_veto, anchor_in_reach, |
|
26 | 27 | ) |
27 | 28 | from nameparser._pipeline._segment import segment |
28 | 29 | from nameparser._pipeline._state import ( |
29 | | - AMBIGUOUS_ACRONYM_TAG, ParseState, WorkToken, |
| 30 | + AMBIGUOUS_ACRONYM_TAG, ParseState, PendingAmbiguity, WorkToken, |
30 | 31 | ) |
31 | 32 | from nameparser._pipeline._tokenize import tokenize |
32 | 33 | from nameparser._pipeline._vocab import is_one_case, is_title_shaped, tag_marker_runs |
@@ -890,56 +891,88 @@ def test_a_title_inside_the_given_parts_run_is_a_title() -> None: |
890 | 891 |
|
891 | 892 | # #610: `trailing_candidates` is the maiden take's choice of which |
892 | 893 | # clause words to read the clause-free view over, and its contract is |
893 | | -# a SUPERSET of what `tail_reading` takes. The grid reads plain names, |
894 | | -# where the candidates' cut must fall at or before every piece the tail |
895 | | -# reading takes (the first name piece aside: in a clause that is the |
896 | | -# word after the marker, which the take always keeps). |
| 894 | +# a SUPERSET of what `tail_reading` takes from that view. Checked AT |
| 895 | +# THE TAKE, on the pieces the take is handed -- before any join, which |
| 896 | +# is where it runs -- by wrapping `_group._maiden_take` and the |
| 897 | +# candidates call inside it: the clause-free name is the head plus every |
| 898 | +# clause word after the first (which the take always keeps), and every |
| 899 | +# piece the tail reading takes from it must fall at or after the cut. |
897 | 900 | _CANDIDATE_HEADS = ("Jane Doe", "J.", "Jane van der Berg", "Mai Le", |
898 | 901 | "abdul Berg", "John") |
899 | 902 | _CANDIDATE_WORDS = ("Smith", "VI", "V", "III", "MA", "Ma", "PhD", "Jr.", |
900 | 903 | "Prof.", "Dr.", "King.", "do", "DO", "de", "X.Y.Z.", |
901 | 904 | "Ph. D.", "i", "Jones") |
902 | 905 |
|
903 | 906 |
|
904 | | -def _candidate_misses() -> list[str]: |
905 | | - misses = [] |
| 907 | +def _candidate_misses(monkeypatch: pytest.MonkeyPatch, |
| 908 | + candidates: Callable[..., int] = trailing_candidates, |
| 909 | + ) -> list[str]: |
| 910 | + misses: list[str] = [] |
| 911 | + seen: list[tuple[int, int]] = [] |
| 912 | + real_take = _group._maiden_take |
| 913 | + |
| 914 | + def cut(lo: int, pieces: Sequence[Sequence[int]], |
| 915 | + ptags: Sequence[Set[str]], |
| 916 | + tokens: Sequence[WorkToken]) -> int: |
| 917 | + c = candidates(lo, pieces, ptags, tokens) |
| 918 | + seen.append((lo, c)) |
| 919 | + return c |
| 920 | + |
| 921 | + def take(pieces: Sequence[Sequence[int]], ptags: Sequence[Set[str]], |
| 922 | + tokens: Sequence[WorkToken], one_case: bool | None, |
| 923 | + site: _group.ClauseSite, |
| 924 | + ambiguities: list[PendingAmbiguity], |
| 925 | + ) -> _group.MaidenIndices | None: |
| 926 | + seen.clear() |
| 927 | + answer = real_take(pieces, ptags, tokens, one_case, site, |
| 928 | + ambiguities) |
| 929 | + for lo, c in seen: |
| 930 | + m = next(v for v in range(1, len(pieces)) |
| 931 | + if _group._is_maiden_marker_piece(pieces[v], tokens)) |
| 932 | + left = list(range(m)) + list(range(lo + 1, len(pieces))) |
| 933 | + view = [pieces[q] for q in left] |
| 934 | + vtags = [ptags[q] for q in left] |
| 935 | + rest = peel_walk(leading_titles(view, vtags, tokens), vtags) |
| 936 | + kept, chained, peel = tail_reading(rest, view, vtags, tokens, |
| 937 | + one_case) |
| 938 | + taken = {left[q] for q in (*kept[peel.names:], *chained) |
| 939 | + if q >= m} |
| 940 | + if any(j < c for j in taken): |
| 941 | + misses.append(" ".join(tokens[i].text for p in pieces |
| 942 | + for i in p)) |
| 943 | + return answer |
| 944 | + |
| 945 | + monkeypatch.setattr(_group, "trailing_candidates", cut) |
| 946 | + monkeypatch.setattr(_group, "_maiden_take", take) |
906 | 947 | for head in _CANDIDATE_HEADS: |
907 | 948 | for n in (1, 2, 3): |
908 | 949 | for tail in itertools.product(_CANDIDATE_WORDS, repeat=n): |
909 | | - text = f"{head} {' '.join(tail)}" |
910 | | - state = _through_group(text) |
911 | | - pieces, ptags = state.pieces[0], state.piece_tags[0] |
912 | | - tokens = state.tokens |
913 | | - lead = leading_titles(pieces, ptags, tokens) |
914 | | - rest = peel_walk(lead, ptags) |
915 | | - if not rest: |
916 | | - continue |
917 | | - kept, chained, peel = tail_reading( |
918 | | - rest, pieces, ptags, tokens, state.one_case) |
919 | | - taken = (set(kept[peel.names:]) | set(chained)) - {rest[0]} |
920 | | - c = trailing_candidates(rest[0], pieces, ptags, tokens) |
921 | | - if any(j < c for j in taken): |
922 | | - misses.append(text) |
| 950 | + _through_group(f"{head} nee Smith {' '.join(tail)}") |
923 | 951 | return misses |
924 | 952 |
|
925 | 953 |
|
926 | | -def test_the_trailing_candidates_cover_what_the_tail_reading_takes() -> None: |
| 954 | +def test_the_trailing_candidates_cover_what_the_tail_reading_takes( |
| 955 | + monkeypatch: pytest.MonkeyPatch) -> None: |
927 | 956 | """M2's take ends the clause at the trailing run the clause-free |
928 | 957 | name reads, which it can only find if every word that run might |
929 | | - hold is in its view. 6 heads x 18 words to a |
930 | | - tail of three, 37,044 parses through `group`, about 2.4s on py3.11 |
| 958 | + hold is in its view. 6 heads x 18 words to a tail of three after |
| 959 | + 'nee Smith', 37,044 parses through `group`, about 2.6s on py3.11 |
931 | 960 | (measured 2026-10-04). |
932 | 961 |
|
933 | 962 | RECORDED NEGATIVE CONTROL, measured 2026-10-04 on this grid: the |
934 | 963 | candidate loop as #601 shipped it (master 8a8459a8), which admitted |
935 | 964 | a roman numeral by its shape only as the clause's LAST word, misses |
936 | | - 288 texts -- every one a numeral in no wordlist ('VI') with a |
937 | | - period-marked title or the merged 'Ph. D.' behind it. The chain |
| 965 | + 432 texts -- every one a numeral in no wordlist ('VI') with |
| 966 | + a period-marked title or the merged 'Ph. D.' behind it. The chain |
938 | 967 | takes the title first, and the walk never holds the flagged |
939 | 968 | 'Ph. D.', so either way the numeral is the walk's last piece and |
940 | 969 | the fork reads it: 'Jane Doe nee Smith VI Prof.' kept maiden 'Smith |
941 | 970 | VI' where 'John Smith VI Prof.' reads suffix 'VI'. A first draft of |
942 | | - the fix covered the title and still missed the 'Ph. D.' half, 111 |
943 | | - texts, which this grid is what found. |
| 971 | + the fix covered the title half and missed the 'Ph. D.' half. |
| 972 | +
|
| 973 | + A first version of this test read plain names through all of |
| 974 | + `group`, so its pieces had the joins applied that the take runs |
| 975 | + before ('de VI Prof.' arrived as one piece); review moved the |
| 976 | + oracle to the take itself. |
944 | 977 | """ |
945 | | - assert _candidate_misses() == [] |
| 978 | + assert _candidate_misses(monkeypatch) == [] |
0 commit comments