From 5faae97567f7b129ba01c9214cf768eb275c7a62 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 17 Oct 2026 18:54:27 -0700 Subject: [PATCH 1/6] Answer "does a name end in this word?" for CJK words in name_usage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A glued honorific peels only if no real name ends in it (rules.md#W2), and the tool could not ask that: its US sources hold no CJK spelling and its Wikidata arm matches labels exactly. Live SPARQL cannot ask it either -- STRENDS over a name class's labels runs past WDQS's 60s limit even with no person join, and full-text search finds a CJK word only where its tokenizer split it out (4 of the family names ending in 殿). So the CJK labels of eleven name-item classes are fetched once into the cache (list each class's items, then their labels by VALUES batch of 1,000; a batch that overruns is halved, and a stopped build resumes), matched locally, and only the matching items' bearers are counted live. The new row also counts people whose family name ends in the word's head and whose given name is its tail (阿部 + 長 for 部長), and shows each matched item's Q-id so a transcription or mislabel can be traced. The listing comes as CSV (73MB of JSON was cut off mid-stream) and is held to a COUNT taken just before it; lexemes, which carry P31 family name too, are skipped. 429s wait out Retry-After. The Census and SSA rows now say they do not apply to a CJK word. Checked against live STRENDS on 20 class x word cells: identical. Refs #707 Co-Authored-By: Claude Opus 5.5 --- tests/v2/test_name_usage_tool.py | 373 ++++++++++++++++++++++++- tools/name_usage.py | 456 +++++++++++++++++++++++++++++-- 2 files changed, 806 insertions(+), 23 deletions(-) diff --git a/tests/v2/test_name_usage_tool.py b/tests/v2/test_name_usage_tool.py index 7a9d5d41..b5cb0e52 100644 --- a/tests/v2/test_name_usage_tool.py +++ b/tests/v2/test_name_usage_tool.py @@ -16,13 +16,14 @@ import io import json import sys +import urllib.error import urllib.parse import urllib.request import zipfile from collections import Counter from pathlib import Path from types import ModuleType -from typing import Any +from typing import Any, Protocol import pytest @@ -323,3 +324,373 @@ def test_main_reports_a_bad_ssa_zip_without_losing_the_other_sources( assert census["family"] == 561 assert (ssa["status"], ssa["given"]) == ("failed", None) assert "not downloaded" not in err # the file exists; it is just wrong + + +# -- the CJK name index (#707) -------------------------------------------- + +@pytest.mark.parametrize(("word", "cjk"), [ + ("关", True), ("씨", True), ("さん", True), ("ｻﾝ", True), + ("佐々木", True), ("Smith", False), ("Sørensen", False), +]) +def test_is_cjk_finds_han_kana_and_hangul(word: str, cjk: bool) -> None: + assert tool.is_cjk(word) is cjk + + +def test_us_sources_say_they_do_not_apply_to_a_cjk_word() -> None: + # A CJK word is not an unlisted name there: the lists hold no + # non-Latin spelling at all, and the report says so in those words. + census = tool.census_usage("議員", tool.read_census(_CENSUS)) + ssa = tool.ssa_usage("議員", {"justice": Counter(M=5)}) + for usage in (census, ssa): + assert usage.status == "unrepresentable" + assert "does not apply to a CJK word" in usage.note + + +def test_labels_query_asks_for_the_batch_by_item_and_filters_to_cjk() -> None: + query = tool.labels_query(["Q1", "Q2"]) + assert "VALUES ?item { wd:Q1 wd:Q2 }" in query + assert "REGEX(STR(?label)" in query + assert "wd:Q101352" in tool.ids_query("Q101352") + + +def _listing(*qids: str) -> list[list[dict[str, str]]]: + """The count query's answer, then the listing's.""" + return [[{"n": str(len(qids))}], + [{"item": f"http://www.wikidata.org/entity/{q}"} for q in qids]] + + +def _labels(*pairs: tuple[str, str]) -> list[dict[str, str]]: + return [{"item": f"http://www.wikidata.org/entity/{q}", "label": label} + for q, label in pairs] + + +def _overrun() -> Exception: + # What WDQS sends at its limit: a 200 whose JSON stops mid-stream. + return json.JSONDecodeError("Expecting ',' delimiter", "{...", 3) + + +@pytest.fixture +def one_class(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + # The listing is fetched as CSV; route it through the _sparql stub so + # one response queue serves every query. + monkeypatch.setattr(tool, "_sparql_csv", + lambda query: tool._bindings(tool._sparql(query))) + monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + monkeypatch.setattr(tool, "NAME_CLASSES", {"Q101352": "family"}) + monkeypatch.setattr(tool, "INDEX_BATCH", 2) + monkeypatch.setattr(tool, "INDEX_MIN_BATCH", 1) + + +def _values(query: str) -> list[str]: + return query.split("VALUES ?item { ")[1].split(" }")[0].split() + + +@pytest.mark.usefixtures("one_class") +def test_a_class_is_listed_once_and_labelled_by_batch( + monkeypatch: pytest.MonkeyPatch) -> None: + queries = _stub_sparql(monkeypatch, [ + *_listing("Q10", "Q9", "Q300", "Q2", "Q9"), # unsorted, one repeat + _labels(("Q2", "向殿")), _labels(("Q10", "井殿")), + ]) + index = tool.load_index(download=True) + assert index is not None + # Sorted by number, not as strings, with the repeat dropped. + assert [_values(q) for q in queries[2:]] == [ + ["wd:Q2", "wd:Q9"], ["wd:Q10", "wd:Q300"]] + assert {(e.qid, e.role, e.label) for e in index.entries} == { + ("Q2", "family", "向殿"), ("Q10", "family", "井殿")} + + +@pytest.mark.usefixtures("one_class") +def test_the_listing_skips_lexemes_and_refuses_anything_else( + monkeypatch: pytest.MonkeyPatch) -> None: + # Lexemes and their senses carry P31 family name too (L1000723 and + # L2873-S1 did, 2026-10-17); no person bears one, and the first two + # builds failed on them. + queries = _stub_sparql(monkeypatch, [ + *_listing("Q1", "L1000723", "L2873-S1", "L5-F2"), _labels()]) + assert tool.load_index(download=True) is not None + assert _values(queries[2]) == ["wd:Q1"] + _stub_sparql(monkeypatch, [[{"n": "1"}], [{"item": "_:b0"}]]) + with pytest.raises(ValueError, match="not an item"): + tool.load_index(download=True, refresh=True) + + +@pytest.mark.usefixtures("one_class") +def test_a_batch_that_overruns_is_halved( + monkeypatch: pytest.MonkeyPatch) -> None: + queries = _stub_sparql(monkeypatch, [ + *_listing("Q1", "Q2"), _overrun(), _labels(("Q1", "向殿")), _labels(), + ]) + index = tool.load_index(download=True) + assert index is not None + assert [_values(q) for q in queries[2:]] == [ + ["wd:Q1", "wd:Q2"], ["wd:Q1"], ["wd:Q2"]] + assert [e.label for e in index.entries] == ["向殿"] + + +@pytest.mark.usefixtures("one_class") +def test_a_resumed_build_queries_only_what_is_missing( + monkeypatch: pytest.MonkeyPatch) -> None: + # An interrupted build keeps its finished batches and its item list, + # and a halved batch is not asked whole again -- it would only + # overrun a second time. + _stub_sparql(monkeypatch, [ + *_listing("Q1", "Q2", "Q3", "Q4"), _overrun(), _labels(("Q1", "向殿")), + OSError("down")]) + with pytest.raises(OSError): + tool.load_index(download=True) + queries = _stub_sparql(monkeypatch, [ + _labels(("Q2", "井殿")), _labels(("Q3", "殿"))]) + index = tool.load_index(download=True) + assert index is not None + assert [_values(q) for q in queries] == [["wd:Q2"], ["wd:Q3", "wd:Q4"]] + assert sorted(e.label for e in index.entries) == ["井殿", "向殿", "殿"] + + +@pytest.mark.usefixtures("one_class") +def test_a_complete_index_loads_offline_and_an_incomplete_one_does_not( + monkeypatch: pytest.MonkeyPatch) -> None: + _stub_sparql(monkeypatch, []) # any query fails the test + assert tool.load_index(download=False) is None + _stub_sparql(monkeypatch, [*_listing("Q1"), _labels(("Q1", "向殿"))]) + fetched = tool.load_index(download=True) + assert fetched is not None and fetched.fetched + _stub_sparql(monkeypatch, []) + index = tool.load_index(download=False) + assert index is not None and index.entries[0].label == "向殿" + + +@pytest.mark.usefixtures("one_class") +def test_refresh_fetches_the_index_again( + monkeypatch: pytest.MonkeyPatch) -> None: + _stub_sparql(monkeypatch, [*_listing("Q1"), _labels(("Q1", "向殿"))]) + tool.load_index(download=True) + _stub_sparql(monkeypatch, [*_listing("Q1"), _labels(("Q1", "井殿"))]) + index = tool.load_index(download=True, refresh=True) + assert index is not None and index.entries[0].label == "井殿" + + +@pytest.mark.usefixtures("one_class") +@pytest.mark.parametrize("failure", [ + urllib.error.HTTPError("u", 400, "Bad Request", {}, io.BytesIO(b"syntax")), # type: ignore[arg-type] + urllib.error.HTTPError("u", 500, "Error", {}, io.BytesIO(b"MalformedQuery")), # type: ignore[arg-type] +]) +def test_an_error_that_is_not_an_overrun_is_not_retried_smaller( + monkeypatch: pytest.MonkeyPatch, failure: Exception) -> None: + queries = _stub_sparql(monkeypatch, [*_listing("Q1", "Q2"), failure]) + with pytest.raises(urllib.error.HTTPError): + tool.load_index(download=True) + assert len(queries) == 3 # count, listing, and the one failure + + +@pytest.mark.usefixtures("one_class") +@pytest.mark.parametrize("failure", [ + urllib.error.HTTPError("u", 504, "Gateway Timeout", {}, io.BytesIO(b"")), # type: ignore[arg-type] + urllib.error.HTTPError("u", 500, "Error", {}, io.BytesIO( # type: ignore[arg-type] + b"java.util.concurrent.TimeoutException")), + TimeoutError("read timed out"), +]) +def test_a_timeout_is_an_overrun( + monkeypatch: pytest.MonkeyPatch, failure: Exception) -> None: + _stub_sparql(monkeypatch, [*_listing("Q1", "Q2"), failure, _labels(), _labels()]) + assert tool.load_index(download=True) is not None + + +@pytest.mark.usefixtures("one_class") +def test_halving_stops_at_the_smallest_batch( + monkeypatch: pytest.MonkeyPatch) -> None: + _stub_sparql(monkeypatch, [*_listing("Q1", "Q2"), _overrun(), _overrun()]) + with pytest.raises(ValueError, match="items 0-1 still overran"): + tool.load_index(download=True) + + +@pytest.mark.usefixtures("one_class") +def test_a_listing_cut_short_is_an_error( + monkeypatch: pytest.MonkeyPatch) -> None: + _stub_sparql(monkeypatch, [[{"n": "5000"}], _listing("Q1", "Q2")[1]]) + with pytest.raises(ValueError, match="returned 2 of 5,000 items"): + tool.load_index(download=True) + + +def test_sparql_csv_reads_the_rows_by_header( + monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(urllib.request, "urlopen", lambda request, timeout: ( + io.BytesIO(b"item\r\nhttp://www.wikidata.org/entity/Q1\r\n"))) + assert tool._sparql_csv("SELECT") == [ + {"item": "http://www.wikidata.org/entity/Q1"}] + + +@pytest.mark.usefixtures("one_class") +def test_an_overrun_listing_is_an_error_not_an_empty_class( + monkeypatch: pytest.MonkeyPatch) -> None: + _stub_sparql(monkeypatch, [_overrun()]) + with pytest.raises(ValueError, match="listing wd:Q101352 items overran"): + tool.load_index(download=True) + + +class _Index(Protocol): + # tools/ is loaded by path, so mypy cannot see NameIndex itself. + def ending_in(self, word: str) -> list[Any]: ... + def boundaries(self, word: str) -> list[Any]: ... + + +def _index(*entries: tuple[str, str, str]) -> _Index: + return tool.NameIndex([tool.IndexEntry(*e) for e in entries], + fetched="2026-10-17") + + +def test_ending_in_matches_labels_that_end_in_the_word_or_equal_it() -> None: + index = _index(("Q1", "family", "向殿"), ("Q2", "family", "殿"), + ("Q3", "family", "殿村"), ("Q4", "given", "二世")) + assert {e.qid for e in index.ending_in("殿")} == {"Q1", "Q2"} + + +def test_boundaries_find_a_family_tail_followed_by_a_whole_given_name() -> None: + # 阿部 + 長 is how 部長 ends a real name without either part ending + # in 部長: the argument #707 makes against peeling it. + index = _index(("Q1", "family", "阿部"), ("Q2", "family", "岡部"), + ("Q3", "given", "長"), ("Q4", "given", "部長代"), + ("Q5", "family", "長")) + splits = index.boundaries("部長") + assert [(s.family_tail, s.given) for s in splits] == [("部", "長")] + assert splits[0].family_items == ["Q1", "Q2"] + assert splits[0].given_items == ["Q3"] + + +def test_ends_with_usage_counts_bearers_of_the_matched_items( + monkeypatch: pytest.MonkeyPatch) -> None: + index = _index(("Q1", "family", "向殿"), ("Q2", "family", "井殿"), + ("Q3", "given", "太郎")) + queries = _stub_sparql(monkeypatch, [ + [{"role": "family", "n": "7"}], + [{"role": "family", "country": "Japan", "n": "6"}, + {"role": "family", "n": "1"}], + ]) + usage = tool.ends_with_usage("殿", index, top=5) + assert "VALUES ?item { wd:Q1 wd:Q2 }" in queries[0] + assert "rdfs:label" not in queries[0].split("UNION")[0] + assert (usage.given, usage.family) == (0, 7) + assert usage.family_countries == [("Japan", 6)] + assert "2 name items end in it: 井殿 Q2, 向殿 Q1" in usage.note + + +def test_ends_with_usage_names_a_single_item_in_the_singular( + monkeypatch: pytest.MonkeyPatch) -> None: + _stub_sparql(monkeypatch, [[{"role": "given", "n": "109"}], []]) + usage = tool.ends_with_usage("二世", _index(("Q18339792", "given", "厄德二世")), + top=5) + assert "1 name item ends in it: 厄德二世 Q18339792" in usage.note + + +@pytest.mark.usefixtures("one_class") +def test_loading_a_cached_index_prints_no_progress( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str]) -> None: + monkeypatch.setattr(tool, "INDEX_BATCH", 1) + _stub_sparql(monkeypatch, [*_listing(*(f"Q{n}" for n in range(1, 51))), + *([_labels()] * 50)]) + tool.load_index(download=True) + assert "50 of 50 items" in capsys.readouterr().err + _stub_sparql(monkeypatch, []) + tool.load_index(download=True) + assert capsys.readouterr().err == "" + + +def test_ends_with_usage_with_no_matching_item_sends_no_query( + monkeypatch: pytest.MonkeyPatch) -> None: + queries = _stub_sparql(monkeypatch, []) + usage = tool.ends_with_usage("議員", _index(("Q1", "family", "向殿")), top=5) + assert queries == [] + assert (usage.status, usage.given, usage.family) == ("ok", 0, 0) + assert "no indexed name item ends in 議員" in usage.note + + +def test_ends_with_usage_sums_chunks_and_says_it_may_double_count( + monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(tool, "ITEM_CHUNK", 1) + index = _index(("Q1", "family", "向殿"), ("Q2", "family", "井殿")) + _stub_sparql(monkeypatch, [ + [{"role": "family", "n": "3"}], [{"role": "family", "n": "3"}], + [{"role": "family", "n": "4"}], [{"role": "family", "n": "4"}], + ]) + usage = tool.ends_with_usage("殿", index, top=5) + assert usage.family == 7 + assert "may count twice" in usage.note + + +def test_ends_with_usage_counts_people_across_a_boundary( + monkeypatch: pytest.MonkeyPatch) -> None: + index = _index(("Q1", "family", "阿部"), ("Q3", "given", "長")) + queries = _stub_sparql(monkeypatch, [ + [{"n": "2", "example": "http://www.wikidata.org/entity/Q99"}], + ]) + usage = tool.ends_with_usage("部長", index, top=5) + assert (usage.given, usage.family) == (0, 0) # nothing ENDS in 部長 + assert "VALUES ?f { wd:Q1 }" in queries[0] + assert "VALUES ?g { wd:Q3 }" in queries[0] + [split] = usage.splits + assert (split.people, split.example) == (2, "Q99") + text = tool.render("部長", [usage]) + assert "family ending 部 + given 長: 2+ people" in text + assert "e.g. 阿部; Q99" in text + + +def test_sparql_waits_out_a_rate_limit_and_retries( + monkeypatch: pytest.MonkeyPatch) -> None: + waits: list[float] = [] + calls: list[int] = [] + + def answer(request: urllib.request.Request, timeout: float) -> io.BytesIO: + calls.append(1) + if len(calls) == 1: + raise urllib.error.HTTPError( + "u", 429, "Too Many Requests", + {"Retry-After": "7"}, io.BytesIO(b"")) # type: ignore[arg-type] + return io.BytesIO(b'{"results": {"bindings": []}}') + monkeypatch.setattr(urllib.request, "urlopen", answer) + monkeypatch.setattr(tool, "_sleep", waits.append) + assert tool._sparql("SELECT") == {"results": {"bindings": []}} + assert waits == [7] + + +def test_main_skips_the_index_for_a_latin_word( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, + capsys: pytest.CaptureFixture[str]) -> None: + monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + monkeypatch.setattr(tool, "load_census", lambda download: None) + monkeypatch.setattr(tool, "load_index", lambda download, refresh=False: ( + pytest.fail("the index was loaded for a Latin word"))) + _stub_sparql(monkeypatch, [[], []]) + assert tool.main(["--json", "knight"]) == 0 + sources = [u["source"] for u in json.loads(capsys.readouterr().out)["knight"]] + assert sources[-1] == "Wikidata people" + + +def test_main_adds_the_ends_with_row_for_a_cjk_word( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, + capsys: pytest.CaptureFixture[str]) -> None: + monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + monkeypatch.setattr(tool, "load_census", lambda download: None) + monkeypatch.setattr(tool, "load_index", lambda download, refresh=False: + _index(("Q1", "family", "向殿"))) + _stub_sparql(monkeypatch, [[], [], [{"role": "family", "n": "1"}], []]) + assert tool.main(["--json", "殿"]) == 0 + rows = json.loads(capsys.readouterr().out)["殿"] + assert rows[-1]["source"] == tool.ENDS_WITH_SOURCE + assert rows[-1]["family"] == 1 + + +def test_main_reports_an_index_that_failed_to_load( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, + capsys: pytest.CaptureFixture[str]) -> None: + monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + monkeypatch.setattr(tool, "load_census", lambda download: None) + + def broken(download: bool, refresh: bool = False) -> None: + raise OSError("down") + monkeypatch.setattr(tool, "load_index", broken) + _stub_sparql(monkeypatch, [[], []]) + assert tool.main(["--json", "殿"]) == 2 + row = json.loads(capsys.readouterr().out)["殿"][-1] + assert (row["status"], row["family"]) == ("failed", None) diff --git a/tools/name_usage.py b/tools/name_usage.py index b711c7f3..d3c481f4 100644 --- a/tools/name_usage.py +++ b/tools/name_usage.py @@ -14,6 +14,7 @@ uv run python tools/name_usage.py McDonald # pass names as written uv run python tools/name_usage.py --no-wikidata knight # offline uv run python tools/name_usage.py --json justice # machine-readable + uv run python tools/name_usage.py 殿 議員 二世 # CJK: ends-with too It reports; it does not decide. Read the counts against the rule's position qualifier: given-name bearers threaten a title or a leading @@ -39,6 +40,15 @@ little and a zero proves nothing. Labels are matched exactly, in the spellings the report lists, so a name with internal capitals is found only when passed as written (`McDonald`, not `mcdonald`). +- Wikidata, name ends with (CC0), for a word holding a Han, kana or + hangul letter only: people bearing a name item whose CJK label ENDS + in the word, which is what a glued honorific's peel risks (#707). The + matching runs against a local index of every CJK label on the + name-item classes, fetched once (resumable; 33 minutes on 2026-10-17) and + refreshed with --refresh-index; the bearers are counted live. It + also counts the names the word ends ACROSS the family/given join, + family ending 部 + given 長 for 部長. A lower bound, like the row + above, and blind to people whose name has no item at all. Both US sources spell names in ASCII letters only, without spaces or punctuation (O'Brien is OBRIEN), so a word is folded the same way @@ -61,14 +71,18 @@ import json import os import re +import shutil import sys +import time import unicodedata +import urllib.error import urllib.parse import urllib.request import zipfile from collections import Counter from collections.abc import Callable from dataclasses import asdict, dataclass, field +from datetime import date from pathlib import Path from typing import Any, Literal, TypeVar @@ -86,6 +100,8 @@ WIKIDATA_ENDPOINT = "https://query.wikidata.org/sparql" USER_AGENT = ("nameparser-name-usage/0.1 " "(https://github.com/derek73/python-nameparser)") +#: Times a rate-limited query is retried after its Retry-After. +SPARQL_RETRIES = 5 #: Label languages a name item is matched in. `mul` is Wikidata's #: language-independent label, which many name items carry; the rest @@ -143,6 +159,9 @@ class Usage: given_stateless: int = 0 family_stateless: int = 0 note: str = "" + #: Where the word can end a name across the family/given boundary, + #: the ends-with row only. + splits: list[Split] = field(default_factory=list) def cache_dir() -> Path: @@ -176,6 +195,13 @@ def us_key(word: str) -> str: return "".join(letters).upper() +def _unrepresentable(source: str, word: str) -> str: + if is_cjk(word): + return (f"does not apply to a CJK word: the {source} list spells " + "names in Latin letters only") + return f"a letter has no ASCII spelling in the {source} list" + + def _lookup_note(word: str, key: str) -> str: return "" if key == word.upper() else f"looked up as {key}" @@ -222,7 +248,7 @@ def census_usage(word: str, table: dict[str, dict[str, str]] | None) -> Usage: key = us_key(word) if not key: usage.status = "unrepresentable" - usage.note = "a letter has no ASCII spelling in the Census list" + usage.note = _unrepresentable("Census", word) return usage row = table.get(key) if row is None: @@ -301,7 +327,7 @@ def ssa_usage(word: str, table: dict[str, Counter[str]] | None) -> Usage: key = us_key(word) if not key: usage.status = "unrepresentable" - usage.note = "a letter has no ASCII spelling in the SSA list" + usage.note = _unrepresentable("SSA", word) return usage sexes = table.get(key.lower()) if not sexes: @@ -330,12 +356,20 @@ def _values(forms: list[str], languages: tuple[str, ...]) -> str: for form in forms for lang in languages) -def _people(forms: list[str], languages: tuple[str, ...]) -> str: +def _label_match(forms: list[str], languages: tuple[str, ...]) -> str: # Every spelling in one VALUES block, so COUNT(DISTINCT ?p) counts a # person reached through two spellings once. + return f"""VALUES ?label {{ {_values(forms, languages)} }} + ?item rdfs:label ?label .""" + + +def _item_match(qids: list[str]) -> str: + return "VALUES ?item { " + " ".join(f"wd:{q}" for q in qids) + " }" + + +def _people(match: str) -> str: return f""" - VALUES ?label {{ {_values(forms, languages)} }} - ?item rdfs:label ?label . + {match} {{ ?p wdt:P735 ?item . BIND("given" AS ?role) }} UNION {{ ?p wdt:P734 ?item . BIND("family" AS ?role) }}""" @@ -344,23 +378,31 @@ def _people(forms: list[str], languages: tuple[str, ...]) -> str: def wikidata_totals_query(forms: list[str], languages: tuple[str, ...] = WIKIDATA_LANGUAGES, ) -> str: - return f""" -SELECT ?role (COUNT(DISTINCT ?p) AS ?n) WHERE {{{_people(forms, languages)} -}} GROUP BY ?role""" + return _totals_query(_label_match(forms, languages)) def wikidata_countries_query(forms: list[str], languages: tuple[str, ...] = WIKIDATA_LANGUAGES, ) -> str: + return _countries_query(_label_match(forms, languages)) + + +def _totals_query(match: str) -> str: + return f""" +SELECT ?role (COUNT(DISTINCT ?p) AS ?n) WHERE {{{_people(match)} +}} GROUP BY ?role""" + + +def _countries_query(match: str) -> str: """People per role and country of citizenship. A country with no English label is named by its Q-id rather than dropped, and an "unknown value" citizenship (a blank-node IRI, not an item) is named as such, so only a person with no P27 at all lands in the empty country. A person with two citizenships counts under both, so these rows may sum past the role's total, which comes from - wikidata_totals_query.""" + _totals_query.""" return f""" -SELECT ?role ?country (COUNT(DISTINCT ?p) AS ?n) WHERE {{{_people(forms, languages)} +SELECT ?role ?country (COUNT(DISTINCT ?p) AS ?n) WHERE {{{_people(match)} OPTIONAL {{ ?p wdt:P27 ?c . OPTIONAL {{ ?c rdfs:label ?cl . FILTER(LANG(?cl) = "en") }} }} BIND(IF(!BOUND(?c), "", @@ -379,28 +421,27 @@ def _bindings(response: dict[str, Any]) -> list[dict[str, str]]: def read_wikidata(totals: list[dict[str, str]], by_country: list[dict[str, str]], top: int) -> Usage: """Raises ValueError on a role the queries cannot produce, rather - than letting it fall through to a count of 0.""" + than letting it fall through to a count of 0. Rows from several + queries over disjoint item sets are summed.""" roles = {"given", "family"} for row in totals + by_country: if row["role"] not in roles: raise ValueError(f"unexpected role {row['role']!r}") counts = {role: 0 for role in roles} for row in totals: - counts[row["role"]] = int(row["n"]) - countries: dict[str, list[tuple[str, int]]] = {role: [] for role in roles} + counts[row["role"]] += int(row["n"]) + countries: dict[str, Counter[str]] = {role: Counter() for role in roles} stateless = {role: 0 for role in roles} for row in by_country: if row.get("country"): - countries[row["role"]].append((row["country"], int(row["n"]))) + countries[row["role"]][row["country"]] += int(row["n"]) else: stateless[row["role"]] += int(row["n"]) - for role in roles: - countries[role].sort(key=lambda item: -item[1]) return Usage( "Wikidata people", given=counts["given"], family=counts["family"], lower_bound=True, - given_countries=countries["given"][:top], - family_countries=countries["family"][:top], + given_countries=countries["given"].most_common(top), + family_countries=countries["family"].most_common(top), given_stateless=stateless["given"], family_stateless=stateless["family"]) @@ -419,18 +460,48 @@ def wikidata_usage(word: str, top: int) -> Usage: return usage +#: Patched out by the tests. +_sleep = time.sleep + + def _sparql(query: str) -> dict[str, Any]: + result: dict[str, Any] = json.loads( + _post(query, "application/sparql-results+json")) + return result + + +def _sparql_csv(query: str) -> list[dict[str, str]]: + """The rows as CSV, for a listing too big for JSON: 700,581 family-name + items are 30MB as CSV and were cut off at 73MB as JSON (2026-10-17).""" + return list(csv.DictReader(io.StringIO(_post(query, "text/csv").decode()))) + + +def _post(query: str, accept: str) -> bytes: # POSTed, not a GET: the VALUES block repeats every spelling once per # label language, so a seven-letter Cyrillic word already pushes the # URL past the endpoint's limit (HTTP 431, #680). body = urllib.parse.urlencode({"query": query}).encode() request = urllib.request.Request(WIKIDATA_ENDPOINT, data=body, headers={ - "Accept": "application/sparql-results+json", + "Accept": accept, "Content-Type": "application/x-www-form-urlencoded; charset=utf-8", "User-Agent": USER_AGENT}) - with urllib.request.urlopen(request, timeout=90) as response: - result: dict[str, Any] = json.load(response) - return result + for attempt in range(SPARQL_RETRIES + 1): + try: + with urllib.request.urlopen(request, timeout=90) as response: + data: bytes = response.read() + return data + except urllib.error.HTTPError as exc: + # WDQS gives each client about a minute of query time a + # minute and answers 429 past it; the index build reaches + # that within a few queries. + if exc.code != 429 or attempt == SPARQL_RETRIES: + raise + retry = exc.headers.get("Retry-After", "") + wait = int(retry) if retry.isdigit() else 60 + print(f"rate-limited by {WIKIDATA_ENDPOINT}; waiting {wait}s", + file=sys.stderr) + _sleep(wait) + raise AssertionError("unreachable") def _download(url: str, path: Path, required_member: str) -> None: @@ -452,6 +523,327 @@ def _download(url: str, path: Path, required_member: str) -> None: tmp.replace(path) +# -- Wikidata CJK name index ---------------------------------------------- +# +# Exact labels answer "is this word a name?"; a glued honorific asks +# whether a name can END in the word (#707). WDQS cannot answer that +# live: STRENDS over every label of every name item ran past its 60s +# limit even with no person join (measured 2026-10-17), and its full-text +# search matches a CJK word only where the tokenizer split it out (4 of +# the ~90 Japanese surnames ending in 殿). So the CJK labels of the name +# classes are fetched once into the cache, a word is matched there, and +# only the bearers of the matching items are counted live. +# +# The fetch lists a class's items, then asks for their labels by VALUES +# batch. Slicing the class inside one query instead (by Q-id suffix) +# scans the whole class every time -- 700,581 family-name items -- so a +# slice of 250 labels overran as surely as one of 25,000. + +#: Name-item classes the index lists, and the role their items play. +#: Read off the P31 statements of known items, 2026-10-17: Japanese and +#: Chinese items usually carry the general class beside their own, but +#: not always, and the undifferentiated kanji surnames (高橋, 阿部 as bare +#: kanji, no reading) carry only theirs. +NAME_CLASSES = { + "Q101352": "family", # family name + "Q19914123": "family", # Japanese family name + "Q88865432": "family", # undifferentiated kanji surname + "Q1093580": "family", # Chinese family name + "Q27951364": "family", # Korean family name + "Q202444": "given", # given name + "Q12308941": "given", # male given name + "Q11879590": "given", # female given name + "Q3409032": "given", # unisex given name + "Q22812564": "given", # Japanese given name + "Q22809413": "given", # Chinese given name +} +INDEX_DIR = "wikidata_cjk_names" +#: Items per label query. Measured 2026-10-17 on family-name items not +#: queried before: 1,000 took 2.2-2.4s and 2,000 took 24s. +INDEX_BATCH = 1000 +#: A batch that overruns is halved; one this small that still overruns +#: is an error, not a reason to keep halving. +INDEX_MIN_BATCH = 50 +#: Items per bearer query; more are counted in several and summed. +ITEM_CHUNK = 500 +ENDS_WITH_SOURCE = "Wikidata people, name ends with" + +#: Kana with the iteration mark 々, the CJK ideograph blocks with their +#: compatibility block, hangul syllables, and halfwidth katakana. One +#: such letter makes a label CJK; the query and is_cjk share it. +_CJK_CHARS = ("々぀-ヿ㐀-鿿가-힯" + "豈-﫿･-ﾟ") +_CJK = re.compile(f"[{_CJK_CHARS}]") + + +def is_cjk(word: str) -> bool: + return _CJK.search(word) is not None + + +@dataclass(frozen=True, slots=True) +class IndexEntry: + qid: str + role: str + label: str + + +@dataclass(slots=True) +class Split: + """People whose family name ends in `family_tail` and whose given + name is `given`, so that the two written together end in the word: + 阿部 + 長 ends in 部長.""" + + family_tail: str + given: str + family_items: list[str] + given_items: list[str] + family_examples: list[str] = field(default_factory=list) + people: int | None = None + example: str = "" + + +@dataclass(slots=True) +class NameIndex: + entries: list[IndexEntry] + fetched: str + + def ending_in(self, word: str) -> list[IndexEntry]: + return [e for e in self.entries if e.label.endswith(word)] + + def boundaries(self, word: str) -> list[Split]: + given: dict[str, set[str]] = {} + for e in self.entries: + if e.role == "given": + given.setdefault(e.label, set()).add(e.qid) + splits = [] + for cut in range(1, len(word)): + tail, rest = word[:cut], word[cut:] + if rest not in given: + continue + family = [e for e in self.entries + if e.role == "family" and e.label.endswith(tail)] + if family: + splits.append(Split( + tail, rest, sorted({e.qid for e in family}), + sorted(given[rest]), + sorted({e.label for e in family})[:3])) + return splits + + +def ids_query(cls: str) -> str: + return f"SELECT ?item WHERE {{ ?item wdt:P31 wd:{cls} }}" + + +def count_query(cls: str) -> str: + return f"SELECT (COUNT(?item) AS ?n) WHERE {{ ?item wdt:P31 wd:{cls} }}" + + +def labels_query(qids: list[str]) -> str: + """The CJK labels of the items, every language.""" + return f""" +SELECT ?item ?label WHERE {{ + {_item_match(qids)} + ?item rdfs:label ?label . + FILTER(REGEX(STR(?label), "[{_CJK_CHARS}]")) +}}""" + + +class _Overrun(Exception): + """The endpoint ran out of time on a query; a smaller one may not.""" + + +class _Missing(Exception): + """Part of the index is not cached and fetching is off.""" + + +def _qid(iri: str) -> str: + qid = iri.rsplit("/", 1)[-1] + if not re.fullmatch(r"Q\d+", qid): + raise ValueError(f"index row names {iri!r}, not an item") + return qid + + +def _index_sparql(query: str, as_csv: bool = False) -> list[dict[str, str]]: + try: + return _sparql_csv(query) if as_csv else _bindings(_sparql(query)) + except urllib.error.HTTPError as exc: + # A 500 is an overrun only when it says so: halving a malformed + # query would send it again and again. + if exc.code in (502, 504) or ( + exc.code == 500 and b"TimeoutException" in exc.read()): + raise _Overrun(f"HTTP {exc.code}") from exc + raise + except (ValueError, TimeoutError, http.client.IncompleteRead) as exc: + # At its limit WDQS cuts a 200 off mid-stream, which surfaces as + # JSON that stops (ValueError) or a short read. + raise _Overrun(f"{type(exc).__name__}: {exc}") from exc + + +def _write(path: Path, data: object) -> None: + tmp = path.with_suffix(".part") + tmp.parent.mkdir(parents=True, exist_ok=True) + tmp.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8") + tmp.replace(path) + + +def _class_ids(cls: str, fetch: bool) -> list[str]: + path = cache_dir() / INDEX_DIR / f"{cls}-ids.json" + if path.exists(): + ids: list[str] = json.loads(path.read_text(encoding="utf-8")) + return ids + if not fetch: + raise _Missing(path.name) + try: + [count] = _index_sparql(count_query(cls)) + rows = _index_sparql(ids_query(cls), as_csv=True) + except _Overrun as exc: + raise ValueError(f"listing wd:{cls} items overran ({exc})") from exc + # CSV has no closing bracket to show a stream cut short, so the + # listing is held to a count taken just before it. The slack is for + # edits landing between the two queries; a cut loses far more. + expected = int(count["n"]) + if len(rows) < expected - expected // 1000: + raise ValueError(f"listing wd:{cls} returned {len(rows):,} of " + f"{expected:,} items; it was cut short") + # Lexemes, their senses and their forms (L1000723, L2873-S1) carry + # P31 family name too, and no person bears one; anything else that + # is not an item is still an error. + ids = sorted({_qid(row["item"]) for row in rows + if not re.search(r"/L\d+(-[SF]\d+)?$", row["item"])}, + key=lambda q: int(q[1:])) + _write(path, ids) + print(f" wd:{cls}: {len(ids):,} items", file=sys.stderr) + return ids + + +def _fetch_batch(cls: str, ids: list[str], start: int, end: int, fetch: bool, + out: list[IndexEntry], dates: set[str]) -> bool: + """Read the labels of ids[start:end] into `out`, querying them if they + are not cached and halving the batch when it overruns. A finished + batch is written at once and a halved one is marked, so a build that + stops resumes where it stopped and never re-asks a halved batch. + True when anything was queried.""" + base = cache_dir() / INDEX_DIR / f"{cls}-{start}-{end}" + done, halved = base.with_suffix(".json"), base.with_suffix(".halved") + if done.exists(): + data = json.loads(done.read_text(encoding="utf-8")) + dates.add(data["fetched"]) + out.extend(IndexEntry(q, NAME_CLASSES[cls], label) + for q, label in data["rows"]) + return False + if not halved.exists(): + if not fetch: + raise _Missing(done.name) + try: + rows = [[_qid(row["item"]), row["label"]] + for row in _index_sparql(labels_query(ids[start:end]))] + except _Overrun as exc: + if end - start <= INDEX_MIN_BATCH: + raise ValueError(f"wd:{cls} items {start}-{end} still " + f"overran ({exc})") from exc + halved.parent.mkdir(parents=True, exist_ok=True) + halved.touch() + else: + today = date.today().isoformat() + _write(done, {"fetched": today, "rows": rows}) + dates.add(today) + out.extend(IndexEntry(q, NAME_CLASSES[cls], label) + for q, label in rows) + return True + middle = (start + end) // 2 + first = _fetch_batch(cls, ids, start, middle, fetch, out, dates) + second = _fetch_batch(cls, ids, middle, end, fetch, out, dates) + return first or second + + +def load_index(download: bool, refresh: bool = False) -> NameIndex | None: + """The cached CJK name index; None when part of it is not cached and + `download` is off. Building it is a one-time fetch of about a + thousand queries, paced by the endpoint's rate limit, and resumable.""" + path = cache_dir() / INDEX_DIR + if refresh and path.exists(): + shutil.rmtree(path) + if download and not path.exists(): + print(f"building the CJK name index in {path} -- a one-time fetch " + "of about half an hour; an interrupted build resumes", + file=sys.stderr) + entries: list[IndexEntry] = [] + dates: set[str] = set() + try: + for cls in NAME_CLASSES: + ids = _class_ids(cls, download) + for start in range(0, len(ids), INDEX_BATCH): + end = min(start + INDEX_BATCH, len(ids)) + fetched = _fetch_batch(cls, ids, start, end, download, + entries, dates) + if fetched and (start // INDEX_BATCH) % 50 == 49: + print(f" wd:{cls}: {end:,} of {len(ids):,} items", + file=sys.stderr) + except _Missing: + return None + return NameIndex(list(dict.fromkeys(entries)), + fetched=min(dates) if dates else "") + + +def _count_items(qids: list[str], top: int) -> tuple[Usage, int]: + chunks = [qids[i:i + ITEM_CHUNK] for i in range(0, len(qids), ITEM_CHUNK)] + totals: list[dict[str, str]] = [] + by_country: list[dict[str, str]] = [] + for chunk in chunks: + totals += _bindings(_sparql(_totals_query(_item_match(chunk)))) + by_country += _bindings(_sparql(_countries_query(_item_match(chunk)))) + return read_wikidata(totals, by_country, top), len(chunks) + + +def _count_split(split: Split) -> None: + family = " ".join(f"wd:{q}" for q in split.family_items) + given = " ".join(f"wd:{q}" for q in split.given_items) + [row] = _bindings(_sparql(f""" +SELECT (COUNT(DISTINCT ?p) AS ?n) (SAMPLE(?p) AS ?example) WHERE {{ + VALUES ?f {{ {family} }} + VALUES ?g {{ {given} }} + ?p wdt:P734 ?f ; wdt:P735 ?g . +}}""")) or [{"n": "0"}] + split.people = int(row["n"]) + split.example = row.get("example", "").rsplit("/", 1)[-1] + + +def ends_with_usage(word: str, index: NameIndex, top: int) -> Usage: + """Bearers of the indexed name items whose label ends in the word, + and of the family/given pairs that end in it together.""" + matches = index.ending_in(word) + qids = sorted({e.qid for e in matches}) + fetched = f"index fetched {index.fetched}" + try: + if qids: + usage, queries = _count_items(qids, top) + # With its Q-id, so an item reached through a transcription or + # a mislabel (厄德二世 is the given name Odo) can be looked up. + named = sorted({(e.label, e.qid) for e in matches}) + shown = ", ".join(f"{label} {qid}" for label, qid in named[:8]) + if len(named) > 8: + shown += ", ..." + twice = (f"summed over {queries} queries, so a person bearing " + "two of these names may count twice" + if queries > 1 else "") + items = "1 name item ends" if len(qids) == 1 else ( + f"{len(qids)} name items end") + usage.note = _join(f"{items} in it: {shown}", + twice, fetched) + else: + usage = Usage("", given=0, family=0, lower_bound=True, + note=_join(f"no indexed name item ends in {word}", + fetched)) + usage.splits = index.boundaries(word) + for split in usage.splits: + _count_split(split) + except (OSError, ValueError, KeyError, http.client.HTTPException) as exc: + return Usage(ENDS_WITH_SOURCE, status="failed", lower_bound=True, + note=f"query failed: {type(exc).__name__}: {exc}") + usage.source = ENDS_WITH_SOURCE + return usage + + # -- output -------------------------------------------------------------- def _attempt(load: Callable[[], T | None]) -> tuple[T | None, str]: @@ -492,6 +884,13 @@ def render(word: str, usages: list[Usage]) -> str: parts.append(f"{stateless} with no citizenship stated") lines.append(f" {'':<{width}} {role} by citizenship: " + ", ".join(parts)) + for split in u.splits: + example = "; ".join(split.family_examples + + ([split.example] if split.example else [])) + lines.append( + f" {'':<{width}} family ending {split.family_tail} + " + f"given {split.given}: {_count(split.people, u)} people" + + (f" (e.g. {example})" if example else "")) return "\n".join(lines) @@ -504,11 +903,18 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--countries", type=int, default=5, metavar="N", help="countries of citizenship to list per role") parser.add_argument("--json", action="store_true") + parser.add_argument("--refresh-index", action="store_true", + help="fetch the CJK name index again") args = parser.parse_args(argv) census, census_error = _attempt( lambda: load_census(download=not args.no_wikidata)) ssa, ssa_error = _attempt(load_ssa) + index: NameIndex | None = None + index_error = "" + if not args.no_wikidata and any(map(is_cjk, args.words)): + index, index_error = _attempt( + lambda: load_index(download=True, refresh=args.refresh_index)) report = {} for word in args.words: @@ -519,6 +925,12 @@ def main(argv: list[str] | None = None) -> int: usages = [census_row, ssa_row] if not args.no_wikidata: usages.append(wikidata_usage(word, args.countries)) + if is_cjk(word): + usages.append( + ends_with_usage(word, index, args.countries) + if index is not None else + Usage(ENDS_WITH_SOURCE, status="failed", lower_bound=True, + note=index_error or "the index did not load")) report[word] = usages if args.json: print(json.dumps({w: [asdict(u) for u in us] for w, us in report.items()}, From e70b834224e2237bdecc135e5002d0311e571980 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 17 Oct 2026 18:54:27 -0700 Subject: [PATCH 2/6] =?UTF-8?q?Peel=20the=20#671=20credential=20words=20of?= =?UTF-8?q?f=20a=20glued=20name,=20except=20=E9=83=A8=E9=95=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 議員/议员, 国会議員/國會議員, 太平紳士/太平绅士, 医学博士/醫學博士, 二世, 社長/課長 and 社长/课长 join GLUED_HONORIFICS, so 田中社長 reads family 田中, suffix 社長 and 陳怡君二世 gives up 二世. Measured with tools/name_usage.py's new ends-with row: no Wikidata name item ends in any of them (二世's one hit is the given name Odo, mislabelled 厄德二世), and no person pairs a family name ending in the word's head with a given name equal to its tail. 部長/部长 stay spaced-only: 長 alone is a borne given name, so 阿部長 can be 阿部 + 長, and the measured zero is a lower bound that does not refute that. Snapshot: four CLDR renderings and 陳怡君二世 lose the glued word to the suffix; cldr_agreement gains one agreeing suffix field and loses none. The JA locale view, measured with the ja extra, does not move. Closes #707 Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- docs/design/decisions.md | 1 + docs/design/rules.md | 27 +++++----- docs/release_log.rst | 2 +- docs/usage.rst | 15 +++--- nameparser/config/suffixes.py | 35 +++++++----- tests/v2/cases.py | 54 +++++++++++++------ tests/v2/test_cases.py | 7 +-- tests/v2/test_parser.py | 26 ++++++--- tools/differential/cldr_agreement.json | 20 +++---- .../differential/cldr_agreement_locales.json | 18 +++---- tools/differential/corpus_cjk.jsonl | 4 ++ tools/differential/corpus_rules.jsonl | 1 + tools/differential/snapshot.jsonl | 14 +++-- 14 files changed, 138 insertions(+), 88 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e74cc1cb..5a783b21 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -253,7 +253,7 @@ Add a dedicated `copy.deepcopy()` round-trip test for it too (see `test_regexes_ **Unknown-key attribute access on `TupleManager`/`RegexTupleManager` warns (1.4, #256) — 2.0 note: the warning became a hard `AttributeError` naming the known keys (shim `TupleManager`); the paragraph below describes the deleted v1 machinery and applies only when reading v1 history.** — a key not currently in the dict emits `DeprecationWarning` naming the miss and the known keys (`_warn_unknown_key`), before falling back to the same `None`/`EMPTY_REGEX` default as before; `.get()` stays silent. Dunder probes (`__deepcopy__`) still raise `AttributeError` outright, and single-underscore probes (`_repr_html_`, IPython's `_ipython_canary_method_should_not_exist_`, etc.) are excluded from the warning too — no real config key starts with `_`, so both guards just keep protocol/introspection probes from misfiring as "typo" warnings. This means internal parser code that reads `self.C.regexes.` unconditionally (e.g. `squash_bidi`'s `bidi`) now warns if a caller's custom `regexes` dict omits that key — a previously-silent partial-override pattern is on the same deprecation path as an actual typo. -**Adding a word to a config set** — first check the *other* sets for the same word (grep `nameparser/config/` or intersect the sets in a `python3 -c`). Real overlaps exist: `do`/`st`/`mc` ∈ `PARTICLES` ∩ `TITLES`/`SUFFIX_ACRONYMS`; `abd` = "ABD" ∈ `SUFFIX_ACRONYMS`; `abu` ∈ `PARTICLES` ∩ `BOUND_GIVEN_NAMES` (position-dependent: leading token → given-name join, mid-name → family-name join). Usually position-dependent and harmless, but can force a guard or an exclusion (the `last_base` all-particles guard; dropping `abd` from `BOUND_GIVEN_NAMES`). Then check the word against real names, not memory: `uv run python tools/name_usage.py ` counts its bearers as a given name and as a family name (US Census surnames, SSA given names, Wikidata with countries of citizenship). Pass a name as it is written (`McDonald`): Wikidata labels match exactly, and no rule rebuilds an internal capital from `mcdonald`. It reports and does not decide — read the counts against the position the set's claim acts on. And check the predicates that test a member's SPELLING rather than its membership: they encode what the set's members happened to look like when they were written. `_pieces.starts_a_credential_run` implemented rules.md#S2's "Latin" as `isascii()`, exact until #661 added `mlađi`, which would have read `mlađi Jones` differently from `stariji Jones`; the design-docs review caught it, after a first draft had credited the test to the wrong clause of S2 and proposed deleting it on a mutation that moved nothing because nothing pinned it. (#679 then dropped the script test on purpose, after measuring what it protected.) Before adding the first member of a new script, or the first accented one, grep the set's consumers for `isascii`, `isalpha` and script tests, and read the rule each one implements. +**Adding a word to a config set** — first check the *other* sets for the same word (grep `nameparser/config/` or intersect the sets in a `python3 -c`). Real overlaps exist: `do`/`st`/`mc` ∈ `PARTICLES` ∩ `TITLES`/`SUFFIX_ACRONYMS`; `abd` = "ABD" ∈ `SUFFIX_ACRONYMS`; `abu` ∈ `PARTICLES` ∩ `BOUND_GIVEN_NAMES` (position-dependent: leading token → given-name join, mid-name → family-name join). Usually position-dependent and harmless, but can force a guard or an exclusion (the `last_base` all-particles guard; dropping `abd` from `BOUND_GIVEN_NAMES`). Then check the word against real names, not memory: `uv run python tools/name_usage.py ` counts its bearers as a given name and as a family name (US Census surnames, SSA given names, Wikidata with countries of citizenship). Pass a name as it is written (`McDonald`): Wikidata labels match exactly, and no rule rebuilds an internal capital from `mcdonald`. It reports and does not decide — read the counts against the position the set's claim acts on. A word that would peel GLUED (`GLUED_HONORIFICS`) asks a different question, whether any name ENDS in it, and for a CJK word the tool answers that too: an ends-with row from a cached index of Wikidata's CJK name labels (built on first use, about 35 minutes), plus the names the word ends across the family/given join (阿部 + 長 for 部長). Read its hits by item before counting them, because labels include transcriptions and mislabels: #707's one hit for 二世 was 厄德二世, the given name Odo with a regnal numeral in its Chinese label, borne by 109 Western people named Odo. And check the predicates that test a member's SPELLING rather than its membership: they encode what the set's members happened to look like when they were written. `_pieces.starts_a_credential_run` implemented rules.md#S2's "Latin" as `isascii()`, exact until #661 added `mlađi`, which would have read `mlađi Jones` differently from `stariji Jones`; the design-docs review caught it, after a first draft had credited the test to the wrong clause of S2 and proposed deleting it on a mutation that moved nothing because nothing pinned it. (#679 then dropped the script test on purpose, after measuring what it protected.) Before adding the first member of a new script, or the first accented one, grep the set's consumers for `isascii`, `isalpha` and script tests, and read the rule each one implements. **Before adding a short/common word to `PARTICLES` globally**, test it mid-string against realistic 3-token names, not just check for English-word collisions: Korean/Vietnamese given names put a short syllable in the middle slot (`Park In Hwan`, `Nguyen To Nga`), and Western names put a bare initial there (`John V. Smith`). A word that looks safe ("nobody is named 'to'") can still swallow a real middle name/initial into the last name once it's a global prefix — confirmed regressions for `to`/`in`/`an`/`ten`/`then` and bare `v` this way (PR #191). diff --git a/docs/design/decisions.md b/docs/design/decisions.md index ebb61150..52f39ea9 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -654,6 +654,7 @@ Declined: - 2026-08 #312 — the peel crosses the family comma and the 间隔号: both answer where a name DIVIDES into surname and given, a question the peel never asks. Provenance matters here: neither gate was argued for the peel specifically — both came with the placement (#312's own framing) — so the crossing was a repair of inherited gates, not a designed-in property. - 2026-08 #319 — a wholly suffix-shaped second run is declined as a peel site (the "田中さん, V." shape), but only when the name's own run offers a site, since a glued honorific is itself part of what makes a run read as suffix-shaped. - 2026-10-17 #671 — SPACED AND AFTER-COMMA ONLY, no glued peel (Derek, 2026-10-17). 議員/议员, 国会議員/國會議員, 太平紳士/太平绅士, 医学博士/醫學博士, 二世 and the job titles 社長/部長/課長 and the simplified 社长/部长/课长 (Derek, 2026-10-17: the characters are shared and the offices are real in both languages; no borne-name data either way, #707) join `SUFFIX_WORDS`; none joins `GLUED_HONORIFICS`. The spaced bar is only that a word standing alone is no name, so they ship; the glued bar ("can it END a name?") needs real-name counts that this change did not get — the Wikidata suffix query timed out and the native-script Wikidata corpus holds none of the words — and 部長 fails it on argument alone (長 is a borne given name, 阿部長). A follow-up can vet 医学博士, 議員 and 太平紳士 one at a time. Consequence measured on `parser_for(locales.ZH)`: after a listed surname each word divides off, `王二世` family 王, suffix 二世 (it read given 二世) and `李部長` family 李, suffix 部長, because the segmenter's division is not a peel and the split-off 二世 is then a whole-token suffix word, the way 王博士 already reads; `陳怡君二世` still reads given 怡君二世 (glued, nothing divides it). ジュニア was already shipped by #661, and #679 declined 二世 as CLDR's generation word ("regnal, and glued"); here it ships spaced-only as the Chinese/Japanese numeral, which does not reverse that. Excluded as the issue stated: 小 (a prefix and a common given-name character) and 冯/馮 for "von" (a top-30 surname). +- 2026-10-17 #707 — the #671 words meet the glued bar, all but 部長/部长 (Derek, 2026-10-17). 議員/议员, 国会議員/國會議員, 太平紳士/太平绅士, 医学博士/醫學博士, 二世, 社長/課長 and 社长/课长 join `GLUED_HONORIFICS`; 部長/部长 stay spaced-only on the #671 argument, since 長 is a borne given name (Wikidata holds the item) and 阿部長 can be 阿部 + 長 — the measurement below found no such bearer, and a zero that is a lower bound does not refute the argument. The measurement is `tools/name_usage.py`'s ends-with row, new in #707: a local index of every CJK label on eleven name-item classes (family name, given name and their Japanese, Chinese, Korean, male, female, unisex and undifferentiated-kanji subclasses; 215,798 labels on 128,695 items, fetched 2026-10-17), matched locally, with the matching items' bearers counted live. Two questions per word: does any name item's label END in it, and does any person bear a family name ending in the word's head and a given name equal to its tail (the 阿部 + 長 shape)? Neither found anything for any spelling, with one exception that is a data error: the only item ending in 二世 is Q18339792, the given name Odo, whose Chinese label carries a regnal numeral (厄德二世), and its 109 bearers are Western people named Odo. Of the boundary shapes, only 二+世, 社+長, 社+长, 部+長 and 部+长 had a family name and a given-name item to pair at all, and each counted 0 people. What makes the zeros evidence: the index agreed exactly with a live `STRENDS` query on all 20 class × word cells it was cross-checked on (five subclasses × 殿, 子, 郎, 田, the largest 403 items); 子 found 439 items and 2,436 given-name bearers, 2,081 of them Japanese; and 田 + 太郎 counted 22 people across the boundary. What limits them: Wikidata's coverage, not the index's — 殿 ends 13 name items where a surname dictionary lists about 90, and Japanese family-name items number 4,919. Two shortcuts were measured and rejected before the index: `STRENDS` over every label of a name class timed out at WDQS's 60s with no person join, and WDQS full-text search found 4 family names ending in 殿, matching only where its tokenizer split the character out. Consequence: the snapshot moved four CLDR renderings, each losing its glued word to `suffix` (`王雅婷婷婷二世議員`, both spellings of `…小惠儀太平紳士`, `馮·陳·怡君·達印·二世醫學博士`), and `陳怡君二世`, which the case table and rules.md W2 pin; cldr_agreement records one suffix field newly agreeing (`二世 醫學博士`, where the peel takes 醫學博士 and the 二世 left behind is a whole-token suffix word) and none regressing. The peel still takes one word: `王雅婷婷婷二世議員` peels 議員 and keeps 二世 in the name. Recompute: `uv run python tools/name_usage.py 議員 议员 国会議員 國會議員 太平紳士 太平绅士 医学博士 醫學博士 二世 社長 部長 課長 社长 部长 课长`; the first run builds the index (about 35 minutes, resumable), and `--refresh-index` rebuilds it. Excluded (the never-given / ambiguous particle line, nameparser/config/particles.py — #360 owns the vocabulary question): diff --git a/docs/design/rules.md b/docs/design/rules.md index f5538407..22aadefa 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -2420,20 +2420,21 @@ W2. Rationale: some East Asian honorifics glue directly onto the end "선생님" → family="선생님" · boundary "王君" → family="王君" · boundary "田中 社長" → suffix="社長" + "田中社長" → suffix="社長" + "陳怡君二世" → suffix="二世" "田中部長" → family="田中部長" · boundary - "陳怡君二世" → family="陳怡君二世" · boundary - Accepted: the credential, office and generational words of #671 - (議員, 国会議員, 太平紳士, 医学博士, 二世, 社長, 部長, 課長 and their - variant spellings) are vocabulary in the spaced and after-comma - positions only and no peel splits them off a glued name: - 陳怡君二世 and 田中部長 keep the word in the name. The glued bar - is "can it END a name?", and none of these was measured against - real names (部長 fails it outright, 阿部長 being 阿部 + 長). The - Han segmenter's own division is not a peel and still exposes - them: under the Chinese pack a listed surname followed by one of - the words divides there and the word reads as a suffix, so - 王二世 reads family 王, suffix 二世 and 李部長 family 李, suffix - 部長, as 王博士 already does. + Accepted: 部長 and its simplified spelling 部长 are vocabulary in + the spaced and after-comma positions only, and no peel splits + them off a glued name: 田中部長 keeps the word in the name, + because 長 alone is a borne given name and 阿部長 can be 阿部 + + 長. The other credential, office and generational words of #671 + (議員, 国会議員, 太平紳士, 医学博士, 二世, 社長, 課長 and their + variant spellings) met the glued bar when measured against real + names (#707) and peel. The Han segmenter's own division is not a + peel and exposes 部長 regardless: under the Chinese pack a + listed surname followed by it divides there and the word reads + as a suffix, so 李部長 reads family 李, suffix 部長, as 王博士 + already does. history: decisions.md#W2 · interacts: W3 · implemented: nameparser/_pipeline/_script_segment.py W3. Rationale: a family name declared by a comma is the writer's diff --git a/docs/release_log.rst b/docs/release_log.rst index 55809d12..973ff583 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -8,7 +8,7 @@ Release Log - **Add junior and senior in other languages as suffixes.** ``Antonio Pavlović mlađi`` gives last ``Pavlović``, suffix ``mlađi``, where 1.4.0 through 2.4.2 gave middle ``Pavlović``, last ``mlađi``. The words come from 45 languages and are read when written as a separate word after the name. :issue:`661`, :issue:`679` - **Remove mai from the default post-nominal acronyms, so a trailing Mai keeps the name.** ``Tran Thi Mai`` read in Vietnamese order (``FAMILY_FIRST_GIVEN_LAST``) gives given ``Mai``, middle ``Thi``, family ``Tran``, where 2.0.0 through 2.4.2 gave suffix ``Mai`` and no given name; ``Nguyen Mai`` gives family ``Mai``, as 1.4.0 did. The cost is that ``John Smith MAI`` gives last ``MAI`` instead of suffix; ``Smith, John, MAI`` and ``John Smith M.A.I.`` still read as credentials. A caller who needs it back adds it: ``Lexicon.default().add(suffix_acronyms={"mai"})``. See the ``suffix-acronym-collisions`` entry of ``docs/design/decisions.md`` (:issue:`687`) - - **Add Chinese and Japanese credential, office and generational words as suffixes.** ``彭德威 议员`` under ``locales.ZH`` gives family ``彭``, given ``德威``, suffix ``议员``, where 2.1 through 2.4.2 gave middle ``议员``; ``王雅婷，議員`` gives suffix ``議員``, where it gave given ``議員``. Read when written as a separate word or after a comma; no peel splits them off a glued name (``陳怡君二世`` is unchanged), though the Chinese pack's surname split exposes them after a listed surname (``王二世``). :issue:`671` + - **Add Chinese and Japanese credential, office and generational words as suffixes.** ``彭德威 议员`` under ``locales.ZH`` gives family ``彭``, given ``德威``, suffix ``议员``, where 2.1 through 2.4.2 gave middle ``议员``; ``田中社長`` gives family ``田中``, suffix ``社長``, where 2.1 through 2.4.2 gave family ``田中社長``. Read written as a separate word, after a comma, or glued to the end of the name, except 部長 and 部长, which are read only as a separate word: ``田中部長`` is unchanged. :issue:`671`, :issue:`707` - **Add the Akan and Indonesian salutations owura and bapak as titles.** ``Bapak Dwi Putro`` gives title ``Bapak``, first ``Dwi``, last ``Putro``, where 1.4.0 through 2.4.2 gave first ``Bapak``, middle ``Dwi``. :issue:`662` - **Fix a title written before a family comma reading as part of the family name.** ``Dr. Smith, John`` gives title ``Dr.``, last ``Smith``, where 1.4.0 through 2.4.2 gave last ``Dr. Smith``. Only listed titles are read there, so ``Xyz. Smith, John`` keeps its family, and a title that is also a surname particle, such as ``St`` or ``Freiherr``, stays in the surname. :issue:`701` - **Add Gov as a title.** ``Gov Smith`` gives title ``Gov``, last ``Smith``, where 1.4.0 through 2.4.2 gave first ``Gov``, and ``Lt. Gov. Smith, John`` gives title ``Lt. Gov.`` with the change above. :issue:`701` diff --git a/docs/usage.rst b/docs/usage.rst index 59d1ab24..34af61c7 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -590,18 +590,19 @@ honorifics rather than parts: ``김민준박사님`` gives up 박사님 entire, not 님 with 박사 left behind. Credential and office words (議員 legislator, 医学博士 doctor of -medicine, 社長 company president, 二世 the second) are recognized in the -spaced form and after a comma only, and are not peeled when glued, -unlike さん. Nobody has checked them against names that end in them, so -the glued form is left whole: +medicine, 社長 company president, 二世 the second) peel when glued as +well: no name Wikidata records ends in one of them. 部長 (department +head) is the exception and is read only when spaced, because 長 alone +is a given name, so 阿部長 can be the surname 阿部 and the given name +長: .. doctest:: - >>> shacho = parse("田中 社長") + >>> shacho = parse("田中社長") >>> shacho.family, shacho.suffix ('田中', '社長') - >>> parse("田中社長").family - '田中社長' + >>> parse("田中部長").family + '田中部長' When a division was a judgment call ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/nameparser/config/suffixes.py b/nameparser/config/suffixes.py index 2c93791f..51d0dc99 100644 --- a/nameparser/config/suffixes.py +++ b/nameparser/config/suffixes.py @@ -165,16 +165,14 @@ 'さま', # ja the kana spelling of 様 'くん', # ja the kana spelling of 君 'ちゃん', # ja familiar/diminutive - # #671: CJK credential, office and generational words. SPACED - # (and after-comma) ONLY -- none is in GLUED_HONORIFICS below. - # The spaced position is a token boundary the writer drew, so - # the vetting bar is just "is a word standing alone ever a - # name?"; the glued bar ("can it END a name?") needs real-name - # data this set was not measured against (decisions.md#W2, #671), - # so no peel splits them off a glued name (陳怡君二世, 田中社長 - # keep the word) and glued peeling is a separate question (#707). - # The Chinese pack's SEGMENTER is not a peel and still exposes - # them after a listed surname: 王二世 and 李部長 divide at the + # #671: CJK credential, office and generational words. The + # spaced position is a token boundary the writer drew, so the + # vetting bar there is just "is a word standing alone ever a + # name?". The glued bar ("can it END a name?") was measured + # separately in #707 (decisions.md#W2), and every entry but + # 部長/部长 passed it and is in GLUED_HONORIFICS below. The + # Chinese pack's SEGMENTER is not a peel and exposes them after a + # listed surname regardless: 王二世 and 李部長 divide at the # surname and the split-off word is then a whole-token suffix # word, as 王博士 already reads. # - 二世: Japanese/Chinese "II" (the generational numeral). @@ -187,8 +185,9 @@ # department head, section chief; Chinese society head, # minister, section chief. The simplified 长 never occurs in # Japanese text, so those entries touch no Japanese name. - # 部長 is the reason glued is out: 長 alone is a borne given - # name (阿部長 = 阿部 + 長). + # 部長/部长 stay spaced-only: 長 alone is a borne given name, + # so 阿部長 can be 阿部 + 長 (#707 declined it on that argument + # though Wikidata held no such bearer). '二世', '議員', '议员', '国会議員', '國會議員', '太平紳士', '太平绅士', @@ -281,6 +280,18 @@ # its Han twin 博士 is not: that collision is Japanese (博士 = # ひろし) and the hangul spelling carries none of it. '씨', '님', '선생님', '교수님', '박사', '박사님', + # #707: the #671 credential, office and generational words. Vetted + # with tools/name_usage.py's ends-with row on 2026-10-17: no + # Wikidata name item with a CJK label ends in any of them (二世's + # one hit, 厄德二世, is the given name Odo with a regnal numeral in + # its label), and no person bears a family name and a given name + # that end in one together. 部長/部长 are absent on the 阿部 + 長 + # argument (above), which a lower-bound zero does not refute. + '二世', + '議員', '议员', '国会議員', '國會議員', + '太平紳士', '太平绅士', + '医学博士', '醫學博士', + '社長', '課長', '社长', '课长', }) """ diff --git a/tests/v2/cases.py b/tests/v2/cases.py index 95043469..a1b4636d 100644 --- a/tests/v2/cases.py +++ b/tests/v2/cases.py @@ -11585,9 +11585,9 @@ def _check_cjk_shape_purity(self) -> None: Case("ja_spaced_diet_member_is_a_suffix", "佐藤恵子 国会議員", {"family": "佐藤恵子", "suffix": "国会議員"}, classification="fix(#671)", - notes="2.4.2 read given '国会議員'. The unspaced twin " - "佐藤恵子国会議員 is unchanged (glued, nothing peels it); " - "family-first comes from the wholly-Han script order"), + notes="2.4.2 read given '国会議員'; family-first comes from " + "the wholly-Han script order. The unspaced twin peels " + "too since #707 (ja_glued_diet_member_peels)"), Case("zh_comma_legislator_is_a_suffix", "王雅婷，議員", {"family": "王雅婷", "suffix": "議員"}, classification="fix(#671)", tolerated=True, @@ -11613,9 +11613,10 @@ def _check_cjk_shape_purity(self) -> None: {"family": "田中部長"}, classification="fix(#271)", notes="the glued bar is unmet: 長 alone is a borne given " - "name (阿部長 = 阿部 + 長), so 部長 is spaced-only. " - "Unchanged from 2.4.2, pinned so GLUED_HONORIFICS " - "cannot gain it unseen"), + "name (阿部長 = 阿部 + 長), so 部長 is spaced-only -- " + "#707 declined it on that argument while peeling its " + "neighbours (ja_glued_job_title_peels). Unchanged from " + "2.4.2, pinned so GLUED_HONORIFICS cannot gain it unseen"), Case("zh_listed_surname_then_new_word_divides", "李部長", {"family": "李", "suffix": "部長"}, locale="zh", classification="fix(#671)", @@ -11623,15 +11624,38 @@ def _check_cjk_shape_purity(self) -> None: "split-off word is a whole-token suffix word, as 王博士 " "already reads; 2.4.2 read given '部長'. 阿部長 stays " "whole because 阿 is no listed surname"), - Case("zh_glued_generational_two_is_not_peeled", "陳怡君二世", - {"family": "陳", "given": "怡君二世"}, - locale="zh", classification="fix(#271)", - notes="the glued form #671 left open: no peel reads it, " - "and 二世 is not vetted against real names. The " - "SEGMENTER's division is another matter: after a " - "listed surname the pack reads 王二世 as family 王, " - "suffix 二世, and 李部長 likewise, because the " - "split-off piece is a whole-token suffix word"), + Case("zh_glued_generational_two_peels", "陳怡君二世", + {"family": "陳", "given": "怡君", "suffix": "二世"}, + locale="zh", classification="fix(#707)", + notes="the glued form #671 left open, vetted in #707: no " + "Wikidata name item ends in 二世 bar a mislabel, so it " + "joined GLUED_HONORIFICS; 2.4.2 read given '怡君二世'. " + "After a listed surname the segmenter already divided " + "it (王二世, family 王, suffix 二世) before any peel"), + Case("ja_glued_diet_member_peels", "佐藤恵子国会議員", + {"family": "佐藤恵子", "suffix": "国会議員"}, + classification="fix(#707)", + notes="2.4.2 read family '佐藤恵子国会議員'. Longest-first: " + "国会議員 peels whole rather than leaving 国会 behind " + "for 議員 to peel off"), + Case("ja_glued_degree_word_peels", "田中医学博士", + {"family": "田中", "suffix": "医学博士"}, + classification="fix(#707)", + notes="2.4.2 read family '田中医学博士'. The four-character " + "word peels where its tail 博士 cannot (ja_glued_degree_" + "stays): 医学博士 ends no name, 博士 is ひろし"), + Case("ja_glued_job_title_peels", "田中社長", + {"family": "田中", "suffix": "社長"}, + classification="fix(#707)", + notes="2.4.2 read family '田中社長'. Unlike 部長, no given " + "name 長 follows a family name ending in 社 in " + "Wikidata, and none is plausible"), + Case("ja_glued_generational_after_katakana_peels", "エリザベス二世", + {"given": "エリザベス", "suffix": "二世"}, + classification="fix(#707)", ambiguities=("given-or-family",), + notes="Elizabeth II as Japanese writes it. 2.4.2 read family " + "'エリザベス二世'; the peeled katakana name keeps the " + "positional default, a lone word read as given"), Case("zh_glued_jun_stays", "王君", {"family": "王君"}, classification="fix(#271)", diff --git a/tests/v2/test_cases.py b/tests/v2/test_cases.py index b8fa1ee1..76bfc60d 100644 --- a/tests/v2/test_cases.py +++ b/tests/v2/test_cases.py @@ -182,9 +182,10 @@ def test_the_field_sweep_sees_the_claims_it_checks() -> None: # 1162 on 2026-10-14: the Агриппина Младшая row (#661, one # given-or-family claim under each order); 1162 -> 1171 on # 2026-10-17: three #701 rows reporting suffix-or-name, under each - # order. - assert claims == 1171, ( - f"the field sweep checks {claims} claims, recorded as 1171 on " + # order; 1171 -> 1174 on 2026-10-17: the エリザベス二世 row (#707, + # one given-or-family claim under each order). + assert claims == 1174, ( + f"the field sweep checks {claims} claims, recorded as 1174 on " f"2026-10-17") diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index eeb7de2d..83d9eee6 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -2106,14 +2106,16 @@ def test_a_decomposed_dotted_letter_is_dotted() -> None: # #671: the credential, office and generational words, one spelling each -# per distinct entry. They are spaced / after-comma vocabulary only; the -# glued bar was not met (#707 revisits it), so none may become a glued -# honorific without this list being edited on purpose. -_CJK_SPACED_WORDS = ( +# per distinct entry, all spaced / after-comma vocabulary. #707 vetted +# the glued bar ("can it END a name?") and every word passed but +# 部長/部长, so the two lists below partition this one: a word moves +# between them only on purpose. +_CJK_GLUED_WORDS = ( "議員", "议员", "国会議員", "國會議員", "太平紳士", "太平绅士", - "医学博士", "醫學博士", "二世", "社長", "部長", "課長", - "社长", "部长", "课长", + "医学博士", "醫學博士", "二世", "社長", "課長", "社长", "课长", ) +_CJK_SPACED_ONLY_WORDS = ("部長", "部长") +_CJK_SPACED_WORDS = _CJK_GLUED_WORDS + _CJK_SPACED_ONLY_WORDS @pytest.mark.parametrize("word", _CJK_SPACED_WORDS) @@ -2129,8 +2131,16 @@ def test_a_cjk_credential_word_is_a_suffix_spaced_and_after_a_comma( assert parser.parse(f", {word}").suffix == word -@pytest.mark.parametrize("word", _CJK_SPACED_WORDS) -def test_a_cjk_credential_word_does_not_peel_off_a_glued_name( +@pytest.mark.parametrize("word", _CJK_GLUED_WORDS) +def test_a_vetted_cjk_credential_word_peels_off_a_glued_name( + word: str) -> None: + parsed = Parser().parse(f"田中{word}") + assert (parsed.family, parsed.suffix) == ("田中", word) + + +@pytest.mark.parametrize("word", _CJK_SPACED_ONLY_WORDS) +def test_a_declined_cjk_credential_word_does_not_peel_off_a_glued_name( word: str) -> None: + # 長 alone is a borne given name, so 阿部長 can be 阿部 + 長 (#707). assert word not in Parser().lexicon.honorific_tails assert Parser().parse(f"田中{word}").suffix == "" diff --git a/tools/differential/cldr_agreement.json b/tools/differential/cldr_agreement.json index 83b07bc3..c494696e 100644 --- a/tools/differential/cldr_agreement.json +++ b/tools/differential/cldr_agreement.json @@ -12982,11 +12982,11 @@ }, "given_names": { "expected": "子媚 惠儀", - "got": "小惠儀太平紳士" + "got": "小惠儀" }, "suffix": { "expected": "小 太平紳士", - "got": "" + "got": "太平紳士" }, "text": "陳子媚先生，小惠儀太平紳士", "title": { @@ -13240,11 +13240,11 @@ }, "given_names": { "expected": "子媚 惠仪", - "got": "小惠仪太平绅士" + "got": "小惠仪" }, "suffix": { "expected": "小 太平绅士", - "got": "" + "got": "太平绅士" }, "text": "陈子媚先生，小惠仪太平绅士", "title": { @@ -13674,11 +13674,7 @@ }, "given_names": { "expected": "怡君 達印", - "got": "陳 怡君 達印 二世醫學博士" - }, - "suffix": { - "expected": "二世 醫學博士", - "got": "" + "got": "陳 怡君 達印" }, "text": "馮·陳·怡君·達印·二世醫學博士", "title": { @@ -13782,11 +13778,11 @@ "zh_Hant/nativeFull/surnameFirst": { "family": { "expected": "王", - "got": "王雅婷婷婷二世議員" + "got": "王雅婷婷婷二世" }, "family_base": { "expected": "王", - "got": "王雅婷婷婷二世議員" + "got": "王雅婷婷婷二世" }, "given_names": { "expected": "雅婷 婷婷", @@ -13794,7 +13790,7 @@ }, "suffix": { "expected": "二世 議員", - "got": "" + "got": "議員" }, "text": "王雅婷婷婷二世議員" }, diff --git a/tools/differential/cldr_agreement_locales.json b/tools/differential/cldr_agreement_locales.json index 6a1987e9..bbe862d8 100644 --- a/tools/differential/cldr_agreement_locales.json +++ b/tools/differential/cldr_agreement_locales.json @@ -640,11 +640,11 @@ }, "given_names": { "expected": "子媚 惠儀", - "got": "小惠儀太平紳士" + "got": "小惠儀" }, "suffix": { "expected": "小 太平紳士", - "got": "" + "got": "太平紳士" }, "text": "陳子媚先生，小惠儀太平紳士", "title": { @@ -839,11 +839,11 @@ }, "given_names": { "expected": "子媚 惠仪", - "got": "小惠仪太平绅士" + "got": "小惠仪" }, "suffix": { "expected": "小 太平绅士", - "got": "" + "got": "太平绅士" }, "text": "陈子媚先生，小惠仪太平绅士", "title": { @@ -1170,11 +1170,7 @@ }, "given_names": { "expected": "怡君 達印", - "got": "陳 怡君 達印 二世醫學博士" - }, - "suffix": { - "expected": "二世 醫學博士", - "got": "" + "got": "陳 怡君 達印" }, "text": "馮·陳·怡君·達印·二世醫學博士", "title": { @@ -1263,11 +1259,11 @@ "zh_Hant/nativeFull/surnameFirst": { "given_names": { "expected": "雅婷 婷婷", - "got": "雅婷婷婷二世議員" + "got": "雅婷婷婷二世" }, "suffix": { "expected": "二世 議員", - "got": "" + "got": "議員" }, "text": "王雅婷婷婷二世議員" }, diff --git a/tools/differential/corpus_cjk.jsonl b/tools/differential/corpus_cjk.jsonl index 9d5cccfa..5bf09ad6 100644 --- a/tools/differential/corpus_cjk.jsonl +++ b/tools/differential/corpus_cjk.jsonl @@ -22,6 +22,7 @@ "みなみ" "ア゙イ タロウ" "ア゛イ タロウ" +"エリザベス二世" "マイケル" "マイケル ア゙イ" "マイケル ジャクソン" @@ -32,6 +33,7 @@ "佐々木 太郎" "佐藤 花子 \"旧姓・田中\" (旧姓 鈴木)" "佐藤恵子 国会議員" +"佐藤恵子国会議員" "司马相如" "夏侯惇" "威廉·莎士比亚" @@ -92,7 +94,9 @@ "田中さん, 様." "田中さん." "田中さん., V." +"田中医学博士" "田中博士" +"田中社長" "田中部長" "諸葛亮" "阿明" diff --git a/tools/differential/corpus_rules.jsonl b/tools/differential/corpus_rules.jsonl index b05e9c36..d306e928 100644 --- a/tools/differential/corpus_rules.jsonl +++ b/tools/differential/corpus_rules.jsonl @@ -572,6 +572,7 @@ "田中さん 様." "田中さん, Dr." "田中さん, PhD" +"田中社長" "田中部長" "陳子媚 二世" "陳怡君二世" diff --git a/tools/differential/snapshot.jsonl b/tools/differential/snapshot.jsonl index 9328c606..9af9e51f 100644 --- a/tools/differential/snapshot.jsonl +++ b/tools/differential/snapshot.jsonl @@ -5366,6 +5366,7 @@ {"name": "ア゛イ タロウ", "given": "ア゛イ", "family": "タロウ", "initials": "ア. タ."} {"name": "アインシュタイン・アルベルト", "order": "FAMILY_FIRST", "given": "アルベルト", "family": "アインシュタイン", "initials": "ア. ア."} {"name": "アルベルト・アインシュタイン", "given": "アルベルト", "family": "アインシュタイン", "initials": "ア. ア."} +{"name": "エリザベス二世", "given": "エリザベス", "suffix": "二世", "ambiguities": ["GIVEN_OR_FAMILY"], "initials": "エ."} {"name": "ジェニファー・ソフィア・フォン・スミス・ジュニア博士", "given": "ソフィア", "middle": "フォン スミス ジュニア博士", "family": "ジェニファー", "initials": "ソ. フ. ス. ジ. ジ."} {"name": "セシリア・ローズ・ブラウン", "given": "セシリア", "middle": "ローズ", "family": "ブラウン", "initials": "セ. ロ. ブ."} {"name": "フォン・スミス・ソフィア・ジェニファー", "order": "FAMILY_FIRST", "given": "スミス", "middle": "ソフィア ジェニファー", "family": "フォン", "initials": "ス. ソ. ジ. フ."} @@ -5395,6 +5396,7 @@ {"name": "佐藤グレース恵子", "order": "FAMILY_FIRST", "family": "佐藤グレース恵子", "initials": "佐."} {"name": "佐藤グレース恵子さん", "order": "FAMILY_FIRST", "family": "佐藤グレース恵子", "suffix": "さん", "initials": "佐."} {"name": "佐藤恵子 国会議員", "family": "佐藤恵子", "suffix": "国会議員", "initials": "佐."} +{"name": "佐藤恵子国会議員", "family": "佐藤恵子", "suffix": "国会議員", "initials": "佐."} {"name": "何炅", "family": "何炅", "initials": "何."} {"name": "何瑞麟", "family": "何瑞麟", "initials": "何."} {"name": "俊年杰思陈", "family": "俊年杰思陈", "initials": "俊."} @@ -5560,7 +5562,7 @@ {"name": "王鑫", "family": "王鑫", "initials": "王."} {"name": "王镇", "family": "王镇", "initials": "王."} {"name": "王雅婷婷婷", "order": "FAMILY_FIRST", "family": "王雅婷婷婷", "initials": "王."} -{"name": "王雅婷婷婷二世議員", "order": "FAMILY_FIRST", "family": "王雅婷婷婷二世議員", "initials": "王."} +{"name": "王雅婷婷婷二世議員", "order": "FAMILY_FIRST", "family": "王雅婷婷婷二世", "suffix": "議員", "initials": "王."} {"name": "王雅婷，議員", "family": "王雅婷", "suffix": "議員", "initials": "王."} {"name": "王雪莹", "family": "王雪莹", "initials": "王."} {"name": "田中 さん", "family": "田中", "suffix": "さん", "initials": "田."} @@ -5587,7 +5589,9 @@ {"name": "田中さん, 様.", "family": "田中", "suffix": "さん, 様.", "initials": "田."} {"name": "田中さん.", "family": "田中", "suffix": "さん.", "initials": "田."} {"name": "田中さん., V.", "given": "V.", "family": "田中", "suffix": "さん.", "initials": "V. 田."} +{"name": "田中医学博士", "family": "田中", "suffix": "医学博士", "initials": "田."} {"name": "田中博士", "family": "田中博士", "initials": "田."} +{"name": "田中社長", "family": "田中", "suffix": "社長", "initials": "田."} {"name": "田中部長", "family": "田中部長", "initials": "田."} {"name": "白文正", "family": "白文正", "initials": "白."} {"name": "石智勇", "family": "石智勇", "initials": "石."} @@ -5633,7 +5637,7 @@ {"name": "陈·陈·怡君·教授，小·达印·博士", "order": "FAMILY_FIRST", "given": "小", "middle": "达印", "family": "陈 陈 怡君", "suffix": "教授, 博士", "initials": "小. 达. 陈. 陈. 怡."} {"name": "陈俊年杰思", "order": "FAMILY_FIRST", "family": "陈俊年杰思", "initials": "陈."} {"name": "陈凯歌", "family": "陈凯歌", "initials": "陈."} -{"name": "陈子媚先生，小惠仪太平绅士", "order": "FAMILY_FIRST", "given": "小惠仪太平绅士", "family": "陈子媚先生", "initials": "小. 陈."} +{"name": "陈子媚先生，小惠仪太平绅士", "order": "FAMILY_FIRST", "given": "小惠仪", "family": "陈子媚先生", "suffix": "太平绅士", "initials": "小. 陈."} {"name": "陈子媚惠仪", "order": "FAMILY_FIRST", "family": "陈子媚惠仪", "initials": "陈."} {"name": "陈静", "family": "陈静", "initials": "陈."} {"name": "陳·怡君·達印", "order": "FAMILY_FIRST", "given": "怡君", "middle": "達印", "family": "陳", "initials": "怡. 達. 陳."} @@ -5641,10 +5645,10 @@ {"name": "陳俊傑", "family": "陳俊傑", "initials": "陳."} {"name": "陳婷", "family": "陳婷", "initials": "陳."} {"name": "陳子媚 二世", "family": "陳子媚", "suffix": "二世", "initials": "陳."} -{"name": "陳子媚先生，小惠儀太平紳士", "order": "FAMILY_FIRST", "given": "小惠儀太平紳士", "family": "陳子媚先生", "initials": "小. 陳."} +{"name": "陳子媚先生，小惠儀太平紳士", "order": "FAMILY_FIRST", "given": "小惠儀", "family": "陳子媚先生", "suffix": "太平紳士", "initials": "小. 陳."} {"name": "陳子媚惠儀", "order": "FAMILY_FIRST", "family": "陳子媚惠儀", "initials": "陳."} {"name": "陳怡君 二世", "family": "陳怡君", "suffix": "二世", "initials": "陳."} -{"name": "陳怡君二世", "family": "陳怡君二世", "initials": "陳."} +{"name": "陳怡君二世", "family": "陳怡君", "suffix": "二世", "initials": "陳."} {"name": "陳怡君，醫學博士", "family": "陳怡君", "suffix": "醫學博士", "initials": "陳."} {"name": "陳明忠", "family": "陳明忠", "initials": "陳."} {"name": "雅婷", "family": "雅婷", "initials": "雅."} @@ -5658,7 +5662,7 @@ {"name": "韩旭", "family": "韩旭", "initials": "韩."} {"name": "韩红", "family": "韩红", "initials": "韩."} {"name": "馮·陳·怡君·達印", "order": "FAMILY_FIRST", "given": "陳", "middle": "怡君 達印", "family": "馮", "initials": "陳. 怡. 達. 馮."} -{"name": "馮·陳·怡君·達印·二世醫學博士", "order": "FAMILY_FIRST", "given": "陳", "middle": "怡君 達印 二世醫學博士", "family": "馮", "initials": "陳. 怡. 達. 二. 馮."} +{"name": "馮·陳·怡君·達印·二世醫學博士", "order": "FAMILY_FIRST", "given": "陳", "middle": "怡君 達印", "family": "馮", "suffix": "二世 醫學博士", "initials": "陳. 怡. 達. 馮."} {"name": "馮定國", "family": "馮定國", "initials": "馮."} {"name": "駒井玲子", "family": "駒井玲子", "initials": "駒."} {"name": "马丁·路德·金", "given": "马丁", "middle": "路德", "family": "金", "initials": "马. 路. 金."} From 05087fe8141b378133f8e0c55240773d70c6fac8 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 17 Oct 2026 19:29:49 -0700 Subject: [PATCH 3/6] Fix three doc claims from the #712 design-docs review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - decisions.md: list 部長/部长 in the honorific_tails Excluded block. Their measurement came back zero like the words that shipped, so without the entry a sweep of the list would add them; the exclusion rests on the 阿部 + 長 argument. - release log: 部長/部长 ARE read after a comma (田中，部長) and under locales.ZH after a listed surname (李部長); the bullet said "only as a separate word". - usage.rst: "no name Wikidata records ends in one of them" was false (厄德二世, the given name Odo); say they are not known to end a name. Co-Authored-By: Claude Opus 5.5 --- docs/design/decisions.md | 1 + docs/release_log.rst | 2 +- docs/usage.rst | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 6ec58482..0a859319 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -962,6 +962,7 @@ Excluded (Lexicon.honorific_tails — a glued tail peels only if it could never - 博士 — 田中博士 is Tanaka Hiroshi as readily as Doctor Tanaka. - 殿 — Japanese surnames end in it (鵜殿, 真殿, four-figure populations); peeling it would cut a real family name in two. - 君 — 王君 is a complete Chinese name; its kana spelling くん does peel. +- 部長, 部长 — 長 alone is a borne given name, so 阿部長 can be 阿部 + 長 (#707). The measurement found no such bearer, as it found none for the words that shipped; the exclusion rests on the argument, which a lower-bound zero does not refute, so a re-measure that again comes back zero is no reason to add them. - जी, and with it the whole spaced trailing set the 2026-09-06 Indic bundle shipped — जी, साहब, साहिब, साहेब, महाराज, সাহেব, বাবু, মহারাজ and Latin rinpoche. None is a honorific_tails entry and none may become one. The evidence is जी's and it carries the set: Banerjee, Mukherjee and Chatterjee end in the substring (बनर्जी/मुखर्जी/चटर्जी), and glued peeling strands a fragment on a bare virama (बनर् + जी). The 殿 criterion, in a non-CJK script — which also shows the criterion is not CJK-specific. The accepted cost is that गांधीजी stays unpeeled, and it is the right trade: an unrecognized honorific costs one field, a strand cuts a real family name in two. Both shapes are pinned in tests/v2/test_locales.py. See #indic-honorifics. Excluded (TITLES): diff --git a/docs/release_log.rst b/docs/release_log.rst index f4c26068..00bef45f 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -8,7 +8,7 @@ Release Log - **Add junior and senior in other languages as suffixes.** ``Antonio Pavlović mlađi`` gives last ``Pavlović``, suffix ``mlađi``, where 1.4.0 through 2.4.2 gave middle ``Pavlović``, last ``mlađi``. The words come from 45 languages and are read when written as a separate word after the name. :issue:`661`, :issue:`679` - **Remove mai from the default post-nominal acronyms, so a trailing Mai keeps the name.** ``Tran Thi Mai`` read in Vietnamese order (``FAMILY_FIRST_GIVEN_LAST``) gives given ``Mai``, middle ``Thi``, family ``Tran``, where 2.0.0 through 2.4.2 gave suffix ``Mai`` and no given name; ``Nguyen Mai`` gives family ``Mai``, as 1.4.0 did. The cost is that ``John Smith MAI`` gives last ``MAI`` instead of suffix; ``Smith, John, MAI`` and ``John Smith M.A.I.`` still read as credentials. A caller who needs it back adds it: ``Lexicon.default().add(suffix_acronyms={"mai"})``. See the ``suffix-acronym-collisions`` entry of ``docs/design/decisions.md`` (:issue:`687`) - - **Add Chinese and Japanese credential, office and generational words as suffixes.** ``彭德威 议员`` under ``locales.ZH`` gives family ``彭``, given ``德威``, suffix ``议员``, where 2.1 through 2.4.2 gave middle ``议员``; ``田中社長`` gives family ``田中``, suffix ``社長``, where 2.1 through 2.4.2 gave family ``田中社長``. Read written as a separate word, after a comma, or glued to the end of the name, except 部長 and 部长, which are read only as a separate word: ``田中部長`` is unchanged. :issue:`671`, :issue:`707` + - **Add Chinese and Japanese credential, office and generational words as suffixes.** ``彭德威 议员`` under ``locales.ZH`` gives family ``彭``, given ``德威``, suffix ``议员``, where 2.1 through 2.4.2 gave middle ``议员``; ``田中社長`` gives family ``田中``, suffix ``社長``, where 2.1 through 2.4.2 gave family ``田中社長``. Read written as a separate word, after a comma, or glued to the end of the name, except that 部長 and 部长 are not split off a glued name (``田中部長`` is unchanged) unless ``locales.ZH`` divides it at a listed surname (``李部長``). :issue:`671`, :issue:`707` - **Add the Akan and Indonesian salutations owura and bapak as titles.** ``Bapak Dwi Putro`` gives title ``Bapak``, first ``Dwi``, last ``Putro``, where 1.4.0 through 2.4.2 gave first ``Bapak``, middle ``Dwi``. :issue:`662` - **Fix a title written before a family comma reading as part of the family name.** ``Dr. Smith, John`` gives title ``Dr.``, last ``Smith``, where 1.4.0 through 2.4.2 gave last ``Dr. Smith``. Only listed titles are read there, so ``Xyz. Smith, John`` keeps its family, and a title that is also a surname particle, such as ``St`` or ``Freiherr``, stays in the surname. :issue:`701` - **Fix a surname particle such as d' or al- reading as a middle name when a listing puts it after the given name.** ``Ivoi, Paul d'`` gives last ``d' Ivoi``, where 1.4.0 through 2.4.2 gave middle ``d'``. The Arabic article reads the same in its plain and assimilated spellings (``al-``, ``az-``, ``aš-``). A particle written joined to the surname, as in ``Paul d'Ivoi``, is unchanged. :issue:`700` diff --git a/docs/usage.rst b/docs/usage.rst index 34af61c7..f1ba218e 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -591,7 +591,7 @@ not 님 with 박사 left behind. Credential and office words (議員 legislator, 医学博士 doctor of medicine, 社長 company president, 二世 the second) peel when glued as -well: no name Wikidata records ends in one of them. 部長 (department +well: they are not known to end a name. 部長 (department head) is the exception and is read only when spaced, because 長 alone is a given name, so 阿部長 can be the surname 阿部 and the given name 長: From 567b94529868f46b8f444b3c65227481d452ddad Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 17 Oct 2026 19:41:26 -0700 Subject: [PATCH 4/6] Record the compound-office split as an accepted cost of #707 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The #707 vetting asked whether a NAME ends in each word and never whether a longer, unlisted office word does. Common ones do, and the peel matches the end of a token spaced or glued, so 衆議院議員, 副社長, 代表取締役社長 and 獣医学博士 split at the listed tail: 田中 副社長 reads given 副, suffix 社長. Kept (Derek, 2026-10-17): before, the whole word stayed in the name, which was no more right. - rules.md W2: a second Accepted clause with two examples - decisions.md#W2: the cost in the #707 entry, with the older 박사 case (김민준의학박사) it had left unrecorded - suffixes.py: the cost beside the #707 entries - AGENTS.md: a third vetting question, "does a common longer word end in it?", since the corpora hold no compound and the snapshot is silent Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- docs/design/decisions.md | 2 +- docs/design/rules.md | 9 +++++++++ nameparser/config/suffixes.py | 3 +++ tools/differential/corpus_rules.jsonl | 2 ++ tools/differential/snapshot.jsonl | 2 ++ 6 files changed, 18 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index dcde9f1c..03c9bec6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -253,7 +253,7 @@ Add a dedicated `copy.deepcopy()` round-trip test for it too (see `test_regexes_ **Unknown-key attribute access on `TupleManager`/`RegexTupleManager` warns (1.4, #256) — 2.0 note: the warning became a hard `AttributeError` naming the known keys (shim `TupleManager`); the paragraph below describes the deleted v1 machinery and applies only when reading v1 history.** — a key not currently in the dict emits `DeprecationWarning` naming the miss and the known keys (`_warn_unknown_key`), before falling back to the same `None`/`EMPTY_REGEX` default as before; `.get()` stays silent. Dunder probes (`__deepcopy__`) still raise `AttributeError` outright, and single-underscore probes (`_repr_html_`, IPython's `_ipython_canary_method_should_not_exist_`, etc.) are excluded from the warning too — no real config key starts with `_`, so both guards just keep protocol/introspection probes from misfiring as "typo" warnings. This means internal parser code that reads `self.C.regexes.` unconditionally (e.g. `squash_bidi`'s `bidi`) now warns if a caller's custom `regexes` dict omits that key — a previously-silent partial-override pattern is on the same deprecation path as an actual typo. -**Adding a word to a config set** — first check the *other* sets for the same word (grep `nameparser/config/` or intersect the sets in a `python3 -c`). Real overlaps exist: `do`/`st`/`mc` ∈ `PARTICLES` ∩ `TITLES`/`SUFFIX_ACRONYMS`; `abd` = "ABD" ∈ `SUFFIX_ACRONYMS`; `abu` ∈ `PARTICLES` ∩ `BOUND_GIVEN_NAMES` (position-dependent: leading token → given-name join, mid-name → family-name join). Usually position-dependent and harmless, but can force a guard or an exclusion (the `last_base` all-particles guard; dropping `abd` from `BOUND_GIVEN_NAMES`). Then check the word against real names, not memory: `uv run python tools/name_usage.py ` counts its bearers as a given name and as a family name (US Census surnames, SSA given names, Wikidata with countries of citizenship). Pass a name as it is written (`McDonald`): Wikidata labels match exactly, and no rule rebuilds an internal capital from `mcdonald`. It reports and does not decide — read the counts against the position the set's claim acts on. A word that would peel GLUED (`GLUED_HONORIFICS`) asks a different question, whether any name ENDS in it, and for a CJK word the tool answers that too: an ends-with row from a cached index of Wikidata's CJK name labels (built on first use, about 35 minutes), plus the names the word ends across the family/given join (阿部 + 長 for 部長). Read its hits by item before counting them, because labels include transcriptions and mislabels: #707's one hit for 二世 was 厄德二世, the given name Odo with a regnal numeral in its Chinese label, borne by 109 Western people named Odo. And check the predicates that test a member's SPELLING rather than its membership: they encode what the set's members happened to look like when they were written. `_pieces.starts_a_credential_run` implemented rules.md#S2's "Latin" as `isascii()`, exact until #661 added `mlađi`, which would have read `mlađi Jones` differently from `stariji Jones`; the design-docs review caught it, after a first draft had credited the test to the wrong clause of S2 and proposed deleting it on a mutation that moved nothing because nothing pinned it. (#679 then dropped the script test on purpose, after measuring what it protected.) Before adding the first member of a new script, or the first accented one, grep the set's consumers for `isascii`, `isalpha` and script tests, and read the rule each one implements. The same goes for the first member holding PUNCTUATION: #700 added `d'` and `az-`, the parse read them right and every snapshot move was an intended one, and `capitalized()` still wrote `D' Ivoi`, because case repair splits a token at its apostrophe or hyphen and asks the vocabulary of the stem. `capitalized()` is invisible to the snapshot (`initials()` is recorded there), so parse such a member's names through `capitalized()` and `capitalized(force=True)` in each case before calling it done. +**Adding a word to a config set** — first check the *other* sets for the same word (grep `nameparser/config/` or intersect the sets in a `python3 -c`). Real overlaps exist: `do`/`st`/`mc` ∈ `PARTICLES` ∩ `TITLES`/`SUFFIX_ACRONYMS`; `abd` = "ABD" ∈ `SUFFIX_ACRONYMS`; `abu` ∈ `PARTICLES` ∩ `BOUND_GIVEN_NAMES` (position-dependent: leading token → given-name join, mid-name → family-name join). Usually position-dependent and harmless, but can force a guard or an exclusion (the `last_base` all-particles guard; dropping `abd` from `BOUND_GIVEN_NAMES`). Then check the word against real names, not memory: `uv run python tools/name_usage.py ` counts its bearers as a given name and as a family name (US Census surnames, SSA given names, Wikidata with countries of citizenship). Pass a name as it is written (`McDonald`): Wikidata labels match exactly, and no rule rebuilds an internal capital from `mcdonald`. It reports and does not decide — read the counts against the position the set's claim acts on. A word that would peel GLUED (`GLUED_HONORIFICS`) asks a different question, whether any name ENDS in it, and for a CJK word the tool answers that too: an ends-with row from a cached index of Wikidata's CJK name labels (built on first use, about 35 minutes), plus the names the word ends across the family/given join (阿部 + 長 for 部長). Then ask a third question the tool cannot: does a common longer word end in it? The peel matches the end of a token, spaced or glued, so a listed tail splits every unlisted compound that ends in it — #707 vetted 議員, 社長 and 医学博士 against names and its review found 衆議院議員, 副社長 and 獣医学博士 cut at the tail (`田中 副社長` → given 副); the corpora hold no compound office word, so the snapshot stays silent. List the common compounds whole or record the split as a cost. Read its hits by item before counting them, because labels include transcriptions and mislabels: #707's one hit for 二世 was 厄德二世, the given name Odo with a regnal numeral in its Chinese label, borne by 109 Western people named Odo. And check the predicates that test a member's SPELLING rather than its membership: they encode what the set's members happened to look like when they were written. `_pieces.starts_a_credential_run` implemented rules.md#S2's "Latin" as `isascii()`, exact until #661 added `mlađi`, which would have read `mlađi Jones` differently from `stariji Jones`; the design-docs review caught it, after a first draft had credited the test to the wrong clause of S2 and proposed deleting it on a mutation that moved nothing because nothing pinned it. (#679 then dropped the script test on purpose, after measuring what it protected.) Before adding the first member of a new script, or the first accented one, grep the set's consumers for `isascii`, `isalpha` and script tests, and read the rule each one implements. The same goes for the first member holding PUNCTUATION: #700 added `d'` and `az-`, the parse read them right and every snapshot move was an intended one, and `capitalized()` still wrote `D' Ivoi`, because case repair splits a token at its apostrophe or hyphen and asks the vocabulary of the stem. `capitalized()` is invisible to the snapshot (`initials()` is recorded there), so parse such a member's names through `capitalized()` and `capitalized(force=True)` in each case before calling it done. **Before adding a short/common word to `PARTICLES` globally**, test it mid-string against realistic 3-token names, not just check for English-word collisions: Korean/Vietnamese given names put a short syllable in the middle slot (`Park In Hwan`, `Nguyen To Nga`), and Western names put a bare initial there (`John V. Smith`). A word that looks safe ("nobody is named 'to'") can still swallow a real middle name/initial into the last name once it's a global prefix — confirmed regressions for `to`/`in`/`an`/`ten`/`then` and bare `v` this way (PR #191). diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 0a859319..67c18280 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -659,7 +659,7 @@ Declined: - 2026-08 #312 — the peel crosses the family comma and the 间隔号: both answer where a name DIVIDES into surname and given, a question the peel never asks. Provenance matters here: neither gate was argued for the peel specifically — both came with the placement (#312's own framing) — so the crossing was a repair of inherited gates, not a designed-in property. - 2026-08 #319 — a wholly suffix-shaped second run is declined as a peel site (the "田中さん, V." shape), but only when the name's own run offers a site, since a glued honorific is itself part of what makes a run read as suffix-shaped. - 2026-10-17 #671 — SPACED AND AFTER-COMMA ONLY, no glued peel (Derek, 2026-10-17). 議員/议员, 国会議員/國會議員, 太平紳士/太平绅士, 医学博士/醫學博士, 二世 and the job titles 社長/部長/課長 and the simplified 社长/部长/课长 (Derek, 2026-10-17: the characters are shared and the offices are real in both languages; no borne-name data either way, #707) join `SUFFIX_WORDS`; none joins `GLUED_HONORIFICS`. The spaced bar is only that a word standing alone is no name, so they ship; the glued bar ("can it END a name?") needs real-name counts that this change did not get — the Wikidata suffix query timed out and the native-script Wikidata corpus holds none of the words — and 部長 fails it on argument alone (長 is a borne given name, 阿部長). A follow-up can vet 医学博士, 議員 and 太平紳士 one at a time. Consequence measured on `parser_for(locales.ZH)`: after a listed surname each word divides off, `王二世` family 王, suffix 二世 (it read given 二世) and `李部長` family 李, suffix 部長, because the segmenter's division is not a peel and the split-off 二世 is then a whole-token suffix word, the way 王博士 already reads; `陳怡君二世` still reads given 怡君二世 (glued, nothing divides it). ジュニア was already shipped by #661, and #679 declined 二世 as CLDR's generation word ("regnal, and glued"); here it ships spaced-only as the Chinese/Japanese numeral, which does not reverse that. Excluded as the issue stated: 小 (a prefix and a common given-name character) and 冯/馮 for "von" (a top-30 surname). -- 2026-10-17 #707 — the #671 words meet the glued bar, all but 部長/部长 (Derek, 2026-10-17). 議員/议员, 国会議員/國會議員, 太平紳士/太平绅士, 医学博士/醫學博士, 二世, 社長/課長 and 社长/课长 join `GLUED_HONORIFICS`; 部長/部长 stay spaced-only on the #671 argument, since 長 is a borne given name (Wikidata holds the item) and 阿部長 can be 阿部 + 長 — the measurement below found no such bearer, and a zero that is a lower bound does not refute the argument. The measurement is `tools/name_usage.py`'s ends-with row, new in #707: a local index of every CJK label on eleven name-item classes (family name, given name and their Japanese, Chinese, Korean, male, female, unisex and undifferentiated-kanji subclasses; 215,798 labels on 128,695 items, fetched 2026-10-17), matched locally, with the matching items' bearers counted live. Two questions per word: does any name item's label END in it, and does any person bear a family name ending in the word's head and a given name equal to its tail (the 阿部 + 長 shape)? Neither found anything for any spelling, with one exception that is a data error: the only item ending in 二世 is Q18339792, the given name Odo, whose Chinese label carries a regnal numeral (厄德二世), and its 109 bearers are Western people named Odo. Of the boundary shapes, only 二+世, 社+長, 社+长, 部+長 and 部+长 had a family name and a given-name item to pair at all, and each counted 0 people. What makes the zeros evidence: the index agreed exactly with a live `STRENDS` query on all 20 class × word cells it was cross-checked on (five subclasses × 殿, 子, 郎, 田, the largest 403 items); 子 found 439 items and 2,436 given-name bearers, 2,081 of them Japanese; and 田 + 太郎 counted 22 people across the boundary. What limits them: Wikidata's coverage, not the index's — 殿 ends 13 name items where a surname dictionary lists about 90, and Japanese family-name items number 4,919. Two shortcuts were measured and rejected before the index: `STRENDS` over every label of a name class timed out at WDQS's 60s with no person join, and WDQS full-text search found 4 family names ending in 殿, matching only where its tokenizer split the character out. Consequence: the snapshot moved four CLDR renderings, each losing its glued word to `suffix` (`王雅婷婷婷二世議員`, both spellings of `…小惠儀太平紳士`, `馮·陳·怡君·達印·二世醫學博士`), and `陳怡君二世`, which the case table and rules.md W2 pin; cldr_agreement records one suffix field newly agreeing (`二世 醫學博士`, where the peel takes 醫學博士 and the 二世 left behind is a whole-token suffix word) and none regressing. The peel still takes one word: `王雅婷婷婷二世議員` peels 議員 and keeps 二世 in the name. Recompute: `uv run python tools/name_usage.py 議員 议员 国会議員 國會議員 太平紳士 太平绅士 医学博士 醫學博士 二世 社長 部長 課長 社长 部长 课长`; the first run builds the index (about 35 minutes, resumable), and `--refresh-index` rebuilds it. +- 2026-10-17 #707 — the #671 words meet the glued bar, all but 部長/部长 (Derek, 2026-10-17). 議員/议员, 国会議員/國會議員, 太平紳士/太平绅士, 医学博士/醫學博士, 二世, 社長/課長 and 社长/课长 join `GLUED_HONORIFICS`; 部長/部长 stay spaced-only on the #671 argument, since 長 is a borne given name (Wikidata holds the item) and 阿部長 can be 阿部 + 長 — the measurement below found no such bearer, and a zero that is a lower bound does not refute the argument. The measurement is `tools/name_usage.py`'s ends-with row, new in #707: a local index of every CJK label on eleven name-item classes (family name, given name and their Japanese, Chinese, Korean, male, female, unisex and undifferentiated-kanji subclasses; 215,798 labels on 128,695 items, fetched 2026-10-17), matched locally, with the matching items' bearers counted live. Two questions per word: does any name item's label END in it, and does any person bear a family name ending in the word's head and a given name equal to its tail (the 阿部 + 長 shape)? Neither found anything for any spelling, with one exception that is a data error: the only item ending in 二世 is Q18339792, the given name Odo, whose Chinese label carries a regnal numeral (厄德二世), and its 109 bearers are Western people named Odo. Of the boundary shapes, only 二+世, 社+長, 社+长, 部+長 and 部+长 had a family name and a given-name item to pair at all, and each counted 0 people. What makes the zeros evidence: the index agreed exactly with a live `STRENDS` query on all 20 class × word cells it was cross-checked on (five subclasses × 殿, 子, 郎, 田, the largest 403 items); 子 found 439 items and 2,436 given-name bearers, 2,081 of them Japanese; and 田 + 太郎 counted 22 people across the boundary. What limits them: Wikidata's coverage, not the index's — 殿 ends 13 name items where a surname dictionary lists about 90, and Japanese family-name items number 4,919. Two shortcuts were measured and rejected before the index: `STRENDS` over every label of a name class timed out at WDQS's 60s with no person join, and WDQS full-text search found 4 family names ending in 殿, matching only where its tokenizer split the character out. Consequence: the snapshot moved four CLDR renderings, each losing its glued word to `suffix` (`王雅婷婷婷二世議員`, both spellings of `…小惠儀太平紳士`, `馮·陳·怡君·達印·二世醫學博士`), and `陳怡君二世`, which the case table and rules.md W2 pin; cldr_agreement records one suffix field newly agreeing (`二世 醫學博士`, where the peel takes 醫學博士 and the 二世 left behind is a whole-token suffix word) and none regressing. The peel still takes one word: `王雅婷婷婷二世議員` peels 議員 and keeps 二世 in the name. Recompute: `uv run python tools/name_usage.py 議員 议员 国会議員 國會議員 太平紳士 太平绅士 医学博士 醫學博士 二世 社長 部長 課長 社长 部长 课长`; the first run builds the index (about 35 minutes, resumable), and `--refresh-index` rebuilds it. ACCEPTED COST, found by the design-docs review and kept (Derek, 2026-10-17): the vetting asked whether a NAME ends in each word and never whether a longer, unlisted OFFICE word does, and common ones do — 衆議院議員 and 参議院議員, 副社長 and 代表取締役社長, 元議員 and 市议员, 副課長, 獣医学博士. The peel matches at the end of the token, spaced or glued, so each splits at the listed tail: `岸田文雄衆議院議員` reads family 岸田文雄衆議院, suffix 議員, and `田中 副社長` given 副, suffix 社長, where before the word stayed whole in the name. Neither reading is right; the compound's head is what the peel leaves behind. The corpora hold no compound office word, which is why the snapshot was silent. Older entries carry the same cost unrecorded until now (`김민준의학박사` reads given 민준의학, suffix 박사). Listing a compound whole would peel it whole, longest first. Excluded (the never-given / ambiguous particle line, nameparser/config/particles.py — #360 owns the vocabulary question): diff --git a/docs/design/rules.md b/docs/design/rules.md index cd672590..90dfb841 100644 --- a/docs/design/rules.md +++ b/docs/design/rules.md @@ -2447,6 +2447,15 @@ W2. Rationale: some East Asian honorifics glue directly onto the end listed surname followed by it divides there and the word reads as a suffix, so 李部長 reads family 李, suffix 部長, as 王博士 already does. + Accepted: a longer office or degree word that ends in a listed + tail is split at it, whether glued or spaced, because the tail + is matched at the end of the token and the longer word is not + listed: 衆議院議員, 副社長, 代表取締役社長 and 獣医学博士 leave + their head (衆議院, 副, 代表取締役, 獣) in the name. Before the + tail was listed the whole word stayed in the name, which was no + more right (#707). + "田中 副社長" → given="副" + "岸田文雄衆議院議員" → family="岸田文雄衆議院" history: decisions.md#W2 · interacts: W3 · implemented: nameparser/_pipeline/_script_segment.py W3. Rationale: a family name declared by a comma is the writer's diff --git a/nameparser/config/suffixes.py b/nameparser/config/suffixes.py index 51d0dc99..12b49f85 100644 --- a/nameparser/config/suffixes.py +++ b/nameparser/config/suffixes.py @@ -287,6 +287,9 @@ # its label), and no person bears a family name and a given name # that end in one together. 部長/部长 are absent on the 阿部 + 長 # argument (above), which a lower-bound zero does not refute. + # Accepted cost: a longer office word ending in one of these is + # split at it (田中 副社長 → given 副, suffix 社長; 獣医学博士 leaves + # 獣), since nothing longer is listed (decisions.md#W2, #707). '二世', '議員', '议员', '国会議員', '國會議員', '太平紳士', '太平绅士', diff --git a/tools/differential/corpus_rules.jsonl b/tools/differential/corpus_rules.jsonl index 175a94f1..dbacbfef 100644 --- a/tools/differential/corpus_rules.jsonl +++ b/tools/differential/corpus_rules.jsonl @@ -568,9 +568,11 @@ "山田 太郎" "山田 花子 旧姓：佐藤" "山田 ﾀﾛｳ" +"岸田文雄衆議院議員" "毛·泽东" "毛泽东" "王君" +"田中 副社長" "田中 社長" "田中. 太郎" "田中、太郎" diff --git a/tools/differential/snapshot.jsonl b/tools/differential/snapshot.jsonl index c662e457..a7601d47 100644 --- a/tools/differential/snapshot.jsonl +++ b/tools/differential/snapshot.jsonl @@ -5482,6 +5482,7 @@ {"name": "山田｢ﾀﾛｰ｣太郎", "given": "太郎", "family": "山田", "nickname": "ﾀﾛｰ", "initials": "太. 山."} {"name": "山田ﾀﾛｳ", "family": "山田ﾀﾛｳ", "initials": "山."} {"name": "岡正雄", "family": "岡正雄", "initials": "岡."} +{"name": "岸田文雄衆議院議員", "family": "岸田文雄衆議院", "suffix": "議員", "initials": "岸."} {"name": "崔晓桐", "family": "崔晓桐", "initials": "崔."} {"name": "布鲁赫·艾达·科妮莉亚·塞萨尔·马丁·冯", "order": "FAMILY_FIRST", "given": "艾达", "middle": "科妮莉亚 塞萨尔 马丁 冯", "family": "布鲁赫", "initials": "艾. 科. 塞. 马. 冯. 布."} {"name": "平賀淳", "family": "平賀淳", "initials": "平."} @@ -5570,6 +5571,7 @@ {"name": "王雅婷，議員", "family": "王雅婷", "suffix": "議員", "initials": "王."} {"name": "王雪莹", "family": "王雪莹", "initials": "王."} {"name": "田中 さん", "family": "田中", "suffix": "さん", "initials": "田."} +{"name": "田中 副社長", "given": "副", "family": "田中", "suffix": "社長", "initials": "副. 田."} {"name": "田中 太郎 旧姓 佐藤 MA", "given": "太郎", "family": "田中", "suffix": "MA", "maiden": "佐藤", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "太. 田."} {"name": "田中 太郎 様", "given": "太郎", "family": "田中", "suffix": "様", "initials": "太. 田."} {"name": "田中 太郎, MA", "given": "太郎", "family": "田中", "suffix": "MA", "ambiguities": ["SUFFIX_OR_NAME"], "initials": "太. 田."} From d6531ef74e78b4ff6e88022cb25ac62caf866009 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 17 Oct 2026 19:58:36 -0700 Subject: [PATCH 5/6] Close name_usage's false-zero paths found in the #712 review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI: since 3.14 an HTTPError left unclosed warns, and the 429 retry dropped each one; it and the overrun path now close the error. False zeros and lost reports: - A class listing no items is refused instead of cached as [], which had made every later ends-with row a silent zero. - Cache files are read through one check: a corrupt or wrongly shaped file raises ValueError naming it and --refresh-index (it raised a KeyError past _attempt, losing every source's answer). A batch records the first and last item it covers and is fetched again when the class's listing no longer matches, rather than answering for other items. - The character class takes supplementary-plane Han (Extension B on) and 〇, and drops the Yijing hexagrams; words are NFC-normalized, so decomposed hangul reaches the row. The cache carries a version and an older one is rebuilt. - A CSV listing that ends mid-line is cut short, not a shorter Q-id. - A failed pairing count fails that pairing (Split.error, exit 2) instead of discarding the row's item counts; pairings are counted in chunks over distinct people, and an empty answer is zero by construction rather than a default. - NAME_CLASSES roles are checked at import; a typo had silently dropped a class from the pairing count. - --refresh-index loads the index for a Latin word too. Tests for each, plus the reviewers' gaps: two classes in one build, every cut of a longer word, the count's slack at both edges, 429 exhaustion and date-form Retry-After, a non-429 error, chunked country sums, 502 and IncompleteRead as overruns, the CSV Accept header. Docs: the corpora now hold the two compound examples; NAME_CLASSES' comment matched to the measured overlap; one build time; 長's item is a Chinese given name; the #671 entry points to #707; index entries, not labels; usage.rst says how 部長 and compounds read. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- docs/design/decisions.md | 4 +- docs/release_log.rst | 2 +- docs/usage.rst | 9 +- tests/v2/test_name_usage_tool.py | 317 ++++++++++++++++++++++++++++++- tools/name_usage.py | 281 +++++++++++++++++++-------- 6 files changed, 523 insertions(+), 92 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 03c9bec6..44d301ea 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -253,7 +253,7 @@ Add a dedicated `copy.deepcopy()` round-trip test for it too (see `test_regexes_ **Unknown-key attribute access on `TupleManager`/`RegexTupleManager` warns (1.4, #256) — 2.0 note: the warning became a hard `AttributeError` naming the known keys (shim `TupleManager`); the paragraph below describes the deleted v1 machinery and applies only when reading v1 history.** — a key not currently in the dict emits `DeprecationWarning` naming the miss and the known keys (`_warn_unknown_key`), before falling back to the same `None`/`EMPTY_REGEX` default as before; `.get()` stays silent. Dunder probes (`__deepcopy__`) still raise `AttributeError` outright, and single-underscore probes (`_repr_html_`, IPython's `_ipython_canary_method_should_not_exist_`, etc.) are excluded from the warning too — no real config key starts with `_`, so both guards just keep protocol/introspection probes from misfiring as "typo" warnings. This means internal parser code that reads `self.C.regexes.` unconditionally (e.g. `squash_bidi`'s `bidi`) now warns if a caller's custom `regexes` dict omits that key — a previously-silent partial-override pattern is on the same deprecation path as an actual typo. -**Adding a word to a config set** — first check the *other* sets for the same word (grep `nameparser/config/` or intersect the sets in a `python3 -c`). Real overlaps exist: `do`/`st`/`mc` ∈ `PARTICLES` ∩ `TITLES`/`SUFFIX_ACRONYMS`; `abd` = "ABD" ∈ `SUFFIX_ACRONYMS`; `abu` ∈ `PARTICLES` ∩ `BOUND_GIVEN_NAMES` (position-dependent: leading token → given-name join, mid-name → family-name join). Usually position-dependent and harmless, but can force a guard or an exclusion (the `last_base` all-particles guard; dropping `abd` from `BOUND_GIVEN_NAMES`). Then check the word against real names, not memory: `uv run python tools/name_usage.py ` counts its bearers as a given name and as a family name (US Census surnames, SSA given names, Wikidata with countries of citizenship). Pass a name as it is written (`McDonald`): Wikidata labels match exactly, and no rule rebuilds an internal capital from `mcdonald`. It reports and does not decide — read the counts against the position the set's claim acts on. A word that would peel GLUED (`GLUED_HONORIFICS`) asks a different question, whether any name ENDS in it, and for a CJK word the tool answers that too: an ends-with row from a cached index of Wikidata's CJK name labels (built on first use, about 35 minutes), plus the names the word ends across the family/given join (阿部 + 長 for 部長). Then ask a third question the tool cannot: does a common longer word end in it? The peel matches the end of a token, spaced or glued, so a listed tail splits every unlisted compound that ends in it — #707 vetted 議員, 社長 and 医学博士 against names and its review found 衆議院議員, 副社長 and 獣医学博士 cut at the tail (`田中 副社長` → given 副); the corpora hold no compound office word, so the snapshot stays silent. List the common compounds whole or record the split as a cost. Read its hits by item before counting them, because labels include transcriptions and mislabels: #707's one hit for 二世 was 厄德二世, the given name Odo with a regnal numeral in its Chinese label, borne by 109 Western people named Odo. And check the predicates that test a member's SPELLING rather than its membership: they encode what the set's members happened to look like when they were written. `_pieces.starts_a_credential_run` implemented rules.md#S2's "Latin" as `isascii()`, exact until #661 added `mlađi`, which would have read `mlađi Jones` differently from `stariji Jones`; the design-docs review caught it, after a first draft had credited the test to the wrong clause of S2 and proposed deleting it on a mutation that moved nothing because nothing pinned it. (#679 then dropped the script test on purpose, after measuring what it protected.) Before adding the first member of a new script, or the first accented one, grep the set's consumers for `isascii`, `isalpha` and script tests, and read the rule each one implements. The same goes for the first member holding PUNCTUATION: #700 added `d'` and `az-`, the parse read them right and every snapshot move was an intended one, and `capitalized()` still wrote `D' Ivoi`, because case repair splits a token at its apostrophe or hyphen and asks the vocabulary of the stem. `capitalized()` is invisible to the snapshot (`initials()` is recorded there), so parse such a member's names through `capitalized()` and `capitalized(force=True)` in each case before calling it done. +**Adding a word to a config set** — first check the *other* sets for the same word (grep `nameparser/config/` or intersect the sets in a `python3 -c`). Real overlaps exist: `do`/`st`/`mc` ∈ `PARTICLES` ∩ `TITLES`/`SUFFIX_ACRONYMS`; `abd` = "ABD" ∈ `SUFFIX_ACRONYMS`; `abu` ∈ `PARTICLES` ∩ `BOUND_GIVEN_NAMES` (position-dependent: leading token → given-name join, mid-name → family-name join). Usually position-dependent and harmless, but can force a guard or an exclusion (the `last_base` all-particles guard; dropping `abd` from `BOUND_GIVEN_NAMES`). Then check the word against real names, not memory: `uv run python tools/name_usage.py ` counts its bearers as a given name and as a family name (US Census surnames, SSA given names, Wikidata with countries of citizenship). Pass a name as it is written (`McDonald`): Wikidata labels match exactly, and no rule rebuilds an internal capital from `mcdonald`. It reports and does not decide — read the counts against the position the set's claim acts on. A word that would peel GLUED (`GLUED_HONORIFICS`) asks a different question, whether any name ENDS in it, and for a CJK word the tool answers that too: an ends-with row from a cached index of Wikidata's CJK name labels (built on first use; 33 minutes on 2026-10-17), plus the names the word ends across the family/given join (阿部 + 長 for 部長). Read the tool's hits by item before counting them, because labels include transcriptions and mislabels: #707's one hit for 二世 was 厄德二世, the given name Odo with a regnal numeral in its Chinese label, borne by 109 Western people named Odo. Then ask a third question the tool cannot: does a common longer word end in it? The peel matches the end of a token, spaced or glued, so a listed tail splits every unlisted compound that ends in it — #707 vetted 議員, 社長 and 医学博士 against names and its review found 衆議院議員, 副社長 and 獣医学博士 cut at the tail (`田中 副社長` → given 副); no corpus held a compound office word until #707's rules.md examples added two, so the snapshot was silent about the class. List the common compounds whole or record the split as a cost. And check the predicates that test a member's SPELLING rather than its membership: they encode what the set's members happened to look like when they were written. `_pieces.starts_a_credential_run` implemented rules.md#S2's "Latin" as `isascii()`, exact until #661 added `mlađi`, which would have read `mlađi Jones` differently from `stariji Jones`; the design-docs review caught it, after a first draft had credited the test to the wrong clause of S2 and proposed deleting it on a mutation that moved nothing because nothing pinned it. (#679 then dropped the script test on purpose, after measuring what it protected.) Before adding the first member of a new script, or the first accented one, grep the set's consumers for `isascii`, `isalpha` and script tests, and read the rule each one implements. The same goes for the first member holding PUNCTUATION: #700 added `d'` and `az-`, the parse read them right and every snapshot move was an intended one, and `capitalized()` still wrote `D' Ivoi`, because case repair splits a token at its apostrophe or hyphen and asks the vocabulary of the stem. `capitalized()` is invisible to the snapshot (`initials()` is recorded there), so parse such a member's names through `capitalized()` and `capitalized(force=True)` in each case before calling it done. **Before adding a short/common word to `PARTICLES` globally**, test it mid-string against realistic 3-token names, not just check for English-word collisions: Korean/Vietnamese given names put a short syllable in the middle slot (`Park In Hwan`, `Nguyen To Nga`), and Western names put a bare initial there (`John V. Smith`). A word that looks safe ("nobody is named 'to'") can still swallow a real middle name/initial into the last name once it's a global prefix — confirmed regressions for `to`/`in`/`an`/`ten`/`then` and bare `v` this way (PR #191). diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 67c18280..12f73c5a 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -658,8 +658,8 @@ Declined: - 2026-08 #308 — an entry peels only where it can never end a name: 씨/님/さん/様/先生 peel; 양/군/氏/博士/殿 stay spaced-only; 君 is in neither set while its kana spelling くん peels. The vocabulary carries the license, so no structural or per-script gate stands over the peel. - 2026-08 #312 — the peel crosses the family comma and the 间隔号: both answer where a name DIVIDES into surname and given, a question the peel never asks. Provenance matters here: neither gate was argued for the peel specifically — both came with the placement (#312's own framing) — so the crossing was a repair of inherited gates, not a designed-in property. - 2026-08 #319 — a wholly suffix-shaped second run is declined as a peel site (the "田中さん, V." shape), but only when the name's own run offers a site, since a glued honorific is itself part of what makes a run read as suffix-shaped. -- 2026-10-17 #671 — SPACED AND AFTER-COMMA ONLY, no glued peel (Derek, 2026-10-17). 議員/议员, 国会議員/國會議員, 太平紳士/太平绅士, 医学博士/醫學博士, 二世 and the job titles 社長/部長/課長 and the simplified 社长/部长/课长 (Derek, 2026-10-17: the characters are shared and the offices are real in both languages; no borne-name data either way, #707) join `SUFFIX_WORDS`; none joins `GLUED_HONORIFICS`. The spaced bar is only that a word standing alone is no name, so they ship; the glued bar ("can it END a name?") needs real-name counts that this change did not get — the Wikidata suffix query timed out and the native-script Wikidata corpus holds none of the words — and 部長 fails it on argument alone (長 is a borne given name, 阿部長). A follow-up can vet 医学博士, 議員 and 太平紳士 one at a time. Consequence measured on `parser_for(locales.ZH)`: after a listed surname each word divides off, `王二世` family 王, suffix 二世 (it read given 二世) and `李部長` family 李, suffix 部長, because the segmenter's division is not a peel and the split-off 二世 is then a whole-token suffix word, the way 王博士 already reads; `陳怡君二世` still reads given 怡君二世 (glued, nothing divides it). ジュニア was already shipped by #661, and #679 declined 二世 as CLDR's generation word ("regnal, and glued"); here it ships spaced-only as the Chinese/Japanese numeral, which does not reverse that. Excluded as the issue stated: 小 (a prefix and a common given-name character) and 冯/馮 for "von" (a top-30 surname). -- 2026-10-17 #707 — the #671 words meet the glued bar, all but 部長/部长 (Derek, 2026-10-17). 議員/议员, 国会議員/國會議員, 太平紳士/太平绅士, 医学博士/醫學博士, 二世, 社長/課長 and 社长/课长 join `GLUED_HONORIFICS`; 部長/部长 stay spaced-only on the #671 argument, since 長 is a borne given name (Wikidata holds the item) and 阿部長 can be 阿部 + 長 — the measurement below found no such bearer, and a zero that is a lower bound does not refute the argument. The measurement is `tools/name_usage.py`'s ends-with row, new in #707: a local index of every CJK label on eleven name-item classes (family name, given name and their Japanese, Chinese, Korean, male, female, unisex and undifferentiated-kanji subclasses; 215,798 labels on 128,695 items, fetched 2026-10-17), matched locally, with the matching items' bearers counted live. Two questions per word: does any name item's label END in it, and does any person bear a family name ending in the word's head and a given name equal to its tail (the 阿部 + 長 shape)? Neither found anything for any spelling, with one exception that is a data error: the only item ending in 二世 is Q18339792, the given name Odo, whose Chinese label carries a regnal numeral (厄德二世), and its 109 bearers are Western people named Odo. Of the boundary shapes, only 二+世, 社+長, 社+长, 部+長 and 部+长 had a family name and a given-name item to pair at all, and each counted 0 people. What makes the zeros evidence: the index agreed exactly with a live `STRENDS` query on all 20 class × word cells it was cross-checked on (five subclasses × 殿, 子, 郎, 田, the largest 403 items); 子 found 439 items and 2,436 given-name bearers, 2,081 of them Japanese; and 田 + 太郎 counted 22 people across the boundary. What limits them: Wikidata's coverage, not the index's — 殿 ends 13 name items where a surname dictionary lists about 90, and Japanese family-name items number 4,919. Two shortcuts were measured and rejected before the index: `STRENDS` over every label of a name class timed out at WDQS's 60s with no person join, and WDQS full-text search found 4 family names ending in 殿, matching only where its tokenizer split the character out. Consequence: the snapshot moved four CLDR renderings, each losing its glued word to `suffix` (`王雅婷婷婷二世議員`, both spellings of `…小惠儀太平紳士`, `馮·陳·怡君·達印·二世醫學博士`), and `陳怡君二世`, which the case table and rules.md W2 pin; cldr_agreement records one suffix field newly agreeing (`二世 醫學博士`, where the peel takes 醫學博士 and the 二世 left behind is a whole-token suffix word) and none regressing. The peel still takes one word: `王雅婷婷婷二世議員` peels 議員 and keeps 二世 in the name. Recompute: `uv run python tools/name_usage.py 議員 议员 国会議員 國會議員 太平紳士 太平绅士 医学博士 醫學博士 二世 社長 部長 課長 社长 部长 课长`; the first run builds the index (about 35 minutes, resumable), and `--refresh-index` rebuilds it. ACCEPTED COST, found by the design-docs review and kept (Derek, 2026-10-17): the vetting asked whether a NAME ends in each word and never whether a longer, unlisted OFFICE word does, and common ones do — 衆議院議員 and 参議院議員, 副社長 and 代表取締役社長, 元議員 and 市议员, 副課長, 獣医学博士. The peel matches at the end of the token, spaced or glued, so each splits at the listed tail: `岸田文雄衆議院議員` reads family 岸田文雄衆議院, suffix 議員, and `田中 副社長` given 副, suffix 社長, where before the word stayed whole in the name. Neither reading is right; the compound's head is what the peel leaves behind. The corpora hold no compound office word, which is why the snapshot was silent. Older entries carry the same cost unrecorded until now (`김민준의학박사` reads given 민준의학, suffix 박사). Listing a compound whole would peel it whole, longest first. +- 2026-10-17 #671 — SPACED AND AFTER-COMMA ONLY, no glued peel (Derek, 2026-10-17). 議員/议员, 国会議員/國會議員, 太平紳士/太平绅士, 医学博士/醫學博士, 二世 and the job titles 社長/部長/課長 and the simplified 社长/部长/课长 (Derek, 2026-10-17: the characters are shared and the offices are real in both languages; no borne-name data either way, #707) join `SUFFIX_WORDS`; none joins `GLUED_HONORIFICS`. The spaced bar is only that a word standing alone is no name, so they ship; the glued bar ("can it END a name?") needs real-name counts that this change did not get — the Wikidata suffix query timed out and the native-script Wikidata corpus holds none of the words — and 部長 fails it on argument alone (長 is a borne given name, 阿部長). A follow-up can vet 医学博士, 議員 and 太平紳士 one at a time. Consequence measured on `parser_for(locales.ZH)`: after a listed surname each word divides off, `王二世` family 王, suffix 二世 (it read given 二世) and `李部長` family 李, suffix 部長, because the segmenter's division is not a peel and the split-off 二世 is then a whole-token suffix word, the way 王博士 already reads; `陳怡君二世` still reads given 怡君二世 (glued, nothing divides it; superseded for the glued question by the #707 entry below). ジュニア was already shipped by #661, and #679 declined 二世 as CLDR's generation word ("regnal, and glued"); here it ships spaced-only as the Chinese/Japanese numeral, which does not reverse that. Excluded as the issue stated: 小 (a prefix and a common given-name character) and 冯/馮 for "von" (a top-30 surname). +- 2026-10-17 #707 — the #671 words meet the glued bar, all but 部長/部长 (Derek, 2026-10-17). 議員/议员, 国会議員/國會議員, 太平紳士/太平绅士, 医学博士/醫學博士, 二世, 社長/課長 and 社长/课长 join `GLUED_HONORIFICS`; 部長/部长 stay spaced-only on the #671 argument, since 長 is a borne given name (Wikidata holds it as a Chinese given name, Q96240389; the Japanese reading rests on the argument) and 阿部長 can be 阿部 + 長 — the measurement below found no such bearer, and a zero that is a lower bound does not refute the argument. The measurement is `tools/name_usage.py`'s ends-with row, new in #707: a local index of every CJK label on eleven name-item classes (family name, given name and their Japanese, Chinese, Korean, male, female, unisex and undifferentiated-kanji subclasses; 215,798 index entries on 128,695 items, 215,696 distinct item-label pairs, fetched 2026-10-17 by the first index, whose character class stopped at the Basic Multilingual Plane), matched locally, with the matching items' bearers counted live. Two questions per word: does any name item's label END in it, and does any person bear a family name ending in the word's head and a given name equal to its tail (the 阿部 + 長 shape)? Neither found anything for any spelling, with one exception that is a data error: the only item ending in 二世 is Q18339792, the given name Odo, whose Chinese label carries a regnal numeral (厄德二世), and its 109 bearers are Western people named Odo. Of the boundary shapes, only 二+世, 社+長, 社+长, 部+長 and 部+长 had a family name and a given-name item to pair at all, and each counted 0 people. What makes the zeros evidence: the index agreed exactly with a live `STRENDS` query on all 20 class × word cells it was cross-checked on (five subclasses × 殿, 子, 郎, 田, the largest 403 items); 子 found 439 items and 2,436 given-name bearers, 2,081 of them Japanese; and 田 + 太郎 counted 22 people across the boundary. What limits them: Wikidata's coverage, not the index's — 殿 ends 13 name items where a surname dictionary lists about 90, and Japanese family-name items number 4,919. Two shortcuts were measured and rejected before the index: `STRENDS` over every label of a name class timed out at WDQS's 60s with no person join, and WDQS full-text search found 4 family names ending in 殿, matching only where its tokenizer split the character out. Consequence: the snapshot moved four CLDR renderings, each losing its glued word to `suffix` (`王雅婷婷婷二世議員`, both spellings of `…小惠儀太平紳士`, `馮·陳·怡君·達印·二世醫學博士`), and `陳怡君二世`, which the case table and rules.md W2 pin; cldr_agreement records one suffix field newly agreeing (`二世 醫學博士`, where the peel takes 醫學博士 and the 二世 left behind is a whole-token suffix word) and none regressing. The peel still takes one word: `王雅婷婷婷二世議員` peels 議員 and keeps 二世 in the name. Recompute: `uv run python tools/name_usage.py 議員 议员 国会議員 國會議員 太平紳士 太平绅士 医学博士 醫學博士 二世 社長 部長 課長 社长 部长 课长`; the first run builds the index (33 minutes on 2026-10-17, resumable), and `--refresh-index` rebuilds it. ACCEPTED COST, found by the design-docs review and kept (Derek, 2026-10-17): the vetting asked whether a NAME ends in each word and never whether a longer, unlisted OFFICE word does, and common ones do — 衆議院議員 and 参議院議員, 副社長 and 代表取締役社長, 元議員 and 市议员, 副課長, 獣医学博士. The peel matches at the end of the token, spaced or glued, so each splits at the listed tail: `岸田文雄衆議院議員` reads family 岸田文雄衆議院, suffix 議員, and `田中 副社長` given 副, suffix 社長, where before the word stayed whole in the name. Neither reading is right; the compound's head is what the peel leaves behind. No corpus held a compound office word when the review found this, which is why the snapshot was silent; the two W2 examples now put `岸田文雄衆議院議員` and `田中 副社長` in corpus_rules.jsonl. Older entries carry the same cost unrecorded until now (`김민준의학박사` reads given 민준의학, suffix 박사). Listing a compound whole would peel it whole, longest first. Excluded (the never-given / ambiguous particle line, nameparser/config/particles.py — #360 owns the vocabulary question): diff --git a/docs/release_log.rst b/docs/release_log.rst index 00bef45f..5136cbc5 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -8,7 +8,7 @@ Release Log - **Add junior and senior in other languages as suffixes.** ``Antonio Pavlović mlađi`` gives last ``Pavlović``, suffix ``mlađi``, where 1.4.0 through 2.4.2 gave middle ``Pavlović``, last ``mlađi``. The words come from 45 languages and are read when written as a separate word after the name. :issue:`661`, :issue:`679` - **Remove mai from the default post-nominal acronyms, so a trailing Mai keeps the name.** ``Tran Thi Mai`` read in Vietnamese order (``FAMILY_FIRST_GIVEN_LAST``) gives given ``Mai``, middle ``Thi``, family ``Tran``, where 2.0.0 through 2.4.2 gave suffix ``Mai`` and no given name; ``Nguyen Mai`` gives family ``Mai``, as 1.4.0 did. The cost is that ``John Smith MAI`` gives last ``MAI`` instead of suffix; ``Smith, John, MAI`` and ``John Smith M.A.I.`` still read as credentials. A caller who needs it back adds it: ``Lexicon.default().add(suffix_acronyms={"mai"})``. See the ``suffix-acronym-collisions`` entry of ``docs/design/decisions.md`` (:issue:`687`) - - **Add Chinese and Japanese credential, office and generational words as suffixes.** ``彭德威 议员`` under ``locales.ZH`` gives family ``彭``, given ``德威``, suffix ``议员``, where 2.1 through 2.4.2 gave middle ``议员``; ``田中社長`` gives family ``田中``, suffix ``社長``, where 2.1 through 2.4.2 gave family ``田中社長``. Read written as a separate word, after a comma, or glued to the end of the name, except that 部長 and 部长 are not split off a glued name (``田中部長`` is unchanged) unless ``locales.ZH`` divides it at a listed surname (``李部長``). :issue:`671`, :issue:`707` + - **Add Chinese and Japanese credential, office and generational words as suffixes.** ``彭德威 议员`` under ``locales.ZH`` gives family ``彭``, given ``德威``, suffix ``议员``, where 2.1 through 2.4.2 gave middle ``议员``; ``田中社長`` gives family ``田中``, suffix ``社長``, where 2.1 through 2.4.2 gave family ``田中社長``. Read when written as a separate word, after a comma, or glued to the end of the name, except that 部長 and 部长 are not split off a glued name (``田中部長`` is unchanged) unless ``locales.ZH`` divides it at a listed surname (``李部長``). :issue:`671`, :issue:`707` - **Add the Akan and Indonesian salutations owura and bapak as titles.** ``Bapak Dwi Putro`` gives title ``Bapak``, first ``Dwi``, last ``Putro``, where 1.4.0 through 2.4.2 gave first ``Bapak``, middle ``Dwi``. :issue:`662` - **Fix a title written before a family comma reading as part of the family name.** ``Dr. Smith, John`` gives title ``Dr.``, last ``Smith``, where 1.4.0 through 2.4.2 gave last ``Dr. Smith``. Only listed titles are read there, so ``Xyz. Smith, John`` keeps its family, and a title that is also a surname particle, such as ``St`` or ``Freiherr``, stays in the surname. :issue:`701` - **Fix a surname particle such as d' or al- reading as a middle name when a listing puts it after the given name.** ``Ivoi, Paul d'`` gives last ``d' Ivoi``, where 1.4.0 through 2.4.2 gave middle ``d'``. The Arabic article reads the same in its plain and assimilated spellings (``al-``, ``az-``, ``aš-``). A particle written joined to the surname, as in ``Paul d'Ivoi``, is unchanged. :issue:`700` diff --git a/docs/usage.rst b/docs/usage.rst index f1ba218e..e98772f0 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -591,10 +591,11 @@ not 님 with 박사 left behind. Credential and office words (議員 legislator, 医学博士 doctor of medicine, 社長 company president, 二世 the second) peel when glued as -well: they are not known to end a name. 部長 (department -head) is the exception and is read only when spaced, because 長 alone -is a given name, so 阿部長 can be the surname 阿部 and the given name -長: +well: they are not known to end a name. A longer office word that +ends in one of them is split at it, so 田中 副社長 (vice-president) +gives the given name 副 and the suffix 社長. 部長 (department head) is +the exception and is not peeled when glued, because 長 alone is a +given name, so 阿部長 can be the surname 阿部 and the given name 長: .. doctest:: diff --git a/tests/v2/test_name_usage_tool.py b/tests/v2/test_name_usage_tool.py index b5cb0e52..a539e0d1 100644 --- a/tests/v2/test_name_usage_tool.py +++ b/tests/v2/test_name_usage_tool.py @@ -13,8 +13,10 @@ _differential_fixtures.py loads compare.py. """ import importlib.util +import http.client import io import json +import unicodedata import sys import urllib.error import urllib.parse @@ -489,6 +491,8 @@ def test_an_error_that_is_not_an_overrun_is_not_retried_smaller( urllib.error.HTTPError("u", 504, "Gateway Timeout", {}, io.BytesIO(b"")), # type: ignore[arg-type] urllib.error.HTTPError("u", 500, "Error", {}, io.BytesIO( # type: ignore[arg-type] b"java.util.concurrent.TimeoutException")), + urllib.error.HTTPError("u", 502, "Bad Gateway", {}, io.BytesIO(b"")), # type: ignore[arg-type] + http.client.IncompleteRead(b""), TimeoutError("read timed out"), ]) def test_a_timeout_is_an_overrun( @@ -622,18 +626,16 @@ def test_ends_with_usage_sums_chunks_and_says_it_may_double_count( def test_ends_with_usage_counts_people_across_a_boundary( monkeypatch: pytest.MonkeyPatch) -> None: index = _index(("Q1", "family", "阿部"), ("Q3", "given", "長")) - queries = _stub_sparql(monkeypatch, [ - [{"n": "2", "example": "http://www.wikidata.org/entity/Q99"}], - ]) + queries = _stub_sparql(monkeypatch, [_people("Q99", "Q7")]) usage = tool.ends_with_usage("部長", index, top=5) assert (usage.given, usage.family) == (0, 0) # nothing ENDS in 部長 assert "VALUES ?f { wd:Q1 }" in queries[0] assert "VALUES ?g { wd:Q3 }" in queries[0] [split] = usage.splits - assert (split.people, split.example) == (2, "Q99") + assert (split.people, split.example) == (2, "Q7") text = tool.render("部長", [usage]) assert "family ending 部 + given 長: 2+ people" in text - assert "e.g. 阿部; Q99" in text + assert "e.g. 阿部; bearer Q7" in text def test_sparql_waits_out_a_rate_limit_and_retries( @@ -694,3 +696,308 @@ def broken(download: bool, refresh: bool = False) -> None: assert tool.main(["--json", "殿"]) == 2 row = json.loads(capsys.readouterr().out)["殿"][-1] assert (row["status"], row["family"]) == ("failed", None) + + +# -- review of #712: the index's failure paths ------------------------------ + +def _people(*qids: str) -> list[dict[str, str]]: + return [{"p": f"http://www.wikidata.org/entity/{q}"} for q in qids] + + +@pytest.mark.usefixtures("one_class") +def test_each_class_keeps_its_own_role_and_batches( + monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(tool, "NAME_CLASSES", + {"Q101352": "family", "Q202444": "given"}) + queries = _stub_sparql(monkeypatch, [ + *_listing("Q1"), _labels(("Q1", "向殿")), + *_listing("Q2"), _labels(("Q2", "太郎")), + ]) + index = tool.load_index(download=True) + assert index is not None + assert {(e.qid, e.role) for e in index.entries} == { + ("Q1", "family"), ("Q2", "given")} + assert [_values(q) for q in queries if "rdfs:label" in q] == [ + ["wd:Q1"], ["wd:Q2"]] + + +@pytest.mark.usefixtures("one_class") +def test_a_class_that_lists_no_items_is_an_error_not_an_empty_index( + monkeypatch: pytest.MonkeyPatch) -> None: + # A mistyped or merged class answers 0 to both queries; caching that + # would make every later ends-with row a silent zero. + _stub_sparql(monkeypatch, [*_listing()]) + with pytest.raises(ValueError, match="wd:Q101352 lists no items"): + tool.load_index(download=True) + assert not (tool.cache_dir() / tool.INDEX_DIR / "Q101352-ids.json").exists() + + +@pytest.mark.usefixtures("one_class") +@pytest.mark.parametrize(("rows", "loads"), [(999, True), (998, False)]) +def test_the_listing_count_allows_a_thousandth_of_slack( + monkeypatch: pytest.MonkeyPatch, rows: int, loads: bool) -> None: + monkeypatch.setattr(tool, "INDEX_BATCH", 5000) + qids = [f"Q{n}" for n in range(1, rows + 1)] + _stub_sparql(monkeypatch, [[{"n": "1000"}], _listing(*qids)[1], _labels()]) + if loads: + assert tool.load_index(download=True) is not None + else: + with pytest.raises(ValueError, match="returned 998 of 1,000"): + tool.load_index(download=True) + + +@pytest.mark.usefixtures("one_class") +@pytest.mark.parametrize(("name", "content"), [ + ("Q101352-ids.json", "["), + ("Q101352-ids.json", "{}"), + ("Q101352-ids.json", '["Q1", "L5"]'), + ("Q101352-ids.json", "[]"), + ("Q101352-0-1.json", '{"rows": []}'), + ("Q101352-0-1.json", '{"fetched": "x", "span": ["Q1", "Q1"], "rows": null}'), + ("Q101352-0-1.json", '{"fetched": "x", "span": ["Q1", "Q1"], "rows": [["Q9", "殿"]]}'), +]) +def test_an_unusable_cache_file_is_named_with_its_remedy( + monkeypatch: pytest.MonkeyPatch, name: str, content: str) -> None: + _stub_sparql(monkeypatch, [*_listing("Q1"), _labels(("Q1", "向殿"))]) + tool.load_index(download=True) + (tool.cache_dir() / tool.INDEX_DIR / name).write_text(content) + _stub_sparql(monkeypatch, []) + with pytest.raises(ValueError, match=f"{name} is unusable.*--refresh-index"): + tool.load_index(download=True) + + +@pytest.mark.usefixtures("one_class") +def test_a_batch_cached_against_another_listing_is_fetched_again( + monkeypatch: pytest.MonkeyPatch) -> None: + # Deleting one class's listing must not leave its batches answering + # for whatever items now sit at the same positions. + _stub_sparql(monkeypatch, [*_listing("Q1", "Q2"), _labels(("Q1", "向殿"))]) + tool.load_index(download=True) + (tool.cache_dir() / tool.INDEX_DIR / "Q101352-ids.json").unlink() + queries = _stub_sparql(monkeypatch, [ + *_listing("Q10", "Q11"), _labels(("Q11", "井殿"))]) + index = tool.load_index(download=True) + assert index is not None + assert [(e.qid, e.label) for e in index.entries] == [("Q11", "井殿")] + assert _values(queries[2]) == ["wd:Q10", "wd:Q11"] + + +@pytest.mark.usefixtures("one_class") +def test_an_index_from_an_older_version_of_the_tool_is_rebuilt( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str]) -> None: + _stub_sparql(monkeypatch, [*_listing("Q1"), _labels(("Q1", "向殿"))]) + tool.load_index(download=True) + (tool.cache_dir() / tool.INDEX_DIR / "version.json").write_text("1") + _stub_sparql(monkeypatch, []) + assert tool.load_index(download=False) is None + capsys.readouterr() + _stub_sparql(monkeypatch, [*_listing("Q1"), _labels(("Q1", "井殿"))]) + index = tool.load_index(download=True) + assert index is not None and index.entries[0].label == "井殿" + assert "older version" in capsys.readouterr().err + + +def test_sparql_csv_refuses_a_body_cut_mid_line( + monkeypatch: pytest.MonkeyPatch) -> None: + # A last line cut inside its IRI (Q1234567 arriving as Q12) is still + # a valid Q-id; the missing newline is the only sign. + monkeypatch.setattr(urllib.request, "urlopen", lambda request, timeout: ( + io.BytesIO(b"item\r\nhttp://www.wikidata.org/entity/Q12"))) + with pytest.raises(ValueError, match="cut short"): + tool._sparql_csv("SELECT") + + +def test_sparql_csv_asks_for_csv(monkeypatch: pytest.MonkeyPatch) -> None: + seen: list[str | None] = [] + + def answer(request: urllib.request.Request, timeout: float) -> io.BytesIO: + seen.append(request.get_header("Accept")) + return io.BytesIO(b"item\r\n") + monkeypatch.setattr(urllib.request, "urlopen", answer) + tool._sparql_csv("SELECT") + assert seen == ["text/csv"] + + +@pytest.mark.parametrize(("word", "cjk"), [ + ("𠮷田", True), # supplementary-plane Han (Extension B) + ("〇", True), + ("䷀", False), # a Yijing hexagram, inside the old range +]) +def test_is_cjk_covers_the_supplementary_han_planes(word: str, cjk: bool) -> None: + assert tool.is_cjk(word) is cjk + + +def test_the_index_query_and_is_cjk_share_one_character_class() -> None: + assert f"[{tool._CJK_CHARS}]" in tool.labels_query(["Q1"]) + + +def test_boundaries_try_every_cut_and_match_the_given_name_whole() -> None: + index = _index(("Q1", "family", "某国会"), ("Q2", "given", "議員"), + ("Q3", "given", "信議員"), ("Q4", "given", "某国"), + ("Q5", "given", "国会議員")) + splits = index.boundaries("国会議員") + # The only split is at the second cut, and 信議員 ends in the tail + # without being it; 某国 is a GIVEN name ending in the head, which is + # no family name; and a word equal to a whole given name is no split. + assert [(s.family_tail, s.given, s.family_items, s.given_items) + for s in splits] == [("国会", "議員", ["Q1"], ["Q2"])] + + +def test_a_failed_bearer_count_fails_the_row( + monkeypatch: pytest.MonkeyPatch) -> None: + _stub_sparql(monkeypatch, [OSError("down")]) + usage = tool.ends_with_usage("殿", _index(("Q1", "family", "向殿")), top=5) + assert (usage.source, usage.status) == (tool.ENDS_WITH_SOURCE, "failed") + assert (usage.given, usage.family) == (None, None) + + +def test_a_failed_boundary_count_keeps_the_rows_other_counts( + monkeypatch: pytest.MonkeyPatch) -> None: + index = _index(("Q1", "family", "阿部"), ("Q2", "given", "長"), + ("Q3", "family", "山部長")) + _stub_sparql(monkeypatch, [[{"role": "family", "n": "7"}], [], + OSError("timed out")]) + usage = tool.ends_with_usage("部長", index, top=5) + assert (usage.status, usage.family) == ("ok", 7) + [split] = usage.splits + assert split.people is None and "timed out" in split.error + assert "family ending 部 + given 長: - people (query failed" in ( + tool.render("部長", [usage])) + + +def test_a_boundary_count_is_chunked_and_counts_a_person_once( + monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(tool, "ITEM_CHUNK", 1) + index = _index(("Q1", "family", "阿部"), ("Q2", "family", "岡部"), + ("Q3", "given", "長")) + queries = _stub_sparql(monkeypatch, [_people("Q9"), _people("Q9", "Q8")]) + usage = tool.ends_with_usage("部長", index, top=5) + assert len(queries) == 2 + assert usage.splits[0].people == 2 + + +def test_an_empty_boundary_answer_is_zero_people( + monkeypatch: pytest.MonkeyPatch) -> None: + _stub_sparql(monkeypatch, [[]]) + usage = tool.ends_with_usage( + "部長", _index(("Q1", "family", "阿部"), ("Q3", "given", "長")), top=5) + assert (usage.splits[0].people, usage.splits[0].example) == (0, "") + + +def test_the_ends_with_note_gives_the_index_size( + monkeypatch: pytest.MonkeyPatch) -> None: + _stub_sparql(monkeypatch, []) + usage = tool.ends_with_usage("議員", _index(("Q1", "family", "向殿")), top=5) + assert "index of 1 entries" in usage.note + + +def test_chunked_country_counts_are_summed( + monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(tool, "ITEM_CHUNK", 1) + index = _index(("Q1", "family", "向殿"), ("Q2", "family", "井殿")) + _stub_sparql(monkeypatch, [ + [{"role": "family", "n": "3"}], [{"role": "family", "country": "Japan", "n": "3"}], + [{"role": "family", "n": "4"}], [{"role": "family", "country": "Japan", "n": "4"}], + ]) + usage = tool.ends_with_usage("殿", index, top=5) + assert usage.family_countries == [("Japan", 7)] + + +def _http_error(code: int, retry_after: str = "") -> urllib.error.HTTPError: + headers = {"Retry-After": retry_after} if retry_after else {} + return urllib.error.HTTPError("u", code, "x", headers, io.BytesIO(b"")) # type: ignore[arg-type] + + +def test_sparql_raises_a_non_rate_limit_error_without_waiting( + monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[int] = [] + waits: list[float] = [] + + def answer(request: urllib.request.Request, timeout: float) -> io.BytesIO: + calls.append(1) + raise _http_error(500) + monkeypatch.setattr(urllib.request, "urlopen", answer) + monkeypatch.setattr(tool, "_sleep", waits.append) + with pytest.raises(urllib.error.HTTPError) as caught: + tool._sparql("SELECT") + caught.value.close() # 3.14 warns on an HTTPError left open + assert (len(calls), waits) == (1, []) + + +def test_sparql_gives_up_after_its_retries_with_the_http_error( + monkeypatch: pytest.MonkeyPatch) -> None: + waits: list[float] = [] + + def answer(request: urllib.request.Request, timeout: float) -> io.BytesIO: + raise _http_error(429, "Fri, 17 Oct 2026 18:00:00 GMT") + monkeypatch.setattr(urllib.request, "urlopen", answer) + monkeypatch.setattr(tool, "_sleep", waits.append) + with pytest.raises(urllib.error.HTTPError) as caught: + tool._sparql("SELECT") + caught.value.close() + # A date-form Retry-After waits the default minute. + assert waits == [60] * tool.SPARQL_RETRIES + + +@pytest.mark.parametrize(("argv", "refresh"), [ + (["--json", "殿"], False), (["--json", "--refresh-index", "殿"], True), + (["--json", "--refresh-index", "knight"], True), +]) +def test_main_passes_refresh_to_the_index( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, + argv: list[str], refresh: bool) -> None: + monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + monkeypatch.setattr(tool, "load_census", lambda download: None) + seen: list[bool] = [] + + def load(download: bool, refresh: bool = False) -> _Index: + seen.append(refresh) + return _index(("Q1", "family", "x")) + monkeypatch.setattr(tool, "load_index", load) + monkeypatch.setattr(tool, "_sparql", lambda query: {"results": {"bindings": []}}) + tool.main(argv) + assert seen == [refresh] + + +def test_main_reads_a_decomposed_word_as_its_composed_twin( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, + capsys: pytest.CaptureFixture[str]) -> None: + # Wikidata labels are NFC; a decomposed hangul word is conjoining + # jamo, which neither is_cjk nor any label would match. + monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + monkeypatch.setattr(tool, "load_census", lambda download: None) + monkeypatch.setattr(tool, "load_index", lambda download, refresh=False: + _index(("Q1", "given", "민준씨"))) + monkeypatch.setattr(tool, "_sparql", lambda query: {"results": {"bindings": []}}) + tool.main(["--json", unicodedata.normalize("NFD", "씨")]) + [rows] = json.loads(capsys.readouterr().out).values() + assert rows[-1]["source"] == tool.ENDS_WITH_SOURCE + + +def test_main_exits_2_when_a_boundary_count_failed( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, + capsys: pytest.CaptureFixture[str]) -> None: + monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + monkeypatch.setattr(tool, "load_census", lambda download: None) + monkeypatch.setattr(tool, "load_index", lambda download, refresh=False: + _index(("Q1", "family", "阿部"), ("Q2", "given", "長"))) + _stub_sparql(monkeypatch, [[], [], OSError("timed out")]) + assert tool.main(["--json", "部長"]) == 2 + + +def test_main_reports_an_unusable_cache_without_losing_the_other_sources( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, + capsys: pytest.CaptureFixture[str]) -> None: + monkeypatch.setenv("NAMEPARSER_TOOLS_CACHE", str(tmp_path)) + monkeypatch.setattr(tool, "NAME_CLASSES", {"Q101352": "family"}) + with zipfile.ZipFile(tmp_path / tool.CENSUS_FILE, "w") as zf: + zf.writestr(tool.CENSUS_MEMBER, _CENSUS) + index_dir = tmp_path / tool.INDEX_DIR + index_dir.mkdir() + (index_dir / "version.json").write_text(str(tool.INDEX_VERSION)) + (index_dir / "Q101352-ids.json").write_text("{") + _stub_sparql(monkeypatch, [[], []]) + assert tool.main(["--json", "殿"]) == 2 + rows = json.loads(capsys.readouterr().out)["殿"] + assert rows[-1]["status"] == "failed" and "unusable" in rows[-1]["note"] diff --git a/tools/name_usage.py b/tools/name_usage.py index d3c481f4..87be7b17 100644 --- a/tools/name_usage.py +++ b/tools/name_usage.py @@ -40,10 +40,10 @@ little and a zero proves nothing. Labels are matched exactly, in the spellings the report lists, so a name with internal capitals is found only when passed as written (`McDonald`, not `mcdonald`). -- Wikidata, name ends with (CC0), for a word holding a Han, kana or - hangul letter only: people bearing a name item whose CJK label ENDS +- Wikidata, name ends with (CC0), only for a word holding at least one + Han, kana or hangul letter: people bearing a name item whose CJK label ENDS in the word, which is what a glued honorific's peel risks (#707). The - matching runs against a local index of every CJK label on the + matching runs against a local index of every CJK rdfs:label (not aliases) on the name-item classes, fetched once (resumable; 33 minutes on 2026-10-17) and refreshed with --refresh-index; the bearers are counted live. It also counts the names the word ends ACROSS the family/given join, @@ -471,9 +471,17 @@ def _sparql(query: str) -> dict[str, Any]: def _sparql_csv(query: str) -> list[dict[str, str]]: - """The rows as CSV, for a listing too big for JSON: 700,581 family-name - items are 30MB as CSV and were cut off at 73MB as JSON (2026-10-17).""" - return list(csv.DictReader(io.StringIO(_post(query, "text/csv").decode()))) + """The rows as CSV, for a listing too big for JSON: the 700,581 + family-name listing rows are 30MB as CSV and were cut off at 73MB as JSON (2026-10-17).""" + body = _post(query, "text/csv").decode() + if body and not body.endswith("\n"): + # A line cut inside its IRI is still a valid Q-id (Q1234567 + # arriving as Q12); the missing line end is the only sign. + raise ValueError("CSV response ends mid-line; it was cut short") + try: + return list(csv.DictReader(io.StringIO(body))) + except csv.Error as exc: + raise ValueError(f"unreadable CSV response: {exc}") from exc def _post(query: str, accept: str) -> bytes: @@ -498,6 +506,7 @@ def _post(query: str, accept: str) -> bytes: raise retry = exc.headers.get("Retry-After", "") wait = int(retry) if retry.isdigit() else 60 + exc.close() # it holds the response; 3.14 warns on a leak print(f"rate-limited by {WIKIDATA_ENDPOINT}; waiting {wait}s", file=sys.stderr) _sleep(wait) @@ -536,15 +545,20 @@ def _download(url: str, path: Path, required_member: str) -> None: # # The fetch lists a class's items, then asks for their labels by VALUES # batch. Slicing the class inside one query instead (by Q-id suffix) -# scans the whole class every time -- 700,581 family-name items -- so a -# slice of 250 labels overran as surely as one of 25,000. +# scans the whole class every time -- 700,581 family-name listing rows, +# 700,304 items once lexemes are dropped -- so a slice of 250 labels +# overran as surely as one of 25,000. + +Role = Literal["given", "family"] #: Name-item classes the index lists, and the role their items play. -#: Read off the P31 statements of known items, 2026-10-17: Japanese and -#: Chinese items usually carry the general class beside their own, but -#: not always, and the undifferentiated kanji surnames (高橋, 阿部 as bare -#: kanji, no reading) carry only theirs. -NAME_CLASSES = { +#: Counted on the index, 2026-10-17: Japanese and Chinese family-name +#: items and Chinese given-name items nearly all carry the general class +#: beside their own, but Japanese given names almost never do (13 of +#: 4,620), nor do most male and female given names, and the +#: undifferentiated kanji surnames (高橋, 阿部 as bare kanji, no +#: reading) carry only theirs -- so every class is listed. +NAME_CLASSES: dict[str, Role] = { "Q101352": "family", # family name "Q19914123": "family", # Japanese family name "Q88865432": "family", # undifferentiated kanji surname @@ -557,7 +571,15 @@ def _download(url: str, path: Path, required_member: str) -> None: "Q22812564": "given", # Japanese given name "Q22809413": "given", # Chinese given name } +# A misspelled role would drop its class out of the boundary count +# without a word, so the roles are checked here, where a typo lands. +assert set(NAME_CLASSES.values()) == {"given", "family"}, NAME_CLASSES + INDEX_DIR = "wikidata_cjk_names" +#: Bumped whenever what the index holds changes (version 2 widened the +#: character class to supplementary-plane Han); a cache built by another +#: version is rebuilt rather than trusted. +INDEX_VERSION = 2 #: Items per label query. Measured 2026-10-17 on family-name items not #: queried before: 1,000 took 2.2-2.4s and 2,000 took 24s. INDEX_BATCH = 1000 @@ -568,11 +590,15 @@ def _download(url: str, path: Path, required_member: str) -> None: ITEM_CHUNK = 500 ENDS_WITH_SOURCE = "Wikidata people, name ends with" -#: Kana with the iteration mark 々, the CJK ideograph blocks with their -#: compatibility block, hangul syllables, and halfwidth katakana. One -#: such letter makes a label CJK; the query and is_cjk share it. -_CJK_CHARS = ("々぀-ヿ㐀-鿿가-힯" - "豈-﫿･-ﾟ") +#: The iteration mark 々 and 〇, kana, the CJK ideograph blocks (Extension +#: A, the unified block, compatibility, and Extensions B onward in the +#: supplementary planes, where rare surname characters live), hangul +#: syllables, and halfwidth katakana. The Yijing hexagrams between +#: Extension A and the unified block are left out. One such letter makes +#: a label or a word CJK; the query and is_cjk share the class. +_CJK_CHARS = ("々〇぀-ヿ㐀-䶿一-鿿" + "가-힯豈-﫿･-ﾟ" + "\U00020000-\U0003134f\U0002f800-\U0002fa1f") _CJK = re.compile(f"[{_CJK_CHARS}]") @@ -583,7 +609,7 @@ def is_cjk(word: str) -> bool: @dataclass(frozen=True, slots=True) class IndexEntry: qid: str - role: str + role: Role label: str @@ -591,7 +617,8 @@ class IndexEntry: class Split: """People whose family name ends in `family_tail` and whose given name is `given`, so that the two written together end in the word: - 阿部 + 長 ends in 部長.""" + 阿部 + 長 ends in 部長. `people` is None until counted, and stays + None, with `error` saying why, when the count failed.""" family_tail: str given: str @@ -599,7 +626,9 @@ class Split: given_items: list[str] family_examples: list[str] = field(default_factory=list) people: int | None = None + #: The Q-id of one bearer, for looking the pairing up. example: str = "" + error: str = "" @dataclass(slots=True) @@ -639,7 +668,8 @@ def count_query(cls: str) -> str: def labels_query(qids: list[str]) -> str: - """The CJK labels of the items, every language.""" + """The CJK rdfs:labels of the items, every language. Aliases and + native-label statements are not read.""" return f""" SELECT ?item ?label WHERE {{ {_item_match(qids)} @@ -669,13 +699,18 @@ def _index_sparql(query: str, as_csv: bool = False) -> list[dict[str, str]]: except urllib.error.HTTPError as exc: # A 500 is an overrun only when it says so: halving a malformed # query would send it again and again. - if exc.code in (502, 504) or ( - exc.code == 500 and b"TimeoutException" in exc.read()): - raise _Overrun(f"HTTP {exc.code}") from exc + try: + overran = exc.code in (502, 504) or ( + exc.code == 500 and b"TimeoutException" in exc.read()) + finally: + exc.close() + if overran: + raise _Overrun(f"HTTP {exc.code}") from None raise except (ValueError, TimeoutError, http.client.IncompleteRead) as exc: # At its limit WDQS cuts a 200 off mid-stream, which surfaces as - # JSON that stops (ValueError) or a short read. + # JSON that stops (ValueError), a CSV line with no end, or a + # short read. raise _Overrun(f"{type(exc).__name__}: {exc}") from exc @@ -686,11 +721,30 @@ def _write(path: Path, data: object) -> None: tmp.replace(path) +def _unusable(path: Path, why: str) -> ValueError: + return ValueError(f"{path} is unusable ({why}); delete it, or run " + "again with --refresh-index") + + +def _read_json(path: Path) -> object: + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise _unusable(path, f"{type(exc).__name__}: {exc}") from exc + + +def _is_qid(value: object) -> bool: + return isinstance(value, str) and re.fullmatch(r"Q\d+", value) is not None + + def _class_ids(cls: str, fetch: bool) -> list[str]: path = cache_dir() / INDEX_DIR / f"{cls}-ids.json" if path.exists(): - ids: list[str] = json.loads(path.read_text(encoding="utf-8")) - return ids + cached = _read_json(path) + if not (isinstance(cached, list) and cached + and all(map(_is_qid, cached))): + raise _unusable(path, "not a non-empty list of Q-ids") + return [str(q) for q in cached] if not fetch: raise _Missing(path.name) try: @@ -698,6 +752,9 @@ def _class_ids(cls: str, fetch: bool) -> list[str]: rows = _index_sparql(ids_query(cls), as_csv=True) except _Overrun as exc: raise ValueError(f"listing wd:{cls} items overran ({exc})") from exc + if rows and "item" not in rows[0]: + raise ValueError(f"listing wd:{cls} has columns {sorted(rows[0])}, " + "not item") # CSV has no closing bracket to show a stream cut short, so the # listing is held to a count taken just before it. The slack is for # edits landing between the two queries; a cut loses far more. @@ -711,11 +768,35 @@ def _class_ids(cls: str, fetch: bool) -> list[str]: ids = sorted({_qid(row["item"]) for row in rows if not re.search(r"/L\d+(-[SF]\d+)?$", row["item"])}, key=lambda q: int(q[1:])) + if not ids: + # A merged or mistyped class answers 0 to both queries; caching + # that would make every later ends-with row a silent zero. + raise ValueError(f"wd:{cls} lists no items; check NAME_CLASSES") _write(path, ids) print(f" wd:{cls}: {len(ids):,} items", file=sys.stderr) return ids +def _cached_batch(path: Path, span: list[str], + ids: list[str]) -> tuple[str, list[list[str]]] | None: + """A batch file's date and rows, or None when it was built against + another listing: batches are named by position, so after a class is + listed again the same position can hold other items.""" + data = _read_json(path) + if not (isinstance(data, dict) and isinstance(data.get("fetched"), str) + and isinstance(data.get("rows"), list)): + raise _unusable(path, "not a batch: needs fetched, span and rows") + if data.get("span") != span: + return None + rows = data["rows"] + batch = set(ids) + for row in rows: + if not (isinstance(row, list) and len(row) == 2 + and row[0] in batch and isinstance(row[1], str)): + raise _unusable(path, f"row {row!r} is not an item of the batch") + return data["fetched"], rows + + def _fetch_batch(cls: str, ids: list[str], start: int, end: int, fetch: bool, out: list[IndexEntry], dates: set[str]) -> bool: """Read the labels of ids[start:end] into `out`, querying them if they @@ -725,48 +806,66 @@ def _fetch_batch(cls: str, ids: list[str], start: int, end: int, fetch: bool, True when anything was queried.""" base = cache_dir() / INDEX_DIR / f"{cls}-{start}-{end}" done, halved = base.with_suffix(".json"), base.with_suffix(".halved") + batch = ids[start:end] + span = [batch[0], batch[-1]] if done.exists(): - data = json.loads(done.read_text(encoding="utf-8")) - dates.add(data["fetched"]) - out.extend(IndexEntry(q, NAME_CLASSES[cls], label) - for q, label in data["rows"]) - return False - if not halved.exists(): - if not fetch: - raise _Missing(done.name) - try: - rows = [[_qid(row["item"]), row["label"]] - for row in _index_sparql(labels_query(ids[start:end]))] - except _Overrun as exc: - if end - start <= INDEX_MIN_BATCH: - raise ValueError(f"wd:{cls} items {start}-{end} still " - f"overran ({exc})") from exc - halved.parent.mkdir(parents=True, exist_ok=True) - halved.touch() - else: - today = date.today().isoformat() - _write(done, {"fetched": today, "rows": rows}) - dates.add(today) + cached = _cached_batch(done, span, batch) + if cached is not None: + dates.add(cached[0]) out.extend(IndexEntry(q, NAME_CLASSES[cls], label) - for q, label in rows) - return True - middle = (start + end) // 2 - first = _fetch_batch(cls, ids, start, middle, fetch, out, dates) - second = _fetch_batch(cls, ids, middle, end, fetch, out, dates) - return first or second + for q, label in cached[1]) + return False + halved.unlink(missing_ok=True) # it was halved for other items + elif halved.exists(): + middle = (start + end) // 2 + first = _fetch_batch(cls, ids, start, middle, fetch, out, dates) + second = _fetch_batch(cls, ids, middle, end, fetch, out, dates) + return first or second + if not fetch: + raise _Missing(done.name) + try: + rows = [[_qid(row["item"]), row["label"]] + for row in _index_sparql(labels_query(batch))] + except _Overrun as exc: + if end - start <= INDEX_MIN_BATCH: + raise ValueError(f"wd:{cls} items {start}-{end} still " + f"overran ({exc})") from exc + halved.parent.mkdir(parents=True, exist_ok=True) + halved.touch() + middle = (start + end) // 2 + _fetch_batch(cls, ids, start, middle, fetch, out, dates) + _fetch_batch(cls, ids, middle, end, fetch, out, dates) + return True + today = date.today().isoformat() + _write(done, {"fetched": today, "span": span, "rows": rows}) + dates.add(today) + out.extend(IndexEntry(q, NAME_CLASSES[cls], label) for q, label in rows) + return True def load_index(download: bool, refresh: bool = False) -> NameIndex | None: """The cached CJK name index; None when part of it is not cached and - `download` is off. Building it is a one-time fetch of about a - thousand queries, paced by the endpoint's rate limit, and resumable.""" + `download` is off. Building it is a one-time fetch of about 900 + queries (2026-10-17), paced by the endpoint's rate limit, and + resumable. Raises ValueError naming any cache file it cannot use.""" path = cache_dir() / INDEX_DIR + version = path / "version.json" if refresh and path.exists(): shutil.rmtree(path) - if download and not path.exists(): + if path.exists() and (not version.exists() + or _read_json(version) != INDEX_VERSION): + if not download: + return None + print(f"the CJK name index in {path} was built by an older version " + "of this tool; rebuilding it", file=sys.stderr) + shutil.rmtree(path) + if not path.exists(): + if not download: + return None print(f"building the CJK name index in {path} -- a one-time fetch " "of about half an hour; an interrupted build resumes", file=sys.stderr) + _write(version, INDEX_VERSION) entries: list[IndexEntry] = [] dates: set[str] = set() try: @@ -785,8 +884,12 @@ def load_index(download: bool, refresh: bool = False) -> NameIndex | None: fetched=min(dates) if dates else "") +def _chunks(qids: list[str]) -> list[list[str]]: + return [qids[i:i + ITEM_CHUNK] for i in range(0, len(qids), ITEM_CHUNK)] + + def _count_items(qids: list[str], top: int) -> tuple[Usage, int]: - chunks = [qids[i:i + ITEM_CHUNK] for i in range(0, len(qids), ITEM_CHUNK)] + chunks = _chunks(qids) totals: list[dict[str, str]] = [] by_country: list[dict[str, str]] = [] for chunk in chunks: @@ -796,24 +899,35 @@ def _count_items(qids: list[str], top: int) -> tuple[Usage, int]: def _count_split(split: Split) -> None: - family = " ".join(f"wd:{q}" for q in split.family_items) + """The distinct bearers of the pairing, the family items asked in + chunks (a one-letter tail such as 部 or 田 matches hundreds) and the + people gathered here, so a person is counted once across chunks.""" given = " ".join(f"wd:{q}" for q in split.given_items) - [row] = _bindings(_sparql(f""" -SELECT (COUNT(DISTINCT ?p) AS ?n) (SAMPLE(?p) AS ?example) WHERE {{ + people: set[str] = set() + for chunk in _chunks(split.family_items): + family = " ".join(f"wd:{q}" for q in chunk) + rows = _bindings(_sparql(f""" +SELECT DISTINCT ?p WHERE {{ VALUES ?f {{ {family} }} VALUES ?g {{ {given} }} ?p wdt:P734 ?f ; wdt:P735 ?g . -}}""")) or [{"n": "0"}] - split.people = int(row["n"]) - split.example = row.get("example", "").rsplit("/", 1)[-1] +}}""")) + people |= {_qid(row["p"]) for row in rows} + split.people = len(people) + split.example = min(people, key=lambda q: int(q[1:])) if people else "" + + +_QUERY_ERRORS = (OSError, ValueError, KeyError, http.client.HTTPException) def ends_with_usage(word: str, index: NameIndex, top: int) -> Usage: """Bearers of the indexed name items whose label ends in the word, - and of the family/given pairs that end in it together.""" + and of the family/given pairs that end in it together. A failed + pairing count fails only that pairing (Split.error).""" matches = index.ending_in(word) qids = sorted({e.qid for e in matches}) - fetched = f"index fetched {index.fetched}" + about = (f"index of {len(index.entries):,} entries fetched " + f"{index.fetched}") try: if qids: usage, queries = _count_items(qids, top) @@ -828,19 +942,21 @@ def ends_with_usage(word: str, index: NameIndex, top: int) -> Usage: if queries > 1 else "") items = "1 name item ends" if len(qids) == 1 else ( f"{len(qids)} name items end") - usage.note = _join(f"{items} in it: {shown}", - twice, fetched) + usage.note = _join(f"{items} in it: {shown}", twice, about) else: usage = Usage("", given=0, family=0, lower_bound=True, note=_join(f"no indexed name item ends in {word}", - fetched)) - usage.splits = index.boundaries(word) - for split in usage.splits: - _count_split(split) - except (OSError, ValueError, KeyError, http.client.HTTPException) as exc: + about)) + except _QUERY_ERRORS as exc: return Usage(ENDS_WITH_SOURCE, status="failed", lower_bound=True, note=f"query failed: {type(exc).__name__}: {exc}") usage.source = ENDS_WITH_SOURCE + usage.splits = index.boundaries(word) + for split in usage.splits: + try: + _count_split(split) + except _QUERY_ERRORS as exc: + split.error = f"query failed: {type(exc).__name__}: {exc}" return usage @@ -885,12 +1001,14 @@ def render(word: str, usages: list[Usage]) -> str: lines.append(f" {'':<{width}} {role} by citizenship: " + ", ".join(parts)) for split in u.splits: - example = "; ".join(split.family_examples - + ([split.example] if split.example else [])) + example = "; ".join( + split.family_examples + + ([f"bearer {split.example}"] if split.example else [])) + detail = split.error or (f"e.g. {example}" if example else "") lines.append( f" {'':<{width}} family ending {split.family_tail} + " f"given {split.given}: {_count(split.people, u)} people" - + (f" (e.g. {example})" if example else "")) + + (f" ({detail})" if detail else "")) return "\n".join(lines) @@ -910,14 +1028,18 @@ def main(argv: list[str] | None = None) -> int: census, census_error = _attempt( lambda: load_census(download=not args.no_wikidata)) ssa, ssa_error = _attempt(load_ssa) + # Wikidata labels are NFC: a decomposed word (hangul as conjoining + # jamo, an accent as a combining mark) would match no label at all. + words = [unicodedata.normalize("NFC", word) for word in args.words] index: NameIndex | None = None index_error = "" - if not args.no_wikidata and any(map(is_cjk, args.words)): + if not args.no_wikidata and (args.refresh_index + or any(map(is_cjk, words))): index, index_error = _attempt( lambda: load_index(download=True, refresh=args.refresh_index)) report = {} - for word in args.words: + for word in words: census_row, ssa_row = census_usage(word, census), ssa_usage(word, ssa) for row, error in ((census_row, census_error), (ssa_row, ssa_error)): if error: @@ -940,7 +1062,8 @@ def main(argv: list[str] | None = None) -> int: if ssa is None and not ssa_error: sys.stdout.flush() # keep the notice after the report, not before it print("\n" + ssa_missing_message(), file=sys.stderr) - failed = any(u.status == "failed" for us in report.values() for u in us) + failed = any(u.status == "failed" or any(s.error for s in u.splits) + for us in report.values() for u in us) return 2 if failed else 0 From 45e245107e973b4d70d0b4acda1da4e162633806 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sat, 17 Oct 2026 20:30:58 -0700 Subject: [PATCH 6/6] Record that the rebuilt CJK index gave the same #707 answers Co-Authored-By: Claude Opus 5.5 --- docs/design/decisions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/design/decisions.md b/docs/design/decisions.md index 12f73c5a..2ce2ef6c 100644 --- a/docs/design/decisions.md +++ b/docs/design/decisions.md @@ -659,7 +659,7 @@ Declined: - 2026-08 #312 — the peel crosses the family comma and the 间隔号: both answer where a name DIVIDES into surname and given, a question the peel never asks. Provenance matters here: neither gate was argued for the peel specifically — both came with the placement (#312's own framing) — so the crossing was a repair of inherited gates, not a designed-in property. - 2026-08 #319 — a wholly suffix-shaped second run is declined as a peel site (the "田中さん, V." shape), but only when the name's own run offers a site, since a glued honorific is itself part of what makes a run read as suffix-shaped. - 2026-10-17 #671 — SPACED AND AFTER-COMMA ONLY, no glued peel (Derek, 2026-10-17). 議員/议员, 国会議員/國會議員, 太平紳士/太平绅士, 医学博士/醫學博士, 二世 and the job titles 社長/部長/課長 and the simplified 社长/部长/课长 (Derek, 2026-10-17: the characters are shared and the offices are real in both languages; no borne-name data either way, #707) join `SUFFIX_WORDS`; none joins `GLUED_HONORIFICS`. The spaced bar is only that a word standing alone is no name, so they ship; the glued bar ("can it END a name?") needs real-name counts that this change did not get — the Wikidata suffix query timed out and the native-script Wikidata corpus holds none of the words — and 部長 fails it on argument alone (長 is a borne given name, 阿部長). A follow-up can vet 医学博士, 議員 and 太平紳士 one at a time. Consequence measured on `parser_for(locales.ZH)`: after a listed surname each word divides off, `王二世` family 王, suffix 二世 (it read given 二世) and `李部長` family 李, suffix 部長, because the segmenter's division is not a peel and the split-off 二世 is then a whole-token suffix word, the way 王博士 already reads; `陳怡君二世` still reads given 怡君二世 (glued, nothing divides it; superseded for the glued question by the #707 entry below). ジュニア was already shipped by #661, and #679 declined 二世 as CLDR's generation word ("regnal, and glued"); here it ships spaced-only as the Chinese/Japanese numeral, which does not reverse that. Excluded as the issue stated: 小 (a prefix and a common given-name character) and 冯/馮 for "von" (a top-30 surname). -- 2026-10-17 #707 — the #671 words meet the glued bar, all but 部長/部长 (Derek, 2026-10-17). 議員/议员, 国会議員/國會議員, 太平紳士/太平绅士, 医学博士/醫學博士, 二世, 社長/課長 and 社长/课长 join `GLUED_HONORIFICS`; 部長/部长 stay spaced-only on the #671 argument, since 長 is a borne given name (Wikidata holds it as a Chinese given name, Q96240389; the Japanese reading rests on the argument) and 阿部長 can be 阿部 + 長 — the measurement below found no such bearer, and a zero that is a lower bound does not refute the argument. The measurement is `tools/name_usage.py`'s ends-with row, new in #707: a local index of every CJK label on eleven name-item classes (family name, given name and their Japanese, Chinese, Korean, male, female, unisex and undifferentiated-kanji subclasses; 215,798 index entries on 128,695 items, 215,696 distinct item-label pairs, fetched 2026-10-17 by the first index, whose character class stopped at the Basic Multilingual Plane), matched locally, with the matching items' bearers counted live. Two questions per word: does any name item's label END in it, and does any person bear a family name ending in the word's head and a given name equal to its tail (the 阿部 + 長 shape)? Neither found anything for any spelling, with one exception that is a data error: the only item ending in 二世 is Q18339792, the given name Odo, whose Chinese label carries a regnal numeral (厄德二世), and its 109 bearers are Western people named Odo. Of the boundary shapes, only 二+世, 社+長, 社+长, 部+長 and 部+长 had a family name and a given-name item to pair at all, and each counted 0 people. What makes the zeros evidence: the index agreed exactly with a live `STRENDS` query on all 20 class × word cells it was cross-checked on (five subclasses × 殿, 子, 郎, 田, the largest 403 items); 子 found 439 items and 2,436 given-name bearers, 2,081 of them Japanese; and 田 + 太郎 counted 22 people across the boundary. What limits them: Wikidata's coverage, not the index's — 殿 ends 13 name items where a surname dictionary lists about 90, and Japanese family-name items number 4,919. Two shortcuts were measured and rejected before the index: `STRENDS` over every label of a name class timed out at WDQS's 60s with no person join, and WDQS full-text search found 4 family names ending in 殿, matching only where its tokenizer split the character out. Consequence: the snapshot moved four CLDR renderings, each losing its glued word to `suffix` (`王雅婷婷婷二世議員`, both spellings of `…小惠儀太平紳士`, `馮·陳·怡君·達印·二世醫學博士`), and `陳怡君二世`, which the case table and rules.md W2 pin; cldr_agreement records one suffix field newly agreeing (`二世 醫學博士`, where the peel takes 醫學博士 and the 二世 left behind is a whole-token suffix word) and none regressing. The peel still takes one word: `王雅婷婷婷二世議員` peels 議員 and keeps 二世 in the name. Recompute: `uv run python tools/name_usage.py 議員 议员 国会議員 國會議員 太平紳士 太平绅士 医学博士 醫學博士 二世 社長 部長 課長 社长 部长 课长`; the first run builds the index (33 minutes on 2026-10-17, resumable), and `--refresh-index` rebuilds it. ACCEPTED COST, found by the design-docs review and kept (Derek, 2026-10-17): the vetting asked whether a NAME ends in each word and never whether a longer, unlisted OFFICE word does, and common ones do — 衆議院議員 and 参議院議員, 副社長 and 代表取締役社長, 元議員 and 市议员, 副課長, 獣医学博士. The peel matches at the end of the token, spaced or glued, so each splits at the listed tail: `岸田文雄衆議院議員` reads family 岸田文雄衆議院, suffix 議員, and `田中 副社長` given 副, suffix 社長, where before the word stayed whole in the name. Neither reading is right; the compound's head is what the peel leaves behind. No corpus held a compound office word when the review found this, which is why the snapshot was silent; the two W2 examples now put `岸田文雄衆議院議員` and `田中 副社長` in corpus_rules.jsonl. Older entries carry the same cost unrecorded until now (`김민준의학박사` reads given 민준의학, suffix 박사). Listing a compound whole would peel it whole, longest first. +- 2026-10-17 #707 — the #671 words meet the glued bar, all but 部長/部长 (Derek, 2026-10-17). 議員/议员, 国会議員/國會議員, 太平紳士/太平绅士, 医学博士/醫學博士, 二世, 社長/課長 and 社长/课长 join `GLUED_HONORIFICS`; 部長/部长 stay spaced-only on the #671 argument, since 長 is a borne given name (Wikidata holds it as a Chinese given name, Q96240389; the Japanese reading rests on the argument) and 阿部長 can be 阿部 + 長 — the measurement below found no such bearer, and a zero that is a lower bound does not refute the argument. The measurement is `tools/name_usage.py`'s ends-with row, new in #707: a local index of every CJK label on eleven name-item classes (family name, given name and their Japanese, Chinese, Korean, male, female, unisex and undifferentiated-kanji subclasses; 215,798 index entries on 128,695 items, 215,696 distinct item-label pairs, fetched 2026-10-17 by the first index, whose character class stopped at the Basic Multilingual Plane; the rebuilt index that takes supplementary-plane Han too (215,820 entries, same day) gave the same answer for every word, 殿 and the boundary pairs below), matched locally, with the matching items' bearers counted live. Two questions per word: does any name item's label END in it, and does any person bear a family name ending in the word's head and a given name equal to its tail (the 阿部 + 長 shape)? Neither found anything for any spelling, with one exception that is a data error: the only item ending in 二世 is Q18339792, the given name Odo, whose Chinese label carries a regnal numeral (厄德二世), and its 109 bearers are Western people named Odo. Of the boundary shapes, only 二+世, 社+長, 社+长, 部+長 and 部+长 had a family name and a given-name item to pair at all, and each counted 0 people. What makes the zeros evidence: the index agreed exactly with a live `STRENDS` query on all 20 class × word cells it was cross-checked on (five subclasses × 殿, 子, 郎, 田, the largest 403 items); 子 found 439 items and 2,436 given-name bearers, 2,081 of them Japanese; and 田 + 太郎 counted 22 people across the boundary. What limits them: Wikidata's coverage, not the index's — 殿 ends 13 name items where a surname dictionary lists about 90, and Japanese family-name items number 4,919. Two shortcuts were measured and rejected before the index: `STRENDS` over every label of a name class timed out at WDQS's 60s with no person join, and WDQS full-text search found 4 family names ending in 殿, matching only where its tokenizer split the character out. Consequence: the snapshot moved four CLDR renderings, each losing its glued word to `suffix` (`王雅婷婷婷二世議員`, both spellings of `…小惠儀太平紳士`, `馮·陳·怡君·達印·二世醫學博士`), and `陳怡君二世`, which the case table and rules.md W2 pin; cldr_agreement records one suffix field newly agreeing (`二世 醫學博士`, where the peel takes 醫學博士 and the 二世 left behind is a whole-token suffix word) and none regressing. The peel still takes one word: `王雅婷婷婷二世議員` peels 議員 and keeps 二世 in the name. Recompute: `uv run python tools/name_usage.py 議員 议员 国会議員 國會議員 太平紳士 太平绅士 医学博士 醫學博士 二世 社長 部長 課長 社长 部长 课长`; the first run builds the index (33 minutes on 2026-10-17, resumable), and `--refresh-index` rebuilds it. ACCEPTED COST, found by the design-docs review and kept (Derek, 2026-10-17): the vetting asked whether a NAME ends in each word and never whether a longer, unlisted OFFICE word does, and common ones do — 衆議院議員 and 参議院議員, 副社長 and 代表取締役社長, 元議員 and 市议员, 副課長, 獣医学博士. The peel matches at the end of the token, spaced or glued, so each splits at the listed tail: `岸田文雄衆議院議員` reads family 岸田文雄衆議院, suffix 議員, and `田中 副社長` given 副, suffix 社長, where before the word stayed whole in the name. Neither reading is right; the compound's head is what the peel leaves behind. No corpus held a compound office word when the review found this, which is why the snapshot was silent; the two W2 examples now put `岸田文雄衆議院議員` and `田中 副社長` in corpus_rules.jsonl. Older entries carry the same cost unrecorded until now (`김민준의학박사` reads given 민준의학, suffix 박사). Listing a compound whole would peel it whole, longest first. Excluded (the never-given / ambiguous particle line, nameparser/config/particles.py — #360 owns the vocabulary question):