From 4bbb16f348f6ae4cb9541930fec2210335ce524e Mon Sep 17 00:00:00 2001 From: ZHIJUN XU <57060481+xzjncu@users.noreply.github.com> Date: Mon, 27 Jul 2026 09:23:11 +0800 Subject: [PATCH 1/2] fix: normalize_title retains Unicode alphanumeric chars via NFKC+casefold; add full CJK name equality check for leading author matching - normalize_title(): replace ASCII-only a-z0-9 filter with unicodedata.normalize("NFKC") + character.isalnum() to retain CJK/Unicode characters - _full_leading_author_matches(): add NFKC+casefold full-name equality check before falling back to per-part comparison - _leading_author_status(): add normalized full-name equality as a match condition alongside family-name key comparison Fixes source_pdf_mismatch false negatives for Chinese-titled papers (e.g. same title/author previously scored title_similarity=0.0, leading_author=conflict). --- skills/deeppapernote/scripts/common.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/skills/deeppapernote/scripts/common.py b/skills/deeppapernote/scripts/common.py index e3ecd10..0e73b22 100644 --- a/skills/deeppapernote/scripts/common.py +++ b/skills/deeppapernote/scripts/common.py @@ -135,7 +135,10 @@ def strip_tags(text: str) -> str: def normalize_title(text: str) -> str: - return re.sub(r"[^a-z0-9\s]", "", normalize_whitespace(text).lower()).strip() + normalized = unicodedata.normalize("NFKC", normalize_whitespace(text)).casefold() + return normalize_whitespace("".join( + character for character in normalized if character.isalnum() or character.isspace() + )) LOCAL_PDF_PREFIX_PATTERN = re.compile(r"^(?:[^-]{1,120})\s+-\s+(?:19|20)\d{2}\s+-\s+") @@ -576,6 +579,10 @@ def _full_leading_author_matches( work_authors = _dedupe_string_list(work_record.get("authors", [])) if not source_authors or not work_authors: return False + source_normalized = unicodedata.normalize("NFKC", source_authors[0]).casefold() + work_normalized = unicodedata.normalize("NFKC", work_authors[0]).casefold() + if source_normalized == work_normalized: + return True source_parts = _author_identity_parts(source_authors[0]) work_parts = _author_identity_parts(work_authors[0]) if len(source_parts) < 2 or len(work_parts) < 2: @@ -598,9 +605,11 @@ def _leading_author_status( work_authors = _dedupe_string_list(work_record.get("authors", [])) if not source_authors or not work_authors: return None + source_normalized = unicodedata.normalize("NFKC", source_authors[0]).casefold() + work_normalized = unicodedata.normalize("NFKC", work_authors[0]).casefold() source_key = _author_key(source_authors[0]) work_key = _author_key(work_authors[0]) - status = "match" if source_key and source_key == work_key else "conflict" + status = "match" if source_normalized == work_normalized or (source_key and source_key == work_key) else "conflict" return { "kind": "leading_author", "status": status, @@ -3990,4 +3999,4 @@ def pick_sentences_by_keywords(text: str, keywords: list[str], *, limit: int = 5 picked.append(normalize_whitespace(sentence)) if len(picked) >= limit: break - return picked + return picked \ No newline at end of file From ceff2878b92284131644bbf12a595d0cafc53f9b Mon Sep 17 00:00:00 2001 From: dingdingcar <1054263492@qq.com> Date: Tue, 28 Jul 2026 15:30:01 +0800 Subject: [PATCH 2/2] fix: contain Unicode identity matching --- skills/deeppapernote/scripts/common.py | 26 +++++---- tests/test_common.py | 74 ++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 10 deletions(-) diff --git a/skills/deeppapernote/scripts/common.py b/skills/deeppapernote/scripts/common.py index 06fcd21..670c9b3 100644 --- a/skills/deeppapernote/scripts/common.py +++ b/skills/deeppapernote/scripts/common.py @@ -146,10 +146,7 @@ def strip_tags(text: str) -> str: def normalize_title(text: str) -> str: - normalized = unicodedata.normalize("NFKC", normalize_whitespace(text)).casefold() - return normalize_whitespace("".join( - character for character in normalized if character.isalnum() or character.isspace() - )) + return re.sub(r"[^a-z0-9\s]", "", normalize_whitespace(text).lower()).strip() LOCAL_PDF_PREFIX_PATTERN = re.compile(r"^(?:[^-]{1,120})\s+-\s+(?:19|20)\d{2}\s+-\s+") @@ -307,6 +304,10 @@ def env_config_value(*names: str, default: str = "") -> str: def title_similarity(a: str, b: str) -> float: + a_identity = normalize_identity_title(a) + b_identity = normalize_identity_title(b) + if a_identity and a_identity == b_identity: + return 1.0 a_norm = normalize_title(a) b_norm = normalize_title(b) if not a_norm or not b_norm: @@ -590,8 +591,8 @@ def _full_leading_author_matches( work_authors = _dedupe_string_list(work_record.get("authors", [])) if not source_authors or not work_authors: return False - source_normalized = unicodedata.normalize("NFKC", source_authors[0]).casefold() - work_normalized = unicodedata.normalize("NFKC", work_authors[0]).casefold() + source_normalized = normalize_identity_title(source_authors[0]) + work_normalized = normalize_identity_title(work_authors[0]) if source_normalized == work_normalized: return True source_parts = _author_identity_parts(source_authors[0]) @@ -616,11 +617,16 @@ def _leading_author_status( work_authors = _dedupe_string_list(work_record.get("authors", [])) if not source_authors or not work_authors: return None - source_normalized = unicodedata.normalize("NFKC", source_authors[0]).casefold() - work_normalized = unicodedata.normalize("NFKC", work_authors[0]).casefold() + source_normalized = normalize_identity_title(source_authors[0]) + work_normalized = normalize_identity_title(work_authors[0]) source_key = _author_key(source_authors[0]) work_key = _author_key(work_authors[0]) - status = "match" if source_normalized == work_normalized or (source_key and source_key == work_key) else "conflict" + status = ( + "match" + if source_normalized == work_normalized + or (source_key and source_key == work_key) + else "conflict" + ) return { "kind": "leading_author", "status": status, @@ -4010,4 +4016,4 @@ def pick_sentences_by_keywords(text: str, keywords: list[str], *, limit: int = 5 picked.append(normalize_whitespace(sentence)) if len(picked) >= limit: break - return picked \ No newline at end of file + return picked diff --git a/tests/test_common.py b/tests/test_common.py index 2e38447..1b02b71 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -406,6 +406,80 @@ def test_local_pdf_good_title_does_not_use_unrelated_external_title() -> None: assert corrected == "" +def test_manifestation_equivalence_accepts_identical_cjk_identity() -> None: + title = "双元联盟网络如何影响突破式创新——技术能力与网络惯例的调节作用" + decision = common.manifestation_equivalence_decision( + {"title": title, "authors": ["朱云鹃"]}, + {"title": title, "authors": ["朱云鹃"]}, + ) + + evidence = {item["kind"]: item for item in decision["evidence"]} + assert decision["status"] == "equivalent" + assert evidence["title_similarity"]["score"] == 1.0 + assert evidence["leading_author"]["status"] == "match" + + +def test_manifestation_equivalence_rejects_distinct_spaced_cjk_authors() -> None: + decision = common.manifestation_equivalence_decision( + {"title": "完全不同论文", "authors": ["李 云鹃"]}, + {"title": "深度学习研究一", "authors": ["朱 云鹃"]}, + ) + + evidence = {item["kind"]: item for item in decision["evidence"]} + assert decision["status"] == "ambiguous" + assert evidence["leading_author"]["status"] == "conflict" + + +def test_manifestation_equivalence_preserves_unicode_combining_marks() -> None: + decision = common.manifestation_equivalence_decision( + {"title": "कतब", "authors": ["करण"]}, + {"title": "किताब", "authors": ["किरण"]}, + ) + + evidence = {item["kind"]: item for item in decision["evidence"]} + assert decision["status"] == "ambiguous" + assert evidence["title_similarity"]["score"] == 0.0 + assert evidence["leading_author"]["status"] == "conflict" + + +def test_ascii_identity_matching_remains_unchanged() -> None: + assert common.normalize_title("Attention: Is All You Need!") == ( + "attention is all you need" + ) + equivalent = common.manifestation_equivalence_decision( + {"title": "attention is all you need", "authors": ["Alice Example"]}, + {"title": "Attention Is All You Need", "authors": ["Alice Example"]}, + ) + ambiguous = common.manifestation_equivalence_decision( + {"title": "Completely Different Paper", "authors": ["Bob Other"]}, + {"title": "Attention Is All You Need", "authors": ["Alice Example"]}, + ) + + assert equivalent["status"] == "equivalent" + assert ambiguous["status"] == "ambiguous" + + +def test_identity_adjudication_accepts_identical_cjk_author() -> None: + title = "双元联盟网络如何影响突破式创新——技术能力与网络惯例的调节作用" + decision = common.adjudicate_identity_observations( + {"title": title, "authors": ["朱云鹃"], "year": "2026"}, + [ + { + "provider": "crossref", + "record": { + "title": title, + "authors": ["朱云鹃"], + "year": "2026", + }, + } + ], + ) + + assert decision["rejected_observations"] == [] + assert len(decision["accepted_observations"]) == 1 + assert decision["accepted_observations"][0]["reason"] == "title_author_year" + + def test_resolve_note_output_mode_falls_back_to_workspace(tmp_path: Path, monkeypatch) -> None: monkeypatch.chdir(tmp_path) config = {