From 332d07c46cd0141b4bb379118db194916974ca73 Mon Sep 17 00:00:00 2001 From: Atsushi Kojima Date: Fri, 21 Aug 2026 15:21:48 +0900 Subject: [PATCH 1/2] feat(search): support CJK queries in ticket and article search MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Search silently dropped every Japanese, Chinese and Korean character. `Search.clean_query()` and `sanitize_query()` stripped anything outside `[a-zA-Z0-9\s]`, so a CJK-only query became an empty string and returned nothing. Sites running in ja/zh/ko could not search their own tickets. - add `helpdesk/search_i18n.py`: NFKC normalisation that keeps letters and digits in any script, CJK detection, and n-gram helpers - index a `cjk_terms` field (2- and 3-grams) for both the Redis and the SQLite backends so substring matches work without a CJK tokenizer - expand CJK runs in the query into 3-grams; Latin terms are untouched - skip the TextBlob noun-phrase fallback for CJK queries (its corpus is English-only and it discarded valid results) - bump the SQLite index filename so existing sites rebuild with the new column - add unit tests for normalisation, detection, n-grams and query expansion Verified with FTS5: "パスワード" matches a document titled "パスワード変更の手順" while an unrelated document does not match. --- helpdesk/api/article.py | 13 +++----- helpdesk/search.py | 36 +++++++++++++++------ helpdesk/search_i18n.py | 63 ++++++++++++++++++++++++++++++++++++ helpdesk/search_sqlite.py | 47 +++++++++++++++++++++++++-- helpdesk/test_search_i18n.py | 58 +++++++++++++++++++++++++++++++++ 5 files changed, 197 insertions(+), 20 deletions(-) create mode 100644 helpdesk/search_i18n.py create mode 100644 helpdesk/test_search_i18n.py diff --git a/helpdesk/api/article.py b/helpdesk/api/article.py index 3368c5c8a6..016985151c 100644 --- a/helpdesk/api/article.py +++ b/helpdesk/api/article.py @@ -1,11 +1,10 @@ -import re - import frappe from textblob import TextBlob from textblob.exceptions import MissingCorpusError from helpdesk.search import NUM_RESULTS from helpdesk.search import search as hd_search +from helpdesk.search_i18n import contains_cjk, normalize_search_text def get_nouns(blob: TextBlob): @@ -34,11 +33,7 @@ def search_with_enough_results( def sanitize_query(query: str) -> str: - q = query.strip().lower() - q = re.sub(r"[^a-z0-9\s]", " ", q) - # Collapse multiple spaces into one - q = re.sub(r"\s+", " ", q) - return q.strip() + return normalize_search_text(query) @frappe.whitelist() @@ -71,8 +66,10 @@ def get_article_stats(article_name: str): @frappe.whitelist() def search(query: str) -> list: query = sanitize_query(query) + if not query: + return [] ret, enough = search_with_enough_results([], query) - if enough: + if enough or contains_cjk(query): return ret blob = TextBlob(query) # fallback if noun_phrases := get_noun_phrases(blob): diff --git a/helpdesk/search.py b/helpdesk/search.py index 2935f76f3e..678b60a0ca 100644 --- a/helpdesk/search.py +++ b/helpdesk/search.py @@ -25,6 +25,12 @@ from redis.commands.search.query import Query from redis.exceptions import ResponseError +from helpdesk.search_i18n import ( + cjk_index_terms, + expand_cjk_query, + normalize_search_text, +) + if TYPE_CHECKING: from helpdesk.helpdesk.doctype.hd_settings.hd_settings import HDSettings @@ -93,8 +99,6 @@ def get_synonym_words() -> list[str]: class Search: - unsafe_chars = re.compile(r"[^a-zA-Z0-9\s]") - def __init__(self, index_name, prefix, schema) -> None: self.redis = frappe.cache() self.index_name = index_name @@ -155,8 +159,10 @@ def search( start=0, page_length=NUM_RESULTS, highlight=False, + prepared=False, ): - query = self.clean_query(query) + if not prepared: + query = self.clean_query(query) query = Query(query).paging(start, page_length) if highlight: query = query.highlight() @@ -178,11 +184,7 @@ def search( return out def clean_query(self, query): - query = query.strip().replace("-*", "*") - query = self.unsafe_chars.sub(" ", query) - # Collapse multiple spaces - query = re.sub(r"\s+", " ", query) - return query.strip().lower() + return normalize_search_text(query) def spellcheck(self, query, **kwargs): return self.redis.ft(self.index_name).spellcheck(query, **kwargs) @@ -231,6 +233,11 @@ def __init__(self): {"name": "subject", "weight": settings.subject_weight or 6}, {"name": "description", "weight": settings.description_weight or 5}, {"name": "headings", "weight": settings.headings_weight or 8}, + { + "name": "cjk_terms", + "weight": settings.description_weight or 5, + "no_stem": True, + }, {"name": "modified", "sortable": True}, {"name": "creation", "sortable": True}, ] @@ -255,6 +262,15 @@ def index_doc(self, doc): "subject": doc.title, "description": strip_html_tags(doc.content), "headings": doc.headings, + "cjk_terms": cjk_index_terms( + " ".join( + ( + doc.title or "", + strip_html_tags(doc.content or ""), + doc.headings or "", + ) + ) + ), "modified": doc.modified, } self.add_document(id, fields) @@ -323,7 +339,7 @@ def get_records(self, doctype): def search(query, qtype: Literal["and", "or"] = "and") -> list[dict[str, list[dict]]]: search = HelpdeskSearch() - query = search.clean_query(query) + query = expand_cjk_query(search.clean_query(query)) query_parts: list[str] = query.split() query = "" sep = " " if qtype == "and" else "|" @@ -339,7 +355,7 @@ def search(query, qtype: Literal["and", "or"] = "and") -> list[dict[str, list[di query += f"{sep}{part}*" query = query.lstrip(sep) # Remove leading separator (| at beginning is invalid) - result = search.search(query, start=0, highlight=True) + result = search.search(query, start=0, highlight=True, prepared=True) groups = {} for r in result.docs: doctype, name = r.id.split(":") diff --git a/helpdesk/search_i18n.py b/helpdesk/search_i18n.py new file mode 100644 index 0000000000..004886cf8e --- /dev/null +++ b/helpdesk/search_i18n.py @@ -0,0 +1,63 @@ +# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and Contributors +# MIT License. See license.txt + +import re +import unicodedata + + +CJK_RUN_RE = re.compile( + r"[\u3005-\u3007\u303b\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]+" +) +WHITESPACE_RE = re.compile(r"\s+") + + +def normalize_search_text(text: str | None) -> str: + """Normalize user input while preserving Japanese letters and numbers.""" + if not text: + return "" + + normalized = unicodedata.normalize("NFKC", str(text)).lower() + sanitized = "".join( + char if char.isalnum() or char.isspace() else " " for char in normalized + ) + return WHITESPACE_RE.sub(" ", sanitized).strip() + + +def contains_cjk(text: str | None) -> bool: + return bool(text and CJK_RUN_RE.search(unicodedata.normalize("NFKC", str(text)))) + + +def cjk_ngrams(text: str | None, sizes: tuple[int, ...] = (2, 3)) -> list[str]: + """Return stable, unique CJK n-grams for substring search.""" + normalized = normalize_search_text(text) + terms = [] + seen = set() + + for run in CJK_RUN_RE.findall(normalized): + for size in sizes: + if len(run) < size: + continue + for offset in range(len(run) - size + 1): + term = run[offset : offset + size] + if term not in seen: + seen.add(term) + terms.append(term) + + return terms + + +def cjk_index_terms(text: str | None) -> str: + return " ".join(cjk_ngrams(text)) + + +def expand_cjk_query(text: str | None) -> str: + """Expand CJK runs into searchable terms without changing Latin terms.""" + normalized = normalize_search_text(text) + + def expand(match: re.Match) -> str: + run = match.group(0) + if len(run) <= 2: + return f" {run} " + return f" {' '.join(cjk_ngrams(run, sizes=(3,)))} " + + return WHITESPACE_RE.sub(" ", CJK_RUN_RE.sub(expand, normalized)).strip() diff --git a/helpdesk/search_sqlite.py b/helpdesk/search_sqlite.py index 5e7897d412..448f0b38c5 100644 --- a/helpdesk/search_sqlite.py +++ b/helpdesk/search_sqlite.py @@ -4,6 +4,13 @@ import frappe from frappe.search.sqlite_search import SQLiteSearch, SQLiteSearchIndexMissingError +from helpdesk.search_i18n import ( + cjk_index_terms, + contains_cjk, + expand_cjk_query, + normalize_search_text, +) + class HelpdeskSearchIndexMissingError(SQLiteSearchIndexMissingError): pass @@ -16,12 +23,14 @@ class HelpdeskSearchIndexMissingError(SQLiteSearchIndexMissingError): class HelpdeskSearch(SQLiteSearch): - INDEX_NAME = "helpdesk_search.db" + # Bumped so existing sites rebuild the index with the CJK terms column. + INDEX_NAME = "helpdesk_search_v2.db" # Resting value: core search() can bail before get_search_filters() runs. is_post_filter_required = False INDEX_SCHEMA = { + "text_fields": ["title", "content", "cjk_terms"], "metadata_fields": [ "agent_group", "customer", @@ -74,7 +83,26 @@ class HelpdeskSearch(SQLiteSearch): } def search(self, query, title_only: bool = False, filters: dict | None = None): - result = super().search(query, title_only=title_only, filters=filters) + japanese_title_search = title_only and contains_cjk(query) + result = super().search( + query, + # Japanese substring terms live in the hidden n-gram field. + title_only=False if japanese_title_search else title_only, + filters=filters, + ) + if japanese_title_search: + query_parts = normalize_search_text(query).split() + result["results"] = [ + row + for row in result["results"] + if all( + part in normalize_search_text(row.get("title", "")) + for part in query_parts + ) + ] + result["summary"]["title_only"] = True + result["summary"]["filtered_matches"] = len(result["results"]) + result["summary"]["returned_matches"] = len(result["results"]) if self.is_post_filter_required: result["results"] = self._drop_unpermitted(result["results"]) result["summary"]["filtered_matches"] = len(result["results"]) @@ -126,6 +154,10 @@ def prepare_document(self, doc): if not document: return None + document["cjk_terms"] = cjk_index_terms( + " ".join((document.get("title", ""), document.get("content", ""))) + ) + if ( doc.doctype == "HD Ticket Comment" and doc.reference_ticket @@ -156,6 +188,17 @@ def prepare_document(self, doc): return document + def _expand_query_with_corrections(self, query): + # Expand Japanese before spelling correction; 2-3 character terms are + # intentionally left untouched by Frappe's English-oriented corrector. + return super()._expand_query_with_corrections(expand_cjk_query(query)) + + def _process_search_results(self, raw_results, query): + results = super()._process_search_results(raw_results, query) + for result in results: + result.pop("cjk_terms", None) + return results + def get_filter_options(self): """Get available filter options for search interface.""" if not self.index_exists(): diff --git a/helpdesk/test_search_i18n.py b/helpdesk/test_search_i18n.py new file mode 100644 index 0000000000..c4a4ff96b6 --- /dev/null +++ b/helpdesk/test_search_i18n.py @@ -0,0 +1,58 @@ +import sqlite3 +import unittest + +from helpdesk.search_i18n import ( + cjk_index_terms, + cjk_ngrams, + contains_cjk, + expand_cjk_query, + normalize_search_text, +) + + +class TestJapaneseSearch(unittest.TestCase): + def test_normalization_preserves_japanese_and_normalizes_width(self): + self.assertEqual( + normalize_search_text(" ﾊﾟｽﾜｰﾄﾞ変更！ VPN-Error "), + "パスワード変更 vpn error", + ) + + def test_cjk_detection(self): + self.assertTrue(contains_cjk("障害 notification")) + self.assertTrue(contains_cjk("時々確認")) + self.assertFalse(contains_cjk("incident notification")) + + def test_index_contains_bigrams_and_trigrams(self): + terms = cjk_ngrams("障害通知") + self.assertEqual(terms, ["障害", "害通", "通知", "障害通", "害通知"]) + + def test_query_uses_trigrams_for_contiguous_phrase(self): + self.assertEqual(expand_cjk_query("VPN 障害通知"), "vpn 障害通 害通知") + self.assertEqual(expand_cjk_query("障害"), "障害") + + def test_ngram_terms_make_japanese_substrings_searchable(self): + connection = sqlite3.connect(":memory:") + self.addCleanup(connection.close) + connection.execute( + "CREATE VIRTUAL TABLE documents USING fts5(content, cjk_terms, " + 'tokenize="unicode61 remove_diacritics 2")' + ) + connection.execute( + "INSERT INTO documents VALUES (?, ?)", + ( + "パスワード変更時に障害通知を確認します", + cjk_index_terms("パスワード変更時に障害通知を確認します"), + ), + ) + + for query in ("パスワード", "変更", "障害通知"): + terms = expand_cjk_query(query).split() + fts_query = " ".join(f'"{term}"' for term in terms) + count = connection.execute( + "SELECT COUNT(*) FROM documents WHERE documents MATCH ?", (fts_query,) + ).fetchone()[0] + self.assertEqual(count, 1, query) + + +if __name__ == "__main__": + unittest.main() From 7a72cfc4bcb426b3b44d0de31c3b2508e908c8f4 Mon Sep 17 00:00:00 2001 From: Atsushi Kojima Date: Fri, 21 Aug 2026 15:47:10 +0900 Subject: [PATCH 2/2] fix(search): rebuild the Redis index when the schema gains a field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SQLite backend gets a new index filename, so upgraded sites rebuild it. The Redis index keeps its name, and `index_exists()` only compared the document count — which stays correct across schema changes. An index built before `cjk_terms` existed therefore looked valid, was never rebuilt, and could not serve the CJK query path. `index_exists()` now also compares the declared schema against the fields reported by FT.INFO and treats a missing field as "index absent". When the attribute shape cannot be parsed it falls back to the previous behaviour rather than forcing a rebuild. Adds tests for the FT.INFO attribute shapes (flat lists, byte strings and mappings) and for the stale-index case. --- helpdesk/search.py | 17 ++++++++++++-- helpdesk/search_i18n.py | 27 ++++++++++++++++++++++ helpdesk/test_search_i18n.py | 43 ++++++++++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+), 2 deletions(-) diff --git a/helpdesk/search.py b/helpdesk/search.py index 678b60a0ca..6188900441 100644 --- a/helpdesk/search.py +++ b/helpdesk/search.py @@ -28,6 +28,7 @@ from helpdesk.search_i18n import ( cjk_index_terms, expand_cjk_query, + indexed_field_names, normalize_search_text, ) @@ -202,14 +203,26 @@ def num_records(self) -> int: num += self.get_count(doctype) return num + def index_fields(self) -> set[str]: + """Field names declared in this class's schema.""" + return {f["name"] for f in self.schema if isinstance(f, dict) and f.get("name")} + def index_exists(self): if hasattr(self, "_index_exists"): return self._index_exists self._index_exists = False with suppress(ResponseError): ftinfo = self.redis.ft(self.index_name).info() - if isclose(int(ftinfo["num_docs"]), self.num_records(), rel_tol=0.1): - self._index_exists = True + if not isclose(int(ftinfo["num_docs"]), self.num_records(), rel_tol=0.1): + return self._index_exists + # The document count alone cannot tell us whether the index was built + # from the current schema. An index created before a field was added + # has the right number of documents but cannot serve queries against + # the new field, so treat it as missing and let it be rebuilt. + existing = indexed_field_names(ftinfo.get("attributes")) + if existing and not self.index_fields() <= existing: + return self._index_exists + self._index_exists = True return self._index_exists diff --git a/helpdesk/search_i18n.py b/helpdesk/search_i18n.py index 004886cf8e..a224e34daa 100644 --- a/helpdesk/search_i18n.py +++ b/helpdesk/search_i18n.py @@ -61,3 +61,30 @@ def expand(match: re.Match) -> str: return f" {' '.join(cjk_ngrams(run, sizes=(3,)))} " return WHITESPACE_RE.sub(" ", CJK_RUN_RE.sub(expand, normalized)).strip() + + +def indexed_field_names(attributes) -> set[str]: + """Field names present in an existing RediSearch index. + + FT.INFO reports attributes either as flat lists + (``["identifier", "title", "attribute", "title", "type", "TEXT", ...]``) + or as mappings, depending on the server and client version. Returning an + empty set means "could not tell", and callers should not treat that as + proof that a field is missing. + """ + names = set() + for attr in attributes or []: + if isinstance(attr, dict): + name = attr.get("identifier") or attr.get("attribute") + if name: + names.add(name.decode() if isinstance(name, bytes) else str(name)) + continue + if isinstance(attr, (list, tuple)): + values = [v.decode() if isinstance(v, bytes) else str(v) for v in attr] + if "identifier" in values: + idx = values.index("identifier") + 1 + if idx < len(values): + names.add(values[idx]) + elif values: + names.add(values[0]) + return names diff --git a/helpdesk/test_search_i18n.py b/helpdesk/test_search_i18n.py index c4a4ff96b6..8148b1f462 100644 --- a/helpdesk/test_search_i18n.py +++ b/helpdesk/test_search_i18n.py @@ -3,6 +3,7 @@ from helpdesk.search_i18n import ( cjk_index_terms, + indexed_field_names, cjk_ngrams, contains_cjk, expand_cjk_query, @@ -54,5 +55,47 @@ def test_ngram_terms_make_japanese_substrings_searchable(self): self.assertEqual(count, 1, query) +class TestIndexFieldDetection(unittest.TestCase): + """An index built before `cjk_terms` existed must be detected as stale. + + FT.INFO only reports the document count, which stays correct across schema + changes; without looking at the attributes an upgraded site keeps serving + an index that cannot match CJK queries. + """ + + def test_flat_attributes_with_identifier(self): + attributes = [ + ["identifier", "subject", "attribute", "subject", "type", "TEXT"], + ["identifier", "description", "attribute", "description", "type", "TEXT"], + ] + self.assertEqual( + indexed_field_names(attributes), {"subject", "description"} + ) + + def test_bytes_and_mapping_attributes(self): + attributes = [ + [b"identifier", b"subject", b"type", b"TEXT"], + {"identifier": "cjk_terms", "type": "TEXT"}, + ] + self.assertEqual(indexed_field_names(attributes), {"subject", "cjk_terms"}) + + def test_legacy_flat_attributes_without_identifier(self): + self.assertEqual( + indexed_field_names([["subject", "type", "TEXT"]]), {"subject"} + ) + + def test_missing_field_is_visible(self): + old_index = indexed_field_names( + [["identifier", "subject"], ["identifier", "description"]] + ) + wanted = {"subject", "description", "cjk_terms"} + self.assertFalse(wanted <= old_index) + + def test_unknown_shape_returns_empty(self): + # Empty means "cannot tell"; callers must not rebuild on that basis. + self.assertEqual(indexed_field_names(None), set()) + self.assertEqual(indexed_field_names([42]), set()) + + if __name__ == "__main__": unittest.main()