feat(search): support CJK queries in ticket and article search

helpdesk/api/article.py

@@ -1,11 +1,10 @@
-import re
-
 import frappe
 from textblob import TextBlob
 from textblob.exceptions import MissingCorpusError
 
 from helpdesk.search import NUM_RESULTS
 from helpdesk.search import search as hd_search
+from helpdesk.search_i18n import contains_cjk, normalize_search_text
 
 
 def get_nouns(blob: TextBlob):
@@ -34,11 +33,7 @@ def search_with_enough_results(
 
 
 def sanitize_query(query: str) -> str:
-    q = query.strip().lower()
-    q = re.sub(r"[^a-z0-9\s]", " ", q)
-    # Collapse multiple spaces into one
-    q = re.sub(r"\s+", " ", q)
-    return q.strip()
+    return normalize_search_text(query)
 
 
 @frappe.whitelist()
@@ -71,8 +66,10 @@ def get_article_stats(article_name: str):
 @frappe.whitelist()
 def search(query: str) -> list:
     query = sanitize_query(query)
+    if not query:
+        return []
     ret, enough = search_with_enough_results([], query)
-    if enough:
+    if enough or contains_cjk(query):
         return ret
     blob = TextBlob(query)  # fallback
     if noun_phrases := get_noun_phrases(blob):

helpdesk/search.py

@@ -25,6 +25,13 @@
 from redis.commands.search.query import Query
 from redis.exceptions import ResponseError
 
+from helpdesk.search_i18n import (
+    cjk_index_terms,
+    expand_cjk_query,
+    indexed_field_names,
+    normalize_search_text,
+)
+
 if TYPE_CHECKING:
     from helpdesk.helpdesk.doctype.hd_settings.hd_settings import HDSettings
 
@@ -93,8 +100,6 @@ def get_synonym_words() -> list[str]:
 
 
 class Search:
-    unsafe_chars = re.compile(r"[^a-zA-Z0-9\s]")
-
     def __init__(self, index_name, prefix, schema) -> None:
         self.redis = frappe.cache()
         self.index_name = index_name
@@ -155,8 +160,10 @@ def search(
         start=0,
         page_length=NUM_RESULTS,
         highlight=False,
+        prepared=False,
     ):
-        query = self.clean_query(query)
+        if not prepared:
+            query = self.clean_query(query)
         query = Query(query).paging(start, page_length)
         if highlight:
             query = query.highlight()
@@ -178,11 +185,7 @@ def search(
         return out
 
     def clean_query(self, query):
-        query = query.strip().replace("-*", "*")
-        query = self.unsafe_chars.sub(" ", query)
-        # Collapse multiple spaces
-        query = re.sub(r"\s+", " ", query)
-        return query.strip().lower()
+        return normalize_search_text(query)
 
     def spellcheck(self, query, **kwargs):
         return self.redis.ft(self.index_name).spellcheck(query, **kwargs)
@@ -200,14 +203,26 @@ def num_records(self) -> int:
             num += self.get_count(doctype)
         return num
 
+    def index_fields(self) -> set[str]:
+        """Field names declared in this class's schema."""
+        return {f["name"] for f in self.schema if isinstance(f, dict) and f.get("name")}
+
     def index_exists(self):
         if hasattr(self, "_index_exists"):
             return self._index_exists
         self._index_exists = False
         with suppress(ResponseError):
             ftinfo = self.redis.ft(self.index_name).info()
-            if isclose(int(ftinfo["num_docs"]), self.num_records(), rel_tol=0.1):
-                self._index_exists = True
+            if not isclose(int(ftinfo["num_docs"]), self.num_records(), rel_tol=0.1):
+                return self._index_exists
+            # The document count alone cannot tell us whether the index was built
+            # from the current schema. An index created before a field was added
+            # has the right number of documents but cannot serve queries against
+            # the new field, so treat it as missing and let it be rebuilt.
+            existing = indexed_field_names(ftinfo.get("attributes"))
+            if existing and not self.index_fields() <= existing:
+                return self._index_exists
+            self._index_exists = True
         return self._index_exists
 
 
@@ -231,6 +246,11 @@ def __init__(self):
             {"name": "subject", "weight": settings.subject_weight or 6},
             {"name": "description", "weight": settings.description_weight or 5},
             {"name": "headings", "weight": settings.headings_weight or 8},
+            {
+                "name": "cjk_terms",
+                "weight": settings.description_weight or 5,
+                "no_stem": True,
+            },
             {"name": "modified", "sortable": True},
             {"name": "creation", "sortable": True},
         ]
@@ -255,6 +275,15 @@ def index_doc(self, doc):
                 "subject": doc.title,
                 "description": strip_html_tags(doc.content),
                 "headings": doc.headings,
+                "cjk_terms": cjk_index_terms(
+                    " ".join(
+                        (
+                            doc.title or "",
+                            strip_html_tags(doc.content or ""),
+                            doc.headings or "",
+                        )
+                    )
+                ),
                 "modified": doc.modified,
             }
             self.add_document(id, fields)
@@ -323,7 +352,7 @@ def get_records(self, doctype):
 
 def search(query, qtype: Literal["and", "or"] = "and") -> list[dict[str, list[dict]]]:
     search = HelpdeskSearch()
-    query = search.clean_query(query)
+    query = expand_cjk_query(search.clean_query(query))
     query_parts: list[str] = query.split()
     query = ""
     sep = " " if qtype == "and" else "|"
@@ -339,7 +368,7 @@ def search(query, qtype: Literal["and", "or"] = "and") -> list[dict[str, list[di
             query += f"{sep}{part}*"
 
     query = query.lstrip(sep)  # Remove leading separator (| at beginning is invalid)
-    result = search.search(query, start=0, highlight=True)
+    result = search.search(query, start=0, highlight=True, prepared=True)
     groups = {}
     for r in result.docs:
         doctype, name = r.id.split(":")

helpdesk/search_i18n.py

@@ -0,0 +1,90 @@
+# Copyright (c) 2026, Frappe Technologies Pvt. Ltd. and Contributors
+# MIT License. See license.txt
+
+import re
+import unicodedata
+
+
+CJK_RUN_RE = re.compile(
+    r"[\u3005-\u3007\u303b\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]+"
+)
+WHITESPACE_RE = re.compile(r"\s+")
+
+
+def normalize_search_text(text: str | None) -> str:
+    """Normalize user input while preserving Japanese letters and numbers."""
+    if not text:
+        return ""
+
+    normalized = unicodedata.normalize("NFKC", str(text)).lower()
+    sanitized = "".join(
+        char if char.isalnum() or char.isspace() else " " for char in normalized
+    )
+    return WHITESPACE_RE.sub(" ", sanitized).strip()
+
+
+def contains_cjk(text: str | None) -> bool:
+    return bool(text and CJK_RUN_RE.search(unicodedata.normalize("NFKC", str(text))))
+
+
+def cjk_ngrams(text: str | None, sizes: tuple[int, ...] = (2, 3)) -> list[str]:
+    """Return stable, unique CJK n-grams for substring search."""
+    normalized = normalize_search_text(text)
+    terms = []
+    seen = set()
+
+    for run in CJK_RUN_RE.findall(normalized):
+        for size in sizes:
+            if len(run) < size:
+                continue
+            for offset in range(len(run) - size + 1):
+                term = run[offset : offset + size]
+                if term not in seen:
+                    seen.add(term)
+                    terms.append(term)
+
+    return terms
+
+
+def cjk_index_terms(text: str | None) -> str:
+    return " ".join(cjk_ngrams(text))
+
+
+def expand_cjk_query(text: str | None) -> str:
+    """Expand CJK runs into searchable terms without changing Latin terms."""
+    normalized = normalize_search_text(text)
+
+    def expand(match: re.Match) -> str:
+        run = match.group(0)
+        if len(run) <= 2:
+            return f" {run} "
+        return f" {' '.join(cjk_ngrams(run, sizes=(3,)))} "
+
+    return WHITESPACE_RE.sub(" ", CJK_RUN_RE.sub(expand, normalized)).strip()
+
+
+def indexed_field_names(attributes) -> set[str]:
+    """Field names present in an existing RediSearch index.
+
+    FT.INFO reports attributes either as flat lists
+    (``["identifier", "title", "attribute", "title", "type", "TEXT", ...]``)
+    or as mappings, depending on the server and client version. Returning an
+    empty set means "could not tell", and callers should not treat that as
+    proof that a field is missing.
+    """
+    names = set()
+    for attr in attributes or []:
+        if isinstance(attr, dict):
+            name = attr.get("identifier") or attr.get("attribute")
+            if name:
+                names.add(name.decode() if isinstance(name, bytes) else str(name))
+            continue
+        if isinstance(attr, (list, tuple)):
+            values = [v.decode() if isinstance(v, bytes) else str(v) for v in attr]
+            if "identifier" in values:
+                idx = values.index("identifier") + 1
+                if idx < len(values):
+                    names.add(values[idx])
+            elif values:
+                names.add(values[0])
+    return names

helpdesk/search_sqlite.py

@@ -4,6 +4,13 @@
 import frappe
 from frappe.search.sqlite_search import SQLiteSearch, SQLiteSearchIndexMissingError
 
+from helpdesk.search_i18n import (
+    cjk_index_terms,
+    contains_cjk,
+    expand_cjk_query,
+    normalize_search_text,
+)
+
 
 class HelpdeskSearchIndexMissingError(SQLiteSearchIndexMissingError):
     pass
@@ -16,12 +23,14 @@ class HelpdeskSearchIndexMissingError(SQLiteSearchIndexMissingError):
 
 
 class HelpdeskSearch(SQLiteSearch):
-    INDEX_NAME = "helpdesk_search.db"
+    # Bumped so existing sites rebuild the index with the CJK terms column.
+    INDEX_NAME = "helpdesk_search_v2.db"
 
     # Resting value: core search() can bail before get_search_filters() runs.
     is_post_filter_required = False
 
     INDEX_SCHEMA = {
+        "text_fields": ["title", "content", "cjk_terms"],
         "metadata_fields": [
             "agent_group",
             "customer",
@@ -74,7 +83,26 @@ class HelpdeskSearch(SQLiteSearch):
     }
 
     def search(self, query, title_only: bool = False, filters: dict | None = None):
-        result = super().search(query, title_only=title_only, filters=filters)
+        japanese_title_search = title_only and contains_cjk(query)
+        result = super().search(
+            query,
+            # Japanese substring terms live in the hidden n-gram field.
+            title_only=False if japanese_title_search else title_only,
+            filters=filters,
+        )
+        if japanese_title_search:
+            query_parts = normalize_search_text(query).split()
+            result["results"] = [
+                row
+                for row in result["results"]
+                if all(
+                    part in normalize_search_text(row.get("title", ""))
+                    for part in query_parts
+                )
+            ]
+            result["summary"]["title_only"] = True
+            result["summary"]["filtered_matches"] = len(result["results"])
+            result["summary"]["returned_matches"] = len(result["results"])
         if self.is_post_filter_required:
             result["results"] = self._drop_unpermitted(result["results"])
             result["summary"]["filtered_matches"] = len(result["results"])
@@ -126,6 +154,10 @@ def prepare_document(self, doc):
         if not document:
             return None
 
+        document["cjk_terms"] = cjk_index_terms(
+            " ".join((document.get("title", ""), document.get("content", "")))
+        )
+
         if (
             doc.doctype == "HD Ticket Comment"
             and doc.reference_ticket
@@ -156,6 +188,17 @@ def prepare_document(self, doc):
 
         return document
 
+    def _expand_query_with_corrections(self, query):
+        # Expand Japanese before spelling correction; 2-3 character terms are
+        # intentionally left untouched by Frappe's English-oriented corrector.
+        return super()._expand_query_with_corrections(expand_cjk_query(query))
+
+    def _process_search_results(self, raw_results, query):
+        results = super()._process_search_results(raw_results, query)
+        for result in results:
+            result.pop("cjk_terms", None)
+        return results
+
     def get_filter_options(self):
         """Get available filter options for search interface."""
         if not self.index_exists():