diff --git a/compendium/docs/de/compendium/docs/index.md b/compendium/docs/de/compendium/docs/index.md index fad9460..25d32bd 100644 --- a/compendium/docs/de/compendium/docs/index.md +++ b/compendium/docs/de/compendium/docs/index.md @@ -1,6 +1,6 @@ --- title: App-interne Dokumentation -translated_from_rev: 82e1353 +translated_from_rev: f837f9e --- Mit Compendium können Frappe-Apps ihre Benutzerdokumentation als Markdown-Dateien in einem `docs/`-Ordner im App-Paket ausliefern. @@ -15,4 +15,6 @@ Mit Compendium können Frappe-Apps ihre Benutzerdokumentation als Markdown-Datei - Ohne `roles` ist eine Seite für alle **Desk User** sichtbar - Später installierte Apps überschreiben Seiten mit gleichem logischem Pfad -Eine Einführung finden Sie unter [Erste Schritte](/app/docs/compendium/docs/getting-started). +Eine Einführung finden Sie unter [Erste Schritte](/app/docs/compendium/docs/getting-started), +und unter [Suche](/app/docs/compendium/docs/search), wie Sie eine Seite über die +Awesome Bar finden. diff --git a/compendium/docs/de/compendium/docs/search.md b/compendium/docs/de/compendium/docs/search.md new file mode 100644 index 0000000..0f55e4e --- /dev/null +++ b/compendium/docs/de/compendium/docs/search.md @@ -0,0 +1,39 @@ +--- +title: Suche +translated_from_rev: f837f9e +--- + +Tippen Sie in die Awesome Bar am oberen Rand des Desk, um die Dokumentation zu +durchsuchen. Die Treffer erscheinen zusammen mit den übrigen Ergebnissen der +Awesome Bar. Wählen Sie einen Treffer aus, um die Seite zu öffnen. + +## Was Compendium durchsucht + +Compendium durchsucht den Titel, den Pfad und den gesamten Text jeder Seite. +Dazu gehören Überschriften, Fließtext, Tabellen, Codeblöcke und der Alternativtext +von Bildern. Linkziele werden nicht durchsucht, denn der Leser sieht sie nicht. + +Compendium durchsucht die Seiten so, wie Sie sie lesen. Gibt es eine Seite nicht +in Ihrer Sprache, durchsucht Compendium die Fassung, die Sie stattdessen erhalten. + +## Wie eine Suchanfrage trifft + +- Ein Wort trifft ab seinem Anfang. `migr` findet eine Seite über Migration. +- Eine Anfrage aus mehreren Wörtern trifft eine Seite, die alle enthält. +- Akzente werden ignoriert. `ubersicht` findet `Übersicht`. +- Die Wortform muss übereinstimmen. `Migrationen` findet keine Seite, die + `Migration` schreibt. + +Angezeigt werden die besten 20 Treffer, der beste zuerst. Ein Treffer im Titel +wird höher bewertet als ein Treffer im Text. Unter jedem Ergebnis zeigt +Compendium die Stelle der Seite, die den Treffer enthält. + +## Berechtigungen + +Die Ergebnisse enthalten nur Seiten, die Sie öffnen dürfen. Eine über `roles` +eingeschränkte Seite bleibt für alle anderen Benutzer verborgen. + +> [!NOTE] +> Der Index erneuert sich, sobald sich eine Markdown-Datei ändert. Suchen Sie +> nach dem Bearbeiten einer Seite erneut, um den neuen Inhalt zu sehen. Ein +> Neustart ist nicht nötig. diff --git a/compendium/docs/en/compendium/docs/index.md b/compendium/docs/en/compendium/docs/index.md index 7161485..fe262de 100644 --- a/compendium/docs/en/compendium/docs/index.md +++ b/compendium/docs/en/compendium/docs/index.md @@ -17,4 +17,5 @@ Compendium lets Frappe apps ship user documentation as Markdown files under a `d - Omit `roles` to allow all **Desk User** accounts - Later installed apps override pages at the same logical path -See [Getting Started](/app/docs/en/compendium/docs/getting-started) for a walkthrough. +See [Getting Started](/app/docs/en/compendium/docs/getting-started) for a walkthrough, +or [Search](/app/docs/en/compendium/docs/search) to find a page from the Awesome Bar. diff --git a/compendium/docs/en/compendium/docs/search.md b/compendium/docs/en/compendium/docs/search.md new file mode 100644 index 0000000..d9d1c02 --- /dev/null +++ b/compendium/docs/en/compendium/docs/search.md @@ -0,0 +1,38 @@ +--- +title: Search +order: 5 +roles: + - System Manager +--- + +Type in the Awesome Bar at the top of Desk to search the documentation. Results +appear together with the other Awesome Bar results. Select one to open the page. + +## What Compendium searches + +Compendium searches the title, the path, and the full text of every page. This +includes headings, body text, tables, code blocks, and the alt text of images. +Link targets are not searched, because the reader does not see them. + +Compendium searches the pages as you read them. If a page has no version in your +language, Compendium searches the version you get instead. + +## How a query matches + +- A word matches at its start. `migr` finds a page about migration. +- A query with several words matches a page that contains all of them. +- Accents are ignored. `ubersicht` finds `Übersicht`. +- A word form must match. `migrating` does not find a page that writes `migrate`. + +The best 20 matches are shown, best first. A match in the title ranks above a +match in the body. Under each result Compendium shows the part of the page that +contains the match. + +## Permissions + +The results only contain pages that you are allowed to open. A page that is +restricted with `roles` stays hidden for all other users. + +> [!NOTE] +> The index updates itself when a Markdown file changes. After you edit a page, +> search again to see the new content. A restart is not needed. diff --git a/compendium/hooks.py b/compendium/hooks.py index fdea89a..fff9f49 100644 --- a/compendium/hooks.py +++ b/compendium/hooks.py @@ -46,6 +46,9 @@ }, ] +# Extra Awesome Bar results: list of dicts with label, description, route, index. +awesomebar_search = ["compendium.search.awesomebar_results"] + # include js, css files in header of web template # web_include_css = "/assets/compendium/css/compendium.css" # web_include_js = "/assets/compendium/js/compendium.js" diff --git a/compendium/search.py b/compendium/search.py new file mode 100644 index 0000000..f88a570 --- /dev/null +++ b/compendium/search.py @@ -0,0 +1,171 @@ +# Copyright (c) 2026, ALYF GmbH and Contributors +# License: MIT. See LICENSE + +import os +import re +import sqlite3 +import threading + +import frappe +from frappe import _ +from frappe.utils import escape_html + +from compendium import docs +from compendium.docs import DEFAULT_LANG, DOCS_FOLDER, is_permitted, normalize_locale + +RESULT_LIMIT = 20 +TOKEN_PATTERN = re.compile(r"\w+") +SNIPPET_TOKENS = 12 +ROLE_SEPARATOR = "\n" +# Columns are (path, title, roles, body); bm25 ranks a title hit above a body hit. +RANK_SQL = """ + SELECT rowid, path, title, roles + FROM pages WHERE pages MATCH ? ORDER BY bm25(pages, 2.0, 10.0, 0.0, 1.0) +""" +SNIPPET_SQL = f""" + SELECT rowid, snippet(pages, 3, '', '', '…', {SNIPPET_TOKENS}) + FROM pages WHERE pages MATCH ? AND rowid IN ({{rowids}}) +""" + +# ponytail: one lock for all indexes; per-index locks if search ever gets hot +INDEX_LOCK = threading.Lock() +INDEXES = {} + + +def awesomebar_results(txt): + """Return matching docs pages for the Awesome Bar `awesomebar_search` hook.""" + match_query = build_match_query(txt) + if not match_query: + return [] + + locale = normalize_locale(frappe.local.lang or DEFAULT_LANG) + results = [] + + for path, title, snippet in search(locale, match_query): + route = f"/app/docs/{locale}/{path}" if path else f"/app/docs/{locale}" + results.append( + { + "label": title, + # the Awesome Bar renders the description as HTML + "description": escape_html(snippet) or path or _("Documentation"), + "route": route, + "index": 50, + } + ) + + return results + + +def build_match_query(txt): + """Turn free text into an FTS5 MATCH expression: every word as a quoted prefix term. + + Quoting keeps words like `and` or `not` from being read as query operators. + """ + return " ".join(f'"{token}"*' for token in TOKEN_PATTERN.findall((txt or "").lower())) + + +def search(locale, match_query): + """The pages this user may read that match the query, best first, each with a preview. + + Ranking and preview are separate passes: `snippet` re-scans a whole page body, so a + query matching every page pays for it once per page. Ranking has to see every match + anyway — a page the reader may open can rank below any number of pages they may not. + """ + with INDEX_LOCK: + index = get_index(locale) + hits = [] + + for rowid, path, title, roles in index.execute(RANK_SQL, (match_query,)): + if not is_permitted(frappe._dict(roles=roles.split(ROLE_SEPARATOR))): + continue + + hits.append((rowid, path, title)) + if len(hits) >= RESULT_LIMIT: + break + + snippets = get_snippets(index, match_query, [rowid for rowid, _path, _title in hits]) + + return [(path, title, snippets.get(rowid, "")) for rowid, path, title in hits] + + +def get_snippets(index, match_query, rowids): + if not rowids: + return {} + + sql = SNIPPET_SQL.format(rowids=", ".join("?" * len(rowids))) + return dict(index.execute(sql, (match_query, *rowids))) + + +def get_index(locale): + """Full-content FTS5 index for one locale, kept in memory for the life of the worker. + + Rebuilt when a Markdown file appears, disappears or changes, so a docs deployment + needs no restart and an author editing a page sees the change on the next search. + Sites share a worker but not their installed apps, hence the site in the key. + """ + key = (frappe.local.site, locale) + fingerprint = get_docs_fingerprint() + cached = INDEXES.get(key) + if cached and cached[0] == fingerprint: + return cached[1] + + index = build_index(locale) + INDEXES[key] = (fingerprint, index) + return index + + +def build_index(locale): + # the index outlives the request, so it outlives the thread that built it + connection = sqlite3.connect(":memory:", check_same_thread=False) + connection.execute( + "CREATE VIRTUAL TABLE pages USING fts5" + "(path, title, roles UNINDEXED, body, tokenize='unicode61 remove_diacritics 2')" + ) + connection.executemany( + "INSERT INTO pages (path, title, roles, body) VALUES (?, ?, ?, ?)", + ( + (page.path, page.title, ROLE_SEPARATOR.join(page.roles), to_plain_text(page.body)) + for page in docs.discover_pages(locale).values() + ), + ) + return connection + + +@frappe.request_cache +def get_docs_fingerprint(): + """Staleness check for the index: every Markdown file with its mtime and size. + + Stat-only, so it costs a fraction of the reading and parsing an index build does. + A file count and a newest mtime would be cheaper but would miss a deployment that + restores the timestamps it found, leaving an edited page unsearchable until restart. + """ + stamps = [] + + for app in docs.get_installed_apps(): + docs_root = os.path.join(docs.get_app_path(app), DOCS_FOLDER) + for basepath, _folders, files in os.walk(docs_root): + for fname in files: + if not fname.endswith(".md"): + continue + + filepath = os.path.join(basepath, fname) + stat = os.stat(filepath) + stamps.append((filepath, stat.st_mtime_ns, stat.st_size)) + + return tuple(sorted(stamps)) + + +def to_plain_text(markdown): + """The words of a page, without the markup around them. + + Markdown source makes both a poor preview and a poor index: a snippet cut out of it + shows syntax mid-sentence, and link targets match queries the reader never sees. + Image alt text is the exception among attributes — it is prose, so it is kept. + """ + from bs4 import BeautifulSoup + + soup = BeautifulSoup(frappe.utils.md_to_html(markdown or ""), "html.parser") + for image in soup.find_all("img"): + image.replace_with(image.get("alt") or "") + + return re.sub(r"\s+", " ", soup.get_text(" ")).strip() diff --git a/compendium/tests/test_docs.py b/compendium/tests/test_docs.py index ac3e5c6..f9e59f4 100644 --- a/compendium/tests/test_docs.py +++ b/compendium/tests/test_docs.py @@ -916,9 +916,7 @@ def __enter__(self): if self.repository: with open(os.path.join(self.tmpdir.name, "pyproject.toml"), "w", encoding="utf-8") as f: - f.write( - '[project]\nname = "example"\n\n[project.urls]\n' f'Repository = "{self.repository}"\n' - ) + f.write(f'[project]\nname = "example"\n\n[project.urls]\nRepository = "{self.repository}"\n') self._patches = [ patch("compendium.docs.get_installed_apps", return_value=["frappe"]), diff --git a/compendium/tests/test_search.py b/compendium/tests/test_search.py new file mode 100644 index 0000000..1162c10 --- /dev/null +++ b/compendium/tests/test_search.py @@ -0,0 +1,210 @@ +import os +from unittest.mock import patch + +import frappe +from frappe.tests.utils import FrappeTestCase + +from compendium.search import awesomebar_results, get_index +from compendium.tests.test_docs import DocsTestEnvironment + + +class TestSearch(FrappeTestCase): + def setUp(self): + frappe.set_user("Administrator") + frappe.local.lang = "en" + self.next_request() + + def next_request(self): + """Drop everything a single request caches, so the next call starts like a fresh one.""" + if hasattr(frappe.local, "request_cache"): + frappe.local.request_cache.clear() + + def tearDown(self): + frappe.set_user("Administrator") + frappe.local.lang = "en" + + def test_empty_query_returns_nothing(self): + self.assertEqual(awesomebar_results(""), []) + self.assertEqual(awesomebar_results(" "), []) + self.assertEqual(awesomebar_results("!?-"), []) + + def test_matches_title_and_builds_docs_route(self): + with DocsTestEnvironment( + { + "en/guides/setup.md": "---\ntitle: Setup Guide\n---\n# Setup\n\nInstall the app.", + "en/other.md": "---\ntitle: Other\n---\n# Other", + } + ): + results = awesomebar_results("setup") + + self.assertEqual(len(results), 1) + self.assertEqual(results[0]["label"], "Setup Guide") + self.assertEqual(results[0]["route"], "/app/docs/en/guides/setup") + self.assertEqual(results[0]["index"], 50) + + def test_matches_path(self): + with DocsTestEnvironment( + { + "en/guides/setup.md": "---\ntitle: Setup Guide\n---\n# Setup", + } + ): + results = awesomebar_results("guides") + + self.assertEqual(results[0]["route"], "/app/docs/en/guides/setup") + + def test_matches_body_content(self): + with DocsTestEnvironment( + { + "en/setup.md": "---\ntitle: Setup\n---\nRun bench migrate to apply patches.", + "en/other.md": "---\ntitle: Other\n---\nNothing to see here.", + } + ): + results = awesomebar_results("patches") + + self.assertEqual([item["label"] for item in results], ["Setup"]) + self.assertIn("patches", results[0]["description"]) + + def test_matches_word_prefix(self): + with DocsTestEnvironment({"en/setup.md": "---\ntitle: Setup\n---\nRun bench migrate."}): + results = awesomebar_results("migr") + + self.assertEqual([item["label"] for item in results], ["Setup"]) + + def test_all_words_must_match(self): + with DocsTestEnvironment( + { + "en/setup.md": "---\ntitle: Setup\n---\nRun bench migrate.", + "en/other.md": "---\ntitle: Other\n---\nRun the tests.", + } + ): + results = awesomebar_results("run migrate") + + self.assertEqual([item["label"] for item in results], ["Setup"]) + + def test_title_hit_ranks_above_body_hit(self): + with DocsTestEnvironment( + { + "en/backup.md": "---\ntitle: Backup\n---\nHow to keep copies.", + "en/setup.md": "---\ntitle: Setup\n---\nTake a backup first.", + } + ): + results = awesomebar_results("backup") + + self.assertEqual([item["label"] for item in results], ["Backup", "Setup"]) + + def test_snippet_is_plain_text(self): + with DocsTestEnvironment( + { + "en/setup.md": ( + "---\ntitle: Setup\n---\n## Bench\n\n" + "*Run* bench [migrate](https://example.com/docs) & wait." + ), + } + ): + results = awesomebar_results("migrate") + + self.assertEqual(results[0]["description"], "Bench Run bench migrate & wait.") + + def test_link_targets_are_not_searchable(self): + with DocsTestEnvironment( + {"en/setup.md": "---\ntitle: Setup\n---\nSee [the manual](https://example.com/hyperion)."} + ): + self.assertEqual(awesomebar_results("hyperion"), []) + self.assertEqual([item["label"] for item in awesomebar_results("manual")], ["Setup"]) + + def test_index_page_route_has_no_trailing_path(self): + with DocsTestEnvironment({"en/index.md": "---\ntitle: Home\n---\n# Home"}): + results = awesomebar_results("home") + + self.assertEqual(results[0]["route"], "/app/docs/en") + + def test_skips_unpermitted_pages(self): + with DocsTestEnvironment( + { + "en/public.md": "---\ntitle: Public Guide\nroles: Desk User\n---\n# Public", + "en/admin.md": "---\ntitle: Admin Guide\nroles: System Manager\n---\n# Admin", + } + ): + frappe.set_user("test@example.com") + with patch("compendium.docs.get_user_roles", return_value=["Desk User"]): + results = awesomebar_results("guide") + + labels = [item["label"] for item in results] + self.assertEqual(labels, ["Public Guide"]) + + def test_index_is_reused_across_requests(self): + with DocsTestEnvironment({"en/setup.md": "---\ntitle: Setup\n---\nRun bench migrate."}): + index = get_index("en") + self.next_request() + self.assertIs(get_index("en"), index) + + def test_index_rebuilds_when_a_page_changes(self): + with DocsTestEnvironment({"en/setup.md": "---\ntitle: Setup\n---\nRun bench migrate."}) as docs_root: + self.assertEqual(awesomebar_results("hyphenation"), []) + + with open(os.path.join(docs_root, "en", "extra.md"), "w", encoding="utf-8") as f: + f.write("---\ntitle: Extra\n---\nAbout hyphenation.") + + self.next_request() + results = awesomebar_results("hyphenation") + + self.assertEqual([item["label"] for item in results], ["Extra"]) + + def test_index_rebuilds_when_a_deployment_restores_timestamps(self): + files = { + "en/setup.md": "---\ntitle: Setup\n---\nRun bench migrate.", + "en/other.md": "---\ntitle: Other\n---\nNothing to see here.", + } + with DocsTestEnvironment(files) as docs_root: + self.assertEqual(awesomebar_results("hyphenation"), []) + + newest = os.stat(os.path.join(docs_root, "en", "other.md")).st_mtime + setup = os.path.join(docs_root, "en", "setup.md") + with open(setup, "w", encoding="utf-8") as f: + f.write("---\ntitle: Setup\n---\nAbout hyphenation.") + # a deployment that preserves timestamps: same file count, same newest mtime + os.utime(setup, (newest - 60, newest - 60)) + + self.next_request() + results = awesomebar_results("hyphenation") + + self.assertEqual([item["label"] for item in results], ["Setup"]) + + def test_permitted_page_ranked_below_unpermitted_ones_survives(self): + files = { + f"en/admin{index}.md": f"---\ntitle: Admin {index}\nroles: System Manager\n---\nAbout hyphenation." + for index in range(25) + } + # padded, so bm25 ranks it below every page the reader may not open + files["en/public.md"] = ( + "---\ntitle: Public\nroles: Desk User\n---\nAbout hyphenation. " + "filler " * 200 + ) + + with DocsTestEnvironment(files): + frappe.set_user("test@example.com") + with patch("compendium.docs.get_user_roles", return_value=["Desk User"]): + results = awesomebar_results("hyphenation") + + self.assertEqual([item["label"] for item in results], ["Public"]) + + def test_caps_results_and_previews_every_one(self): + files = { + f"en/page{index}.md": f"---\ntitle: Page {index}\n---\nAbout hyphenation." for index in range(25) + } + with DocsTestEnvironment(files): + results = awesomebar_results("hyphenation") + + self.assertEqual(len(results), 20) + self.assertTrue(all(item["description"] == "About hyphenation." for item in results)) + + def test_finds_image_alt_text_and_code_blocks(self): + with DocsTestEnvironment( + { + "en/setup.md": ( + "---\ntitle: Setup\n---\n" + "![The hyphenation dialog](dialog.png)\n\n```\nbench build --hard\n```" + ) + } + ): + self.assertEqual([item["label"] for item in awesomebar_results("hyphenation")], ["Setup"]) + self.assertEqual([item["label"] for item in awesomebar_results("bench build")], ["Setup"])