diff --git a/DistFiles/localization/en/Bloom.xlf b/DistFiles/localization/en/Bloom.xlf index 812224f220b6..6bd3ba716b0e 100644 --- a/DistFiles/localization/en/Bloom.xlf +++ b/DistFiles/localization/en/Bloom.xlf @@ -384,6 +384,103 @@ ID: CollectionSettingsDialog.AdvancedTab.Experimental.SpreadsheetImportExport This is the label of a checkbox which, if enabled, allows a menu command to be used that can import Bloom books from an Excel spreadsheet or to an Excel spreadsheet. + + AI Source Translation + ID: CollectionSettingsDialog.AdvancedTab.Experimental.AiSourceBubbles + Label of the experimental-features checkbox that turns on AI source-language translation for book source bubbles. + + + AI Source Translation + ID: CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.SectionLabel + Heading of the settings group where the AI source-translation providers (SIL Alpha2, DeepL, Google Translate) are configured. + + + Target Language + ID: CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.TargetLanguageLabel + + + Choose one of this collection's languages, or Other... to select another language. + ID: CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.TargetLanguageDescription + Currently unused (orphaned from an earlier design); left in place rather than deleted. + + + Other... + ID: CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.OtherLanguageLabel + Currently unused (orphaned from an earlier design); left in place rather than deleted. + + + DeepL + ID: CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.DeepLEnabledLabel + Label of the checkbox that enables the DeepL translation engine for AI source bubbles. "DeepL" is a product name and must not be translated. + + + API Key + ID: CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.DeepLApiKeyLabel + Label of the API-key field within the DeepL section of AI Source Translation settings (the section heading already identifies it as DeepL). + + + The key needs the 'translate:text' and 'languages:read' permissions. + ID: CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.DeepLApiKeyDescription + Description subtitle under the DeepL API key field in AI Source Translation settings. 'translate:text' and 'languages:read' are DeepL permission-scope identifiers and must not be translated. + + + Google Translate + ID: CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.GoogleEnabledLabel + Label of the checkbox that enables the Google Translate engine for AI source bubbles. "Google Translate" is a product name and must not be translated. + + + Google service account email + ID: CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.GoogleServiceAccountEmailLabel + + + Google private key + ID: CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.GooglePrivateKeyLabel + + + The service account needs access to the Cloud Translation API (the 'Cloud Translation API User' role). + ID: CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.GoogleServiceAccountDescription + Description subtitle under the Google service account fields in AI Source Translation settings. 'Cloud Translation API' and 'Cloud Translation API User' are Google product/role names and must not be translated. + + + SIL Alpha2 + ID: CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.Alpha2EnabledLabel + Label of the checkbox that enables the SIL Alpha2 translation engine for AI source bubbles. "SIL Alpha2" is a product name and must not be translated. + + + API Key + ID: CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.Alpha2ApiKeyLabel + Label of the API-key field within the SIL Alpha2 section of AI Source Translation settings (the section heading already identifies it as SIL Alpha2). + + + Translation Test + ID: CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.TranslationTestLabel + Label of the row showing a live translation-test result for one AI source bubbles engine (DeepL, Google Translate, or SIL Alpha2); this same label is reused once per engine. + + + Source Language + ID: CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.Alpha2SourceLanguageLabel + Label of the SIL Alpha2 source-language chooser in AI Source Bubbles settings: the language Alpha2 translates FROM (Alpha2 needs a fixed source language, unlike the other engines). + + + {0} does not support translating to {1}, so it will be skipped. + ID: CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.TargetLanguageNotSupported + Amber note under an AI Source Bubbles engine when it can't translate into the chosen target language. {0} is the translation engine's name (e.g. "DeepL"), which must not be translated; {1} is the target language name. + + + no enabled service supports this + ID: CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.NoServiceSupportsLanguage + Shown in parentheses next to the currently-selected target language in the AI Source Bubbles target-language dropdown when none of the enabled translation services offers that language. + + + Select at least one translation provider to get a list of languages it supports. + ID: CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.NoProviderSelected + Message shown below the (disabled) AI Source Bubbles target-language field when no translation provider is enabled yet, so there is no language list to choose from. + + + SIL Alpha2 shows the languages it can translate from {0}. + ID: CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.Alpha2LanguagesNote + Explanatory note below the AI Source Bubbles target-language dropdown, shown when SIL Alpha2 is an enabled provider. It clarifies that Alpha2's offered languages depend on its configured source language. {0} is the source language's name (e.g. "English"); "SIL Alpha2" is a product name and must not be translated. + Bloom is switching the default font for "{0}" to the new "Andika New Basic". ID: CollectionSettingsDialog.AndikaNewBasicUpdate1 @@ -1349,6 +1446,36 @@ Add Page... ID: EditTab.AddPageDialog.Title + + Cancelled. + ID: EditTab.AiTranslation.Cancelled + Status line in the AI translation progress dialog after the user clicks Cancel; the book is left unchanged. + + + Translating + ID: EditTab.AiTranslation.DialogTitle + Title of the progress dialog shown while AI engines translate the book's text before it opens for editing. + + + {0}: done. + ID: EditTab.AiTranslation.EngineDone + Progress line in the AI translation dialog. {0} is replaced with the translation engine's name (e.g. "DeepL"), which must not be translated. + + + {0}: {1} + ID: EditTab.AiTranslation.EngineError + Error line in the AI translation dialog. {0} is the translation engine's name (e.g. "DeepL"); {1} is an error message. Only the separator punctuation can be localized. + + + {0}: skipped {1} text box(es) with no {2} text. + ID: EditTab.AiTranslation.EngineSkippedNoSourceText + Progress line in the AI translation dialog, shown when an engine with a fixed source language (e.g. SIL Alpha2) had to skip text boxes that have no text in that language. {0} is the translation engine's name (e.g. "SIL Alpha2"), which must not be translated; {1} is the number of text boxes skipped; {2} is the source language tag (e.g. "en"). + + + {0}: translating {1} text box(es)... + ID: EditTab.AiTranslation.EngineTranslating + Progress line in the AI translation dialog. {0} is the translation engine's name (e.g. "DeepL"), which must not be translated; {1} is the number of text boxes being translated. + If you need somewhere to put more information about the book, you can use this page, which is the inside of the back cover. ID: EditTab.BackMatter.InsideBackCoverTextPrompt diff --git a/DistFiles/localization/en/BloomLowPriority.xlf b/DistFiles/localization/en/BloomLowPriority.xlf index 41d3d319086f..d5472c84be87 100644 --- a/DistFiles/localization/en/BloomLowPriority.xlf +++ b/DistFiles/localization/en/BloomLowPriority.xlf @@ -353,6 +353,16 @@ ID: CollectionTab.BookMenu.MoveToCurrentCollection {0} will be replaced with the name of the collection that the user currently has open for editing. + + Remove AI Source Translations + ID: CollectionTab.BookMenu.RemoveAiSourceTranslations + A book right-click menu command in the Collection tab. It deletes the AI-generated translations that appear as source bubbles when editing. "AI" stands for artificial intelligence. + + + Bloom removed the AI source translations from this book. + ID: CollectionTab.BookMenu.RemovedAiSourceTranslations + Confirmation message shown after the "Remove AI Source Translations" menu command finishes. "Bloom" is the product name and must not be translated; "AI" stands for artificial intelligence. + diff --git a/PAPERCUTS.md b/PAPERCUTS.md index 54622f83be77..f2323ff0d08f 100644 --- a/PAPERCUTS.md +++ b/PAPERCUTS.md @@ -19,6 +19,52 @@ House rules: --- +## 2026-07-15 — Orphaned `node.exe` dev stacks pile up until `go.sh` can't launch +- **Cut:** Each `go.sh` spawns a dev stack (`dev.mjs` → vite + ~7 `onchange`/`watchLess` + watchers + `watchBloomExe`). On Windows, closing a terminal or Ctrl-C doesn't reliably tear + down that child tree, so stacks orphan across worktrees/sessions and `node.exe` accumulates + (saw ~40 live: 3 worktrees + a dead-parent chrome-devtools-mcp daemon). Under that load + `go.mjs`'s Vite health gate times out — it needs **2 consecutive sub-3s** `/@vite/client` + responses, and Vite binds **IPv6-only (`[::1]`, 127.0.0.1 refused)** on this machine, so + there's zero margin — and every launch fails with "Vite … never became reachable," i.e. it + "piles up until none work." `go.mjs`'s startup sweep only reaps *this* worktree's stale + procs, and the chrome-devtools-mcp daemon has a **watchdog that respawns it after kill** + (self-healing zombie). +- **Idea:** Add a zombie-reaper keyed on *liveness of the controller*, not worktree path: a + vite/watcher subtree with no living `go.mjs`/`watchBloomExe` ancestor (or a proc whose + parent is dead) is a zombie → auto-kill. Run it on `go.sh` startup across **all** Bloom + worktree stacks (safe because it never touches subtrees with a live controller), and/or ship + a standalone `pnpm reap`. Separately harden the health probe so a slow-but-listening Vite + passes: bind Vite on 127.0.0.1 too, accept a single success, and scale the timeout with + detected load. For chrome-devtools-mcp, kill the watchdog first (else it resurrects the + daemon) — memory note already says prefer the CLI over the MCP. +- **Context:** BL-16549AiSourceBubbles; repeated `./go.sh` failures. See `go.mjs` + `waitForViteClient` / `startDevServerOnPort` and `processTree.mjs` + `sweepStaleWorktreeNodeProcesses`. + +- Running `dotnet test` (or any BloomExe build) while a `./go.sh` / `dotnet watch` + Bloom is live fails at the copy-to-output step: the running process locks both + `output/Debug/AnyCPU/Bloom.exe` (native apphost) and, once hot-reload deltas have + been applied, `Bloom.dll` too (MSB3026/MSB3027 "being used by another process"). + Compilation itself succeeds; only the copy fails, so the tests never run. + Workaround that neither kills the running instance nor touches the locked output: + redirect the whole build to a scratch dir and skip the apphost, e.g. + `dotnet test src/BloomTests/BloomTests.csproj --filter ... -p:UseAppHost=false -p:OutDir=`. + +## 2026-07-15 — config-r draws a divider between every direct group child; label is string-typed +- **Cut:** `@sillsdev/config-r`'s `ConfigrGroup` (in a focused page) inserts a horizontal + divider between *every* direct child of the group. So an engine block written as a + `` followed by a separate `{enabled && <>...fields...}` gets an unwanted + line between the checkbox and its own settings. Also, `IConfigrProps.label` is typed `string`, + so you can't cleanly put a logo/node before a label. +- **Workaround:** Wrap each engine's checkbox + conditional fields in a single fragment so the + group sees one child per engine (dividers land only *between* engines). For a logo-in-label, + pass a ReactNode cast `as unknown as string` — config-r renders `label` straight into MUI + `ListItemText` `primary`, which accepts a node, so it works at runtime. +- **Idea:** Ask config-r for a `label?: React.ReactNode` type and/or a per-row `hideDivider` + (or a "subgroup" that suppresses internal dividers). See `AiTranslationSettingsGroup.tsx`. +- **Context:** BL-16549AiSourceBubbles AI Source Bubbles settings. + ## 2026-07-13 — pnpm-lock.yaml reformats wholesale on any install (format drift) - **Cut:** The committed `src/BloomBrowserUI/pnpm-lock.yaml` (on master too) is in an older pnpm serialization style (double-quoted `lockfileVersion`, 4-space indent, and it diff --git a/src/BloomBrowserUI/bookEdit/js/collectionSettings.d.ts b/src/BloomBrowserUI/bookEdit/js/collectionSettings.d.ts index b67b525e3b79..c0f021820259 100644 --- a/src/BloomBrowserUI/bookEdit/js/collectionSettings.d.ts +++ b/src/BloomBrowserUI/bookEdit/js/collectionSettings.d.ts @@ -9,6 +9,7 @@ interface ICollectionSettings { defaultSourceLanguage2: string; currentCollectionLanguage2: string; currentCollectionLanguage3: string; + allowAiSourceBubbles: boolean; browserRoot: string; topics: string[]; } diff --git a/src/BloomBrowserUI/bookEdit/sourceBubbles/BloomSourceBubbles.tsx b/src/BloomBrowserUI/bookEdit/sourceBubbles/BloomSourceBubbles.tsx index 518bb5056c11..b6be05b378d4 100644 --- a/src/BloomBrowserUI/bookEdit/sourceBubbles/BloomSourceBubbles.tsx +++ b/src/BloomBrowserUI/bookEdit/sourceBubbles/BloomSourceBubbles.tsx @@ -9,7 +9,10 @@ // The actual function is injected by C#. /// import { renderRoot } from "../../utils/reactRender"; +import * as ReactDOM from "react-dom"; +import * as React from "react"; import $ from "jquery"; +import AutoAwesomeIcon from "@mui/icons-material/AutoAwesome"; import theOneLocalizationManager from "../../lib/localizationManager/localizationManager"; import StyleEditor from "../StyleEditor/StyleEditor"; import bloomQtipUtils from "../js/bloomQtipUtils"; @@ -17,8 +20,14 @@ import "../../lib/jquery.easytabs.js"; //load into global space import BloomHintBubbles from "../js/BloomHintBubbles"; import { postJson, postString } from "../../utils/bloomApi"; import CopyContentButton from "../../react_components/CopyContentButton"; +import { + getAiProviderDisplayName, + getLanguageNameFromBrowser, + isAiLanguageTag, + removeAiTranslationDivsFromClone, +} from "./aiTranslationDisplay"; -declare function GetSettings(): any; // C# (or test code) injects this +declare function GetSettings(): ICollectionSettings; export default class BloomSourceBubbles { //:empty is not quite enough... we don't want to show bubbles if all there is is an empty paragraph @@ -29,6 +38,67 @@ export default class BloomSourceBubbles { return $.trim($(obj).text()).length === 0; } + private static maybeRememberSourceBubbleLanguage(langTag: string): void { + // Note (BL-16549): this remembers whatever tab was activated as the preferred source + // language, including AI tabs (synthetic tags like "es-x-ai-deepl"). Devin flagged that + // clicking an AI tab therefore displaces the remembered *real* source language. We are + // deliberately leaving that as-is for now: whether an AI tab should be remembered is a + // product decision, not clearly a bug (a user who just chose to view the AI translation may + // well want it to persist). Revisit if that turns out to be undesirable. + postString("editView/sourceTextTab", langTag); + } + + private static getLanguageDisplayName(langTag: string): string { + const aiSplitMarker = "-x-ai-"; + const aiSplitIndex = langTag.indexOf(aiSplitMarker); + if (aiSplitIndex < 0) { + return ( + theOneLocalizationManager.getLanguageName(langTag) || langTag + ); + } + + const targetLanguageTag = langTag.substring(0, aiSplitIndex); + const providerId = langTag.substring( + aiSplitIndex + aiSplitMarker.length, + ); + // getLanguageName echoes the tag back when it doesn't know the language, so treat + // that as "unknown" and let the browser's Intl database have a try. + const bloomLanguageName = + theOneLocalizationManager.getLanguageName(targetLanguageTag); + const targetLanguageName = + (bloomLanguageName !== targetLanguageTag + ? bloomLanguageName + : undefined) || + getLanguageNameFromBrowser(targetLanguageTag) || + targetLanguageTag; + const providerDisplayName = getAiProviderDisplayName(providerId); + return `AI ${targetLanguageName} (${providerDisplayName})`; + } + + private static appendSourceTabLabel( + anchor: HTMLAnchorElement, + langTag: string, + localizedLanguageName: string, + ): void { + if (!isAiLanguageTag(langTag)) { + anchor.textContent = localizedLanguageName; + return; + } + + // TODO (BL-16549 follow-up): migrate to React 18 createRoot. Kept on the synchronous + // ReactDOM.render here deliberately for now -- these tab-label anchors are rebuilt each + // time bubbles are constructed, and createRoot's async render / one-root-per-container + // rules need their own change; behavior is unchanged in the meantime. + // eslint-disable-next-line react/no-deprecated + ReactDOM.render( + <> + + {localizedLanguageName} + , + anchor, + ); + } + // This is the method that should be called from bloomEditing to create tabbed source bubbles // for translation. // param 'group' is a .bloom-translationGroup DIV @@ -94,9 +164,11 @@ export default class BloomSourceBubbles { newLangTag?: string, ): JQuery { if (group.classList.contains("bloom-no-source-bubble")) return $(); + const liveGroup = $(group); + // Copy source texts out to their own div, where we can make a bubble with tabs out of them // We do this because if we made a bubble out of the div, that would suck up the vernacular editable area, too, - const divForBubble = $(group).clone(); + const divForBubble = liveGroup.clone(); divForBubble.removeAttr("style"); divForBubble.removeClass(); //remove them all divForBubble.addClass("ui-sourceTextsForBubble"); @@ -104,6 +176,9 @@ export default class BloomSourceBubbles { divForBubble.find("label.bubble").each((index, element) => { $(element).remove(); }); + if (!GetSettings().allowAiSourceBubbles) { + removeAiTranslationDivsFromClone(divForBubble); + } //make the source texts in the bubble read-only and remove any user font size adjustments divForBubble.find("textarea, div").each(function (): boolean { @@ -172,8 +247,7 @@ export default class BloomSourceBubbles { const langTag = sourceElement.getAttribute("lang"); if (langTag) { const localizedLanguageName = - theOneLocalizationManager.getLanguageName(langTag) || - langTag; + BloomSourceBubbles.getLanguageDisplayName(langTag); // This is bizarre. The href ought to be referring to the element with the specified ID, // which should be the tab CONTENT that should be shown for this language. But we have modified // easytabs (see above) so that the target (main page content div) for a tab is the element whose @@ -181,22 +255,27 @@ export default class BloomSourceBubbles { // Even more bizarrely, we make the id of the list item have that value also, so that // the apparent target of the is the
  • it resides inside. Not sure why this is // helpful. - $(list).append( - '
  • ' + - localizedLanguageName + - "
    • ", + const liElement = document.createElement("li"); + liElement.id = langTag; + liElement.title = langTag; + const anchor = document.createElement("a"); + anchor.className = "sourceTextTab"; + anchor.href = `#${langTag}`; + BloomSourceBubbles.appendSourceTabLabel( + anchor, + langTag, + localizedLanguageName, ); + liElement.append(anchor); + list.append(liElement); ( list.get(0) as HTMLElement ).lastElementChild?.firstElementChild?.addEventListener( "click", - () => postString("editView/sourceTextTab", langTag), + () => + BloomSourceBubbles.maybeRememberSourceBubbleLanguage( + langTag, + ), ); // BL-8174: Add a tooltip with the language tag to the item // BL-15212: we no longer want the tag here, just on the language-name label @@ -304,7 +383,9 @@ export default class BloomSourceBubbles { if (indexA >= 0) return -1; if (indexB >= 0) return 1; // Neither in preferred list - maintain alphabetical order - return langA < langB ? (langA > langB ? 1 : 0) : -1; + if (langA < langB) return -1; + if (langA > langB) return 1; + return 0; }); return $(itemArray); @@ -427,7 +508,23 @@ export default class BloomSourceBubbles { } private static styledSelectChangeHandler(event) { - const newLangTag = event.target.href.split("#")[1]; + const anchor = + event.target instanceof Element + ? event.target.closest("a.sourceTextTab") + : undefined; + const currentTarget = + event.currentTarget instanceof Element + ? event.currentTarget + : undefined; + const href = + anchor?.getAttribute("href") ?? + currentTarget + ?.querySelector("a.sourceTextTab") + ?.getAttribute("href"); + if (!href) { + return; + } + const newLangTag = href.split("#")[1]; // Figure out which qtip we're in and go find the associated bloom-translationGroup const qtip = $(event.target).closest(".qtip").attr("id"); @@ -442,7 +539,7 @@ export default class BloomSourceBubbles { group[0], newLangTag, ); - postString("editView/sourceTextTab", newLangTag); + BloomSourceBubbles.maybeRememberSourceBubbleLanguage(newLangTag); if (divForBubble.length !== 0) { BloomHintBubbles.addHintBubbles( group.get(0), diff --git a/src/BloomBrowserUI/bookEdit/sourceBubbles/SourceBubblesSpec.ts b/src/BloomBrowserUI/bookEdit/sourceBubbles/SourceBubblesSpec.ts index e3f31a24e80e..0c325f5dd553 100644 --- a/src/BloomBrowserUI/bookEdit/sourceBubbles/SourceBubblesSpec.ts +++ b/src/BloomBrowserUI/bookEdit/sourceBubbles/SourceBubblesSpec.ts @@ -1,16 +1,34 @@ /// /// -import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; import $ from "jquery"; + +vi.mock("../../utils/bloomApi", async (importOriginal) => { + const actual = + await importOriginal(); + return { + ...actual, + postJson: vi.fn(), + postString: vi.fn(), + }; +}); + +import { postString } from "../../utils/bloomApi"; import BloomSourceBubbles from "./BloomSourceBubbles"; +const mockedPostString = vi.mocked(postString); + describe("SourceBubbles", () => { + const originalGetSettings = (window as any).GetSettings; + // reset fixture beforeEach(() => { $("body").html(""); + mockedPostString.mockReset(); }); afterEach(() => { $("body").html(""); + (window as any).GetSettings = originalGetSettings; }); it("Run MakeSourceTextDivForGroup with pre-defined settings", () => { // TODO: Testing is a bit hampered by not being able (currently) to put test values @@ -243,6 +261,40 @@ describe("SourceBubbles", () => { expect(listItems[1].getAttribute("id")).toBe("es"); // alphabetical }); + it("MakeSourceTextDivForGroup keeps non-preferred languages in alphabetical order", () => { + const testHtml = $( + [ + "
    ", + "
    English text
    ", + "
    Swahili text
    ", + "
    French text
    ", + "
    Spanish text
    ", + "
    ", + ].join("\n"), + ); + $("body").append(testHtml); + + const oldGetSettings = (window as any).GetSettings; + (window as any).GetSettings = () => ({ + defaultSourceLanguage: "", + defaultSourceLanguage2: "", + currentCollectionLanguage2: "", + currentCollectionLanguage3: "", + }); + + const result = BloomSourceBubbles.MakeSourceTextDivForGroup( + $("body").find("#testTarget")[0], + ); + + (window as any).GetSettings = oldGetSettings; + + const listItems = result.find("nav ul li"); + expect(listItems.length).toBe(3); + expect(listItems[0].getAttribute("id")).toBe("es"); + expect(listItems[1].getAttribute("id")).toBe("fr"); + expect(listItems[2].getAttribute("id")).toBe("sw"); + }); + it("MakeSourceTextDivForGroup handles when defaultSourceLanguage2 equals defaultSourceLanguage", () => { const testHtml = $( [ @@ -278,4 +330,282 @@ describe("SourceBubbles", () => { expect(listItems[1].getAttribute("id")).toBe("tpi"); // collection lang 2 expect(listItems[2].getAttribute("id")).toBe("es"); // alphabetical }); + + // --- AI source bubble display tests --- + // + // AI-translated source divs (lang tag like "fr-x-ai-deepl") are written into the book + // by a C# batch process that runs before the book is opened for editing. The front end + // is display-only: it never requests a translation, never writes an AI div, and never + // removes one from the live .bloom-translationGroup. These tests exercise only that + // display behavior. + + it("MakeSourceTextDivForGroup shows one tab per AI-provider div, alongside the normal source tabs", () => { + const testHtml = $( + [ + "
    ", + "
    English text
    ", + "
    Spanish text
    ", + "
    Texte français DeepL
    ", + "
    Texte français Google
    ", + "
    ", + ].join("\n"), + ); + $("body").append(testHtml); + + // sanity check: both AI divs are actually present with distinct text before we build the bubble + expect( + $("body").find("#testTarget div[lang='fr-x-ai-deepl']").text(), + ).toBe("Texte français DeepL"); + expect( + $("body").find("#testTarget div[lang='fr-x-ai-google']").text(), + ).toBe("Texte français Google"); + + (window as any).GetSettings = () => ({ + defaultSourceLanguage: "en", + defaultSourceLanguage2: "", + currentCollectionLanguage2: "", + currentCollectionLanguage3: "", + allowAiSourceBubbles: true, + }); + + const result = BloomSourceBubbles.MakeSourceTextDivForGroup( + $("body").find("#testTarget")[0], + ); + + // English is vernacular (no tab); es, fr-x-ai-deepl and fr-x-ai-google each get a tab + const listItems = result.find("nav ul li"); + expect(listItems.length).toBe(3); + expect(result.find("li#es").length).toBe(1); + expect(result.find("li#fr-x-ai-deepl").length).toBe(1); + expect(result.find("li#fr-x-ai-google").length).toBe(1); + expect( + result.find("div.source-text[lang='fr-x-ai-deepl']")[0] + .childNodes[0].textContent, + ).toBe("Texte français DeepL"); + expect( + result.find("div.source-text[lang='fr-x-ai-google']")[0] + .childNodes[0].textContent, + ).toBe("Texte français Google"); + }); + + it("getLanguageDisplayName labels an AI tab with the language name and mapped provider name", () => { + const getLanguageDisplayName = (BloomSourceBubbles as any) + .getLanguageDisplayName as (langTag: string) => string; + + expect(getLanguageDisplayName("fr-x-ai-deepl")).toBe( + "AI français (DeepL)", + ); + expect(getLanguageDisplayName("fr-x-ai-google")).toBe( + "AI français (Google Translate)", + ); + expect(getLanguageDisplayName("fr-x-ai-alpha2")).toBe( + "AI français (SIL Alpha2)", + ); + // an unrecognized provider id is shown as-is + expect(getLanguageDisplayName("fr-x-ai-mystery")).toBe( + "AI français (mystery)", + ); + }); + + it("getLanguageDisplayName falls back to the browser's language names for tags Bloom doesn't know", () => { + const getLanguageDisplayName = (BloomSourceBubbles as any) + .getLanguageDisplayName as (langTag: string) => string; + + // "de" is not one of the mocked collection languages, so Bloom's own lookup fails; + // Intl.DisplayNames should supply "German" rather than showing the raw tag. + expect(getLanguageDisplayName("de-x-ai-deepl")).toBe( + "AI German (DeepL)", + ); + }); + + it("MakeSourceTextDivForGroup shows the AI icon only on AI tabs, with a provider-labeled tab", () => { + const testHtml = $( + [ + "
    ", + "
    English text
    ", + "
    French text
    ", + "
    Texte français DeepL
    ", + "
    ", + ].join("\n"), + ); + $("body").append(testHtml); + + (window as any).GetSettings = () => ({ + defaultSourceLanguage: "en", + defaultSourceLanguage2: "", + currentCollectionLanguage2: "", + currentCollectionLanguage3: "", + allowAiSourceBubbles: true, + }); + + const result = BloomSourceBubbles.MakeSourceTextDivForGroup( + $("body").find("#testTarget")[0], + ); + + const aiTab = result.find("li#fr-x-ai-deepl a.sourceTextTab"); + expect(aiTab.length).toBe(1); + expect(aiTab.find("svg[data-testid='AutoAwesomeIcon']").length).toBe(1); + expect(aiTab.text().trim()).toBe("AI français (DeepL)"); + + const nonAiTab = result.find("li#fr a.sourceTextTab"); + expect(nonAiTab.length).toBe(1); + expect(nonAiTab.find("svg[data-testid='AutoAwesomeIcon']").length).toBe( + 0, + ); + }); + + it("MakeSourceTextDivForGroup hides AI tabs when allowAiSourceBubbles is false, without touching the live group", () => { + const testHtml = $( + [ + "
    ", + "
    English text
    ", + "
    Spanish text
    ", + "
    Texte français DeepL
    ", + "
    ", + ].join("\n"), + ); + $("body").append(testHtml); + const liveGroup = $("body").find("#testTarget"); + + // sanity check: the AI div is present in the live book DOM before we build the bubble + expect(liveGroup.find("div[lang='fr-x-ai-deepl']").length).toBe(1); + + (window as any).GetSettings = () => ({ + defaultSourceLanguage: "en", + defaultSourceLanguage2: "", + currentCollectionLanguage2: "", + currentCollectionLanguage3: "", + allowAiSourceBubbles: false, + }); + + const result = BloomSourceBubbles.MakeSourceTextDivForGroup( + liveGroup[0], + ); + + expect(result.find("li#fr-x-ai-deepl").length).toBe(0); + expect(result.find("div[lang='fr-x-ai-deepl']").length).toBe(0); + expect(result.find("li#es").length).toBe(1); + // the book's own DOM must be untouched even though the setting is off + expect(liveGroup.find("div[lang='fr-x-ai-deepl']").length).toBe(1); + expect(liveGroup.find("div[lang='fr-x-ai-deepl']").text()).toBe( + "Texte français DeepL", + ); + }); + + it("MakeSourceTextDivForGroup does not show an empty AI div as a tab", () => { + const testHtml = $( + [ + "
    ", + "
    English text
    ", + "
    Spanish text
    ", + "
    ", + "
    ", + ].join("\n"), + ); + $("body").append(testHtml); + + // sanity check: the AI div really is empty before we build the bubble + expect( + $("body") + .find("#testTarget div[lang='fr-x-ai-deepl']") + .text() + .trim(), + ).toBe(""); + + (window as any).GetSettings = () => ({ + defaultSourceLanguage: "en", + defaultSourceLanguage2: "", + currentCollectionLanguage2: "", + currentCollectionLanguage3: "", + allowAiSourceBubbles: true, + }); + + const result = BloomSourceBubbles.MakeSourceTextDivForGroup( + $("body").find("#testTarget")[0], + ); + + expect(result.find("li#fr-x-ai-deepl").length).toBe(0); + expect(result.find("div[lang='fr-x-ai-deepl']").length).toBe(0); + expect(result.find("li#es").length).toBe(1); + }); + + it("AI source bubble tabs are remembered as the default source language", () => { + const aiLanguageTag = "id-x-ai-deepl"; + const testHtml = $( + [ + "
    ", + "
    Tok Pisin text
    ", + `
    Bahasa Indonesia
    `, + "
    ", + ].join("\n"), + ); + $("body").append(testHtml); + (window as any).GetSettings = () => ({ + defaultSourceLanguage: "tpi", + defaultSourceLanguage2: "", + currentCollectionLanguage2: "", + currentCollectionLanguage3: "", + allowAiSourceBubbles: true, + }); + + const result = BloomSourceBubbles.MakeSourceTextDivForGroup( + $("body").find("#testTarget")[0], + ); + + const aiTab = result.find(`li#${CSS.escape(aiLanguageTag)} a`); + expect(aiTab.length).toBe(1); + aiTab.get(0)?.dispatchEvent(new MouseEvent("click", { bubbles: true })); + + expect(mockedPostString).toHaveBeenCalledWith( + "editView/sourceTextTab", + aiLanguageTag, + ); + }); + + it("styled dropdown handles clicks on the dropdown list item", () => { + const qtipId = "qtip-0"; + const groupHtml = $( + [ + `
    `, + "
    Tok Pisin text
    ", + "
    French text
    ", + "
    Spanish text
    ", + "
    ", + ].join("\n"), + ); + const qtip = $(`
    `); + const divForBubble = BloomSourceBubbles.CreateDropdownIfNecessary( + $( + [ + "
    ", + " ", + "
    Spanish text
    ", + "
    French text
    ", + "
    Tok Pisin text
    ", + "
    ", + ].join("\n"), + ), + ); + qtip.append(divForBubble); + $("body").append(groupHtml); + $("body").append(qtip); + const dropdownItem = divForBubble.find(".dropdown-list li[lang='es']"); + + expect(dropdownItem.length).toBe(1); + + dropdownItem + .get(0) + ?.dispatchEvent(new MouseEvent("click", { bubbles: true })); + + expect(mockedPostString).toHaveBeenCalledWith( + "editView/sourceTextTab", + "es", + ); + }); }); diff --git a/src/BloomBrowserUI/bookEdit/sourceBubbles/aiTranslationDisplay.ts b/src/BloomBrowserUI/bookEdit/sourceBubbles/aiTranslationDisplay.ts new file mode 100644 index 000000000000..965f18e2380b --- /dev/null +++ b/src/BloomBrowserUI/bookEdit/sourceBubbles/aiTranslationDisplay.ts @@ -0,0 +1,53 @@ +// Small helpers for displaying AI-translated content in the source bubbles UI. The +// translations themselves are produced by a C# batch process (see AiTranslationBookUpdater) +// before the book is opened for editing; this module only concerns itself with how the +// front end labels and filters that AI-generated content once it's already in the DOM. +import $ from "jquery"; + +/// True if the given lang tag (e.g. "fr-x-ai-deepl") is one Bloom's AI translation machinery +/// wrote, per the "-x-ai" private-use convention (see AiTranslationService.GetAiLanguageTag). +export function isAiLanguageTag(languageTag: string | undefined): boolean { + return !!languageTag && languageTag.includes("-x-ai"); +} + +/// Maps a translation provider id (e.g. "deepl") to the human-facing name shown in AI tab +/// labels. Kept in sync with AiTranslationService.GetProviderDisplayName on the C# side. +export function getAiProviderDisplayName(providerId: string): string { + switch (providerId) { + case "deepl": + return "DeepL"; + case "google": + return "Google Translate"; + case "alpha2": + return "SIL Alpha2"; + default: + return providerId; + } +} + +/// The AI target language is typically not one of the book's languages, so Bloom's own +/// language name lookup often doesn't know it; the browser's Intl database usually does. +export function getLanguageNameFromBrowser( + langTag: string, +): string | undefined { + try { + return new Intl.DisplayNames([navigator.language || "en"], { + type: "language", + }).of(langTag); + } catch { + // Intl.DisplayNames throws on structurally invalid tags; fall back to the raw tag. + return undefined; + } +} + +/// Strips AI-translated divs (lang tag contains "-x-ai-") out of a cloned source-bubble div, +/// used when the user has turned AI source bubbles off. Only mutates the clone passed in; +/// the live .bloom-translationGroup in the book is never touched here. +export function removeAiTranslationDivsFromClone(divForBubble: JQuery): void { + divForBubble.find("div[lang]").each((index, element) => { + const langTag = element.getAttribute("lang") || ""; + if (isAiLanguageTag(langTag)) { + element.remove(); + } + }); +} diff --git a/src/BloomBrowserUI/collection/AdvancedSettingsPanel.test.tsx b/src/BloomBrowserUI/collection/AdvancedSettingsPanel.test.tsx new file mode 100644 index 000000000000..f822dfcb77ac --- /dev/null +++ b/src/BloomBrowserUI/collection/AdvancedSettingsPanel.test.tsx @@ -0,0 +1,717 @@ +// This test harness deliberately mounts/unmounts via the React 17-style ReactDOM.render / +// unmountComponentAtNode API (synchronous, simple for these unit tests). Disable the +// React-18-deprecation rule for the file rather than migrate the harness to createRoot here. +/* eslint-disable react/no-deprecated */ +import * as React from "react"; +import ReactDOM from "react-dom"; +import { act } from "react-dom/test-utils"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const { + mockPostJson, + mockPostJsonAsync, + configrPaneState, + initialAdvancedSettingsData, +} = vi.hoisted(() => ({ + mockPostJson: vi.fn(), + mockPostJsonAsync: vi.fn(), + configrPaneState: { + lastInitialValues: undefined as Record | undefined, + }, + initialAdvancedSettingsData: { + values: { + autoUpdate: true, + showExperimentalBookSources: false, + allowTeamCollection: false, + allowAppBuilder: false, + allowAiSourceBubbles: false, + aiTranslation: { + targetLanguageTag: "en", + engines: [ + { + providerId: "deepl", + enabled: false, + apiKey: "", + serviceAccountEmail: "", + privateKey: "", + validation: { + succeeded: false, + message: "", + upToDate: false, + }, + }, + { + providerId: "google", + enabled: false, + apiKey: "", + serviceAccountEmail: "", + privateKey: "", + validation: { + succeeded: false, + message: "", + upToDate: false, + }, + }, + { + providerId: "alpha2", + enabled: false, + apiKey: "", + serviceAccountEmail: "", + privateKey: "", + validation: { + succeeded: false, + message: "", + upToDate: false, + }, + }, + ], + }, + showQrCode: true, + qrcodeCaption: "caption", + } as Record, + showAutoUpdate: true, + showExperimentalBookSourcesOption: false, + allowTeamCollectionEnabled: true, + }, +})); + +vi.mock("../utils/bloomApi", async (importOriginal) => { + const actual = await importOriginal(); + + return { + ...actual, + get: ( + endpoint: string, + callback: (result: { data: unknown }) => void, + ) => { + if (endpoint === "settings/advancedProgramSettings") { + callback({ data: initialAdvancedSettingsData }); + return; + } + + throw new Error(`Unexpected GET endpoint: ${endpoint}`); + }, + postJson: mockPostJson, + postJsonAsync: mockPostJsonAsync, + }; +}); + +vi.mock("../react_components/featureStatus", () => ({ + useGetFeatureStatus: () => ({ enabled: true }), +})); + +vi.mock("../react_components/l10nHooks", () => ({ + useL10n: (englishText: string) => englishText, +})); + +vi.mock("../react_components/requiresSubscription", () => ({ + BloomSubscriptionIndicatorIconAndText: () => null, +})); + +vi.mock("../utils/WireUpWinform", () => ({ + WireUpForWinforms: vi.fn(), +})); + +vi.mock("@sillsdev/config-r", () => ({ + ConfigrPane: (props: { + children: React.ReactNode; + initialValues: Record; + onChange: (settings: unknown) => void; + }) => { + configrPaneState.lastInitialValues = props.initialValues; + + return ( +
    + + + + + + + {props.children} +
    + ); + }, + ConfigrPage: (props: React.PropsWithChildren) => ( +
    {props.children}
    + ), + ConfigrGroup: (props: React.PropsWithChildren) => ( +
    {props.children}
    + ), + ConfigrBoolean: (props: { path: string; label: string }) => ( +
    {props.label}
    + ), + ConfigrInput: (props: { path: string; label: string }) => ( +
    {props.label}
    + ), + ConfigrSelect: () => null, + ConfigrCustomObjectInput: (props: { + control: React.FunctionComponent<{ + value: unknown; + disabled?: boolean; + onChange: (value: unknown) => void; + }>; + overrideValue?: unknown; + }) => { + const Control = props.control; + return ( + {}} /> + ); + }, + ConfigrCustomStringInput: (props: { + control: React.FunctionComponent<{ + value: string; + disabled?: boolean; + onChange: (value: string) => void; + }>; + }) => { + const Control = props.control; + return {}} />; + }, +})); + +import { AdvancedSettingsPanel } from "./AdvancedSettingsPanel"; +import { parseSupportedTargetLanguageOptions } from "./AiTranslationSettingsGroup"; + +describe("AdvancedSettingsPanel", () => { + let container: HTMLDivElement; + + const click = (selector: string) => { + const button = container.querySelector(selector) as HTMLButtonElement; + expect(button).not.toBeNull(); + act(() => { + button.click(); + }); + }; + + beforeEach(() => { + vi.useFakeTimers(); + container = document.createElement("div"); + document.body.appendChild(container); + mockPostJson.mockReset(); + mockPostJsonAsync.mockReset(); + configrPaneState.lastInitialValues = undefined; + initialAdvancedSettingsData.values = { + autoUpdate: true, + showExperimentalBookSources: false, + allowTeamCollection: false, + allowAppBuilder: false, + allowAiSourceBubbles: false, + aiTranslation: { + targetLanguageTag: "en", + engines: [ + { + providerId: "deepl", + enabled: false, + apiKey: "", + serviceAccountEmail: "", + privateKey: "", + validation: { + succeeded: false, + message: "", + upToDate: false, + }, + }, + { + providerId: "google", + enabled: false, + apiKey: "", + serviceAccountEmail: "", + privateKey: "", + validation: { + succeeded: false, + message: "", + upToDate: false, + }, + }, + { + providerId: "alpha2", + enabled: false, + apiKey: "", + serviceAccountEmail: "", + privateKey: "", + validation: { + succeeded: false, + message: "", + upToDate: false, + }, + }, + ], + }, + showQrCode: true, + qrcodeCaption: "caption", + }; + mockPostJsonAsync.mockImplementation( + async (endpoint: string, body?: unknown) => { + if (endpoint === "settings/validateAiTranslationEngine") { + const providerId = (body as { providerId: string }) + .providerId; + return { + data: { + succeeded: true, + message: `translated by ${providerId}`, + }, + }; + } + + if (endpoint === "settings/aiTranslationSupportedLanguages") { + return { + data: { + languages: [ + { + tag: "es", + name: "Spanish", + providerIds: ["deepl", "google"], + }, + { + tag: "fra", + name: "French (fra)", + providerIds: ["deepl"], + }, + ], + }, + }; + } + + if ( + endpoint === "settings/aiTranslationAlpha2SourceLanguages" + ) { + return { data: { languages: [] } }; + } + + throw new Error(`Unexpected async POST endpoint: ${endpoint}`); + }, + ); + }); + + afterEach(() => { + ReactDOM.unmountComponentAtNode(container); + container.remove(); + document.body.innerHTML = ""; + vi.useRealTimers(); + }); + + it("shows an engine's credential fields only once it is enabled", async () => { + await act(async () => { + ReactDOM.render(, container); + }); + + expect( + container.querySelector( + '[data-testid="configr-input-aiTranslationGoogleServiceAccountEmail"]', + ), + ).toBeNull(); + + click('[data-testid="enable-google"]'); + + expect( + container.querySelector( + '[data-testid="configr-input-aiTranslationGoogleServiceAccountEmail"]', + ), + ).not.toBeNull(); + expect( + container.querySelector( + '[data-testid="configr-input-aiTranslationGooglePrivateKey"]', + ), + ).not.toBeNull(); + // DeepL and Alpha2 were never enabled, so their credential fields stay hidden. + expect( + container.querySelector( + '[data-testid="configr-input-aiTranslationDeepLApiKey"]', + ), + ).toBeNull(); + expect( + container.querySelector( + '[data-testid="configr-input-aiTranslationAlpha2ApiKey"]', + ), + ).toBeNull(); + // The three enable toggles are always present, regardless of enabled state. + expect( + container.querySelector( + '[data-testid="configr-boolean-aiTranslationDeepLEnabled"]', + ), + ).not.toBeNull(); + expect( + container.querySelector( + '[data-testid="configr-boolean-aiTranslationAlpha2Enabled"]', + ), + ).not.toBeNull(); + }); + + it("posts the store payload in the pinned nested wire shape", async () => { + await act(async () => { + ReactDOM.render(, container); + }); + + click('[data-testid="enable-all-three"]'); + + expect(mockPostJson).toHaveBeenCalledWith( + "settings/advancedProgramSettings", + expect.objectContaining({ + allowAiSourceBubbles: true, + aiTranslation: { + targetLanguageTag: "es", + engines: [ + // Google's serviceAccountEmail/privateKey must NOT leak onto the other + // engines' records (doing so wiped their validation when Google creds + // were edited); non-google engines send empty Google fields. + expect.objectContaining({ + providerId: "deepl", + enabled: true, + apiKey: "deepl-key", + serviceAccountEmail: "", + privateKey: "", + }), + expect.objectContaining({ + providerId: "google", + enabled: true, + serviceAccountEmail: "service-account@example.com", + privateKey: + "-----BEGIN PRIVATE KEY-----\nabc\n-----END PRIVATE KEY-----", + }), + expect.objectContaining({ + providerId: "alpha2", + enabled: true, + apiKey: "alpha2-key", + serviceAccountEmail: "", + privateKey: "", + }), + ], + }, + }), + ); + + // The flat, Configr-internal AI keys must not leak into the wire payload. + const [, wirePayload] = mockPostJson.mock.calls[0]; + expect(wirePayload).not.toHaveProperty("aiTranslationGoogleEnabled"); + expect(wirePayload).not.toHaveProperty( + "aiTranslationTargetLanguageTag", + ); + }); + + it("debounces per-engine validation and posts only the providerId", async () => { + await act(async () => { + ReactDOM.render(, container); + }); + + click('[data-testid="enable-google"]'); + + // Only google is enabled+credentialed, so only it should be probed. + await act(async () => { + await vi.advanceTimersByTimeAsync(601); + }); + + expect(mockPostJsonAsync).toHaveBeenCalledWith( + "settings/validateAiTranslationEngine", + { providerId: "google" }, + ); + expect(mockPostJsonAsync).not.toHaveBeenCalledWith( + "settings/validateAiTranslationEngine", + { providerId: "deepl" }, + ); + expect(container.textContent).toContain("translated by google"); + + click('[data-testid="set-french-target"]'); + + expect(container.textContent).toContain("Testing translation..."); + + await act(async () => { + await vi.advanceTimersByTimeAsync(601); + }); + + expect(mockPostJsonAsync).toHaveBeenCalledWith( + "settings/validateAiTranslationEngine", + { providerId: "google" }, + ); + }); + + it("clears validation immediately when the target language is removed, and waits for one before re-validating", async () => { + initialAdvancedSettingsData.values = { + ...initialAdvancedSettingsData.values, + allowAiSourceBubbles: true, + aiTranslation: { + targetLanguageTag: "es", + engines: [ + { + providerId: "deepl", + enabled: false, + apiKey: "", + serviceAccountEmail: "", + privateKey: "", + validation: { + succeeded: false, + message: "", + upToDate: false, + }, + }, + { + providerId: "google", + enabled: true, + apiKey: "", + serviceAccountEmail: "service-account@example.com", + privateKey: + "-----BEGIN PRIVATE KEY-----\nabc\n-----END PRIVATE KEY-----", + validation: { + succeeded: true, + message: "translated by google", + upToDate: true, + }, + }, + { + providerId: "alpha2", + enabled: false, + apiKey: "", + serviceAccountEmail: "", + privateKey: "", + validation: { + succeeded: false, + message: "", + upToDate: false, + }, + }, + ], + }, + }; + + await act(async () => { + ReactDOM.render(, container); + }); + + expect(container.textContent).toContain("translated by google"); + + click('[data-testid="enable-google-without-target"]'); + + expect(container.textContent).not.toContain("translated by google"); + expect(container.textContent).not.toContain("Testing translation..."); + + await act(async () => { + await vi.advanceTimersByTimeAsync(601); + }); + + expect(mockPostJsonAsync).not.toHaveBeenCalledWith( + "settings/validateAiTranslationEngine", + { providerId: "google" }, + ); + + click('[data-testid="set-french-target"]'); + + expect(container.textContent).toContain("Testing translation..."); + + await act(async () => { + await vi.advanceTimersByTimeAsync(601); + }); + + expect(mockPostJsonAsync).toHaveBeenCalledWith( + "settings/validateAiTranslationEngine", + { providerId: "google" }, + ); + }); + + it("fetches the union of provider-supported languages when the ready engine set changes", async () => { + await act(async () => { + ReactDOM.render(, container); + }); + + click('[data-testid="enable-deepl-and-google"]'); + + const targetLanguageSelect = container.querySelector( + '[data-testid="ai-translation-target-language-select"]', + ) as HTMLElement; + expect(targetLanguageSelect).not.toBeNull(); + + await act(async () => { + targetLanguageSelect.dispatchEvent( + new Event("mousedown", { bubbles: true }), + ); + }); + + expect(mockPostJsonAsync).toHaveBeenCalledWith( + "settings/aiTranslationSupportedLanguages", + expect.anything(), + ); + + expect( + parseSupportedTargetLanguageOptions({ + languages: [ + { tag: "es", name: "Spanish (es)", providerIds: ["deepl"] }, + ], + }), + ).toEqual([{ value: "es", label: "Spanish", providerIds: ["deepl"] }]); + }); + + it("round-trips alpha2 sourceLanguageTag through the wire payload", async () => { + await act(async () => { + ReactDOM.render(, container); + }); + + click('[data-testid="enable-alpha2-with-source"]'); + + expect(mockPostJson).toHaveBeenCalledWith( + "settings/advancedProgramSettings", + expect.objectContaining({ + aiTranslation: expect.objectContaining({ + engines: expect.arrayContaining([ + expect.objectContaining({ + providerId: "alpha2", + enabled: true, + apiKey: "alpha2-key", + sourceLanguageTag: "fr", + }), + ]), + }), + }), + ); + // The non-alpha2 engines must send an empty source language. + const [, wirePayload] = mockPostJson.mock.calls[0]; + const engines = ( + wirePayload as { + aiTranslation: { + engines: Array<{ + providerId: string; + sourceLanguageTag: string; + }>; + }; + } + ).aiTranslation.engines; + expect( + engines.find((e) => e.providerId === "deepl")?.sourceLanguageTag, + ).toBe(""); + }); + + it("shows the amber 'will be skipped' note when an engine doesn't support the target", async () => { + mockPostJsonAsync.mockImplementation( + async (endpoint: string, body?: unknown) => { + if (endpoint === "settings/validateAiTranslationEngine") { + const providerId = (body as { providerId: string }) + .providerId; + return { + data: { + succeeded: false, + targetLanguageNotSupported: true, + message: `${providerId} cannot do this language`, + }, + }; + } + if (endpoint === "settings/aiTranslationSupportedLanguages") { + return { data: { languages: [] } }; + } + if ( + endpoint === "settings/aiTranslationAlpha2SourceLanguages" + ) { + return { data: { languages: [] } }; + } + throw new Error(`Unexpected async POST endpoint: ${endpoint}`); + }, + ); + + await act(async () => { + ReactDOM.render(, container); + }); + + click('[data-testid="enable-google"]'); + + await act(async () => { + await vi.advanceTimersByTimeAsync(601); + }); + + expect(container.textContent).toContain( + "does not support translating to", + ); + // The red "Translation test failed" text must NOT be shown for the not-supported case. + expect(container.textContent).not.toContain("Translation test failed"); + }); +}); diff --git a/src/BloomBrowserUI/collection/AdvancedSettingsPanel.tsx b/src/BloomBrowserUI/collection/AdvancedSettingsPanel.tsx index 4e3c6b4a8255..52373527d24c 100644 --- a/src/BloomBrowserUI/collection/AdvancedSettingsPanel.tsx +++ b/src/BloomBrowserUI/collection/AdvancedSettingsPanel.tsx @@ -11,15 +11,50 @@ import { get, postJson } from "../utils/bloomApi"; import { kBloomBlue } from "../bloomMaterialUITheme"; import { tabMargins } from "./commonTabSettings"; import { WireUpForWinforms } from "../utils/WireUpWinform"; -import { BloomSubscriptionIndicatorIconAndText } from "../react_components/requiresSubscription"; import { useGetFeatureStatus } from "../react_components/featureStatus"; +import { BloomSubscriptionIndicatorIconAndText } from "../react_components/requiresSubscription"; import { useL10n } from "../react_components/l10nHooks"; +import { + AiTranslationProviderId, + AiTranslationLanguageControlsContext, + IAiTranslationEngineValidation, + IAiTranslationSettings, + IAiTranslationWireSettings, + buildAiTranslationWirePayload, + extractAiTranslationFlatSettings, + flattenAiTranslationWireSettings, + omitAiTranslationFlatSettings, + useAiTranslationSettingsGroup, +} from "./AiTranslationSettingsGroup"; + +// The shape actually sent/received over the wire for settings/advancedProgramSettings: the +// flat, non-AI settings plus a nested "aiTranslation" object matching the backend contract. +interface IAdvancedSettingsWire { + autoUpdate?: boolean; + showExperimentalBookSources?: boolean; + allowTeamCollection?: boolean; + allowAppBuilder?: boolean; + allowAiSourceBubbles?: boolean; + showQrCode?: boolean; + qrcodeCaption?: string; + aiTranslation?: IAiTranslationWireSettings; +} -interface IAdvancedSettings { +interface IAdvancedSettingsApiData { + values: IAdvancedSettingsWire; + showAutoUpdate?: boolean; + showExperimentalBookSourcesOption?: boolean; + allowTeamCollectionEnabled?: boolean; +} + +// The flattened shape Configr actually edits: non-AI settings plus one boolean+credential set +// per AI engine (see AiTranslationSettingsGroup for why it's flattened rather than nested). +interface IAdvancedSettings extends IAiTranslationSettings { autoUpdate?: boolean; showExperimentalBookSources?: boolean; allowTeamCollection?: boolean; allowAppBuilder?: boolean; + allowAiSourceBubbles?: boolean; showQrCode?: boolean; qrcodeCaption?: string; } @@ -35,6 +70,15 @@ export const AdvancedSettingsPanel: React.FunctionComponent = () => { showExperimentalBookSourcesOption, setShowExperimentalBookSourcesOption, ] = React.useState(false); + const [ + aiTranslationInitialValidations, + setAiTranslationInitialValidations, + ] = + React.useState< + Partial< + Record + > + >(); const advancedProgramSettingsLabel = useL10n( "Advanced Program Settings", @@ -64,6 +108,78 @@ export const AdvancedSettingsPanel: React.FunctionComponent = () => { "App Builder", "CollectionSettingsDialog.AdvancedTab.Experimental.AppBuilder", ); + const aiSourceBubblesLabel = useL10n( + "AI Source Translation", + "CollectionSettingsDialog.AdvancedTab.Experimental.AiSourceBubbles", + ); + const aiTranslationSectionLabel = useL10n( + "AI Source Translation", + "CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.SectionLabel", + ); + const aiTranslationTargetLanguageLabel = useL10n( + "Target Language", + "CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.TargetLanguageLabel", + ); + const aiTranslationDeepLEnabledLabel = useL10n( + "DeepL", + "CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.DeepLEnabledLabel", + ); + const aiTranslationDeepLApiKeyLabel = useL10n( + "API Key", + "CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.DeepLApiKeyLabel", + ); + const aiTranslationDeepLApiKeyDescription = useL10n( + "The key needs the 'translate:text' and 'languages:read' permissions.", + "CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.DeepLApiKeyDescription", + ); + const aiTranslationGoogleEnabledLabel = useL10n( + "Google Translate", + "CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.GoogleEnabledLabel", + ); + const aiTranslationGoogleServiceAccountEmailLabel = useL10n( + "Google Service Account Email", + "CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.GoogleServiceAccountEmailLabel", + ); + const aiTranslationGoogleServiceAccountDescription = useL10n( + "The service account needs access to the Cloud Translation API (the 'Cloud Translation API User' role).", + "CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.GoogleServiceAccountDescription", + ); + const aiTranslationGooglePrivateKeyLabel = useL10n( + "Google Service Account Private Key", + "CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.GooglePrivateKeyLabel", + ); + const aiTranslationAlpha2EnabledLabel = useL10n( + "SIL Alpha2", + "CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.Alpha2EnabledLabel", + ); + const aiTranslationAlpha2ApiKeyLabel = useL10n( + "API Key", + "CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.Alpha2ApiKeyLabel", + ); + const aiTranslationAlpha2SourceLanguageLabel = useL10n( + "Source Language", + "CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.Alpha2SourceLanguageLabel", + ); + const aiTranslationTranslationTestLabel = useL10n( + "Translation Test", + "CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.TranslationTestLabel", + ); + const aiTranslationTargetLanguageNotSupportedTemplate = useL10n( + "{0} does not support translating to {1}, so it will be skipped.", + "CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.TargetLanguageNotSupported", + ); + const aiTranslationNoServiceSupportsLanguageNote = useL10n( + "no enabled service supports this", + "CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.NoServiceSupportsLanguage", + ); + const aiTranslationNoProviderSelectedNote = useL10n( + "Select at least one translation provider to get a list of languages it supports.", + "CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.NoProviderSelected", + ); + const aiTranslationAlpha2LanguagesNoteTemplate = useL10n( + "SIL Alpha2 shows the languages it can translate from {0}.", + "CollectionSettingsDialog.AdvancedTab.AiSourceBubbles.Alpha2LanguagesNote", + ); const qrCodesLabel = useL10n( "QR Codes", "CollectionSettingsDialog.AdvancedTab.QrCodes", @@ -93,6 +209,11 @@ export const AdvancedSettingsPanel: React.FunctionComponent = () => { appBuilderFeatureStatus === undefined ? false : appBuilderFeatureStatus.enabled; + const aiSourceBubblesFeatureStatus = useGetFeatureStatus("AiSourceBubbles"); + const aiSourceBubblesOptionEnabled = + aiSourceBubblesFeatureStatus === undefined + ? false + : aiSourceBubblesFeatureStatus.enabled; const canChangeTeamCollectionOption = allowTeamCollectionEnabled !== false; const normalizeConfigrSettings = React.useCallback( @@ -110,7 +231,6 @@ export const AdvancedSettingsPanel: React.FunctionComponent = () => { [], ); - // Load current advanced settings from the host dialog so Config-r starts with matching values. React.useEffect(() => { get("settings/advancedProgramSettings", (result) => { if (!result || !result.data || result.data === "{}") { @@ -120,17 +240,77 @@ export const AdvancedSettingsPanel: React.FunctionComponent = () => { if (typeof result.data === "string") { data = JSON.parse(result.data); } - setSettings(data["values"]); - setShowAutoUpdate(data["showAutoUpdate"] ?? false); + const advancedSettingsData = data as IAdvancedSettingsApiData; + const { aiTranslation, ...restOfWireValues } = + advancedSettingsData.values; + const { flatSettings, initialValidations } = + flattenAiTranslationWireSettings(aiTranslation); + setSettings({ + ...restOfWireValues, + ...flatSettings, + } as IAdvancedSettings); + setShowAutoUpdate(advancedSettingsData.showAutoUpdate ?? false); setAllowTeamCollectionEnabled( - data["allowTeamCollectionEnabled"] ?? false, + advancedSettingsData.allowTeamCollectionEnabled ?? false, ); setShowExperimentalBookSourcesOption( - data["showExperimentalBookSourcesOption"] ?? false, + advancedSettingsData.showExperimentalBookSourcesOption ?? false, ); + setAiTranslationInitialValidations(initialValidations); }); }, []); + // Memoized so the object reference is stable across renders that don't actually change + // the AI settings; the group's validation/language-fetch effects key off this reference. + // We key the memo on the *serialized* AI settings rather than the whole `settings` object: + // extractAiTranslationFlatSettings returns a fresh object every call, so keying on `settings` + // handed out a new reference whenever ANY unrelated setting changed. That silently cancelled an + // in-flight engine validation and left it stuck showing "Testing translation..." forever (the + // per-engine validation effect early-returns when its probe key is unchanged, so it never + // restarted or cleared the pending flag). Keying on the serialized value fixes that. + const extractedAiTranslationFlatSettings = settings + ? extractAiTranslationFlatSettings( + settings as unknown as Record, + ) + : undefined; + const aiTranslationFlatSettingsKey = JSON.stringify( + extractedAiTranslationFlatSettings ?? null, + ); + const aiTranslationFlatSettings = React.useMemo( + () => extractedAiTranslationFlatSettings, + // eslint-disable-next-line react-hooks/exhaustive-deps + [aiTranslationFlatSettingsKey], + ); + + const { + group: aiTranslationSettingsGroup, + languageControlsData: aiTranslationLanguageControlsData, + } = useAiTranslationSettingsGroup({ + settings: aiTranslationFlatSettings, + initialValidations: aiTranslationInitialValidations, + groupLabel: aiTranslationSectionLabel, + targetLanguageLabel: aiTranslationTargetLanguageLabel, + deepLEnabledLabel: aiTranslationDeepLEnabledLabel, + deepLApiKeyLabel: aiTranslationDeepLApiKeyLabel, + deepLApiKeyDescription: aiTranslationDeepLApiKeyDescription, + googleEnabledLabel: aiTranslationGoogleEnabledLabel, + googleServiceAccountEmailLabel: + aiTranslationGoogleServiceAccountEmailLabel, + googleServiceAccountDescription: + aiTranslationGoogleServiceAccountDescription, + googlePrivateKeyLabel: aiTranslationGooglePrivateKeyLabel, + alpha2EnabledLabel: aiTranslationAlpha2EnabledLabel, + alpha2ApiKeyLabel: aiTranslationAlpha2ApiKeyLabel, + alpha2SourceLanguageLabel: aiTranslationAlpha2SourceLanguageLabel, + translationTestLabel: aiTranslationTranslationTestLabel, + targetLanguageNotSupportedTemplate: + aiTranslationTargetLanguageNotSupportedTemplate, + noServiceSupportsLanguageNote: + aiTranslationNoServiceSupportsLanguageNote, + noProviderSelectedNote: aiTranslationNoProviderSelectedNote, + alpha2LanguagesNoteTemplate: aiTranslationAlpha2LanguagesNoteTemplate, + }); + return (
    { `} > {settings && ( - ["initialValues"] - } - themeOverrides={{ - palette: { - primary: { main: kBloomBlue }, - }, - }} - onChange={(newSettings) => { - const normalized = - normalizeConfigrSettings(newSettings); - if (normalized) { - setSettings(normalized); - postJson( - "settings/advancedProgramSettings", - normalized, - ); - } - }} + // Provide the target-language control's data here, above the Configr pane, so the + // stable module-scope AiTranslationTargetLanguageControl (rendered somewhere inside + // the pane) can read it via context without being redefined each render. + - - - {showAutoUpdate && ( - - )} - - - - - - - {showExperimentalBookSourcesOption && ( + ["initialValues"] + } + themeOverrides={{ + palette: { + primary: { main: kBloomBlue }, + }, + }} + onChange={(newSettings) => { + const normalized = + normalizeConfigrSettings(newSettings); + if (normalized) { + setSettings(normalized); + const aiFlatSettings = + extractAiTranslationFlatSettings( + normalized as unknown as Record< + string, + unknown + >, + ); + const wirePayload = { + ...omitAiTranslationFlatSettings( + normalized as unknown as Record< + string, + unknown + >, + ), + aiTranslation: + buildAiTranslationWirePayload( + aiFlatSettings, + ), + }; + postJson( + "settings/advancedProgramSettings", + wirePayload, + ); + } + }} + > + + + {showAutoUpdate && ( + + )} + + + - )} -
    - {" "} + + + {showExperimentalBookSourcesOption && ( + + )}
    - {" "} +
    + +
    +
    +
    + +
    + +
    -
    -
    -
    - +
    + +
    -
    -     -     -     +     + {settings.allowAiSourceBubbles && + aiTranslationSettingsGroup} +     +     + )}
    ); diff --git a/src/BloomBrowserUI/collection/AiTranslationProviderLogos.tsx b/src/BloomBrowserUI/collection/AiTranslationProviderLogos.tsx new file mode 100644 index 000000000000..c471133560ed --- /dev/null +++ b/src/BloomBrowserUI/collection/AiTranslationProviderLogos.tsx @@ -0,0 +1,62 @@ +import { css } from "@emotion/react"; +import * as React from "react"; +import { AiTranslationProviderId } from "./AiTranslationSettingsGroup"; + +// Small provider-brand marks shown next to each language in the target-language dropdown to +// indicate which enabled services offer that language (in place of spelling the names out). +// The DeepL and Google marks are the services' own icons; the Alpha2 mark is SIL's glyph. All are +// embedded as data URIs so the panel stays self-contained (no bundler asset-path concerns). + +// DeepL's icon (deepl-24x24.png). +const kDeepLLogoDataUri = + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABgAAAAYCAYAAADgdz34AAAACXBIWXMAAAsTAAALEwEAmpwYAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAOdEVYdFNvZnR3YXJlAEZpZ21hnrGWYwAAARhJREFUeAG1lIENgjAQRX+MAzgCbsAGMAIbyAboBOIGboAb4AbECXADdALcAO9CEdK0cBT8yQuQ0PvN9fcAuSKiUhywokKiIBqNkvCwQDviaiisk7kYHYlaULyjUmsmFaqfG0dGzydeUFjnZ7IZGAQW4yfaHqeQKzQZ2MS9fRMX4oWZkhhE6ukTJxeTThnkhxir9xLmUGSuBkMjX63j+1LbDCQtMslDH4oP2iAYtcV8ccEUfVL2g3cnAy54Q3vYd+JBJKpoNLVYGlNOz1595xjZsYtBJw/CeWNTCHNiOCG86wqyhPH/0dgu58RVp4BwdAeYN1VL2GfZqOIJI25HgoXyYG/b4uK6UQ7L3F9TMfq2/cWg0xmCC/cFAYS4WXqkl2cAAAAASUVORK5CYII="; + +// Google Translate's icon (google-24x24.png). +const kGoogleLogoDataUri = + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABgAAAAYCAYAAADgdz34AAAACXBIWXMAAAsTAAALEwEAmpwYAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAOdEVYdFNvZnR3YXJlAEZpZ21hnrGWYwAAAtlJREFUeAGtVU1ME1EQnnm7NaJpKBETJCLLrerBlijhYHQBPZiYWA8mxhApZzXi1cS0jVw8qQdOHrZ6ES9aE4k/CSy9eSBh40XjhQUTCzGBTUCp3e6Ob2mRpezSJfBddjPz5vvmzXvzBqEOfnRHZSK8ggQSAsQcGxHpgGggUM5kkO/4/E33i0c/x9yZaJIhpghAgjogwKzF7IyX0BaBQndUKhMqPEqGnYDAENqOZVpff3ziK+CQWzaqQbL21gDdKq3GOzTdWLexwOSETpDGU5rEcNjwJqceN7kDcf3HshmvN0legfuiJ4dWv0zl3cGFGE8oBClCTG6Q+5xBof+oXP4eVrc4CbOm+edebVZuzHUdT3ByzYv8v0BxAtTiVKO88q4FqFitmg25tqmvV2GXwFWV328bZtY4l0JgPGsHi3/LJerwy2onEJkFMareJdZkQtOdGfj9/nC2eWRe9wq4dv9TTECMQBCIoi7yIp1y27DBgnBiPgcj3jGCID7mRy9DEFh22im4VGs3CWZhL4AgMS/7QQGWYI/gCOi1xpJdedR2C94fhmgDzLrfi2VbhLGVVpnf8LeeUbaVQRCzW8iYzWMgudlIOpIKkb92pSSFcgPcWuiEgrXfIAjFtcEPOgTE9QfjSq0AAsoMe8Bw3pexlSMwUOhyyB1fBMFUApOnVKmWnEN/+bA3v3bItxdOZ4YXT8Ayie4FcqfSV1dkjZxI9XBNVnZRRVzpe8M/CY+FOjFIawPjz93GmCJHGIhDYrnxbtvPG5FQeaP3OKlBpVJ89NElHd0BCGyauyXwBWnOqASnd2ijf0JmI7T8ugwHiu2VVURDr4YvPK2KbcqKz12mbi/ij0NLZ6F58Vx6dLgv49rNZlREBKf2MuwQfJ6ktcGJjNvmO/RjSm+SO1NBdsMbahIFOz19U83X+rBecPzFxfNkWwk+Mnl3Ez9J5yWlyvhkqKFAuen+Cd+36x/CsiwGSaBwpgAAAABJRU5ErkJggg=="; + +// SIL's glyph (tiny-SIL.svg), used for Alpha2 in the compact dropdown. +const kSilGlyphDataUri = + "data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iNjY4IiBoZWlnaHQ9Ijc1MSIgdmlld0JveD0iMCAwIDY2OCA3NTEiIGZpbGw9Im5vbmUiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CjxwYXRoIGQ9Ik00NDAuOTggNTYzLjkzVjI3Mi4zNEw1MDIuMjYgMjcyLjMyVjUwNi4yN0w2NjggNTA2LjQxVjBIMC4wMTAwMDk4VjUwNi40MkwxNjYuOTcgNTA2LjM3QzIwMS42NSA1MDYuMzcgMjA5LjIxIDQ5Mi4zOCAyMDkuNDcgNDgwLjYzQzIwOS45OSA0NTYuNTggMTk2LjkxIDQ0Ni45NyAxNzEuNzEgNDMwLjZDMTI0LjQyIDQwMS43OCAxMTcuMzYgMzY4LjE0IDExNy4zNiAzNTAuNjZDMTE3LjM2IDI5MS4wNCAxNjIuNzggMjY0LjM2IDIwNy43NyAyNjQuMzZDMjI1LjU3IDI2NC4zNiAyNTUuMTkgMjcxLjMyIDI2NS44MiAyNzYuMDNMMjU1LjYzIDMzMC4wNEMyNDcuNTMgMzI1LjU4IDIzNi4xMiAzMjEuMDIgMjIxLjAzIDMyMS4wMkMxOTMuNDkgMzIxLjAyIDE4My42NyAzMzUuMDEgMTgzLjY3IDM0OC4xMkMxODMuNjcgMzY4LjU1IDE5My41NyAzNzUuNDggMjI0Ljg0IDM5NC4yOUMyNTkuNDQgNDE1LjI5IDI3Ni45OCA0NDIuOTQgMjc2Ljk4IDQ3Ni40NEMyNzYuOTggNTI3Ljk4IDIzOC45OSA1NjIuNjggMTgyLjQ3IDU2Mi44MkwwIDU2Mi45Vjc1MC4zNUg2NjcuOTlWNTYzLjkzSDQ0MC45OFpNMzgxLjM0IDU2My45M0gzMjMuODZWMjcyLjUySDM4MS4zNFY1NjMuOTNaIiBmaWxsPSIjMjE1RUFDIi8+Cjwvc3ZnPgo="; + +// SIL's full "SIL" wordmark with the Tai Heritage Pro script sample (SIL-Blue-Tai-Heritage.svg), +// shown before the Alpha2 enable label. +const kSilWordmarkDataUri = + "data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMTM5NSIgaGVpZ2h0PSI3NTIiIHZpZXdCb3g9IjAgMCAxMzk1IDc1MiIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPGcgY2xpcC1wYXRoPSJ1cmwoI2NsaXAwXzQyNzRfMTM2KSI+CjxwYXRoIGQ9Ik02NjcuNTQgMC44MzAwNzhINTgyLjYxSDAuODkwMTM3VjI1OS42MVY0NDcuNjVWNzUwLjc4SDY2OC4zNVYwLjgzMDA3OEg2NjcuNTRaIiBmaWxsPSIjMEVBNURGIi8+CjxwYXRoIGQ9Ik00MC43NzAxIDM0Mi41NkM1My44MzAxIDMxNiA2Ny40MDAxIDI5My4wNCA4MS41MTAxIDI3My42OEM5NS42MTAxIDI1NC4zMiAxMDkuNzkgMjM5LjI0IDEyNC4wNSAyMjguNDRDMTM4LjMgMjE3LjY0IDE1MS43NCAyMTIuMjMgMTY0LjM0IDIxMi4yM0MxNzAuMDQgMjEyLjIzIDE3NC44NCAyMTMuMjggMTc4Ljc1IDIxNS4zOEMxODIuNjUgMjE3LjQ4IDE4NS44IDIyMC4zMyAxODguMiAyMjMuOTNDMTkwLjYgMjI3LjUzIDE5Mi4zMiAyMzEuNTggMTkzLjM4IDIzNi4wOEMxOTQuNDMgMjQwLjU4IDE5NC45NiAyNDUuMjQgMTk0Ljk2IDI1MC4wNEMxOTQuOTYgMjc0LjA1IDE5My4zMSAyOTcuODMgMTkwLjAxIDMyMS4zOUMxODYuNyAzNDQuOTUgMTgzLjEgMzY4LjUxIDE3OS4yMSAzOTIuMDdDMTc1LjMxIDQxNS42MyAxNzEuNzEgNDM5LjE5IDE2OC40MSA0NjIuNzVDMTY1LjEgNDg2LjMxIDE2My40NiA1MTAuMSAxNjMuNDYgNTM0LjFDMTYzLjQ2IDU0OS4xMSAxNjUuMjYgNTYzLjI5IDE2OC44NiA1NzYuNjRDMTcyLjQ2IDU5MCAxNzguMDEgNjAxLjc4IDE4NS41MiA2MTEuOThDMTkzLjAyIDYyMi4xOSAyMDIuNjMgNjMwLjIxIDIxNC4zMyA2MzYuMDZDMjI2LjA0IDY0MS45MSAyNDAuMTQgNjQ0Ljg0IDI1Ni42NSA2NDQuODRDMjcwLjc1IDY0NC44NCAyODQuMTggNjQyLjgxIDI5Ni45NCA2MzguNzZDMzA5LjY5IDYzNC43MSAzMjEuOTIgNjI5LjUzIDMzMy42MyA2MjMuMjNDMzQ1LjMzIDYxNi45MyAzNTYuNzQgNjA5Ljk1IDM2Ny44NCA2MDIuM0MzNzguOTQgNTk0LjY1IDM4OS44MiA1ODcuMzcgNDAwLjQ4IDU4MC40N0M0MTEuMTMgNTczLjU3IDQyMS43MSA1NjcuNDIgNDMyLjIyIDU2Mi4wMUM0NDIuNzIgNTU2LjYxIDQ1My4zOCA1NTIuODYgNDY0LjE4IDU1MC43NkM0NjkuMjggNTYyLjE3IDQ3NS41IDU3My4yNyA0ODIuODYgNTg0LjA3QzQ5MC4yMSA1OTQuODcgNDk4LjY5IDYwNC40OCA1MDguMjkgNjEyLjg4QzUxNy44OSA2MjEuMjkgNTI4LjQ3IDYyOC4xMiA1NDAuMDMgNjMzLjM2QzU1MS41OCA2MzguNjEgNTY0LjExIDY0MS4yNCA1NzcuNjIgNjQxLjI0QzU5Ny4xMiA2NDEuMjQgNjExLjQ1IDYzNS42MSA2MjAuNjEgNjI0LjM2QzYyOS43NiA2MTMuMTEgNjM0LjM0IDU5OC4xOCA2MzQuMzQgNTc5LjU3QzYzNC4zNCA1NjYuMDcgNjMwLjY2IDU1My41NCA2MjMuMzEgNTQxLjk4QzYxNS45NSA1MzAuNDMgNjA2LjY1IDUyMC4zIDU5NS40IDUxMS41OUM1ODQuMTUgNTAyLjg5IDU3MS44NCA0OTUuOTEgNTU4LjQ5IDQ5MC42NkM1NDUuMTMgNDg1LjQxIDUzMi40NSA0ODIuNDkgNTIwLjQ1IDQ4MS44OEM1MTkuODUgNDcwLjQ4IDUxOS4zMiA0NTkuMjIgNTE4Ljg3IDQ0OC4xMkM1MTguNDIgNDM3LjAyIDUxOC4xOSA0MjYuMjIgNTE4LjE5IDQxNS43MUM1MTguMTkgMzY0LjY5IDUyMi45MiAzMTcuMiA1MzIuMzcgMjczLjIzQzU0MS44MiAyMjkuMjcgNTU2LjUyIDE4Ni40MiA1NzYuNDkgMTQ0LjcxQzU5Ni40NSAxMDMgNjIyLjAzIDYxLjI4MDEgNjUzLjI0IDE5LjU2MDFDNjU3Ljg3IDEzLjM3MDEgNjYyLjY1IDcuMTIwMDggNjY3LjU1IDAuODMwMDc4SDU4Mi42MkM1NTkuNDkgMzIuMTMwMSA1MzkuMTMgNjQuODYwMSA1MjEuNTYgOTkuMDEwMUM0OTcuNCAxNDUuOTggNDc4Ljg2IDE5NS43MyA0NjUuOTYgMjQ4LjI0QzQ1My4wNSAzMDAuNzYgNDQ2LjYgMzU2LjEzIDQ0Ni42IDQxNC4zNUM0NDYuNiA0MzcuMTYgNDQ3LjUgNDYwLjI3IDQ0OS4zIDQ4My42OEM0MzguNzkgNDg1LjQ4IDQyNy43NiA0ODguODYgNDE2LjIxIDQ5My44MUM0MDQuNjUgNDk4Ljc2IDM5My4wMyA1MDQuMzkgMzgxLjMyIDUxMC42OUMzNjkuNjIgNTE2Ljk5IDM1OC4xNCA1MjMuNiAzNDYuODggNTMwLjVDMzM1LjYzIDUzNy40MSAzMjUuMDUgNTQzLjcxIDMxNS4xNCA1NDkuNDFDMzA1LjI0IDU1NS4xMSAyOTYuMTYgNTU5Ljg0IDI4Ny45MSA1NjMuNTlDMjc5LjY1IDU2Ny4zNSAyNzIuODMgNTY5LjIyIDI2Ny40MyA1NjkuMjJDMjU2LjMyIDU2OS4yMiAyNDguODIgNTY1LjM5IDI0NC45MiA1NTcuNzRDMjQxLjAyIDU1MC4wOSAyMzkuMDcgNTM5LjUxIDIzOS4wNyA1MjZDMjM5LjA3IDUwOS41IDI0MC40MiA0OTAuNzQgMjQzLjEyIDQ2OS43M0MyNDUuODIgNDQ4LjczIDI0OC44OSA0MjYuNjcgMjUyLjM1IDQwMy41NkMyNTUuOCAzODAuNDUgMjU4Ljg4IDM1Ni45NyAyNjEuNTggMzMzLjExQzI2NC4yOCAzMDkuMjUgMjY1LjYzIDI4Ni4yMiAyNjUuNjMgMjY0LjAxQzI2NS42MyAyNDkuNiAyNjQuMDUgMjM1LjEzIDI2MC45IDIyMC41N0MyNTcuNzUgMjA2LjAyIDI1Mi42NCAxOTIuODggMjQ1LjU5IDE4MS4xOEMyMzguNTQgMTY5LjQ4IDIyOS4zMSAxNTkuOTUgMjE3LjkgMTUyLjU5QzIwNi40OSAxNDUuMjQgMTkyLjM5IDE0MS41NiAxNzUuNTggMTQxLjU2QzE0Ni4xNiAxNDEuNTYgMTE4Ljg2IDE0OS4zNyA5My42NTAxIDE2NC45N0M2OC40NDAxIDE4MC41OCA0NS40ODAxIDIwMS4yMSAyNC43NzAxIDIyNi44N0MxNi40MjAxIDIzNy4yMSA4LjQ2MDEyIDI0OC4xMyAwLjg3MDExNyAyNTkuNjJWNDQ3LjY2QzIuNjIwMTIgNDQxLjUzIDQuNDIwMTEgNDM1LjQ0IDYuMzEwMTEgNDI5LjQ1QzE2LjIzMDEgMzk4LjA4IDI3LjcxMDEgMzY5LjEyIDQwLjc3MDEgMzQyLjU2Wk01NDYuOTggNTQ2LjI2QzU1My44OCA1NDcuNDYgNTYwLjMzIDU0OS40OSA1NjYuMzQgNTUyLjM0QzU3Mi4zNCA1NTUuMiA1NzcuMzcgNTU5LjAyIDU4MS40MiA1NjMuODJDNTg1LjQ3IDU2OC42MiA1ODcuNSA1NzQuNDggNTg3LjUgNTgxLjM4QzU4Ny41IDU4NC42OSA1ODYuNDUgNTg3LjE2IDU4NC4zNSA1ODguODFDNTgyLjI1IDU5MC40NiA1NzguMzQgNTkxLjI5IDU3Mi42NSA1OTEuMjlDNTY3Ljg1IDU5MS4yOSA1NjIuOTcgNTg5LjY0IDU1OC4wMiA1ODYuMzRDNTUzLjA3IDU4My4wNCA1NDguNDkgNTc5LjA3IDU0NC4yOSA1NzQuNDFDNTQwLjA4IDU2OS43NiA1MzYuNTYgNTY0LjczIDUzMy43MSA1NTkuMzNDNTMwLjg1IDU1My45MyA1MjkuMTMgNTQ4Ljk4IDUyOC41MyA1NDQuNDdDNTMzLjkzIDU0NC40NiA1NDAuMDggNTQ1LjA2IDU0Ni45OCA1NDYuMjZaIiBmaWxsPSIjNEFCQ0U3Ii8+CjxwYXRoIGQ9Ik0xMzk0LjE3IDAuNzkwMDM5SDcyNi4xOFY3NTEuMTRIMTM5NC4xN1YwLjc5MDAzOVoiIGZpbGw9IndoaXRlIi8+CjxwYXRoIGQ9Ik0xMTY3LjE1IDU2NC43MlYyNzMuMTNMMTIyOC40MyAyNzMuMTFWNTA3LjA2TDEzOTQuMTcgNTA3LjJWMC43OTAwMzlINzI2LjE4VjUwNy4yMUw4OTMuMTQgNTA3LjE2QzkyNy44MiA1MDcuMTYgOTM1LjM4IDQ5My4xNyA5MzUuNjQgNDgxLjQyQzkzNi4xNiA0NTcuMzcgOTIzLjA4IDQ0Ny43NiA4OTcuODggNDMxLjM5Qzg1MC41OSA0MDIuNTcgODQzLjUzIDM2OC45MyA4NDMuNTMgMzUxLjQ1Qzg0My41MyAyOTEuODMgODg4Ljk1IDI2NS4xNSA5MzMuOTQgMjY1LjE1Qzk1MS43NCAyNjUuMTUgOTgxLjM2IDI3Mi4xMSA5OTEuOTkgMjc2LjgyTDk4MS44IDMzMC44M0M5NzMuNyAzMjYuMzcgOTYyLjI5IDMyMS44MSA5NDcuMiAzMjEuODFDOTE5LjY2IDMyMS44MSA5MDkuODQgMzM1LjggOTA5Ljg0IDM0OC45MUM5MDkuODQgMzY5LjM0IDkxOS43NCAzNzYuMjcgOTUxLjAxIDM5NS4wOEM5ODUuNjEgNDE2LjA4IDEwMDMuMTUgNDQzLjczIDEwMDMuMTUgNDc3LjIzQzEwMDMuMTUgNTI4Ljc3IDk2NS4xNiA1NjMuNDcgOTA4LjY0IDU2My42MUw3MjYuMTcgNTYzLjY5Vjc1MS4xNEgxMzk0LjE2VjU2NC43MkgxMTY3LjE1Wk0xMTA3LjUxIDU2NC43MkgxMDUwLjAzVjI3My4zMUgxMTA3LjUxVjU2NC43MloiIGZpbGw9IiMyMTVFQUMiLz4KPHBhdGggZD0iTTEzNTAuNTQgNjg3LjhDMTM2MS44IDY4Ny44IDEzNzAuNjcgNjk2LjU1IDEzNzAuNjcgNzA3LjY4QzEzNzAuNjcgNzE5LjA2IDEzNjEuNzkgNzI3LjgxIDEzNTAuNDEgNzI3LjgxQzEzMzkuMDMgNzI3LjgxIDEzMzAuMDIgNzE5LjA2IDEzMzAuMDIgNzA3LjY4QzEzMzAuMDIgNjk2LjU1IDEzMzkuMDMgNjg3LjggMTM1MC40MSA2ODcuOEgxMzUwLjU0Wk0xMzUwLjI5IDY5MS4zQzEzNDEuMjkgNjkxLjMgMTMzNC40MSA2OTguNjggMTMzNC40MSA3MDcuNjhDMTMzNC40MSA3MTYuOTQgMTM0MS4yOSA3MjQuMTkgMTM1MC41NCA3MjQuMTlDMTM1OS41NCA3MjQuMzEgMTM2Ni4xNyA3MTYuOTQgMTM2Ni4xNyA3MDcuODFDMTM2Ni4xNyA2OTguNjggMTM1OS41NCA2OTEuMyAxMzUwLjQxIDY5MS4zSDEzNTAuMjlaTTEzNDYuNzkgNzE4LjY5SDEzNDIuOTJWNjk3LjhDMTM0NS4wNCA2OTcuNDIgMTM0Ny4zIDY5Ny4xNyAxMzUwLjE3IDY5Ny4xN0MxMzUzLjY3IDY5Ny4xNyAxMzU1LjQyIDY5Ny44IDEzNTYuOCA2OTguOEMxMzU4LjA1IDY5OS42OCAxMzU4LjkzIDcwMS4zIDEzNTguOTMgNzAzLjE4QzEzNTguOTMgNzA1LjgxIDEzNTcuMDUgNzA3LjQzIDEzNTQuOCA3MDguMThWNzA4LjQzQzEzNTYuNjggNzA5LjE4IDEzNTcuNjggNzEwLjgxIDEzNTguMyA3MTMuNDNDMTM1OC45MyA3MTYuNjggMTM1OS4zIDcxNy45MyAxMzU5LjggNzE4LjY4SDEzNTUuOEMxMzU1LjE3IDcxNy45MyAxMzU0LjY3IDcxNi4wNSAxMzU0LjE3IDcxMy4xOEMxMzUzLjY3IDcxMC45MyAxMzUyLjI5IDcwOS45MyAxMzQ5LjI5IDcwOS45M0gxMzQ2Ljc5VjcxOC42OVpNMTM0Ni43OSA3MDYuOTNIMTM0OS40MkMxMzUyLjI5IDcwNi45MyAxMzU0LjggNzA2LjA1IDEzNTQuOCA3MDMuNTVDMTM1NC44IDcwMS41NSAxMzUzLjU1IDcwMC4xNyAxMzQ5LjggNzAwLjE3QzEzNDguMyA3MDAuMTcgMTM0Ny40MiA3MDAuMjkgMTM0Ni44IDcwMC40MlY3MDYuOTNIMTM0Ni43OVoiIGZpbGw9IndoaXRlIi8+CjwvZz4KPGRlZnM+CjxjbGlwUGF0aCBpZD0iY2xpcDBfNDI3NF8xMzYiPgo8cmVjdCB3aWR0aD0iMTM5NSIgaGVpZ2h0PSI3NTIiIGZpbGw9IndoaXRlIi8+CjwvY2xpcFBhdGg+CjwvZGVmcz4KPC9zdmc+Cg=="; + +const providerLogoDataUris: Record = { + deepl: kDeepLLogoDataUri, + google: kGoogleLogoDataUri, + alpha2: kSilGlyphDataUri, +}; + +// Renders the brand mark for one translation provider, titled with its display name so hovering +// (and screen readers, via the alt/title) still reveals which service it is. +export const AiTranslationProviderLogo: React.FunctionComponent<{ + providerId: AiTranslationProviderId; + title: string; +}> = (props) => ( + {props.title} +); + +// The SIL "SIL" wordmark (with a Tai Heritage script sample), shown before the Alpha2 enable label. +export const SilAlpha2WordmarkLogo: React.FunctionComponent = () => ( + SIL +); diff --git a/src/BloomBrowserUI/collection/AiTranslationSettingsGroup.tsx b/src/BloomBrowserUI/collection/AiTranslationSettingsGroup.tsx new file mode 100644 index 000000000000..d1d09f416592 --- /dev/null +++ b/src/BloomBrowserUI/collection/AiTranslationSettingsGroup.tsx @@ -0,0 +1,1439 @@ +import { css } from "@emotion/react"; +import * as React from "react"; +import { + ConfigrBoolean, + ConfigrCustomObjectInput, + ConfigrGroup, + ConfigrInput, +} from "@sillsdev/config-r"; +import { MenuItem, TextField } from "@mui/material"; +import InfoOutlinedIcon from "@mui/icons-material/InfoOutlined"; +import { postJsonAsync } from "../utils/bloomApi"; +import { + AiTranslationProviderLogo, + SilAlpha2WordmarkLogo, +} from "./AiTranslationProviderLogos"; + +// The order in which provider logos appear as columns in the target-language dropdown, and the +// order the engine sections appear in the settings group. Alpha2 leads (it is the SIL-hosted +// service this feature is built around), followed by the third-party engines. +const kProviderDisplayOrder: AiTranslationProviderId[] = [ + "alpha2", + "deepl", + "google", +]; + +export type AiTranslationProviderId = "deepl" | "google" | "alpha2"; + +export interface ITargetLanguageOption { + value: string; + label: string; + providerIds: AiTranslationProviderId[]; +} + +export interface IAiTranslationEngineValidation { + succeeded: boolean; + // True when validation failed only because the provider doesn't support the chosen target + // language; the engine is auto-skipped rather than treated as broken. + targetLanguageNotSupported: boolean; + message: string; + upToDate: boolean; +} + +// The flat shape our React/Configr state uses. One boolean+credential set per engine, +// plus the target language shared by all engines. This is translated to/from the nested +// wire contract (IAiTranslationWireSettings) at the AdvancedSettingsPanel API boundary. +export interface IAiTranslationSettings { + aiTranslationTargetLanguageTag?: string; + aiTranslationDeepLEnabled?: boolean; + aiTranslationDeepLApiKey?: string; + aiTranslationGoogleEnabled?: boolean; + aiTranslationGoogleServiceAccountEmail?: string; + aiTranslationGooglePrivateKey?: string; + aiTranslationAlpha2Enabled?: boolean; + aiTranslationAlpha2ApiKey?: string; + aiTranslationAlpha2SourceLanguageTag?: string; +} + +// Every key of IAiTranslationSettings, used to split it out of / merge it into the +// larger IAdvancedSettings object that AdvancedSettingsPanel round-trips with the API. +export const aiTranslationFlatSettingsKeys = [ + "aiTranslationTargetLanguageTag", + "aiTranslationDeepLEnabled", + "aiTranslationDeepLApiKey", + "aiTranslationGoogleEnabled", + "aiTranslationGoogleServiceAccountEmail", + "aiTranslationGooglePrivateKey", + "aiTranslationAlpha2Enabled", + "aiTranslationAlpha2ApiKey", + "aiTranslationAlpha2SourceLanguageTag", +] as const; + +export interface IAiTranslationWireEngineSettings { + providerId: AiTranslationProviderId; + enabled: boolean; + apiKey: string; + serviceAccountEmail: string; + privateKey: string; + sourceLanguageTag: string; + validation: IAiTranslationEngineValidation; +} + +// The exact shape posted/received under the "aiTranslation" key of the advanced-settings payload. +export interface IAiTranslationWireSettings { + targetLanguageTag: string; + engines: IAiTranslationWireEngineSettings[]; +} + +interface IEngineFieldSpec { + providerId: AiTranslationProviderId; + enabledPath: keyof IAiTranslationSettings; + credentialPaths: Array; + // Non-credential settings (e.g. alpha2's source language) that still affect validation and the + // supported-language lists, so they belong in the probe/language-config keys but NOT in + // credentialPaths (changing them must not be treated as a credential change that would, say, + // leak between engines). + configPaths?: Array; +} + +const deepLFieldSpec: IEngineFieldSpec = { + providerId: "deepl", + enabledPath: "aiTranslationDeepLEnabled", + credentialPaths: ["aiTranslationDeepLApiKey"], +}; + +const googleFieldSpec: IEngineFieldSpec = { + providerId: "google", + enabledPath: "aiTranslationGoogleEnabled", + credentialPaths: [ + "aiTranslationGoogleServiceAccountEmail", + "aiTranslationGooglePrivateKey", + ], +}; + +const alpha2FieldSpec: IEngineFieldSpec = { + providerId: "alpha2", + enabledPath: "aiTranslationAlpha2Enabled", + credentialPaths: ["aiTranslationAlpha2ApiKey"], + configPaths: ["aiTranslationAlpha2SourceLanguageTag"], +}; + +// Always exactly these three engines, in this order, matching the pinned backend contract. +const aiTranslationEngineSpecs: IEngineFieldSpec[] = [ + deepLFieldSpec, + googleFieldSpec, + alpha2FieldSpec, +]; + +interface IAiTranslationSupportedLanguagesResponse { + languages?: unknown[]; + message?: string; +} + +function readField( + candidate: Record, + camelCaseName: string, + pascalCaseName: string, +): T | undefined { + return (candidate[camelCaseName] ?? candidate[pascalCaseName]) as + | T + | undefined; +} + +// Parses the response of settings/aiTranslationSupportedLanguages. Tolerant of PascalCase +// field names since we don't control the exact casing the backend's JSON serializer produces. +export function parseSupportedTargetLanguageOptions( + data?: IAiTranslationSupportedLanguagesResponse, +): ITargetLanguageOption[] { + const rawLanguages = data?.languages; + if (!Array.isArray(rawLanguages)) { + return []; + } + + return rawLanguages + .map((language) => { + const candidate = language as Record; + const value = readField(candidate, "tag", "Tag") ?? ""; + const rawLabel = + readField(candidate, "name", "Name") ?? value; + const providerIds = + readField( + candidate, + "providerIds", + "ProviderIds", + ) ?? []; + const labelSuffix = ` (${value})`; + const label = rawLabel.endsWith(labelSuffix) + ? rawLabel.substring(0, rawLabel.length - labelSuffix.length) + : rawLabel; + + if (!value) { + return undefined; + } + + return { value, label, providerIds }; + }) + .filter((language): language is ITargetLanguageOption => !!language); +} + +function parseAiTranslationEngineValidation( + data: unknown, +): IAiTranslationEngineValidation | undefined { + if (!data || typeof data !== "object") { + return undefined; + } + + const candidate = data as Record; + return { + succeeded: + readField(candidate, "succeeded", "Succeeded") ?? false, + targetLanguageNotSupported: + readField( + candidate, + "targetLanguageNotSupported", + "TargetLanguageNotSupported", + ) ?? false, + message: readField(candidate, "message", "Message") ?? "", + // A freshly-run validation is by definition current for the settings that produced it. + upToDate: true, + }; +} + +// Splits the nested wire settings for one engine's "validation" sub-object into our +// flat, tolerant-of-casing shape. +function parseWireEngineValidation( + data: unknown, +): IAiTranslationEngineValidation | undefined { + if (!data || typeof data !== "object") { + return undefined; + } + const candidate = data as Record; + return { + succeeded: + readField(candidate, "succeeded", "Succeeded") ?? false, + targetLanguageNotSupported: + readField( + candidate, + "targetLanguageNotSupported", + "TargetLanguageNotSupported", + ) ?? false, + message: readField(candidate, "message", "Message") ?? "", + upToDate: + readField(candidate, "upToDate", "UpToDate") ?? false, + }; +} + +// Converts the nested wire contract (as received from settings/advancedProgramSettings) +// into our flat Configr-friendly settings plus a per-provider map of initial validation state. +export function flattenAiTranslationWireSettings( + wire: IAiTranslationWireSettings | undefined, +): { + flatSettings: IAiTranslationSettings; + initialValidations: Partial< + Record + >; +} { + const wireCandidate = (wire ?? {}) as unknown as Record; + const targetLanguageTag = + readField( + wireCandidate, + "targetLanguageTag", + "TargetLanguageTag", + ) ?? ""; + const rawEngines = + readField(wireCandidate, "engines", "Engines") ?? []; + + const flatSettings: IAiTranslationSettings = { + aiTranslationTargetLanguageTag: targetLanguageTag, + }; + const initialValidations: Partial< + Record + > = {}; + + aiTranslationEngineSpecs.forEach((spec) => { + const rawEngine = rawEngines + .map((engine) => engine as Record) + .find( + (engine) => + readField(engine, "providerId", "ProviderId") === + spec.providerId, + ); + + const enabled = + (rawEngine && + readField(rawEngine, "enabled", "Enabled")) ?? + false; + flatSettings[spec.enabledPath] = enabled as never; + + if (spec.providerId === "google") { + flatSettings.aiTranslationGoogleServiceAccountEmail = + (rawEngine && + readField( + rawEngine, + "serviceAccountEmail", + "ServiceAccountEmail", + )) ?? + ""; + flatSettings.aiTranslationGooglePrivateKey = + (rawEngine && + readField(rawEngine, "privateKey", "PrivateKey")) ?? + ""; + } else { + const apiKey = + (rawEngine && + readField(rawEngine, "apiKey", "ApiKey")) ?? + ""; + flatSettings[spec.credentialPaths[0]] = apiKey as never; + } + + if (spec.providerId === "alpha2") { + flatSettings.aiTranslationAlpha2SourceLanguageTag = + (rawEngine && + readField( + rawEngine, + "sourceLanguageTag", + "SourceLanguageTag", + )) ?? + ""; + } + + const validation = parseWireEngineValidation( + rawEngine && readField(rawEngine, "validation", "Validation"), + ); + if (validation) { + initialValidations[spec.providerId] = validation; + } + }); + + return { flatSettings, initialValidations }; +} + +// Converts our flat Configr-friendly settings back into the nested wire contract for posting +// to settings/advancedProgramSettings. The validation sub-object is server-computed and ignored +// on store, so we send it back empty. +export function buildAiTranslationWirePayload( + flat: IAiTranslationSettings, +): IAiTranslationWireSettings { + return { + targetLanguageTag: flat.aiTranslationTargetLanguageTag ?? "", + engines: aiTranslationEngineSpecs.map((spec) => ({ + providerId: spec.providerId, + enabled: !!flat[spec.enabledPath], + apiKey: + spec.providerId === "google" + ? "" + : ((flat[spec.credentialPaths[0]] as string) ?? ""), + // Google's credentials belong only on the google engine record. Sending them on the + // deepl/alpha2 records too made the backend see those engines' credentials "change" + // whenever Google's were edited, wiping their validation (and silently dropping them + // from the active engines) -- and stored Google's private key in all three records. + serviceAccountEmail: + spec.providerId === "google" + ? (flat.aiTranslationGoogleServiceAccountEmail ?? "") + : "", + privateKey: + spec.providerId === "google" + ? (flat.aiTranslationGooglePrivateKey ?? "") + : "", + // Only alpha2 uses a fixed source language; the other engines send it empty. + sourceLanguageTag: + spec.providerId === "alpha2" + ? (flat.aiTranslationAlpha2SourceLanguageTag ?? "") + : "", + validation: { + succeeded: false, + targetLanguageNotSupported: false, + message: "", + upToDate: false, + }, + })), + }; +} + +// Pulls the flat AI-translation keys out of a larger settings object (e.g. IAdvancedSettings). +export function extractAiTranslationFlatSettings( + source: Record, +): IAiTranslationSettings { + const flat: Record = {}; + aiTranslationFlatSettingsKeys.forEach((key) => { + flat[key] = source[key]; + }); + return flat as IAiTranslationSettings; +} + +// The inverse of extractAiTranslationFlatSettings: returns a shallow copy of source with the +// flat AI-translation keys removed, ready to have a nested "aiTranslation" key added back in. +export function omitAiTranslationFlatSettings< + T extends Record, +>(source: T): T { + const clone: Record = { ...source }; + aiTranslationFlatSettingsKeys.forEach((key) => { + delete clone[key]; + }); + return clone as T; +} + +function isEngineEnabled( + settings: IAiTranslationSettings | undefined, + spec: IEngineFieldSpec, +): boolean { + return !!settings?.[spec.enabledPath]; +} + +function hasEngineCredentials( + settings: IAiTranslationSettings | undefined, + spec: IEngineFieldSpec, +): boolean { + return spec.credentialPaths.every( + (path) => !!(settings?.[path] as string | undefined)?.trim(), + ); +} + +function isEngineReady( + settings: IAiTranslationSettings | undefined, + spec: IEngineFieldSpec, +): boolean { + return ( + isEngineEnabled(settings, spec) && hasEngineCredentials(settings, spec) + ); +} + +function getReadyProviderIds( + settings: IAiTranslationSettings | undefined, +): AiTranslationProviderId[] { + return aiTranslationEngineSpecs + .filter((spec) => isEngineReady(settings, spec)) + .map((spec) => spec.providerId); +} + +function getEngineProbeKey( + settings: IAiTranslationSettings | undefined, + spec: IEngineFieldSpec, +): string { + return JSON.stringify({ + enabled: isEngineEnabled(settings, spec), + targetLanguageTag: settings?.aiTranslationTargetLanguageTag ?? "", + credentials: spec.credentialPaths.map((path) => settings?.[path] ?? ""), + // e.g. alpha2's source language: a change here should re-run validation (the pair changed). + config: (spec.configPaths ?? []).map((path) => settings?.[path] ?? ""), + }); +} + +function getSupportedLanguagesConfigKey( + settings: IAiTranslationSettings | undefined, +): string { + return JSON.stringify( + aiTranslationEngineSpecs.map((spec) => ({ + providerId: spec.providerId, + ready: isEngineReady(settings, spec), + credentials: spec.credentialPaths.map( + (path) => settings?.[path] ?? "", + ), + // e.g. alpha2's source language changes which target languages it can offer. + config: (spec.configPaths ?? []).map( + (path) => settings?.[path] ?? "", + ), + })), + ); +} + +// Debounces live validation for a single engine: whenever its enabled state, credentials, or the +// shared target language change (and it has everything it needs), waits ~600ms and then posts +// settings/validateAiTranslationEngine for just that engine. +function useAiTranslationEngineValidation( + spec: IEngineFieldSpec, + settings: IAiTranslationSettings | undefined, + initialValidation: IAiTranslationEngineValidation | undefined, +): { + validation: IAiTranslationEngineValidation | undefined; + isPending: boolean; +} { + const [validation, setValidation] = React.useState< + IAiTranslationEngineValidation | undefined + >(initialValidation?.upToDate ? initialValidation : undefined); + const [isPending, setIsPending] = React.useState(false); + const lastProbeKeyRef = React.useRef(""); + const latestSettingsRef = React.useRef(settings); + latestSettingsRef.current = settings; + + // Effect justified: synchronizes this engine's displayed validation with the initialValidation + // prop, which arrives asynchronously from the server (an external source) after the settings + // load. When an up-to-date validation loads we adopt it and seed the "last probed" key so we + // don't immediately re-probe; otherwise we clear it. That is external-state synchronization, + // which is what an Effect is for. + React.useEffect(() => { + setValidation( + initialValidation?.upToDate ? initialValidation : undefined, + ); + const loadedProbeKey = getEngineProbeKey( + latestSettingsRef.current, + spec, + ); + if (initialValidation?.upToDate && initialValidation.message) { + lastProbeKeyRef.current = loadedProbeKey; + } else { + lastProbeKeyRef.current = ""; + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [initialValidation]); + + // Effect justified: debounced live validation against the external translation service. When + // this engine's enabled state, credentials, or the target language change, we wait ~600ms of + // quiet and then probe the provider over the network and reflect the result, cancelling the + // pending probe (timer) on further change or unmount. Talking to that external service with + // proper cleanup is a legitimate use of an Effect. + React.useEffect(() => { + if (!isEngineEnabled(settings, spec)) { + setIsPending(false); + return; + } + + const probeKey = getEngineProbeKey(settings, spec); + if (probeKey === lastProbeKeyRef.current) { + return; + } + + lastProbeKeyRef.current = probeKey; + + if ( + !hasEngineCredentials(settings, spec) || + !settings?.aiTranslationTargetLanguageTag?.trim() + ) { + setIsPending(false); + setValidation(undefined); + return; + } + + setIsPending(true); + setValidation(undefined); + + let cancelled = false; + const timeoutId = window.setTimeout(() => { + void (async () => { + try { + const response = await postJsonAsync( + "settings/validateAiTranslationEngine", + { providerId: spec.providerId }, + ); + if (cancelled) { + return; + } + setValidation( + parseAiTranslationEngineValidation(response?.data), + ); + } finally { + if (!cancelled) { + setIsPending(false); + } + } + })(); + }, 600); + + return () => { + cancelled = true; + window.clearTimeout(timeoutId); + }; + }, [settings, spec]); + + return { validation, isPending }; +} + +// Fills placeholders {0}, {1}, ... in a localized template with the given values. +function formatTemplate(template: string, ...values: string[]): string { + return values.reduce( + (text, value, index) => text.split(`{${index}}`).join(value), + template, + ); +} + +function getEngineValidationDisplay( + isPending: boolean, + validation: IAiTranslationEngineValidation | undefined, + engineDisplayName: string, + targetLanguageLabel: string, + targetLanguageNotSupportedTemplate: string, +): { text: string; color: string } { + if (isPending) { + return { text: "Testing translation...", color: "#555" }; + } + + // The engine works, it just can't do this language, so it will be auto-skipped: an amber note, + // not a red failure. + if (validation?.targetLanguageNotSupported) { + return { + text: formatTemplate( + targetLanguageNotSupportedTemplate, + engineDisplayName, + targetLanguageLabel, + ), + color: "#b26a00", + }; + } + + if (!validation?.message) { + return { text: "", color: "#555" }; + } + + return { + text: validation.succeeded + ? `"Today a reader, tomorrow a leader." --> ${validation.message}` + : `Translation test failed: ${validation.message}`, + color: validation.succeeded ? "#2e7d32" : "#b3261e", + }; +} + +// Shared by all three engines: reads its text/color/testId from overrideValue rather than the +// usual value/onChange, since this row is purely a read-only status display. +const EngineValidationStatusControl: React.FunctionComponent<{ + value: { text: string; color: string; testId: string }; +}> = (controlProps) => { + const display = controlProps.value; + return ( +
    + {display.text} +
    + ); +}; + +// The live data the module-scope language controls (target-language and alpha2 source-language) +// need, supplied via context (below) so the controls can live at module scope. Everything here is +// derived from the hook's state and refreshed each render; the controls read it through the context. +export interface IAiTranslationLanguageControlsData { + usesEngineManagedTargetLanguages: boolean; + supportedTargetLanguages: ITargetLanguageOption[]; + supportedLanguagesMessage: string; + isLoadingSupportedLanguages: boolean; + languageOptionsVersion: number; + loadSupportedLanguages: () => Promise; + readyProviderIds: AiTranslationProviderId[]; + engineDisplayNames: Record; + noServiceSupportsLanguageNote: string; + // Shown (disabled state) when no provider is ready yet, in place of a language list. + noProviderSelectedNote: string; + // Explains that the Alpha2 language list is those it can translate from its source language; + // empty unless Alpha2 is a ready provider. + alpha2LanguagesNote: string; + // Alpha2 source-language chooser data: + alpha2SourceLanguages: ITargetLanguageOption[]; + alpha2SourceLanguagesMessage: string; + isLoadingAlpha2SourceLanguages: boolean; + alpha2SourceLanguageOptionsVersion: number; + loadAlpha2SourceLanguages: () => Promise; +} + +// Provided by AdvancedSettingsPanel (wrapping the Configr pane) so the module-scope language +// controls can reach the hook's live data without being redefined on every render. +export const AiTranslationLanguageControlsContext = React.createContext< + IAiTranslationLanguageControlsData | undefined +>(undefined); + +// The target-language chooser passed to ConfigrCustomObjectInput's `control` prop. It MUST live at +// module scope (not inside useAiTranslationSettingsGroup): a component defined inside the hook gets +// a new identity on every render, which makes React unmount/remount it -- closing the dropdown and +// losing focus whenever any other settings field changes. It gets its selection via the usual +// value/onChange, and everything else (options, load callback, etc.) from context. +// The target-language picker is kept narrow (250px), so long language names are clipped to a +// fixed number of characters with an ellipsis; the full name and code are shown in a tooltip. +const kLanguagePickerWidthPx = 250; +const kMaxLanguageLabelChars = 16; + +function clipLanguageLabel(label: string): string { + return label.length > kMaxLanguageLabelChars + ? label.slice(0, kMaxLanguageLabelChars).trimEnd() + "…" + : label; +} + +// The tooltip shown on a language option / the chosen value: full name plus its language code. +function getLanguageTooltip(option: ITargetLanguageOption): string { + return `${option.label} (${option.value})`; +} + +const AiTranslationTargetLanguageControl: React.FunctionComponent<{ + value: string; + disabled?: boolean; + onChange: (value: string) => void; +}> = (controlProps) => { + const data = React.useContext(AiTranslationLanguageControlsContext)!; + + if (!data.usesEngineManagedTargetLanguages) { + // No provider is ready yet, so we have no language list to offer. Rather than let the user + // type a free-form tag (which no enabled service could act on), disable the field and + // explain what to do to populate it. + return ( +
    + +
    + {data.noProviderSelectedNote} +
    +
    + ); + } + + // Which ready providers actually contribute a language to the current list. We reserve a logo + // column only for these, so the columns take exactly the width the selected-and-relevant + // providers need -- not a slot for every provider that merely could exist. + const providersWithSupportedLanguages = new Set(); + data.supportedTargetLanguages.forEach((option) => + option.providerIds.forEach((id) => + providersWithSupportedLanguages.add(id), + ), + ); + + // The relevant ready providers, in a stable order, become the logo columns each language row + // lines up against (a per-provider slot, filled when that provider supports the language). Only + // shown when 2+ such providers exist -- with one there is nothing to distinguish. + const logoColumnProviderIds = kProviderDisplayOrder.filter( + (id) => + data.readyProviderIds.includes(id) && + providersWithSupportedLanguages.has(id), + ); + const showLogoColumns = logoColumnProviderIds.length >= 2; + + const currentValue = controlProps.value || ""; + const knownOptions = data.supportedTargetLanguages.some( + (option) => option.value === currentValue, + ) + ? data.supportedTargetLanguages + : currentValue + ? [ + ...data.supportedTargetLanguages, + { + value: currentValue, + label: currentValue, + providerIds: [], + }, + ] + : data.supportedTargetLanguages; + + return ( + // Constrain the width so a long error message wraps within the dialog instead of + // forcing it to scroll horizontally (the message is rendered below, not as the + // TextField's helperText, so we can style it as a red, wrapping error). +
    + { + controlProps.onChange(event.target.value); + }} + SelectProps={{ + onOpen: () => { + void data.loadSupportedLanguages(); + }, + }} + inputProps={{ + "data-testid": "ai-translation-target-language-select", + "data-language-options-version": + data.languageOptionsVersion, + }} + > + + {data.isLoadingSupportedLanguages && ( + + Loading languages... + + )} + {knownOptions.map((option) => { + // The synthetic current-value option (no supporting providers) is the only one + // that shows the "no enabled service supports this language" text note. + const noServiceNote = + option.providerIds.length === 0 + ? data.noServiceSupportsLanguageNote + : ""; + return ( + + + {clipLanguageLabel(option.label)} + + {showLogoColumns && ( + + {/* One fixed-width slot per ready provider so the logos line + up in columns down the list; the slot is filled only when + this language is supported by that provider. */} + {logoColumnProviderIds.map((id) => ( + + {option.providerIds.includes( + id, + ) && ( + + )} + + ))} + + )} + {noServiceNote && ( + + ({noServiceNote}) + + )} + + ); + })} + + {data.supportedLanguagesMessage && ( +
    + {data.supportedLanguagesMessage} +
    + )} +
    + ); +}; + +// The alpha2 source-language chooser passed to ConfigrCustomObjectInput's `control` prop. Like +// AiTranslationTargetLanguageControl, it MUST live at module scope so React doesn't unmount/remount +// it on every render (which would close the dropdown and lose focus). It reads its options and load +// callback from the shared context and its selection via the usual value/onChange. When the endpoint +// returns no options (no target chosen yet, or alpha2 not configured) it falls back to a free-text +// field, defaulting to "en". +const AiTranslationAlpha2SourceLanguageControl: React.FunctionComponent<{ + value: string; + disabled?: boolean; + onChange: (value: string) => void; +}> = (controlProps) => { + const data = React.useContext(AiTranslationLanguageControlsContext)!; + const currentValue = controlProps.value || "en"; + + if (data.alpha2SourceLanguages.length === 0) { + return ( + { + controlProps.onChange(event.target.value); + }} + inputProps={{ + "data-testid": + "ai-translation-alpha2-source-language-input", + }} + /> + ); + } + + const knownOptions = data.alpha2SourceLanguages.some( + (option) => option.value === currentValue, + ) + ? data.alpha2SourceLanguages + : [ + ...data.alpha2SourceLanguages, + { value: currentValue, label: currentValue, providerIds: [] }, + ]; + + return ( +
    + { + controlProps.onChange(event.target.value); + }} + SelectProps={{ + onOpen: () => { + void data.loadAlpha2SourceLanguages(); + }, + }} + inputProps={{ + "data-testid": + "ai-translation-alpha2-source-language-select", + "data-language-options-version": + data.alpha2SourceLanguageOptionsVersion, + }} + > + {data.isLoadingAlpha2SourceLanguages && ( + + Loading languages... + + )} + {knownOptions.map((option) => ( + + {option.label} + + ))} + + {data.alpha2SourceLanguagesMessage && ( +
    + {data.alpha2SourceLanguagesMessage} +
    + )} + {/* Alpha2's available target languages depend on this source language, so the + explanation lives right under the source picker. */} + {data.alpha2LanguagesNote && ( +
    + + {data.alpha2LanguagesNote} +
    + )} +
    + ); +}; + +export const useAiTranslationSettingsGroup = (props: { + settings: IAiTranslationSettings | undefined; + initialValidations?: Partial< + Record + >; + groupLabel: string; + targetLanguageLabel: string; + deepLEnabledLabel: string; + deepLApiKeyLabel: string; + deepLApiKeyDescription: string; + googleEnabledLabel: string; + googleServiceAccountEmailLabel: string; + googleServiceAccountDescription: string; + googlePrivateKeyLabel: string; + alpha2EnabledLabel: string; + alpha2ApiKeyLabel: string; + alpha2SourceLanguageLabel: string; + translationTestLabel: string; + targetLanguageNotSupportedTemplate: string; + noServiceSupportsLanguageNote: string; + noProviderSelectedNote: string; + // Template with a single {0} placeholder for the Alpha2 source-language name. + alpha2LanguagesNoteTemplate: string; +}): { + group: React.ReactElement; + languageControlsData: IAiTranslationLanguageControlsData; +} => { + const deepLValidation = useAiTranslationEngineValidation( + deepLFieldSpec, + props.settings, + props.initialValidations?.deepl, + ); + const googleValidation = useAiTranslationEngineValidation( + googleFieldSpec, + props.settings, + props.initialValidations?.google, + ); + // The live "Translation Test" is intentionally NOT run for Alpha2 (its display row is hidden + // below). Unlike DeepL/Google, an Alpha2 validation isn't a cheap single call: it has to spin + // up a whole translation system (create a text collection, kick off and poll a translation + // job, then tear it down), which is far too expensive to fire on every credential/target + // change. So we skip the alpha2 validation hook entirely for now. + + const [supportedTargetLanguages, setSupportedTargetLanguages] = + React.useState([]); + const [supportedLanguagesMessage, setSupportedLanguagesMessage] = + React.useState(""); + const [isLoadingSupportedLanguages, setIsLoadingSupportedLanguages] = + React.useState(false); + const [languageOptionsVersion, setLanguageOptionsVersion] = + React.useState(0); + const lastSupportedLanguagesConfigKeyRef = React.useRef(""); + + const loadSupportedLanguages = React.useCallback(async () => { + const languageConfigKey = getSupportedLanguagesConfigKey( + props.settings, + ); + if (getReadyProviderIds(props.settings).length === 0) { + setSupportedTargetLanguages([]); + setSupportedLanguagesMessage(""); + lastSupportedLanguagesConfigKeyRef.current = languageConfigKey; + return; + } + + if ( + languageConfigKey === lastSupportedLanguagesConfigKeyRef.current && + supportedTargetLanguages.length > 0 + ) { + return; + } + + setIsLoadingSupportedLanguages(true); + setSupportedLanguagesMessage(""); + try { + const response = await postJsonAsync( + "settings/aiTranslationSupportedLanguages", + props.settings, + ); + const data = response?.data as + | IAiTranslationSupportedLanguagesResponse + | undefined; + const languages = parseSupportedTargetLanguageOptions(data); + setSupportedTargetLanguages(languages); + setSupportedLanguagesMessage(data?.message ?? ""); + lastSupportedLanguagesConfigKeyRef.current = languageConfigKey; + setLanguageOptionsVersion((value) => value + 1); + } finally { + setIsLoadingSupportedLanguages(false); + } + }, [props.settings, supportedTargetLanguages.length]); + + // Effect justified: keeps the cached supported-languages list in sync with the engine + // configuration. The list is fetched from the providers (external services) and keyed to a + // specific credential/target configuration; when that configuration changes, the cached list + // and message no longer apply, so we clear them (the refetch is triggered by the effect below). + // This synchronizes cached external data with its inputs, which warrants an Effect. + React.useEffect(() => { + const currentLanguageConfigKey = getSupportedLanguagesConfigKey( + props.settings, + ); + if ( + currentLanguageConfigKey !== + lastSupportedLanguagesConfigKeyRef.current + ) { + setSupportedTargetLanguages([]); + setSupportedLanguagesMessage(""); + } + }, [props.settings]); + + // Provider-backed target-language lists depend on external credentials and should be ready + // as soon as any engine's config becomes usable. + React.useEffect(() => { + if (getReadyProviderIds(props.settings).length === 0) { + return; + } + + void loadSupportedLanguages(); + }, [loadSupportedLanguages, props.settings]); + + // --- Alpha2 source-language list (depends on the chosen target language) --- + const [alpha2SourceLanguages, setAlpha2SourceLanguages] = React.useState< + ITargetLanguageOption[] + >([]); + const [alpha2SourceLanguagesMessage, setAlpha2SourceLanguagesMessage] = + React.useState(""); + const [isLoadingAlpha2SourceLanguages, setIsLoadingAlpha2SourceLanguages] = + React.useState(false); + const [ + alpha2SourceLanguageOptionsVersion, + setAlpha2SourceLanguageOptionsVersion, + ] = React.useState(0); + const lastAlpha2SourceLanguagesConfigKeyRef = React.useRef(""); + + // The alpha2 source list depends on the chosen target and alpha2's own credentials. This key + // captures those inputs so we refetch only when they change. + const alpha2SourceLanguagesConfigKey = JSON.stringify({ + enabled: isEngineEnabled(props.settings, alpha2FieldSpec), + targetLanguageTag: props.settings?.aiTranslationTargetLanguageTag ?? "", + apiKey: props.settings?.aiTranslationAlpha2ApiKey ?? "", + }); + + const loadAlpha2SourceLanguages = React.useCallback(async () => { + const configKey = JSON.stringify({ + enabled: isEngineEnabled(props.settings, alpha2FieldSpec), + targetLanguageTag: + props.settings?.aiTranslationTargetLanguageTag ?? "", + apiKey: props.settings?.aiTranslationAlpha2ApiKey ?? "", + }); + if ( + !isEngineEnabled(props.settings, alpha2FieldSpec) || + !props.settings?.aiTranslationAlpha2ApiKey?.trim() || + !props.settings?.aiTranslationTargetLanguageTag?.trim() + ) { + setAlpha2SourceLanguages([]); + setAlpha2SourceLanguagesMessage(""); + lastAlpha2SourceLanguagesConfigKeyRef.current = configKey; + return; + } + + if ( + configKey === lastAlpha2SourceLanguagesConfigKeyRef.current && + alpha2SourceLanguages.length > 0 + ) { + return; + } + + setIsLoadingAlpha2SourceLanguages(true); + setAlpha2SourceLanguagesMessage(""); + try { + const response = await postJsonAsync( + "settings/aiTranslationAlpha2SourceLanguages", + props.settings, + ); + const data = response?.data as + | IAiTranslationSupportedLanguagesResponse + | undefined; + const languages = parseSupportedTargetLanguageOptions(data); + setAlpha2SourceLanguages(languages); + setAlpha2SourceLanguagesMessage(data?.message ?? ""); + lastAlpha2SourceLanguagesConfigKeyRef.current = configKey; + setAlpha2SourceLanguageOptionsVersion((value) => value + 1); + } finally { + setIsLoadingAlpha2SourceLanguages(false); + } + }, [props.settings, alpha2SourceLanguages.length]); + + // Effect justified: keeps the cached alpha2 source-language list in sync with its inputs (the + // target language and alpha2's credentials), which come from an external service. When those + // change the cached list no longer applies, so clear it; the refetch is triggered below. This + // is external-data synchronization, which warrants an Effect. + React.useEffect(() => { + if ( + alpha2SourceLanguagesConfigKey !== + lastAlpha2SourceLanguagesConfigKeyRef.current + ) { + setAlpha2SourceLanguages([]); + setAlpha2SourceLanguagesMessage(""); + } + }, [alpha2SourceLanguagesConfigKey]); + + // Fetch the alpha2 source list as soon as alpha2 is configured and a target is chosen. + React.useEffect(() => { + if ( + !isEngineEnabled(props.settings, alpha2FieldSpec) || + !props.settings?.aiTranslationAlpha2ApiKey?.trim() || + !props.settings?.aiTranslationTargetLanguageTag?.trim() + ) { + return; + } + void loadAlpha2SourceLanguages(); + }, [loadAlpha2SourceLanguages, props.settings]); + + const readyProviderIds = getReadyProviderIds(props.settings); + const usesEngineManagedTargetLanguages = readyProviderIds.length > 0; + + const engineDisplayNames: Record = { + deepl: props.deepLEnabledLabel, + google: props.googleEnabledLabel, + alpha2: props.alpha2EnabledLabel, + }; + + // The human-readable label of the currently-chosen target language, used in the per-engine + // "does not support ⟨language⟩" note; falls back to the tag if we have no matching option. + const currentTargetTag = + props.settings?.aiTranslationTargetLanguageTag ?? ""; + const currentTargetLabel = + supportedTargetLanguages.find( + (option) => option.value === currentTargetTag, + )?.label ?? currentTargetTag; + + // Alpha2's target languages are those it can translate FROM its configured source language + // (default English), so when Alpha2 is a ready provider we explain that against the list. Use + // the source language's human label when we have it, else its tag. + const alpha2SourceTag = + props.settings?.aiTranslationAlpha2SourceLanguageTag || "en"; + const alpha2SourceLabel = + alpha2SourceLanguages.find((option) => option.value === alpha2SourceTag) + ?.label ?? alpha2SourceTag; + const alpha2LanguagesNote = readyProviderIds.includes("alpha2") + ? formatTemplate(props.alpha2LanguagesNoteTemplate, alpha2SourceLabel) + : ""; + + // Bundle the controls' live inputs for the context. The controls themselves are stable + // module-scope components; this data is what changes over time. + const languageControlsData: IAiTranslationLanguageControlsData = { + usesEngineManagedTargetLanguages, + supportedTargetLanguages, + supportedLanguagesMessage, + isLoadingSupportedLanguages, + languageOptionsVersion, + loadSupportedLanguages, + readyProviderIds, + engineDisplayNames, + noServiceSupportsLanguageNote: props.noServiceSupportsLanguageNote, + noProviderSelectedNote: props.noProviderSelectedNote, + alpha2LanguagesNote, + alpha2SourceLanguages, + alpha2SourceLanguagesMessage, + isLoadingAlpha2SourceLanguages, + alpha2SourceLanguageOptionsVersion, + loadAlpha2SourceLanguages, + }; + + // Each engine's enable toggle is branded with its logo before the label. ConfigrBoolean types + // `label` as string, but config-r renders it straight into MUI's ListItemText `primary`, which + // accepts any node -- so the cast is safe here and lets us prepend the logo. + const brandedLabel = (logo: React.ReactNode, text: string) => + ( + + {logo} + {text} + + ) as unknown as string; + + const alpha2EnabledLabelWithLogo = brandedLabel( + , + props.alpha2EnabledLabel, + ); + const deepLEnabledLabelWithLogo = brandedLabel( + , + props.deepLEnabledLabel, + ); + const googleEnabledLabelWithLogo = brandedLabel( + , + props.googleEnabledLabel, + ); + + // Each engine's enable toggle and its (conditional) settings are wrapped in a single fragment + // so config-r treats the whole engine as one group child: it draws a separator line only + // BETWEEN engines, not between an engine's checkbox and its own settings. + const group = ( + + {/* Target language is the first thing to choose, so it leads the group. */} + + path="aiTranslationTargetLanguageTag" + control={AiTranslationTargetLanguageControl} + label={props.targetLanguageLabel} + /> + {/* Alpha2 (SIL's own service) leads the engine list. */} + <> + + {props.settings?.aiTranslationAlpha2Enabled && ( + <> + + + path="aiTranslationAlpha2SourceLanguageTag" + control={AiTranslationAlpha2SourceLanguageControl} + label={props.alpha2SourceLanguageLabel} + /> + {/* The "Translation Test" row is intentionally omitted for Alpha2: running + a live test means spinning up a whole Alpha2 translation system (create + a text collection, start and poll a translation job, then tear it down), + which is far too expensive to run on every settings change. Disabled and + hidden for now; the deepl/google tests remain since those are cheap. */} + + )} + + <> + + {props.settings?.aiTranslationDeepLEnabled && ( + <> + + + path="aiTranslationDeepLValidationDisplay" + control={EngineValidationStatusControl} + label={props.translationTestLabel} + overrideValue={{ + ...getEngineValidationDisplay( + deepLValidation.isPending, + deepLValidation.validation, + props.deepLEnabledLabel, + currentTargetLabel, + props.targetLanguageNotSupportedTemplate, + ), + testId: "ai-translation-deepl-validation-status", + }} + /> + + )} + + <> + + {props.settings?.aiTranslationGoogleEnabled && ( + <> + + + + path="aiTranslationGoogleValidationDisplay" + control={EngineValidationStatusControl} + label={props.translationTestLabel} + overrideValue={{ + ...getEngineValidationDisplay( + googleValidation.isPending, + googleValidation.validation, + props.googleEnabledLabel, + currentTargetLabel, + props.targetLanguageNotSupportedTemplate, + ), + testId: "ai-translation-google-validation-status", + }} + /> + + )} + + + ); + + return { group, languageControlsData }; +}; diff --git a/src/BloomBrowserUI/collection/LanguageChooserDialog.tsx b/src/BloomBrowserUI/collection/LanguageChooserDialog.tsx index ac5810ce6eef..99a75e1df468 100644 --- a/src/BloomBrowserUI/collection/LanguageChooserDialog.tsx +++ b/src/BloomBrowserUI/collection/LanguageChooserDialog.tsx @@ -61,6 +61,7 @@ export function getLanguageData( export const LanguageChooserDialog: React.FunctionComponent<{ initialLanguageTag?: string; initialCustomName?: string; + onOk?: (languageData: ILanguageData) => void; dialogEnvironment?: IBloomDialogEnvironmentParams; }> = (props) => { const { showDialog, closeDialog, propsForBloomDialog } = @@ -108,10 +109,12 @@ export const LanguageChooserDialog: React.FunctionComponent<{ ); function onOk(languageSelection: IOrthography, languageTag: string) { - postData( - "settings/changeLanguage", - getLanguageData(languageTag, languageSelection), - ); + const languageData = getLanguageData(languageTag, languageSelection); + if (props.onOk) { + props.onOk(languageData); + } else { + postData("settings/changeLanguage", languageData); + } closeDialog(); } @@ -125,8 +128,12 @@ export const LanguageChooserDialog: React.FunctionComponent<{ return ( - +
    + +
    ); }; @@ -174,12 +193,14 @@ let show: () => void = () => { export function showLanguageChooserDialog( initialLanguageTag?: string, initialCustomName?: string, + onOk?: (languageData: ILanguageData) => void, ) { try { renderRootSync( , getModalContainer(), ); diff --git a/src/BloomBrowserUI/collectionsTab/BookButton.tsx b/src/BloomBrowserUI/collectionsTab/BookButton.tsx index 023c71218ce3..308a113e1fd8 100644 --- a/src/BloomBrowserUI/collectionsTab/BookButton.tsx +++ b/src/BloomBrowserUI/collectionsTab/BookButton.tsx @@ -209,6 +209,15 @@ export const BookButton: React.FunctionComponent<{ requiresSavePermission: true, // marginal, but it does change the content of the book folder hide: () => !props.collection.isEditableCollection, }, + { + label: "Remove AI Source Translations", + l10nId: "CollectionTab.BookMenu.RemoveAiSourceTranslations", + command: "bookCommand/removeAiSourceTranslations", + requiresSavePermission: true, // changes the content of the book + // Deliberately NOT gated by subscription/feature: a user whose subscription no longer + // includes AI Source Bubbles must still be able to strip AI content out of a book. + hide: () => !props.collection.isEditableCollection, + }, ]; const editableCollectionName = useApiString( diff --git a/src/BloomBrowserUI/react_components/Progress/ProgressDialog.tsx b/src/BloomBrowserUI/react_components/Progress/ProgressDialog.tsx index 3c45842cfdaa..5c2e85cdca88 100644 --- a/src/BloomBrowserUI/react_components/Progress/ProgressDialog.tsx +++ b/src/BloomBrowserUI/react_components/Progress/ProgressDialog.tsx @@ -22,6 +22,7 @@ import { import { DialogCancelButton, DialogCloseButton, + DialogOkButton, } from "../BloomDialog/commonDialogComponents"; import { WireUpForWinforms } from "../../utils/WireUpWinform"; import { @@ -37,6 +38,9 @@ export interface IProgressDialogProps { // defaults to "never" showReportButton?: "always" | "if-error" | "never"; showCancelButton?: boolean; + // When the work finishes, dismiss with an "OK" button instead of the default "Close" button. + // (The Cancel button, if any, is only shown while the work is in progress either way.) + showOkButtonWhenDone?: boolean; open: boolean; // Controls whether or not the dialog is open (visible). (Theoretically, initial value should match dialogEnvironment.initiallyOpen, but not strictly necessary) onClose: () => void; // Callback fired when the component requests to be closed. @@ -199,6 +203,31 @@ export const ProgressDialog: React.FunctionComponent = ( titleColor = "white"; } + // The primary button shown once the work has finished (while it is still running we instead + // show Cancel, or a hidden placeholder -- see the bottom buttons below). A fatal error forces a + // Quit button; otherwise the caller chooses OK (showOkButtonWhenDone) or the default Close. + let donePrimaryButton: JSX.Element; + if (sawFatalError) { + donePrimaryButton = ( + + Quit + + ); + } else if (props.showOkButtonWhenDone) { + donePrimaryButton = ( + + ); + } else { + donePrimaryButton = ; + } + return ( = ( )} - {sawFatalError ? ( - - Quit - - ) : ( - - )} + {donePrimaryButton} ) : // if we're not done, show the cancel button if that was called for... props.showCancelButton ? ( @@ -420,6 +436,7 @@ interface IEmbeddedProgressDialogConfig { // defaults means "never" showReportButton?: "always" | "if-error" | "never"; showCancelButton?: boolean; + showOkButtonWhenDone?: boolean; } // Simply render one of these, with no props, at the top level of any document where the diff --git a/src/BloomExe/AiTranslation/AiTranslationBookScanner.cs b/src/BloomExe/AiTranslation/AiTranslationBookScanner.cs new file mode 100644 index 000000000000..ce5b62432ae7 --- /dev/null +++ b/src/BloomExe/AiTranslation/AiTranslationBookScanner.cs @@ -0,0 +1,720 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using Bloom.Book; +using Bloom.Collection; +using Bloom.SafeXml; + +namespace Bloom.AiTranslation +{ + /// + /// One eligible translation group found by AiTranslationBookScanner.Scan(): the chosen default + /// source text to translate, the text of every candidate source language in the group, and + /// whether a current (up to date) translation already exists per engine. + /// + public class AiTranslationGroupInfo + { + /// The bloom-translationGroup element this info describes. + public SafeXmlElement GroupElement { get; } + + /// + /// The language tag of the bloom-editable chosen (by source-language priority) as the + /// default translation source. This is what the automatic-source engines (deepl/google) use. + /// + public string SourceLanguageTag { get; } + + /// The trimmed text of the default (priority-chosen) source bloom-editable. + public string SourceText { get; } + + /// + /// The trimmed text of every non-AI, non-"z", non-empty bloom-editable in this group, keyed + /// by its lang attribute (in document order). Used by fixed-source engines (alpha2) to look + /// up whether the group has text in the engine's configured source language. + /// + public IReadOnlyDictionary TextsByLanguage { get; } + + /// True if this is the book's data-book="bookTitle" translation group. + public bool IsBookTitle { get; } + + /// + /// Creates group info for one eligible translation group. See + /// AiTranslationBookScanner.Scan() for how the default source language/text, the + /// per-language texts, and IsBookTitle are determined. + /// + public AiTranslationGroupInfo( + SafeXmlElement groupElement, + string sourceLanguageTag, + string sourceText, + IReadOnlyDictionary textsByLanguage, + bool isBookTitle + ) + { + GroupElement = groupElement; + SourceLanguageTag = sourceLanguageTag; + SourceText = sourceText; + TextsByLanguage = + textsByLanguage ?? new Dictionary(StringComparer.OrdinalIgnoreCase); + IsBookTitle = isBookTitle; + } + + /// + /// Resolves the source language tag and text this engine should translate from for this + /// group. Automatic-source engines (deepl/google) always use the group's priority-chosen + /// default. Fixed-source engines (alpha2) require text in their configured source language + /// (matched on normalized primary subtag, so an "en" setting matches an "en-US" editable); + /// if the group has no such text, this returns false and the group is ineligible for that + /// engine. + /// + public bool TryGetSourceForEngine( + AiTranslationEngineSettings engine, + out string sourceLanguageTag, + out string sourceText + ) + { + var fixedSource = AiTranslationBookScanner.GetFixedSourceLanguageTagOrNull(engine); + if (fixedSource == null) + { + sourceLanguageTag = SourceLanguageTag; + sourceText = SourceText; + return !string.IsNullOrWhiteSpace(SourceText); + } + + var wantedPrimarySubtag = AiTranslationBookScanner.GetPrimarySubtag(fixedSource); + // Prefer an exact-normalized match, then fall back to any editable sharing the primary + // subtag (in document order), so a fixed "en" source can use an "en-US" editable. + foreach (var pair in TextsByLanguage) + { + if ( + AiTranslationBookScanner.GetPrimarySubtag(pair.Key) == wantedPrimarySubtag + && !string.IsNullOrWhiteSpace(pair.Value) + ) + { + sourceLanguageTag = pair.Key; + sourceText = pair.Value; + return true; + } + } + + sourceLanguageTag = null; + sourceText = null; + return false; + } + + /// + /// The fingerprint a translation div for the given engine must currently have, based on the + /// engine's resolved source language/text for this group and the engine's AI language tag. + /// Returns null if the engine has no usable source for this group (a fixed-source engine + /// whose source language is absent), meaning any existing div for it is stale. + /// + public string GetExpectedFingerprint( + AiTranslationEngineSettings engine, + string targetLanguageTag + ) + { + if (!TryGetSourceForEngine(engine, out var sourceLanguageTag, out var sourceText)) + return null; + + var aiTag = AiTranslationService.GetAiLanguageTag(targetLanguageTag, engine.ProviderId); + return AiTranslationBookScanner.ComputeFingerprint( + sourceLanguageTag, + sourceText, + aiTag + ); + } + + /// + /// True if this group does NOT need (re)translation for the given engine: either the engine + /// has no usable source for it (so there is nothing to translate), or it already has a + /// non-empty translation div for the engine's AI language tag whose fingerprint matches the + /// engine's current resolved source. + /// + public bool HasCurrentTranslation( + AiTranslationEngineSettings engine, + string targetLanguageTag + ) + { + var expectedFingerprint = GetExpectedFingerprint(engine, targetLanguageTag); + if (expectedFingerprint == null) + return true; // no source for this engine -> nothing to (re)translate. + + var aiTag = AiTranslationService.GetAiLanguageTag(targetLanguageTag, engine.ProviderId); + var existing = GroupElement + .SafeSelectElements($"div[@lang='{aiTag}']") + .FirstOrDefault(); + if (existing == null) + return false; + + var text = (existing.InnerText ?? "").Trim(); + if (string.IsNullOrEmpty(text)) + return false; + + return existing.GetAttribute("data-ai-fingerprint") == expectedFingerprint; + } + } + + /// + /// One group's resolved translation work for a specific engine: the group plus the source + /// language/text that engine should translate from (which, for fixed-source engines, can differ + /// from the group's default source). Returned by AiTranslationBookScan.GroupsNeedingTranslation. + /// + public class AiTranslationGroupSource + { + /// The group needing translation. + public AiTranslationGroupInfo Group { get; } + + /// The language tag the engine should translate from for this group. + public string SourceLanguageTag { get; } + + /// The source text the engine should translate for this group. + public string SourceText { get; } + + /// Pairs a group with the engine-resolved source language/text. + public AiTranslationGroupSource( + AiTranslationGroupInfo group, + string sourceLanguageTag, + string sourceText + ) + { + Group = group; + SourceLanguageTag = sourceLanguageTag; + SourceText = sourceText; + } + } + + /// + /// The result of AiTranslationBookScanner.Scan(): every eligible translation group found, in + /// document order. + /// + public class AiTranslationBookScan + { + /// All eligible groups found, in document order. + public List Groups { get; } + + private readonly string _targetLanguageTag; + + /// Wraps the groups found by a scan, remembering the target language they were scanned for. + public AiTranslationBookScan(List groups, string targetLanguageTag) + { + Groups = groups; + _targetLanguageTag = targetLanguageTag; + } + + /// + /// The ordered subset of Groups that still need a (re)translation for the given engine, + /// each paired with the source language/text that engine should translate from. Groups the + /// engine cannot translate (a fixed-source engine whose source language is absent from the + /// group) are excluded, as are groups that already have a current translation. + /// + public List GroupsNeedingTranslation( + AiTranslationEngineSettings engine + ) + { + var result = new List(); + foreach (var group in Groups) + { + if ( + !group.TryGetSourceForEngine( + engine, + out var sourceLanguageTag, + out var sourceText + ) + ) + continue; + if (group.HasCurrentTranslation(engine, _targetLanguageTag)) + continue; + result.Add(new AiTranslationGroupSource(group, sourceLanguageTag, sourceText)); + } + return result; + } + + /// + /// How many otherwise-eligible groups this engine must skip because it has no source text + /// for them. Only fixed-source engines (alpha2) can skip groups this way -- an + /// automatic-source engine always has the group's default source -- so this is always 0 for + /// deepl/google. Used only to leave a trace in the progress log; the book-level skip itself + /// is silent. + /// + public int CountGroupsSkippedForEngine(AiTranslationEngineSettings engine) + { + if (AiTranslationBookScanner.GetFixedSourceLanguageTagOrNull(engine) == null) + return 0; + return Groups.Count(group => !group.TryGetSourceForEngine(engine, out _, out _)); + } + } + + /// + /// Scans a book's DOM for translation groups eligible for AI translation, and + /// applies or removes AI-generated translation divs. This class does no network calls and no + /// orchestration; it is purely the scan + DOM read/write layer that the book updater (Phase 3) + /// drives around calls to AiTranslationService.TranslateSegmentsAsync. + /// + public class AiTranslationBookScanner + { + private const string kAiLangTagFragment = "-x-ai"; + private const string kBloomEditableClass = "bloom-editable"; + private const string kAiTranslationClass = "bloom-ai-translation"; + private const string kBookTitleDataBookValue = "bookTitle"; + private const string kZeroLang = "z"; + + private readonly HtmlDom _bookDom; + private readonly string _targetLanguageTag; + private readonly List _enabledEngines; + private readonly IReadOnlyList _sourceLanguagePriorities; + + /// + /// Creates a scanner for one book's DOM. sourceLanguagePriorities is the ordered list of + /// candidate language tags to prefer when choosing which bloom-editable's text is the + /// translation source for a group (e.g. the user's last-viewed source languages, then the + /// collection's L2/L3 tags, then "en"). + /// + public AiTranslationBookScanner( + HtmlDom bookDom, + string targetLanguageTag, + IEnumerable enabledEngines, + IReadOnlyList sourceLanguagePriorities + ) + { + _bookDom = bookDom ?? throw new ArgumentNullException(nameof(bookDom)); + _targetLanguageTag = targetLanguageTag; + _enabledEngines = ( + enabledEngines ?? Enumerable.Empty() + ).ToList(); + _sourceLanguagePriorities = sourceLanguagePriorities ?? Array.Empty(); + } + + /// + /// Scans all pages' translation groups, in document order, for groups eligible for AI + /// translation (see TryBuildGroupInfo for eligibility rules), choosing a + /// source text for each from sourceLanguagePriorities. + /// + public AiTranslationBookScan Scan() + { + var groups = new List(); + foreach (var page in SafeXmlElement.GetAllDivsWithClass(_bookDom.Body, "bloom-page")) + { + foreach ( + var group in SafeXmlElement.GetAllDivsWithClass(page, "bloom-translationGroup") + ) + { + var info = TryBuildGroupInfo(group); + if (info != null) + groups.Add(info); + } + } + + return new AiTranslationBookScan(groups, _targetLanguageTag); + } + + /// + /// Writes (creating or replacing) the AI translation div for one group+engine with the + /// given translated text and a matching data-ai-fingerprint. For bookTitle groups, also + /// writes/refreshes the matching entry in #bloomDataDiv (creates nothing if the data div + /// doesn't exist; real books always have one, but we don't want to throw on a test DOM + /// that omits it). + /// + public void ApplyTranslation( + AiTranslationGroupInfo group, + AiTranslationEngineSettings engine, + string translatedText + ) + { + var aiTag = AiTranslationService.GetAiLanguageTag( + _targetLanguageTag, + engine.ProviderId + ); + // Fingerprint against the engine's own resolved source (which, for a fixed-source engine + // like alpha2, may differ from the group's default source) so it matches what + // GroupsNeedingTranslation/RemoveStaleAiDivs compute for the same engine. A group is only + // ever applied for an engine that has a source for it, so this must resolve. + if (!group.TryGetSourceForEngine(engine, out var sourceLanguageTag, out var sourceText)) + { + throw new InvalidOperationException( + "ApplyTranslation called for an engine with no source for the group." + ); + } + var fingerprint = ComputeFingerprint(sourceLanguageTag, sourceText, aiTag); + + WriteChildDiv( + group.GroupElement, + aiTag, + translatedText, + div => + { + div.AddClass(kBloomEditableClass); + div.AddClass(kAiTranslationClass); + div.SetAttribute("data-ai-fingerprint", fingerprint); + } + ); + + if (!group.IsBookTitle) + return; + + var dataDiv = FindBloomDataDiv(); + if (dataDiv == null) + return; // real books always have one; nothing to do if a test DOM omits it. + + WriteChildDiv( + dataDiv, + aiTag, + translatedText, + div => div.SetAttribute("data-book", kBookTitleDataBookValue), + matchAttribute: "data-book", + matchValue: kBookTitleDataBookValue + ); + } + + /// + /// Removes AI translation divs that no longer belong: any div whose lang contains "-x-ai" + /// under a group that is no longer eligible for AI translation at all (orphaned), or under + /// an eligible group but for a disabled engine/inactive target language, or with a + /// fingerprint that no longer matches the group's current source text (the source was + /// edited since translation). Also cleans matching #bloomDataDiv bookTitle entries. Returns + /// the total number of divs removed. + /// + public int RemoveStaleAiDivs() + { + var activeTags = new HashSet( + _enabledEngines.Select(e => + AiTranslationService.GetAiLanguageTag(_targetLanguageTag, e.ProviderId) + ), + StringComparer.OrdinalIgnoreCase + ); + var validBookTitleLangs = new HashSet(StringComparer.OrdinalIgnoreCase); + var removedCount = 0; + + foreach (var page in SafeXmlElement.GetAllDivsWithClass(_bookDom.Body, "bloom-page")) + { + foreach ( + var group in SafeXmlElement.GetAllDivsWithClass(page, "bloom-translationGroup") + ) + { + // Rebuild eligibility from scratch (not just the eligible groups from Scan()) + // so we also catch AI divs left behind in groups that are no longer eligible. + var groupInfo = TryBuildGroupInfo(group); + foreach (var aiDiv in GetAiChildDivs(group)) + { + var lang = aiDiv.GetAttribute("lang"); + if (ShouldRemoveGroupAiDiv(groupInfo, aiDiv, lang, activeTags)) + { + group.RemoveChild(aiDiv); + removedCount++; + } + else if (groupInfo.IsBookTitle) + { + validBookTitleLangs.Add(lang); + } + } + } + } + + var dataDiv = FindBloomDataDiv(); + if (dataDiv != null) + { + foreach (var aiEntry in GetAiChildDivs(dataDiv)) + { + var isBookTitleEntry = + aiEntry.GetAttribute("data-book") == kBookTitleDataBookValue; + var lang = aiEntry.GetAttribute("lang"); + // Any non-bookTitle AI entry in the data div is never written by this class, + // so it's inherently stale/orphaned. A bookTitle entry is only valid if its + // matching page-level translation div is still current (see loop above). + if (!isBookTitleEntry || !validBookTitleLangs.Contains(lang)) + { + dataDiv.RemoveChild(aiEntry); + removedCount++; + } + } + } + + return removedCount; + } + + /// + /// Removes every AI-generated translation div from the book -- all "-x-ai" divs under any + /// bloom-translationGroup and all matching #bloomDataDiv entries -- regardless of engine, + /// target language, or staleness. Unlike RemoveStaleAiDivs, this also removes current, + /// in-use translations: it is the "remove all AI source translations" operation, and does + /// not depend on which engines/target language the scanner was constructed with. Returns + /// the total number of divs removed. + /// + public int RemoveAllAiDivs() + { + var removedCount = 0; + + foreach (var page in SafeXmlElement.GetAllDivsWithClass(_bookDom.Body, "bloom-page")) + { + foreach ( + var group in SafeXmlElement.GetAllDivsWithClass(page, "bloom-translationGroup") + ) + { + foreach (var aiDiv in GetAiChildDivs(group)) + { + group.RemoveChild(aiDiv); + removedCount++; + } + } + } + + var dataDiv = FindBloomDataDiv(); + if (dataDiv != null) + { + foreach (var aiEntry in GetAiChildDivs(dataDiv)) + { + dataDiv.RemoveChild(aiEntry); + removedCount++; + } + } + + return removedCount; + } + + /// + /// Computes the fingerprint stored in data-ai-fingerprint, used to detect a stale AI + /// translation: the first 16 hex characters of the SHA-256 hash of + /// "{sourceLanguageTag}\n{sourceText}\n{aiLanguageTag}". + /// + public static string ComputeFingerprint( + string sourceLanguageTag, + string sourceText, + string aiLanguageTag + ) + { + var input = $"{sourceLanguageTag}\n{sourceText}\n{aiLanguageTag}"; + var hash = SHA256.HashData(Encoding.UTF8.GetBytes(input)); + return Convert.ToHexString(hash).Substring(0, 16); + } + + /// + /// Builds group info for one translation group, or returns null if the group should be + /// skipped: explicitly excluded (bloom-no-source-bubble / bloom-readOnlyInTranslationMode), + /// tied to book data other than the title, or lacking any usable source text. + /// + private AiTranslationGroupInfo TryBuildGroupInfo(SafeXmlElement group) + { + if ( + group.HasClass("bloom-no-source-bubble") + || group.HasClass("bloom-readOnlyInTranslationMode") + ) + return null; + + var editables = SafeXmlElement.GetAllDivsWithClass(group, kBloomEditableClass); + + var isBookTitle = false; + foreach (var editable in editables) + { + if (!editable.HasAttribute("data-book")) + continue; + if (editable.GetAttribute("data-book") == kBookTitleDataBookValue) + isBookTitle = true; + else + return null; // tied to some other book-data item; not ours to translate. + } + + var sourceDiv = ChooseSourceDiv(editables); + if (sourceDiv == null) + return null; + + var sourceText = (sourceDiv.InnerText ?? "").Trim(); + if (string.IsNullOrWhiteSpace(sourceText)) + return null; + + return new AiTranslationGroupInfo( + group, + sourceDiv.GetAttribute("lang"), + sourceText, + BuildTextsByLanguage(editables), + isBookTitle + ); + } + + /// + /// Builds the lang -> trimmed-text map of every candidate source editable in a group: those + /// with a lang attribute that is not an AI ("-x-ai") tag, not the "z" placeholder language, + /// and whose text is non-empty. Preserves document order; when a lang appears more than once, + /// the first non-empty occurrence wins. + /// + private static IReadOnlyDictionary BuildTextsByLanguage( + IEnumerable editables + ) + { + var textsByLanguage = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var editable in editables) + { + if (!editable.HasAttribute("lang")) + continue; + var lang = editable.GetAttribute("lang"); + if (lang.Contains(kAiLangTagFragment) || lang == kZeroLang) + continue; + var text = (editable.InnerText ?? "").Trim(); + if (string.IsNullOrWhiteSpace(text)) + continue; + if (!textsByLanguage.ContainsKey(lang)) + textsByLanguage[lang] = text; + } + + return textsByLanguage; + } + + /// + /// Returns the fixed source language tag an engine must translate from, or null if the + /// engine chooses its source automatically per group. Only the alpha2 provider is + /// fixed-source (its model selection is per source/target pair); its blank source defaults + /// to English via GetEffectiveSourceLanguageTag. + /// + internal static string GetFixedSourceLanguageTagOrNull(AiTranslationEngineSettings engine) + { + if (AiTranslationService.NormalizeProviderId(engine.ProviderId) == "alpha2") + return engine.GetEffectiveSourceLanguageTag(); + return null; + } + + /// + /// The lowercased primary (language) subtag of a Bloom language tag, used to match a fixed + /// source language against a group's editables at primary-subtag granularity (e.g. an "en" + /// setting matches an "en-US" editable). + /// + internal static string GetPrimarySubtag(string languageTag) + { + if (string.IsNullOrWhiteSpace(languageTag)) + return string.Empty; + return AiTranslationService + .NormalizeBloomLanguageTag(languageTag) + .Split('-')[0] + .ToLowerInvariant(); + } + + /// + /// Chooses which bloom-editable's text is the translation source: the first non-empty + /// match walking sourceLanguagePriorities in order (skipping AI and "z" language divs), + /// falling back to the first non-empty, non-AI, non-"z" editable if no priority matches. + /// + private SafeXmlElement ChooseSourceDiv(IEnumerable editables) + { + var candidates = editables + .Where(e => e.HasAttribute("lang")) + .Where(e => !e.GetAttribute("lang").Contains(kAiLangTagFragment)) + .Where(e => e.GetAttribute("lang") != kZeroLang) + .ToList(); + + foreach (var priorityLang in _sourceLanguagePriorities) + { + var match = candidates.FirstOrDefault(e => + e.GetAttribute("lang") == priorityLang + && !string.IsNullOrWhiteSpace(e.InnerText) + ); + if (match != null) + return match; + } + + return candidates.FirstOrDefault(e => !string.IsNullOrWhiteSpace(e.InnerText)); + } + + /// + /// Direct child divs of parent whose lang attribute contains "-x-ai" (AI-generated content + /// of any provider/target, active or not). + /// + private static List GetAiChildDivs(SafeXmlElement parent) + { + return parent + .SafeSelectElements($"div[@lang and contains(@lang, '{kAiLangTagFragment}')]") + .ToList(); + } + + /// + /// Decides whether one existing AI div under a translation group should be removed: + /// always if the group is no longer eligible for AI translation at all (its AI content is + /// orphaned), otherwise if its language isn't currently active (disabled engine or changed + /// target language), or its fingerprint no longer matches the source the div's OWN engine + /// would currently translate from (which, for a fixed-source engine, is that engine's + /// configured source; a missing fixed source makes the div stale). + /// + private bool ShouldRemoveGroupAiDiv( + AiTranslationGroupInfo groupInfo, + SafeXmlElement aiDiv, + string lang, + HashSet activeTags + ) + { + if (groupInfo == null) + return true; + + if (!activeTags.Contains(lang)) + return true; + + var text = (aiDiv.InnerText ?? "").Trim(); + if (string.IsNullOrEmpty(text)) + return true; + + var engine = ResolveEngineFromAiLang(lang); + if (engine == null) + return true; // no matching active engine (shouldn't happen given activeTags check). + + var expectedFingerprint = groupInfo.GetExpectedFingerprint(engine, _targetLanguageTag); + if (expectedFingerprint == null) + return true; // this engine has no source for the group now -> the div is stale. + + return aiDiv.GetAttribute("data-ai-fingerprint") != expectedFingerprint; + } + + /// + /// Finds the enabled engine that produced an AI div, by reading the provider id out of the + /// div's AI language tag (e.g. "es-x-ai-alpha2" -> the alpha2 engine). Returns null if no + /// enabled engine matches. + /// + private AiTranslationEngineSettings ResolveEngineFromAiLang(string lang) + { + if (string.IsNullOrEmpty(lang)) + return null; + var markerIndex = lang.IndexOf( + kAiLangTagFragment + "-", + StringComparison.OrdinalIgnoreCase + ); + if (markerIndex < 0) + return null; + var providerId = AiTranslationService.NormalizeProviderId( + lang.Substring(markerIndex + kAiLangTagFragment.Length + 1) + ); + return _enabledEngines.FirstOrDefault(e => + AiTranslationService.NormalizeProviderId(e.ProviderId) == providerId + ); + } + + /// Finds the book's #bloomDataDiv element, or null if it isn't present. + private SafeXmlElement FindBloomDataDiv() + { + return _bookDom.RawDom.SelectSingleNode("//div[@id='bloomDataDiv']") as SafeXmlElement; + } + + /// + /// Creates or replaces the direct child div of parent with the given lang: removes any + /// existing match (by lang, or by matchAttribute/matchValue when given, for cases like + /// #bloomDataDiv where multiple data-book items could share a lang), then appends a fresh + /// div with that lang and the given text (set via InnerText so markup in translations can't + /// inject HTML), letting configureAttributes add any extra attributes/classes. + /// + private static void WriteChildDiv( + SafeXmlElement parent, + string lang, + string text, + Action configureAttributes, + string matchAttribute = null, + string matchValue = null + ) + { + var xpath = + matchAttribute == null + ? $"div[@lang='{lang}']" + : $"div[@lang='{lang}' and @{matchAttribute}='{matchValue}']"; + var existing = parent.SafeSelectElements(xpath).FirstOrDefault(); + if (existing != null) + parent.RemoveChild(existing); + + var newDiv = parent.AppendChild("div"); + newDiv.SetAttribute("lang", lang); + configureAttributes(newDiv); + newDiv.InnerText = text; + } + } +} diff --git a/src/BloomExe/AiTranslation/AiTranslationBookUpdater.cs b/src/BloomExe/AiTranslation/AiTranslationBookUpdater.cs new file mode 100644 index 000000000000..82eeb57f4afc --- /dev/null +++ b/src/BloomExe/AiTranslation/AiTranslationBookUpdater.cs @@ -0,0 +1,424 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using System.Windows.Forms; +using Bloom.Api; +using Bloom.Book; +using Bloom.Collection; +using Bloom.MiscUI; +using Bloom.Properties; +using Bloom.SubscriptionAndFeatures; +using Bloom.web; +using L10NSharp; +using SIL.Reporting; + +namespace Bloom.AiTranslation +{ + /// + /// Delegate matching AiTranslationService.TranslateSegmentsAsync's signature, so tests can + /// substitute a fake translator without any network calls or a real AiTranslationService. + /// + public delegate Task AiTranslateSegmentsDelegate( + AiTranslationEngineSettings engine, + string[] segments, + string sourceLanguageTag, + CancellationToken ct + ); + + /// + /// One engine's outcome from the parallel translation phase of AiTranslationBookUpdater: either + /// the (group, translatedText) pairs to apply, or the reason the engine failed. Nothing here is + /// written to the book DOM until every engine's work has settled -- see + /// AiTranslationBookUpdater.ApplyOutcomes. + /// + public class AiTranslationEngineOutcome + { + /// The engine this outcome is for. + public AiTranslationEngineSettings Engine { get; } + + /// The (group, translatedText) pairs this engine produced, not yet applied to the DOM. + public List<(AiTranslationGroupInfo Group, string TranslatedText)> Translations { get; } = + new List<(AiTranslationGroupInfo, string)>(); + + /// Null if this engine's translation succeeded; otherwise the error message. + public string ErrorMessage { get; set; } + + /// True if this engine completed without error. + public bool Succeeded => ErrorMessage == null; + + /// Creates an (initially empty, successful) outcome for the given engine. + public AiTranslationEngineOutcome(AiTranslationEngineSettings engine) + { + Engine = engine; + } + } + + /// + /// Orchestrates whole-book AI translation: scans a book for translation work + /// across the collection's active engines and, if any is found (missing/stale translations for + /// an active engine, or orphaned AI content that needs cleanup), blocks behind a modal progress + /// dialog while the engines translate in parallel, then applies every result, removes stale AI + /// content, and saves the book. Called once per book, right before the user starts editing it -- + /// see EditingModel.OnBecomeVisible. If nothing needs doing (the normal case once a book is up to + /// date), this returns immediately: no dialog, no network calls, no book changes. + /// + public class AiTranslationBookUpdater + { + private readonly AiTranslationService _aiTranslationService; + private readonly CollectionSettings _collectionSettings; + private readonly IBloomWebSocketServer _webSocketServer; + + /// + /// Creates an updater for the current collection (AiTranslationService and CollectionSettings + /// are both scoped one-per-collection by the normal Autofac project scope). + /// + public AiTranslationBookUpdater( + AiTranslationService aiTranslationService, + CollectionSettings collectionSettings, + BloomWebSocketServer webSocketServer + ) + { + _aiTranslationService = aiTranslationService; + _collectionSettings = collectionSettings; + _webSocketServer = webSocketServer; + } + + /// + /// Scans the given book and, if any active engine needs to translate something or stale AI + /// content needs removing, blocks (showing a modal progress dialog) until the work is done, + /// applied, and saved -- or the user cancels, in which case the book is left completely + /// untouched. Safe and near-instant to call when nothing needs doing (the normal case once a + /// book is up to date): no network calls are made unless there is confirmed work. + /// + public void UpdateBookIfNeeded(Book.Book book) + { + var featureStatus = FeatureStatus.GetFeatureStatus( + _collectionSettings.Subscription, + FeatureName.AiSourceBubbles + ); + var targetLanguageTag = _collectionSettings.AiTranslationTargetLanguageTag; + if ( + !featureStatus.Visible + || !featureStatus.Enabled + || string.IsNullOrWhiteSpace(targetLanguageTag) + ) + return; + + var activeEngines = AiTranslationService.GetActiveEngines(_collectionSettings); + if (activeEngines.Count == 0) + return; + + var sourceLanguagePriorities = GetSourceLanguagePriorities(); + var bookDom = book.OurHtmlDom; + var scanner = new AiTranslationBookScanner( + bookDom, + targetLanguageTag, + activeEngines, + sourceLanguagePriorities + ); + var scan = scanner.Scan(); + + var hasTranslationWork = activeEngines.Any(engine => + scan.GroupsNeedingTranslation(engine).Count > 0 + ); + var staleDivCount = CountStaleAiDivsWithoutMutating( + bookDom, + targetLanguageTag, + activeEngines, + sourceLanguagePriorities + ); + if (!hasTranslationWork && staleDivCount == 0) + return; // nothing to do; this is the normal case once a book is up to date. + + RunWithProgressDialog(book, scanner, scan, activeEngines); + } + + /// + /// Runs the parallel translation phase (via RunEnginesAsync) behind a modal progress dialog, + /// then applies the results, removes stale AI content, and saves the book -- unless the user + /// cancelled, in which case nothing is applied, cleaned up, or saved. + /// + private void RunWithProgressDialog( + Book.Book book, + AiTranslationBookScanner scanner, + AiTranslationBookScan scan, + List activeEngines + ) + { + BrowserProgressDialog.DoWorkWithProgressDialog( + _webSocketServer, + MakeDialog, + (progress, worker) => + { + using (var cts = new CancellationTokenSource()) + using ( + new System.Threading.Timer( + _ => + { + if (worker.CancellationPending) + cts.Cancel(); + }, + null, + 0, + 150 + ) + ) + { + List outcomes; + try + { + outcomes = RunEnginesAsync( + scan, + activeEngines, + _aiTranslationService.TranslateSegmentsAsync, + progress, + cts.Token + ) + .GetAwaiter() + .GetResult(); + } + catch (OperationCanceledException) + { + progress.Message( + "EditTab.AiTranslation.Cancelled", + "Cancelled.", + useL10nIdPrefix: false + ); + return false; // book left completely untouched; close the dialog. + } + + ApplyOutcomes(scanner, outcomes); + var removedStaleCount = scanner.RemoveStaleAiDivs(); + book.Save(); + Logger.WriteEvent( + $"AI translation: translated {outcomes.Sum(o => o.Translations.Count)} text box(es) " + + $"across {outcomes.Count} engine(s); removed {removedStaleCount} stale AI div(s)." + ); + + // Always keep the dialog open once the run finishes (whether or not any + // engine failed) so the user can read the per-engine result lines and + // dismiss it with the OK button when ready. The Cancel button is only + // present while work is in progress; on finishing, the dialog swaps it for + // OK (and, if there were errors, also shows the Report button). + return true; + } + }, + owner: Shell.GetShellOrOtherOpenForm() + ); + } + + /// + /// Builds the modal Form used to host the progress dialog. This uses the older Form-based + /// BrowserProgressDialog overload (ShowDialog(), which blocks the caller until the dialog + /// closes) rather than the newer overload that embeds a progress dialog in an already-open web + /// page. UpdateBookIfNeeded runs synchronously from EditingModel.OnBecomeVisible, before any + /// Edit tab page -- and thus before any EmbeddedProgressDialog -- has been loaded, so there is + /// no host page available at this point in the app lifecycle. The Form-based overload's + /// ShowDialog() blocking behavior is also exactly what this feature needs: editing must not + /// start until translation has completed, been cancelled, or failed. + /// + private static Form MakeDialog() + { + var dlg = new ReactDialog( + "progressDialogBundle", + new + { + title = LocalizationManager.GetDynamicString( + appId: "Bloom", + id: "EditTab.AiTranslation.DialogTitle", + englishText: "Translating" + ), + titleColor = "white", + titleBackgroundColor = Palette.kBloomBlueHex, + showReportButton = "if-error", + showCancelButton = true, + // When the run finishes, dismiss with an OK button rather than the generic + // Close button (see ProgressDialog.tsx). Until then, only Cancel is shown. + showOkButtonWhenDone = true, + }, + "Translating" + ); + dlg.SetScaledSize(620, 400); + return dlg; + } + + /// + /// Runs every engine that has translation work, in parallel, collecting (group, translatedText) + /// results in memory without touching the book DOM (see ApplyOutcomes for that). Each engine's + /// segments are grouped by source language so translate is called once per language per engine. + /// A per-engine failure is isolated into that engine's outcome and does not affect the others; + /// cancellation (via ct) aborts every engine and propagates out of this method entirely, since a + /// cancelled run must not apply anything. + /// + internal static async Task> RunEnginesAsync( + AiTranslationBookScan scan, + IEnumerable engines, + AiTranslateSegmentsDelegate translate, + IWebSocketProgress progress, + CancellationToken ct + ) + { + var workByEngine = engines + .Select(engine => new + { + Engine = engine, + Groups = scan.GroupsNeedingTranslation(engine), + SkippedCount = scan.CountGroupsSkippedForEngine(engine), + }) + .Where(w => w.Groups.Count > 0) + .ToList(); + + var tasks = workByEngine.Select(w => + TranslateOneEngineAsync(w.Engine, w.Groups, w.SkippedCount, translate, progress, ct) + ); + var outcomes = await Task.WhenAll(tasks); + return outcomes.ToList(); + } + + /// + /// Translates one engine's groups, grouped by the engine-resolved source language (so a + /// fixed-source engine like alpha2 translates from its configured source while deepl/google + /// translate from each group's default source), isolating any failure into the returned + /// outcome (cancellation is the one exception: it is rethrown so it aborts the whole run + /// rather than being recorded as this engine's failure). skippedCount is how many eligible + /// groups this engine had to skip for lack of source text; when non-zero it is noted in the + /// progress log. + /// + private static async Task TranslateOneEngineAsync( + AiTranslationEngineSettings engine, + List groups, + int skippedCount, + AiTranslateSegmentsDelegate translate, + IWebSocketProgress progress, + CancellationToken ct + ) + { + var outcome = new AiTranslationEngineOutcome(engine); + var engineName = AiTranslationService.GetProviderDisplayName(engine.ProviderId); + try + { + progress.MessageWithParams( + "EditTab.AiTranslation.EngineTranslating", + "{0} is the translation engine's name (e.g. DeepL); {1} is how many text boxes are being translated", + "{0}: translating {1} text box(es)...", + ProgressKind.Progress, + engineName, + groups.Count + ); + + foreach (var languageGroup in groups.GroupBy(g => g.SourceLanguageTag)) + { + ct.ThrowIfCancellationRequested(); + var groupList = languageGroup.ToList(); + var segments = groupList.Select(g => g.SourceText).ToArray(); + var translated = await translate(engine, segments, languageGroup.Key, ct); + for (var i = 0; i < groupList.Count; i++) + { + outcome.Translations.Add((groupList[i].Group, translated[i])); + } + } + + if (skippedCount > 0) + { + progress.MessageWithParams( + "EditTab.AiTranslation.EngineSkippedNoSourceText", + "{0} is the translation engine's name (e.g. SIL Alpha2), which must not be translated; {1} is a count of text boxes; {2} is a language tag (e.g. \"en\")", + "{0}: skipped {1} text box(es) with no {2} text.", + ProgressKind.Progress, + engineName, + skippedCount, + engine.GetEffectiveSourceLanguageTag() + ); + } + + progress.MessageWithParams( + "EditTab.AiTranslation.EngineDone", + "{0} is the translation engine's name (e.g. DeepL)", + "{0}: done.", + ProgressKind.Progress, + engineName + ); + } + catch (OperationCanceledException) + { + throw; // cancellation aborts the whole run; not a per-engine failure. + } + catch (Exception ex) + { + outcome.ErrorMessage = ex.Message; + progress.MessageWithParams( + "EditTab.AiTranslation.EngineError", + "{0} is the translation engine's name (e.g. DeepL); {1} is the error message", + "{0}: {1}", + ProgressKind.Error, + engineName, + ex.Message + ); + } + + return outcome; + } + + /// + /// Writes every collected translation to the book DOM via scanner.ApplyTranslation. Called only + /// after all engines have settled, and only when the run was not cancelled. + /// + internal static void ApplyOutcomes( + AiTranslationBookScanner scanner, + IEnumerable outcomes + ) + { + foreach (var outcome in outcomes) + { + foreach (var (group, translatedText) in outcome.Translations) + { + scanner.ApplyTranslation(group, outcome.Engine, translatedText); + } + } + } + + /// + /// Counts how many stale AI divs RemoveStaleAiDivs() would remove, without mutating the real + /// book DOM: runs the (otherwise identical) removal against a throwaway clone. + /// + internal static int CountStaleAiDivsWithoutMutating( + HtmlDom bookDom, + string targetLanguageTag, + IEnumerable activeEngines, + IReadOnlyList sourceLanguagePriorities + ) + { + var probeDom = new HtmlDom(bookDom.RawDom.Clone()); + var probeScanner = new AiTranslationBookScanner( + probeDom, + targetLanguageTag, + activeEngines, + sourceLanguagePriorities + ); + return probeScanner.RemoveStaleAiDivs(); + } + + /// + /// The ordered list of candidate source languages to prefer when a translation group has more + /// than one bloom-editable to choose from: the user's last-viewed source languages, then the + /// collection's L2/L3 tags, then English, deduped and with blanks removed. + /// + private IReadOnlyList GetSourceLanguagePriorities() + { + return new[] + { + Settings.Default.LastSourceLanguageViewed, + Settings.Default.LastSourceLanguageViewed2, + _collectionSettings.Language2Tag, + _collectionSettings.Language3Tag, + "en", + } + .Where(tag => !string.IsNullOrEmpty(tag)) + .Distinct() + .ToList(); + } + } +} diff --git a/src/BloomExe/AiTranslation/AiTranslationEngineSettings.cs b/src/BloomExe/AiTranslation/AiTranslationEngineSettings.cs new file mode 100644 index 000000000000..63840e702bb5 --- /dev/null +++ b/src/BloomExe/AiTranslation/AiTranslationEngineSettings.cs @@ -0,0 +1,96 @@ +namespace Bloom.AiTranslation +{ + /// + /// Per-engine configuration and validation state for one AI translation provider + /// ("deepl", "google", or "alpha2"). A CollectionSettings always has exactly one of these for + /// each known provider id; see CollectionSettings.EnsureAiTranslationEngines(). + /// + public class AiTranslationEngineSettings + { + /// + /// The provider this engine configuration is for: "deepl", "google", or "alpha2". + /// + public string ProviderId; + + /// + /// Whether this engine is turned on for whole-book batch translation. + /// + public bool Enabled; + + /// + /// API key used by the DeepL and Alpha2 providers. + /// + public string ApiKey = ""; + + /// + /// Service account email used by the Google provider. + /// + public string ServiceAccountEmail = ""; + + /// + /// Service account private key used by the Google provider. + /// + public string PrivateKey = ""; + + /// + /// The source language (Bloom/BCP-47 tag) to translate FROM. Only meaningful for the + /// alpha2 provider, whose model selection is per source/target pair; deepl/google choose + /// their source automatically per translation group and ignore this. A blank value means + /// English ("en"); use GetEffectiveSourceLanguageTag() to read it with that default applied. + /// + public string SourceLanguageTag = ""; + + /// + /// Fingerprint (see AiTranslationService.GetEngineFingerprint) of this engine's + /// configuration and the target language at the time it was last validated. + /// + public string ValidatedConfigurationFingerprint = ""; + + /// + /// Whether the last validation attempt for this engine succeeded. + /// + public bool LastValidationSucceeded; + + /// + /// True if the last validation failed specifically because this engine's provider does + /// not support the chosen target language (as opposed to a credential/network error). When + /// set, the settings UI shows a "does not support ⟨language⟩, will be skipped" note and the + /// engine is simply excluded from translation rather than treated as misconfigured. + /// + public bool LastValidationTargetLanguageNotSupported; + + /// + /// The message (success text or error) from the last validation attempt. + /// + public string LastValidationMessage = ""; + + /// + /// The effective source language tag to translate from: the configured SourceLanguageTag + /// if non-blank, otherwise English ("en"). Only the alpha2 provider consults this. + /// + public string GetEffectiveSourceLanguageTag() + { + return string.IsNullOrWhiteSpace(SourceLanguageTag) ? "en" : SourceLanguageTag.Trim(); + } + + /// + /// Creates a deep (independent) copy of this engine's settings. + /// + public AiTranslationEngineSettings Clone() + { + return new AiTranslationEngineSettings + { + ProviderId = ProviderId, + Enabled = Enabled, + ApiKey = ApiKey, + ServiceAccountEmail = ServiceAccountEmail, + PrivateKey = PrivateKey, + SourceLanguageTag = SourceLanguageTag, + ValidatedConfigurationFingerprint = ValidatedConfigurationFingerprint, + LastValidationSucceeded = LastValidationSucceeded, + LastValidationTargetLanguageNotSupported = LastValidationTargetLanguageNotSupported, + LastValidationMessage = LastValidationMessage, + }; + } + } +} diff --git a/src/BloomExe/AiTranslation/AiTranslationService.cs b/src/BloomExe/AiTranslation/AiTranslationService.cs new file mode 100644 index 000000000000..c09ad06fe1b4 --- /dev/null +++ b/src/BloomExe/AiTranslation/AiTranslationService.cs @@ -0,0 +1,647 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Bloom.Collection; +using Bloom.Utils; +using Newtonsoft.Json.Linq; +using SIL.WritingSystems; + +namespace Bloom.AiTranslation +{ + /// + /// Result of validating one AI translation engine's configuration. + /// + public class AiTranslationValidationResult + { + public bool Succeeded { get; set; } + + /// + /// True when validation failed specifically because the provider does not support the + /// chosen target language (rather than a credential/network problem). In that case the + /// engine is auto-skipped for translation rather than reported as broken. + /// + public bool TargetLanguageNotSupported { get; set; } + + public string ConfigurationFingerprint { get; set; } + public string Message { get; set; } + } + + /// + /// Coordinates collection-backed AI translation across the 0..n engines + /// configured on a collection (deepl, google, alpha2). Translation is always batched: callers + /// pass an array of text segments per engine and get back translations in the same order. + /// + public class AiTranslationService + { + public const string kValidationProbeText = "Today a reader, tomorrow a leader."; + + private static readonly HttpClient _httpClient = new HttpClient(); + private readonly CollectionSettings _collectionSettings; + private readonly Dictionary _providers; + + public AiTranslationService(CollectionSettings collectionSettings) + : this(collectionSettings, null) { } + + internal AiTranslationService( + CollectionSettings collectionSettings, + Dictionary providers + ) + { + _collectionSettings = collectionSettings; + _providers = + providers + ?? new Dictionary(StringComparer.OrdinalIgnoreCase) + { + { "deepl", new DeepLTranslationProvider() }, + { "google", new GoogleTranslationProvider() }, + { "alpha2", new Alpha2TranslationProvider() }, + }; + } + + /// + /// Translates all segments via the given engine, chunking requests to the provider's + /// per-request segment-count and byte-size limits. Chunks are sent sequentially; the + /// returned array preserves segment order and count, and this throws if any chunk's + /// response doesn't have exactly as many translations as segments sent. + /// + public async Task TranslateSegmentsAsync( + AiTranslationEngineSettings engine, + string[] segments, + string sourceLanguageTag, + CancellationToken ct + ) + { + if (engine == null) + throw new ArgumentNullException(nameof(engine)); + if (segments == null) + throw new ArgumentNullException(nameof(segments)); + + var targetLanguageTag = NormalizeBloomLanguageTag( + _collectionSettings.AiTranslationTargetLanguageTag + ); + if (string.IsNullOrWhiteSpace(targetLanguageTag)) + { + throw new InvalidOperationException( + "Set a target language tag in Collection Settings > AI Source Bubbles." + ); + } + + var provider = GetProvider(engine.ProviderId); + var results = new string[segments.Length]; + var resultIndex = 0; + foreach ( + var chunk in ChunkSegments( + segments, + provider.MaxSegmentsPerRequest, + provider.MaxRequestBytes + ) + ) + { + var translatedChunk = await provider.TranslateBatchAsync( + engine, + chunk, + sourceLanguageTag, + targetLanguageTag, + _httpClient, + ct + ); + if (translatedChunk.Length != chunk.Length) + { + throw new InvalidOperationException( + $"{provider.ProviderId} returned {translatedChunk.Length} translations for a chunk of {chunk.Length} segments." + ); + } + Array.Copy(translatedChunk, 0, results, resultIndex, translatedChunk.Length); + resultIndex += translatedChunk.Length; + } + + return results; + } + + /// + /// Splits segments into chunks that respect both a maximum segment count and a maximum + /// total UTF-8 byte size per chunk. A single segment whose own byte size exceeds the + /// byte cap is still sent, alone, in its own chunk. + /// + internal static IEnumerable ChunkSegments( + string[] segments, + int maxSegmentsPerRequest, + int maxRequestBytes + ) + { + var currentChunk = new List(); + var currentBytes = 0; + foreach (var segment in segments) + { + var segmentBytes = Encoding.UTF8.GetByteCount(segment ?? string.Empty); + var wouldExceedCount = currentChunk.Count >= maxSegmentsPerRequest; + var wouldExceedBytes = + currentChunk.Count > 0 && currentBytes + segmentBytes > maxRequestBytes; + if (wouldExceedCount || wouldExceedBytes) + { + yield return currentChunk.ToArray(); + currentChunk = new List(); + currentBytes = 0; + } + + currentChunk.Add(segment); + currentBytes += segmentBytes; + } + + if (currentChunk.Count > 0) + { + yield return currentChunk.ToArray(); + } + } + + /// + /// Validates one engine's configuration, credentials, and target language. First, if the + /// provider can cheaply list its supported target languages and the chosen target is + /// definitively absent, it short-circuits with TargetLanguageNotSupported (the engine is + /// fine, it just can't do this language, so it will be auto-skipped rather than reported as + /// broken). Otherwise it runs a probe translation. The probe uses the engine's effective + /// source language (only alpha2 differs from the "en" default); the probe text is always + /// English and only proves that the pair/model and credentials work, not translation quality. + /// + public async Task ValidateEngineAsync( + AiTranslationEngineSettings engine, + CancellationToken ct + ) + { + var targetLanguageTag = _collectionSettings.AiTranslationTargetLanguageTag; + var normalizedTarget = NormalizeBloomLanguageTag(targetLanguageTag); + + var unsupportedResult = await CheckTargetLanguageSupportedAsync( + engine, + normalizedTarget, + targetLanguageTag, + ct + ); + if (unsupportedResult != null) + return unsupportedResult; + + var translations = await TranslateSegmentsAsync( + engine, + new[] { kValidationProbeText }, + engine.GetEffectiveSourceLanguageTag(), + ct + ); + + return new AiTranslationValidationResult + { + Succeeded = true, + ConfigurationFingerprint = GetEngineFingerprint(engine, targetLanguageTag), + Message = translations[0], + }; + } + + /// + /// If the provider can list its supported target languages and the chosen target is + /// definitively NOT among them, returns a failed result flagged TargetLanguageNotSupported; + /// otherwise (target is supported, list is empty, or the list couldn't be fetched) returns + /// null so the caller falls through to the normal probe. Never throws: a failure to fetch + /// the list is treated as "unknown", deferring to the probe which surfaces real errors. + /// + private async Task CheckTargetLanguageSupportedAsync( + AiTranslationEngineSettings engine, + string normalizedTarget, + string targetLanguageTag, + CancellationToken ct + ) + { + if (string.IsNullOrWhiteSpace(normalizedTarget)) + return null; + + List supportedTargets; + try + { + var provider = GetProvider(engine.ProviderId); + supportedTargets = await provider.GetSupportedTargetLanguagesAsync( + engine, + BuildAlpha2SourceLanguageTags(engine), + _httpClient, + ct + ); + } + catch (Exception e) + when (e is HttpRequestException + || e is InvalidOperationException + || e is ArgumentException + || e is CryptographicException + || e is Newtonsoft.Json.JsonException + ) + { + // Couldn't determine support; let the probe be the judge. + return null; + } + + if (supportedTargets == null || supportedTargets.Count == 0) + return null; + + var isSupported = supportedTargets.Any(o => + string.Equals( + NormalizeBloomLanguageTag(o.Value), + normalizedTarget, + StringComparison.OrdinalIgnoreCase + ) + ); + if (isSupported) + return null; + + return new AiTranslationValidationResult + { + Succeeded = false, + TargetLanguageNotSupported = true, + ConfigurationFingerprint = GetEngineFingerprint(engine, targetLanguageTag), + // This message is a plain-English fallback for persistence/logs; the settings UI + // shows its own localized "does not support ⟨language⟩, will be skipped" note driven + // by the TargetLanguageNotSupported flag. + Message = + $"{GetProviderDisplayName(engine.ProviderId)} does not support translating to '{normalizedTarget}'.", + }; + } + + /// + /// Gets the union of target languages supported by all ENABLED engines on the collection, + /// deduped by language tag (each option records which of those engines' providers support it). + /// For source-dependent providers (alpha2) the list is looked up for that engine's own + /// configured source language only (see BuildAlpha2SourceLanguageTags). + /// + public async Task> GetSupportedTargetLanguagesAsync( + CancellationToken ct + ) + { + var optionsByTag = new Dictionary( + StringComparer.OrdinalIgnoreCase + ); + var failures = new List(); + foreach (var engine in _collectionSettings.AiTranslationEngines.Where(e => e.Enabled)) + { + var provider = GetProvider(engine.ProviderId); + var likelySourceLanguageTags = BuildAlpha2SourceLanguageTags(engine); + List options; + try + { + options = await provider.GetSupportedTargetLanguagesAsync( + engine, + likelySourceLanguageTags, + _httpClient, + ct + ); + } + catch (Exception e) + when (e is HttpRequestException + || e is InvalidOperationException + || e is ArgumentException + || e is CryptographicException + || e is Newtonsoft.Json.JsonException + ) + { + // One engine failing to list its languages -- e.g. a DeepL key that can + // translate but lacks the "languages:read" scope -- must not blank out the + // whole union. Other engines can still contribute the list, and this engine may + // still translate fine. Remember the failure only so we can report it if NO + // engine manages to return any languages. + failures.Add($"{GetProviderDisplayName(provider.ProviderId)}: {e.Message}"); + continue; + } + + foreach (var option in options) + { + if (optionsByTag.TryGetValue(option.Value, out var existing)) + { + existing.ProviderIds.Add(provider.ProviderId); + } + else + { + optionsByTag[option.Value] = option; + } + } + } + + // Only surface an error when every enabled engine failed to provide any languages; as + // long as one succeeded, the union we return is enough to populate the dropdown. + if (optionsByTag.Count == 0 && failures.Count > 0) + throw new InvalidOperationException(string.Join(" ", failures)); + + return optionsByTag + .Values.OrderBy(o => o.Label, StringComparer.CurrentCultureIgnoreCase) + .ToList(); + } + + /// + /// Lists the source languages the given engine's provider can translate FROM into the + /// collection's target language. Only the alpha2 provider (whose source/target pairing is + /// meaningful) returns anything; other providers return an empty list. Used to populate the + /// Alpha2 source-language chooser in Collection Settings. + /// + public async Task> GetSupportedSourceLanguagesAsync( + AiTranslationEngineSettings engine, + string targetLanguageTag, + CancellationToken ct + ) + { + var provider = GetProvider(engine.ProviderId); + if (provider is Alpha2TranslationProvider alpha2) + { + return await alpha2.GetSupportedSourceLanguagesAsync( + engine, + targetLanguageTag, + _httpClient, + ct + ); + } + + return new List(); + } + + /// + /// Builds the Bloom language tag used for AI content in a translation group for the given engine. + /// + public string GetAiLanguageTagForEngine(AiTranslationEngineSettings engine) + { + return GetAiLanguageTag( + _collectionSettings.AiTranslationTargetLanguageTag, + engine.ProviderId + ); + } + + /// + /// Returns the human-facing name of a translation engine (e.g. "DeepL"), for use in + /// progress messages and other UI. Engine/product names are not localized. + /// + public static string GetProviderDisplayName(string providerId) + { + switch (NormalizeProviderId(providerId)) + { + case "deepl": + return "DeepL"; + case "google": + return "Google Translate"; + case "alpha2": + return "SIL Alpha2"; + default: + return providerId; + } + } + + /// + /// Builds the Bloom language tag used for AI content in a translation group. + /// + public static string GetAiLanguageTag(string targetLanguageTag, string providerId) + { + var normalizedTarget = NormalizeBloomLanguageTag(targetLanguageTag); + var normalizedProvider = NormalizeProviderId(providerId); + if ( + string.IsNullOrWhiteSpace(normalizedTarget) + || string.IsNullOrWhiteSpace(normalizedProvider) + ) + { + return string.Empty; + } + + return $"{normalizedTarget}-x-ai-{normalizedProvider}"; + } + + /// + /// Normalizes the provider id used in settings, API payloads, and AI language tags. + /// + public static string NormalizeProviderId(string providerId) + { + if (string.IsNullOrWhiteSpace(providerId)) + return string.Empty; + + var normalized = providerId.Trim().ToLowerInvariant(); + return normalized switch + { + "googletranslate" => "google", + _ => normalized, + }; + } + + /// + /// Normalizes a Bloom language tag for provider requests while preserving region when available. + /// + public static string NormalizeBloomLanguageTag(string languageTag) + { + if (string.IsNullOrWhiteSpace(languageTag)) + return string.Empty; + + var trimmed = languageTag.Trim(); + var privateUseIndex = trimmed.IndexOf("-x-", StringComparison.OrdinalIgnoreCase); + if (privateUseIndex >= 0) + { + trimmed = trimmed.Substring(0, privateUseIndex); + } + + trimmed = MiscUtils.NormalizeLanguageTagCapitalization(trimmed); + if ( + !IetfLanguageTag.TryGetParts( + trimmed, + out var language, + out var script, + out var region, + out var variant + ) || string.IsNullOrWhiteSpace(language) + ) + { + return trimmed; + } + + language = language.ToLowerInvariant(); + if (!string.IsNullOrWhiteSpace(region)) + { + return $"{language}-{region.ToUpperInvariant()}"; + } + + return language; + } + + /// + /// Extracts the Google Cloud project id from a service account email. + /// + public static string GetGoogleProjectIdFromServiceAccountEmail(string serviceAccountEmail) + { + if (string.IsNullOrWhiteSpace(serviceAccountEmail)) + { + throw new InvalidOperationException( + "Set a Google service account email in Collection Settings > AI Source Bubbles." + ); + } + + var trimmedEmail = serviceAccountEmail.Trim(); + var atIndex = trimmedEmail.IndexOf('@'); + if (atIndex < 0 || atIndex == trimmedEmail.Length - 1) + { + throw new InvalidOperationException( + "Google service account email is not in the expected format." + ); + } + + var domain = trimmedEmail.Substring(atIndex + 1); + const string kExpectedSuffix = ".iam.gserviceaccount.com"; + if (!domain.EndsWith(kExpectedSuffix, StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException( + "Google service account email must end with .iam.gserviceaccount.com." + ); + } + + var projectId = domain.Substring(0, domain.Length - kExpectedSuffix.Length); + if (string.IsNullOrWhiteSpace(projectId)) + { + throw new InvalidOperationException( + "Google service account email does not contain a project id." + ); + } + + return projectId; + } + + /// + /// The subset of a collection's configured engines that are actually ready to translate with: + /// enabled, and last validated against the exact configuration (provider, target language, + /// credentials) they currently have. This is the single source of truth for "is this engine + /// active" -- both the editor's allowAiSourceBubbles flag (see RuntimeInformationInjector) and + /// the whole-book batch updater (AiTranslationBookUpdater) must agree on it. + /// + public static List GetActiveEngines( + CollectionSettings collectionSettings + ) + { + var targetLanguageTag = collectionSettings.AiTranslationTargetLanguageTag; + return collectionSettings + .AiTranslationEngines.Where(engine => + engine.Enabled + && engine.LastValidationSucceeded + && string.Equals( + engine.ValidatedConfigurationFingerprint, + GetEngineFingerprint(engine, targetLanguageTag), + StringComparison.Ordinal + ) + ) + .ToList(); + } + + /// + /// Builds a stable fingerprint of one engine's provider, target language, and + /// provider-specific credentials, without storing raw secrets. + /// + public static string GetEngineFingerprint( + AiTranslationEngineSettings engine, + string targetLanguageTag + ) + { + if (engine == null) + throw new ArgumentNullException(nameof(engine)); + + var normalizedProvider = NormalizeProviderId(engine.ProviderId); + var normalizedTargetLanguageTag = NormalizeBloomLanguageTag(targetLanguageTag); + // Only alpha2 actually uses SourceLanguageTag, but we fold it in uniformly: deepl/google + // leave it blank, so it normalizes to "" and doesn't perturb their fingerprint. Every + // engine re-validates once after this field was added, which is fine (feature unreleased). + var normalizedSourceLanguageTag = NormalizeBloomLanguageTag(engine.SourceLanguageTag); + var credentialKey = normalizedProvider switch + { + "google" => + $"{engine.ServiceAccountEmail?.Trim()}\n{AiTranslationProviderHelpers.NormalizeGooglePrivateKey(engine.PrivateKey)}", + _ => engine.ApiKey?.Trim() ?? string.Empty, + }; + var fingerprintInput = + $"{normalizedProvider}\n{normalizedTargetLanguageTag}\n{normalizedSourceLanguageTag}\n{credentialKey}"; + return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(fingerprintInput))); + } + + /// + /// Builds the deduped, normalized, blank-filtered list of source languages a + /// source-dependent provider (alpha2) should probe when listing its supported targets: + /// the engine's configured (effective) source, English, and the collection's languages. + /// + /// + /// The source language(s) to look up an engine's supported TARGET languages against. Only + /// alpha2 consumes this (its supported targets are per source language). We deliberately use + /// ONLY the engine's own configured source (defaulting to English when blank) -- NOT a union + /// with English and the collection languages. Alpha2 unions its results across every source + /// passed here, and because English can reach a near-superset of targets, including it made + /// the target list look the same no matter which source the user chose (BL-16549). Returned + /// as a single-element list to match the provider's IReadOnlyList<string> parameter. + /// + private static IReadOnlyList BuildAlpha2SourceLanguageTags( + AiTranslationEngineSettings engine + ) + { + var tag = NormalizeBloomLanguageTag(engine.GetEffectiveSourceLanguageTag()); + return string.IsNullOrWhiteSpace(tag) + ? (IReadOnlyList)Array.Empty() + : new[] { tag }; + } + + private IAiTranslationProvider GetProvider(string providerId) + { + var normalizedProviderId = NormalizeProviderId(providerId); + if (string.IsNullOrWhiteSpace(normalizedProviderId)) + { + throw new InvalidOperationException( + "An AI translation engine must have a provider id." + ); + } + + if (!_providers.TryGetValue(normalizedProviderId, out var provider)) + { + throw new InvalidOperationException( + $"Unsupported AI provider '{normalizedProviderId}'." + ); + } + + return provider; + } + } + + internal static class AiTranslationProviderHelpers + { + internal static string NormalizeGooglePrivateKey(string privateKey) + { + return (privateKey ?? string.Empty).Replace("\\r", "").Replace("\\n", "\n").Trim(); + } + + internal static void EnsureSuccess( + HttpResponseMessage response, + string responseContent, + string providerName + ) + { + if (response.IsSuccessStatusCode) + return; + + var providerMessage = responseContent; + try + { + var json = JObject.Parse(responseContent); + providerMessage = + json["message"]?.Value() + ?? json["error"]?.Value() + ?? json["error"]?["message"]?.Value() + ?? json["detail"]?[0]?["msg"]?.Value() + ?? responseContent; + } + catch + { + // Keep the original response text when it isn't JSON. + } + + throw new InvalidOperationException( + $"{providerName} request failed: {(int)response.StatusCode} {response.ReasonPhrase}. {providerMessage}".Trim() + ); + } + + internal static string Base64UrlEncode(byte[] bytes) + { + return Convert.ToBase64String(bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_'); + } + } +} diff --git a/src/BloomExe/AiTranslation/Alpha2TranslationProvider.cs b/src/BloomExe/AiTranslation/Alpha2TranslationProvider.cs new file mode 100644 index 000000000000..0d8600210d1a --- /dev/null +++ b/src/BloomExe/AiTranslation/Alpha2TranslationProvider.cs @@ -0,0 +1,718 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Bloom.Collection; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using SIL.WritingSystems; + +namespace Bloom.AiTranslation +{ + /// + /// AI translation provider backed by the SIL Alpha2 text-collection API + /// (https://alpha2.multilingualai.com). Unlike DeepL/Google, translation here is + /// collection-based and asynchronous: segments are uploaded as a "text collection", a + /// translation job is kicked off against a chosen model, and the caller polls until every + /// segment's translation is complete. + /// + internal sealed class Alpha2TranslationProvider : IAiTranslationProvider + { + private const string kApiBaseUrl = "https://alpha2.multilingualai.com/api"; + + // How often to re-poll a text collection for completed translations, and how long to + // keep polling before giving up. Observed live round trips for tiny batches complete + // within one poll (~4s); larger/production batches are untested at scale, hence the + // generous overall timeout. + private static readonly TimeSpan kPollInterval = TimeSpan.FromSeconds(3); + private static readonly TimeSpan kPollTimeout = TimeSpan.FromMinutes(5); + + // The /v2/translation_languages endpoint is queried once per (apiKey, src|trg iso3) key. + // AiTranslationService is constructed fresh per API call, so this cache MUST be static to + // survive across calls; a short TTL keeps it from going stale while still collapsing the + // burst of queries a single dropdown-open produces. + private static readonly TimeSpan kLanguageCacheTtl = TimeSpan.FromMinutes(10); + private static readonly ConcurrentDictionary< + string, + (DateTime Expiry, List Languages) + > _languageCache = + new ConcurrentDictionary)>(); + + // Lazily-built reverse map from ISO 639-3 code to the shortest registered Bloom/BCP-47 + // language tag, so alpha2's "fra" can dedup with DeepL's "fr" in the service-level union + // (which keys on the option value). See MapIso6393ToBloomTag. + private static readonly Lazy> _iso6393ToBloomTag = new Lazy< + Dictionary + >(BuildIso6393ToBloomTagMap); + + public string ProviderId => "alpha2"; + public int MaxSegmentsPerRequest => 500; + public int MaxRequestBytes => 400_000; + + /// + /// Lists the target languages Alpha2 can translate INTO from the given likely sources. Since + /// Alpha2's supported targets are per source language, this queries + /// /v2/translation_languages?src={iso3} once for each distinct (mappable) likely source and + /// unions the results, deduped by the reverse-mapped Bloom tag. Sources that can't be mapped + /// to ISO 639-3 are skipped; a query that fails for one source doesn't fail the whole list + /// (unless every source query fails); an empty result for a source is not an error. + /// + public async Task> GetSupportedTargetLanguagesAsync( + AiTranslationEngineSettings engine, + IReadOnlyList likelySourceLanguageTags, + HttpClient httpClient, + CancellationToken ct + ) + { + EnsureCredentials(engine); + return await GetLanguagesForAxisAsync( + httpClient, + engine.ApiKey.Trim(), + "src", + likelySourceLanguageTags, + ct + ); + } + + /// + /// Lists the source languages Alpha2 can translate FROM into the given target language, via + /// /v2/translation_languages?trg={iso3}. Used by the Alpha2 source-language chooser in + /// Collection Settings. Returns an empty list if the target can't be mapped to ISO 639-3. + /// + public async Task> GetSupportedSourceLanguagesAsync( + AiTranslationEngineSettings engine, + string targetLanguageTag, + HttpClient httpClient, + CancellationToken ct + ) + { + EnsureCredentials(engine); + return await GetLanguagesForAxisAsync( + httpClient, + engine.ApiKey.Trim(), + "trg", + new[] { targetLanguageTag }, + ct + ); + } + + /// + /// Shared implementation for both directions: maps each pivot language tag to ISO 639-3 + /// (skipping unmappable ones), queries /v2/translation_languages with the given axis + /// parameter ("src" or "trg") once per distinct pivot, and unions the results deduped by + /// reverse-mapped Bloom tag. Throws only if every pivot query failed. + /// + private static async Task> GetLanguagesForAxisAsync( + HttpClient httpClient, + string apiKey, + string axisParam, + IReadOnlyList pivotLanguageTags, + CancellationToken ct + ) + { + var pivotIso3Codes = new List(); + foreach (var pivotTag in pivotLanguageTags ?? Array.Empty()) + { + try + { + var iso3 = MapToIso6393(pivotTag); + if (!pivotIso3Codes.Contains(iso3)) + pivotIso3Codes.Add(iso3); + } + catch (InvalidOperationException) + { + // An unmappable pivot language (e.g. a collection language with no ISO 639-3 + // code) must not break the whole list; just skip it. + } + } + + var optionsByTag = new Dictionary( + StringComparer.OrdinalIgnoreCase + ); + var failures = new List(); + var anySucceeded = false; + foreach (var iso3 in pivotIso3Codes) + { + try + { + var languages = await GetLanguagesForKeyAsync( + httpClient, + apiKey, + axisParam, + iso3, + ct + ); + anySucceeded = true; + foreach (var option in languages) + { + if (!optionsByTag.ContainsKey(option.Value)) + optionsByTag[option.Value] = option; + } + } + catch (Exception e) + when (e is HttpRequestException || e is InvalidOperationException) + { + failures.Add(e.Message); + } + } + + // If we had pivots to query but every one failed, surface the error; otherwise return + // whatever union we managed to build (possibly empty, which is a valid answer). + if (!anySucceeded && failures.Count > 0) + throw new InvalidOperationException(string.Join(" ", failures)); + + return optionsByTag + .Values.OrderBy(o => o.Label, StringComparer.CurrentCultureIgnoreCase) + .ToList(); + } + + /// + /// Fetches (and caches for kLanguageCacheTtl) the languages Alpha2 reports for one + /// axis/iso3 key. Only successful responses are cached. + /// + private static async Task> GetLanguagesForKeyAsync( + HttpClient httpClient, + string apiKey, + string axisParam, + string iso3, + CancellationToken ct + ) + { + var cacheKey = $"{apiKey}\n{axisParam}:{iso3}"; + if ( + _languageCache.TryGetValue(cacheKey, out var cached) + && cached.Expiry > DateTime.UtcNow + ) + { + return cached.Languages; + } + + using var request = new HttpRequestMessage( + HttpMethod.Get, + $"{kApiBaseUrl}/v2/translation_languages?{axisParam}={Uri.EscapeDataString(iso3)}" + ); + request.Headers.Add("api_key", apiKey); + + using var response = await httpClient.SendAsync(request, ct); + var responseContent = await response.Content.ReadAsStringAsync(); + AiTranslationProviderHelpers.EnsureSuccess(response, responseContent, "Alpha2"); + + var languages = ParseTranslationLanguages(responseContent); + _languageCache[cacheKey] = (DateTime.UtcNow + kLanguageCacheTtl, languages); + return languages; + } + + /// + /// Parses a /v2/translation_languages response body -- a JSON array of + /// {iso, name, display, models:[...]} objects -- into target-language options, mapping each + /// entry's ISO 639-3 code back to the shortest registered Bloom tag. Kept internal and + /// HTTP-free so tests can exercise the parsing/reverse-mapping without mocking the network. + /// + internal static List ParseTranslationLanguages( + string responseContent + ) + { + var array = JArray.Parse(responseContent); + var options = new List(); + var seenTags = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var entry in array) + { + var iso = entry["iso"]?.Value(); + if (string.IsNullOrWhiteSpace(iso)) + continue; + + var bloomTag = MapIso6393ToBloomTag(iso); + if (!seenTags.Add(bloomTag)) + continue; + + var name = entry["name"]?.Value(); + options.Add( + new AiTranslationTargetLanguageOption + { + Value = bloomTag, + Label = string.IsNullOrWhiteSpace(name) ? bloomTag : name, + ProviderIds = new List { "alpha2" }, + } + ); + } + + options.Sort( + (first, second) => + StringComparer.CurrentCultureIgnoreCase.Compare(first.Label, second.Label) + ); + return options; + } + + /// + /// Reverse of MapToIso6393: maps an ISO 639-3 code (as Alpha2 returns) to the shortest + /// registered Bloom/BCP-47 language tag (e.g. "fra" -> "fr"), so an Alpha2 language dedups + /// with the same language from DeepL/Google in the settings union. An iso3 with no shorter + /// registered tag (the usual case for languages that only have a 3-letter code) passes + /// through unchanged. + /// + internal static string MapIso6393ToBloomTag(string iso3) + { + if (string.IsNullOrWhiteSpace(iso3)) + return iso3; + + var normalized = iso3.Trim().ToLowerInvariant(); + if (_iso6393ToBloomTag.Value.TryGetValue(normalized, out var bloomTag)) + return bloomTag; + + return normalized; + } + + /// + /// Builds the ISO-639-3 -> shortest-registered-Bloom-tag map once. When several registered + /// tags share an ISO 639-3 code, the shortest wins (ties broken lexicographically) so we + /// prefer the familiar 2-letter tag (e.g. "fr" over "fra"). + /// + private static Dictionary BuildIso6393ToBloomTagMap() + { + var map = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var language in StandardSubtags.RegisteredLanguages) + { + var iso3 = language.Iso3Code; + var code = language.Code; + if (string.IsNullOrWhiteSpace(iso3) || string.IsNullOrWhiteSpace(code)) + continue; + + var key = iso3.ToLowerInvariant(); + var candidate = code.ToLowerInvariant(); + if ( + !map.TryGetValue(key, out var existing) + || candidate.Length < existing.Length + || ( + candidate.Length == existing.Length + && string.CompareOrdinal(candidate, existing) < 0 + ) + ) + { + map[key] = candidate; + } + } + + return map; + } + + /// + /// Translates a batch of segments via Alpha2's text-collection flow: create a source + /// collection, kick off a translation against the first available model for the + /// language pair, poll until every segment is complete, then delete both the source and + /// output collections. + /// + public async Task TranslateBatchAsync( + AiTranslationEngineSettings engine, + string[] segments, + string sourceLanguageTag, + string targetLanguageTag, + HttpClient httpClient, + CancellationToken ct + ) + { + EnsureCredentials(engine); + + var apiKey = engine.ApiKey.Trim(); + var srcIso3 = MapToIso6393(sourceLanguageTag); + var trgIso3 = MapToIso6393(targetLanguageTag); + + var modelId = await GetModelIdAsync(httpClient, apiKey, srcIso3, trgIso3, ct); + var sourceCollectionId = await CreateTextCollectionAsync( + httpClient, + apiKey, + srcIso3, + segments, + ct + ); + long? outputCollectionId = null; + string[] translations; + try + { + outputCollectionId = await RequestTranslationAsync( + httpClient, + apiKey, + sourceCollectionId, + trgIso3, + modelId, + ct + ); + + translations = await PollForTranslationsAsync( + httpClient, + apiKey, + sourceCollectionId, + trgIso3, + modelId, + segments.Length, + ct + ); + } + catch + { + // Best-effort cleanup on the failure path: a collection is still a real, + // storage-consuming resource even when translation failed, but a delete failure + // here must never replace/mask the original, more informative exception. + // Use CancellationToken.None: this path also runs when the user cancels (ct is + // already tripped), and passing ct would make the very first cleanup request throw + // OperationCanceledException, silently orphaning the collections we came here to delete. + await TryDeleteBestEffortAsync( + httpClient, + apiKey, + sourceCollectionId, + CancellationToken.None + ); + if (outputCollectionId.HasValue) + { + await TryDeleteBestEffortAsync( + httpClient, + apiKey, + outputCollectionId.Value, + CancellationToken.None + ); + } + throw; + } + + // Both the source collection and the translate-created output collection are real, + // storage-consuming resources on the Alpha2 side; clean up both. On this success + // path a delete failure is a genuine problem, so let it propagate normally. + await DeleteTextCollectionAsync(httpClient, apiKey, sourceCollectionId, ct); + if (outputCollectionId.HasValue) + { + await DeleteTextCollectionAsync(httpClient, apiKey, outputCollectionId.Value, ct); + } + + return translations; + } + + /// + /// Maps a Bloom language tag (BCP-47, e.g. "en", "en-US", or already ISO 639-3 like + /// "eng") to the ISO 639-3 code Alpha2 requires. A tag whose primary subtag is already + /// three letters is passed through as-is; otherwise it is looked up in libpalaso's + /// registered-languages table. Throws if the tag can't be mapped. + /// + internal static string MapToIso6393(string bloomLanguageTag) + { + if (string.IsNullOrWhiteSpace(bloomLanguageTag)) + { + throw new InvalidOperationException("Alpha2 requires a language tag."); + } + + var normalized = AiTranslationService.NormalizeBloomLanguageTag(bloomLanguageTag); + var primarySubtag = normalized.Split('-')[0].ToLowerInvariant(); + if (primarySubtag.Length == 3) + { + return primarySubtag; + } + + if ( + !StandardSubtags.RegisteredLanguages.TryGet(primarySubtag, out var languageSubtag) + || string.IsNullOrWhiteSpace(languageSubtag.Iso3Code) + ) + { + throw new InvalidOperationException( + $"Cannot map language tag '{bloomLanguageTag}' to an ISO 639-3 code for Alpha2." + ); + } + + return languageSubtag.Iso3Code; + } + + /// + /// Picks the first translation model Alpha2 reports for the given language pair. Throws + /// if no model supports the pair. + /// + private static async Task GetModelIdAsync( + HttpClient httpClient, + string apiKey, + string srcIso3, + string trgIso3, + CancellationToken ct + ) + { + using var request = new HttpRequestMessage( + HttpMethod.Get, + $"{kApiBaseUrl}/v2/translation_models?src={Uri.EscapeDataString(srcIso3)}&trg={Uri.EscapeDataString(trgIso3)}" + ); + request.Headers.Add("api_key", apiKey); + + using var response = await httpClient.SendAsync(request, ct); + var responseContent = await response.Content.ReadAsStringAsync(); + AiTranslationProviderHelpers.EnsureSuccess(response, responseContent, "Alpha2"); + + var models = JArray.Parse(responseContent); + if (models.Count == 0) + { + throw new InvalidOperationException( + $"Alpha2 has no translation model for {srcIso3} -> {trgIso3}." + ); + } + + return models[0]["id"].Value(); + } + + /// + /// Creates a source text collection holding one text per segment, preserving internal + /// newlines within a segment (using the `texts` array, not the newline-splitting `text` + /// field). Returns the new collection's id. + /// + private static async Task CreateTextCollectionAsync( + HttpClient httpClient, + string apiKey, + string srcIso3, + string[] segments, + CancellationToken ct + ) + { + var requestBody = new + { + name = $"bloom-{Guid.NewGuid():N}", + language = srcIso3, + texts = segments, + }; + + using var request = new HttpRequestMessage( + HttpMethod.Post, + $"{kApiBaseUrl}/v2/text_collections" + ); + request.Headers.Add("api_key", apiKey); + request.Content = new StringContent( + JsonConvert.SerializeObject(requestBody), + Encoding.UTF8, + "application/json" + ); + + using var response = await httpClient.SendAsync(request, ct); + var responseContent = await response.Content.ReadAsStringAsync(); + AiTranslationProviderHelpers.EnsureSuccess(response, responseContent, "Alpha2"); + + var json = JObject.Parse(responseContent); + return json["id"].Value(); + } + + /// + /// Kicks off an asynchronous translation job for a source collection. This creates a + /// separate output collection on the Alpha2 side (returned here so it can be deleted + /// later); the source collection's own texts remain untouched. + /// + private static async Task RequestTranslationAsync( + HttpClient httpClient, + string apiKey, + long sourceCollectionId, + string trgIso3, + int modelId, + CancellationToken ct + ) + { + using var request = new HttpRequestMessage( + HttpMethod.Post, + $"{kApiBaseUrl}/v2/text_collections/{sourceCollectionId}/translate?target_language={Uri.EscapeDataString(trgIso3)}&model_id={modelId}" + ); + request.Headers.Add("api_key", apiKey); + request.Content = new StringContent("{}", Encoding.UTF8, "application/json"); + + using var response = await httpClient.SendAsync(request, ct); + var responseContent = await response.Content.ReadAsStringAsync(); + AiTranslationProviderHelpers.EnsureSuccess(response, responseContent, "Alpha2"); + + var json = JObject.Parse(responseContent); + return json["id"]?.Value(); + } + + /// + /// Polls the source collection's texts (with translations included) until every segment + /// reaches a "complete" status, then returns the translated strings in original + /// submission order. The in-progress statuses "pending" and "running" (and a momentarily + /// missing status) are treated as not-yet-done and keep the poll going. Throws if any + /// segment reaches an unrecognized status, if the response's text count doesn't match the + /// segment count, or if the overall poll timeout elapses. + /// + private static async Task PollForTranslationsAsync( + HttpClient httpClient, + string apiKey, + long sourceCollectionId, + string trgIso3, + int modelId, + int expectedCount, + CancellationToken ct + ) + { + var deadline = DateTime.UtcNow + kPollTimeout; + while (true) + { + var texts = await GetTextsWithTranslationsAsync( + httpClient, + apiKey, + sourceCollectionId, + trgIso3, + modelId, + ct + ); + // The endpoint returns texts newest-first; sort ascending by id to recover the + // original submission order (ids are assigned monotonically at creation time). + var sortedTexts = texts.OrderBy(t => t["id"]?.Value() ?? 0L).ToList(); + + // Observed live: immediately after the translate call returns, a poll can see + // fewer texts than were submitted (the per-text translation rows aren't + // queryable yet for a brief moment). Treat missing texts as still-pending rather + // than an error; a genuinely wrong count only becomes an error once nothing is + // pending anymore (or the poll times out). + if (sortedTexts.Count > expectedCount) + { + throw new InvalidOperationException( + $"Alpha2 returned {sortedTexts.Count} texts for {expectedCount} segments." + ); + } + + var pendingCount = expectedCount - sortedTexts.Count; + foreach (var text in sortedTexts) + { + var translation = (text["translations"] as JArray)?.FirstOrDefault(); + var status = translation?["translation_status"]?.Value(); + // Complete segments are done; in-progress ones keep the poll going; anything + // else throws (see IsTranslationComplete). + if (!IsTranslationComplete(status)) + { + pendingCount++; + } + } + + if (pendingCount == 0) + { + return sortedTexts + .Select(t => + (t["translations"] as JArray) + ?.FirstOrDefault() + ?["text"]?.Value() ?? string.Empty + ) + .ToArray(); + } + + if (DateTime.UtcNow >= deadline) + { + throw new TimeoutException( + $"Alpha2 translation timed out after {kPollTimeout.TotalMinutes} minutes with {pendingCount} of {expectedCount} texts still pending." + ); + } + + await Task.Delay(kPollInterval, ct); + } + } + + /// + /// Classifies a segment's Alpha2 translation_status while polling. Returns true when the + /// segment is finished ("complete"). Returns false for the in-progress statuses Alpha2 + /// walks a translation through — "pending" (queued) and "running" (translating) — as well + /// as a momentarily-missing status seen right after the translate call kicks off. Throws + /// for any other (unrecognized) status, which we treat as a hard failure rather than + /// silently waiting forever on a segment that will never complete. + /// + internal static bool IsTranslationComplete(string status) + { + if (string.Equals(status, "complete", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + if ( + string.IsNullOrEmpty(status) + || string.Equals(status, "pending", StringComparison.OrdinalIgnoreCase) + || string.Equals(status, "running", StringComparison.OrdinalIgnoreCase) + ) + { + return false; + } + + throw new InvalidOperationException( + $"Alpha2 translation failed for a segment: unexpected status '{status}'." + ); + } + + /// + /// Fetches the source collection's texts along with their translations for the given + /// target language/model, used both while polling and (implicitly) once complete. + /// + private static async Task GetTextsWithTranslationsAsync( + HttpClient httpClient, + string apiKey, + long sourceCollectionId, + string trgIso3, + int modelId, + CancellationToken ct + ) + { + using var request = new HttpRequestMessage( + HttpMethod.Get, + $"{kApiBaseUrl}/v2/text_collections/{sourceCollectionId}/texts?include_translations=true&target_language={Uri.EscapeDataString(trgIso3)}&translation_model_id={modelId}" + ); + request.Headers.Add("api_key", apiKey); + + using var response = await httpClient.SendAsync(request, ct); + var responseContent = await response.Content.ReadAsStringAsync(); + AiTranslationProviderHelpers.EnsureSuccess(response, responseContent, "Alpha2"); + + return JArray.Parse(responseContent); + } + + /// + /// Deletes a text collection. Note the delete endpoint uses the singular + /// "text_collection" path, unlike every other Alpha2 endpoint which uses the plural + /// "text_collections". + /// + private static async Task DeleteTextCollectionAsync( + HttpClient httpClient, + string apiKey, + long collectionId, + CancellationToken ct + ) + { + using var request = new HttpRequestMessage( + HttpMethod.Delete, + $"{kApiBaseUrl}/v2/text_collection/{collectionId}" + ); + request.Headers.Add("api_key", apiKey); + + using var response = await httpClient.SendAsync(request, ct); + var responseContent = await response.Content.ReadAsStringAsync(); + AiTranslationProviderHelpers.EnsureSuccess(response, responseContent, "Alpha2"); + } + + /// + /// Deletes a text collection, swallowing any failure. Used only when cleaning up after a + /// translation that has already failed, where a secondary delete failure would otherwise + /// mask the original, more informative exception. + /// + private static async Task TryDeleteBestEffortAsync( + HttpClient httpClient, + string apiKey, + long collectionId, + CancellationToken ct + ) + { + try + { + await DeleteTextCollectionAsync(httpClient, apiKey, collectionId, ct); + } + catch + { + // Deliberately swallowed: see summary above. + } + } + + /// + /// Validates that an API key is configured. + /// + private static void EnsureCredentials(AiTranslationEngineSettings engine) + { + if (string.IsNullOrWhiteSpace(engine.ApiKey)) + { + throw new InvalidOperationException( + "Set an Alpha2 API key in Collection Settings > AI Source Bubbles." + ); + } + } + } +} diff --git a/src/BloomExe/AiTranslation/DeepLTranslationProvider.cs b/src/BloomExe/AiTranslation/DeepLTranslationProvider.cs new file mode 100644 index 000000000000..27f7ffa92add --- /dev/null +++ b/src/BloomExe/AiTranslation/DeepLTranslationProvider.cs @@ -0,0 +1,153 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Bloom.Collection; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace Bloom.AiTranslation +{ + /// + /// AI translation provider backed by the DeepL API. + /// + internal sealed class DeepLTranslationProvider : IAiTranslationProvider + { + public string ProviderId => "deepl"; + public int MaxSegmentsPerRequest => 50; + public int MaxRequestBytes => 120_000; + + public async Task> GetSupportedTargetLanguagesAsync( + AiTranslationEngineSettings engine, + IReadOnlyList likelySourceLanguageTags, + HttpClient httpClient, + CancellationToken ct + ) + { + // DeepL's target list is source-independent, so likelySourceLanguageTags is ignored. + EnsureCredentials(engine); + + using var request = new HttpRequestMessage( + HttpMethod.Get, + GetApiBaseUrl(engine.ApiKey) + "/v2/languages?type=target" + ); + request.Headers.Authorization = new AuthenticationHeaderValue( + "DeepL-Auth-Key", + engine.ApiKey.Trim() + ); + + using var response = await httpClient.SendAsync(request, ct); + var responseContent = await response.Content.ReadAsStringAsync(); + AiTranslationProviderHelpers.EnsureSuccess(response, responseContent, "DeepL"); + + var languages = JArray.Parse(responseContent); + var options = new List(); + foreach (var languageToken in languages) + { + var languageCode = languageToken["language"]?.Value(); + if (string.IsNullOrWhiteSpace(languageCode)) + { + continue; + } + + var normalizedLanguageCode = AiTranslationService.NormalizeBloomLanguageTag( + languageCode + ); + var name = languageToken["name"]?.Value() ?? normalizedLanguageCode; + options.Add( + new AiTranslationTargetLanguageOption + { + Value = normalizedLanguageCode, + Label = name, + ProviderIds = new List { ProviderId }, + } + ); + } + + options.Sort( + (first, second) => + StringComparer.CurrentCultureIgnoreCase.Compare(first.Label, second.Label) + ); + return options; + } + + public async Task TranslateBatchAsync( + AiTranslationEngineSettings engine, + string[] segments, + string sourceLanguageTag, + string targetLanguageTag, + HttpClient httpClient, + CancellationToken ct + ) + { + EnsureCredentials(engine); + + var requestBody = new + { + text = segments, + source_lang = NormalizeDeepLLanguageTag(sourceLanguageTag), + target_lang = NormalizeDeepLLanguageTag(targetLanguageTag), + }; + + using var request = new HttpRequestMessage(HttpMethod.Post, GetEndpoint(engine.ApiKey)); + request.Content = new StringContent( + JsonConvert.SerializeObject(requestBody), + Encoding.UTF8, + "application/json" + ); + request.Headers.Authorization = new AuthenticationHeaderValue( + "DeepL-Auth-Key", + engine.ApiKey.Trim() + ); + + using var response = await httpClient.SendAsync(request, ct); + var responseContent = await response.Content.ReadAsStringAsync(); + AiTranslationProviderHelpers.EnsureSuccess(response, responseContent, "DeepL"); + + var responseJson = JObject.Parse(responseContent); + var translations = responseJson["translations"] as JArray; + if (translations == null || translations.Count != segments.Length) + { + throw new InvalidOperationException( + $"DeepL returned {translations?.Count ?? 0} translations for {segments.Length} segments." + ); + } + + return translations.Select(t => t["text"]?.Value() ?? string.Empty).ToArray(); + } + + private static void EnsureCredentials(AiTranslationEngineSettings engine) + { + if (string.IsNullOrWhiteSpace(engine.ApiKey)) + { + throw new InvalidOperationException( + "Set a DeepL API key in Collection Settings > AI Source Bubbles." + ); + } + } + + private static string GetEndpoint(string apiKey) + { + return GetApiBaseUrl(apiKey) + "/v2/translate"; + } + + private static string GetApiBaseUrl(string apiKey) + { + return apiKey.Trim().EndsWith(":fx", StringComparison.OrdinalIgnoreCase) + ? "https://api-free.deepl.com" + : "https://api.deepl.com"; + } + + private static string NormalizeDeepLLanguageTag(string languageTag) + { + return AiTranslationService + .NormalizeBloomLanguageTag(languageTag) + .Replace('_', '-') + .ToUpperInvariant(); + } + } +} diff --git a/src/BloomExe/AiTranslation/GoogleTranslationProvider.cs b/src/BloomExe/AiTranslation/GoogleTranslationProvider.cs new file mode 100644 index 000000000000..8ee24628dcf0 --- /dev/null +++ b/src/BloomExe/AiTranslation/GoogleTranslationProvider.cs @@ -0,0 +1,250 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Bloom.Collection; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace Bloom.AiTranslation +{ + /// + /// AI translation provider backed by the Google Cloud Translation API, + /// authenticated via a service account. + /// + internal sealed class GoogleTranslationProvider : IAiTranslationProvider + { + private const string kScope = "https://www.googleapis.com/auth/cloud-translation"; + private const string kTokenEndpoint = "https://oauth2.googleapis.com/token"; + private const string kTranslateEndpoint = + "https://translation.googleapis.com/language/translate/v2"; + private const string kSupportedLanguagesEndpointTemplate = + "https://translation.googleapis.com/v3/projects/{0}/locations/global/supportedLanguages?display_language_code=en"; + + public string ProviderId => "google"; + public int MaxSegmentsPerRequest => 128; + public int MaxRequestBytes => 100_000; + + public async Task> GetSupportedTargetLanguagesAsync( + AiTranslationEngineSettings engine, + IReadOnlyList likelySourceLanguageTags, + HttpClient httpClient, + CancellationToken ct + ) + { + // Google's target list is source-independent, so likelySourceLanguageTags is ignored. + EnsureCredentials(engine); + + var accessToken = await GetAccessTokenAsync(engine, httpClient, ct); + var projectId = AiTranslationService.GetGoogleProjectIdFromServiceAccountEmail( + engine.ServiceAccountEmail + ); + using var request = new HttpRequestMessage( + HttpMethod.Get, + string.Format(kSupportedLanguagesEndpointTemplate, Uri.EscapeDataString(projectId)) + ); + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", accessToken); + + using var response = await httpClient.SendAsync(request, ct); + var responseContent = await response.Content.ReadAsStringAsync(); + AiTranslationProviderHelpers.EnsureSuccess( + response, + responseContent, + "Google Translate" + ); + + var responseJson = JObject.Parse(responseContent); + var languages = responseJson["languages"] as JArray; + var options = new List(); + if (languages == null) + { + return options; + } + + foreach (var languageToken in languages) + { + var supportsTarget = languageToken["supportTarget"]?.Value() ?? false; + if (!supportsTarget) + { + continue; + } + + var languageCode = languageToken["languageCode"]?.Value(); + if (string.IsNullOrWhiteSpace(languageCode)) + { + continue; + } + + var normalizedLanguageCode = AiTranslationService.NormalizeBloomLanguageTag( + languageCode + ); + var displayName = + languageToken["displayName"]?.Value() ?? normalizedLanguageCode; + options.Add( + new AiTranslationTargetLanguageOption + { + Value = normalizedLanguageCode, + Label = displayName, + ProviderIds = new List { ProviderId }, + } + ); + } + + options.Sort( + (first, second) => + StringComparer.CurrentCultureIgnoreCase.Compare(first.Label, second.Label) + ); + return options; + } + + public async Task TranslateBatchAsync( + AiTranslationEngineSettings engine, + string[] segments, + string sourceLanguageTag, + string targetLanguageTag, + HttpClient httpClient, + CancellationToken ct + ) + { + EnsureCredentials(engine); + + var accessToken = await GetAccessTokenAsync(engine, httpClient, ct); + var fields = new List>(); + foreach (var segment in segments) + { + fields.Add(new KeyValuePair("q", segment)); + } + fields.Add(new KeyValuePair("target", targetLanguageTag)); + fields.Add(new KeyValuePair("format", "text")); + var normalizedSourceLanguage = AiTranslationService.NormalizeBloomLanguageTag( + sourceLanguageTag + ); + if (!string.IsNullOrWhiteSpace(normalizedSourceLanguage)) + { + fields.Add(new KeyValuePair("source", normalizedSourceLanguage)); + } + + using var request = new HttpRequestMessage(HttpMethod.Post, kTranslateEndpoint) + { + Content = new FormUrlEncodedContent(fields), + }; + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", accessToken); + + using var response = await httpClient.SendAsync(request, ct); + var responseContent = await response.Content.ReadAsStringAsync(); + AiTranslationProviderHelpers.EnsureSuccess( + response, + responseContent, + "Google Translate" + ); + + var responseJson = JObject.Parse(responseContent); + var translations = responseJson["data"]?["translations"] as JArray; + if (translations == null || translations.Count != segments.Length) + { + throw new InvalidOperationException( + $"Google Translate returned {translations?.Count ?? 0} translations for {segments.Length} segments." + ); + } + + return translations + .Select(t => + WebUtility.HtmlDecode(t["translatedText"]?.Value() ?? string.Empty) + ) + .ToArray(); + } + + private static void EnsureCredentials(AiTranslationEngineSettings engine) + { + if (string.IsNullOrWhiteSpace(engine.ServiceAccountEmail)) + { + throw new InvalidOperationException( + "Set a Google service account email in Collection Settings > AI Source Bubbles." + ); + } + if (string.IsNullOrWhiteSpace(engine.PrivateKey)) + { + throw new InvalidOperationException( + "Set a Google service account private key in Collection Settings > AI Source Bubbles." + ); + } + } + + private static async Task GetAccessTokenAsync( + AiTranslationEngineSettings engine, + HttpClient httpClient, + CancellationToken ct + ) + { + var now = DateTimeOffset.UtcNow; + var jwtHeader = AiTranslationProviderHelpers.Base64UrlEncode( + Encoding.UTF8.GetBytes( + JsonConvert.SerializeObject(new { alg = "RS256", typ = "JWT" }) + ) + ); + var jwtPayload = AiTranslationProviderHelpers.Base64UrlEncode( + Encoding.UTF8.GetBytes( + JsonConvert.SerializeObject( + new + { + iss = engine.ServiceAccountEmail, + scope = kScope, + aud = kTokenEndpoint, + iat = now.ToUnixTimeSeconds(), + exp = now.AddMinutes(59).ToUnixTimeSeconds(), + } + ) + ) + ); + var signingInput = $"{jwtHeader}.{jwtPayload}"; + var signedJwt = $"{signingInput}.{SignJwt(signingInput, engine.PrivateKey)}"; + + using var tokenRequest = new HttpRequestMessage(HttpMethod.Post, kTokenEndpoint) + { + Content = new FormUrlEncodedContent( + new[] + { + new KeyValuePair( + "grant_type", + "urn:ietf:params:oauth:grant-type:jwt-bearer" + ), + new KeyValuePair("assertion", signedJwt), + } + ), + }; + + using var tokenResponse = await httpClient.SendAsync(tokenRequest, ct); + var tokenContent = await tokenResponse.Content.ReadAsStringAsync(); + AiTranslationProviderHelpers.EnsureSuccess(tokenResponse, tokenContent, "Google OAuth"); + + var tokenJson = JObject.Parse(tokenContent); + var accessToken = tokenJson["access_token"]?.Value(); + if (string.IsNullOrWhiteSpace(accessToken)) + { + throw new InvalidOperationException("Google OAuth returned no access token."); + } + + return accessToken; + } + + private static string SignJwt(string signingInput, string privateKey) + { + using var rsa = RSA.Create(); + rsa.ImportFromPem( + AiTranslationProviderHelpers.NormalizeGooglePrivateKey(privateKey).ToCharArray() + ); + var signature = rsa.SignData( + Encoding.UTF8.GetBytes(signingInput), + HashAlgorithmName.SHA256, + RSASignaturePadding.Pkcs1 + ); + return AiTranslationProviderHelpers.Base64UrlEncode(signature); + } + } +} diff --git a/src/BloomExe/AiTranslation/IAiTranslationProvider.cs b/src/BloomExe/AiTranslation/IAiTranslationProvider.cs new file mode 100644 index 000000000000..7d8b15bca42e --- /dev/null +++ b/src/BloomExe/AiTranslation/IAiTranslationProvider.cs @@ -0,0 +1,73 @@ +using System.Collections.Generic; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Newtonsoft.Json; + +namespace Bloom.AiTranslation +{ + /// + /// Option surfaced to the Collection Settings target-language picker. + /// + public class AiTranslationTargetLanguageOption + { + [JsonProperty("value")] + public string Value { get; set; } + + [JsonProperty("label")] + public string Label { get; set; } + + /// + /// The ids of the (enabled) providers that were found to support this target language. + /// + [JsonProperty("providerIds")] + public List ProviderIds { get; set; } = new List(); + } + + /// + /// Contract implemented by each AI translation backend (DeepL, Google, Alpha2, ...). All + /// translation is batched: callers pass an array of text segments and get back translations + /// in the same order, chunked to fit this provider's per-request limits. + /// + internal interface IAiTranslationProvider + { + string ProviderId { get; } + + /// + /// Max number of text segments per HTTP request (DeepL: 50, Google: 128). + /// + int MaxSegmentsPerRequest { get; } + + /// + /// Conservative max total UTF-8 bytes of segment text per request (DeepL: 120_000, Google: 100_000). + /// + int MaxRequestBytes { get; } + + /// + /// Lists the target languages this provider supports for the given engine. + /// likelySourceLanguageTags is a hint used only by providers (alpha2) whose supported-target + /// set depends on the source language: the caller passes the sources it is likely to + /// translate from (the engine's configured source, "en", and the collection's languages). + /// Providers that don't need it (deepl/google) ignore the hint. + /// + Task> GetSupportedTargetLanguagesAsync( + AiTranslationEngineSettings engine, + IReadOnlyList likelySourceLanguageTags, + HttpClient httpClient, + CancellationToken ct + ); + + /// + /// Translates segments preserving order and count; implementations must throw if the + /// service returns a different number of translations than segments. + /// + Task TranslateBatchAsync( + AiTranslationEngineSettings engine, + string[] segments, + string sourceLanguageTag, + string targetLanguageTag, + HttpClient httpClient, + CancellationToken ct + ); + } +} diff --git a/src/BloomExe/Book/Book.cs b/src/BloomExe/Book/Book.cs index 621e2d7bf032..0d47998bc5a4 100644 --- a/src/BloomExe/Book/Book.cs +++ b/src/BloomExe/Book/Book.cs @@ -5,6 +5,7 @@ using System.IO; using System.Linq; using System.Net; +using System.Net.Http; using System.Security.Cryptography; using System.Text; using System.Text.RegularExpressions; diff --git a/src/BloomExe/Book/RuntimeInformationInjector.cs b/src/BloomExe/Book/RuntimeInformationInjector.cs index 40eded781fbe..48c0c9466bcb 100644 --- a/src/BloomExe/Book/RuntimeInformationInjector.cs +++ b/src/BloomExe/Book/RuntimeInformationInjector.cs @@ -2,9 +2,11 @@ using System.Collections.Generic; using System.Linq; using System.Xml; +using Bloom.AiTranslation; using Bloom.Collection; using Bloom.Properties; using Bloom.SafeXml; +using Bloom.SubscriptionAndFeatures; using L10NSharp; using Newtonsoft.Json; using SIL.IO; @@ -459,7 +461,7 @@ IFileLocator fileLocator var element = pageDom.RawDom.CreateElement("script"); element.SetAttribute("type", "text/javascript"); element.SetAttribute("id", "ui-settings"); - var d = new Dictionary(); + var d = new Dictionary(); //d.Add("urlOfUIFiles", "file:///" + fileLocator.LocateDirectory("ui", "ui files directory")); if (!String.IsNullOrEmpty(Settings.Default.LastSourceLanguageViewed)) @@ -490,6 +492,22 @@ IFileLocator fileLocator .ToLocalhost() ); + var aiSourceBubblesFeatureStatus = FeatureStatus.GetFeatureStatus( + bookData.CollectionSettings.Subscription, + FeatureName.AiSourceBubbles + ); + // At least one engine must be enabled and validated, with that validation still + // matching its current configuration (provider, target language, credentials). + var anyEngineReady = AiTranslationService + .GetActiveEngines(bookData.CollectionSettings) + .Any(); + d.Add( + "allowAiSourceBubbles", + aiSourceBubblesFeatureStatus.Visible + && aiSourceBubblesFeatureStatus.Enabled + && anyEngineReady + ); + element.InnerText = String.Format( "function GetSettings() {{ return {0};}}", JsonConvert.SerializeObject(d) diff --git a/src/BloomExe/Collection/CollectionSettings.cs b/src/BloomExe/Collection/CollectionSettings.cs index 1a55db3ea6e7..bc8181af2b69 100644 --- a/src/BloomExe/Collection/CollectionSettings.cs +++ b/src/BloomExe/Collection/CollectionSettings.cs @@ -7,6 +7,7 @@ using System.Text; using System.Xml.Linq; using System.Xml.Serialization; +using Bloom.AiTranslation; using Bloom.Api; using Bloom.Book; using Bloom.MiscUI; @@ -72,6 +73,44 @@ public class CollectionSettings // if this is null, relevant code uses the default, so we don't have to initialize it here public string BadgeQrCodeLabel; + /// + /// The provider ids for AI translation, in the fixed order used for settings + /// persistence and the settings UI. AiTranslationEngines always has exactly one + /// entry for each of these, in this order; see EnsureAiTranslationEngines(). + /// + public static readonly string[] kAiTranslationProviderIds = { "deepl", "google", "alpha2" }; + + /// + /// The single target language shared by all AI translation engines. + /// + public string AiTranslationTargetLanguageTag = ""; + + /// + /// Per-engine AI translation configuration. Normalized (see + /// EnsureAiTranslationEngines()) to contain exactly one entry for each provider in + /// kAiTranslationProviderIds, in that order. + /// + public List AiTranslationEngines = + new List(); + + /// + /// Ensures AiTranslationEngines contains exactly one entry for each provider in + /// kAiTranslationProviderIds, in that order, preserving any existing settings for + /// providers already present and adding defaults for any that are missing. + /// + public void EnsureAiTranslationEngines() + { + var normalized = new List(); + foreach (var providerId in kAiTranslationProviderIds) + { + var existing = AiTranslationEngines?.Find(e => e.ProviderId == providerId); + normalized.Add( + existing ?? new AiTranslationEngineSettings { ProviderId = providerId } + ); + } + AiTranslationEngines = normalized; + } + public static readonly Dictionary CssNumberStylesToCultureOrDigits = new Dictionary() { @@ -198,6 +237,7 @@ public CollectionSettings() AudioRecordingMode = TalkingBookApi.AudioRecordingMode.Sentence; AudioRecordingTrimEndMilliseconds = kDefaultAudioRecordingTrimEndMilliseconds; BooksOnWebGoal = kDefaultBooksOnWebGoal; + EnsureAiTranslationEngines(); } public static void CreateNewCollection(NewCollectionSettings collectionInfo) @@ -400,6 +440,41 @@ public void Save() xml.Add(BulkPublishBloomPubSettings.ToXElement()); xml.Add(new XElement("ShowBlorgLanguageQrCode", ShowBlorgLanguageQrCode)); xml.Add(new XElement("BadgeQrCodeLabel", BadgeQrCodeLabel)); + xml.Add(new XElement("AiTranslationTargetLanguageTag", AiTranslationTargetLanguageTag)); + EnsureAiTranslationEngines(); + foreach (var engine in AiTranslationEngines) + { + var engineElement = new XElement("AiTranslationEngine"); + engineElement.Add(new XAttribute("id", engine.ProviderId)); + engineElement.Add(new XAttribute("enabled", engine.Enabled)); + engineElement.Add( + new XAttribute("validationSucceeded", engine.LastValidationSucceeded) + ); + engineElement.Add( + new XAttribute( + "validationTargetLanguageNotSupported", + engine.LastValidationTargetLanguageNotSupported + ) + ); + engineElement.Add( + new XAttribute( + "validatedFingerprint", + engine.ValidatedConfigurationFingerprint ?? "" + ) + ); + engineElement.Add(new XElement("ApiKey", engine.ApiKey ?? "")); + engineElement.Add( + new XElement("ServiceAccountEmail", engine.ServiceAccountEmail ?? "") + ); + engineElement.Add(new XElement("PrivateKey", engine.PrivateKey ?? "")); + engineElement.Add( + new XElement("SourceLanguageTag", engine.SourceLanguageTag ?? "") + ); + engineElement.Add( + new XElement("ValidationMessage", engine.LastValidationMessage ?? "") + ); + xml.Add(engineElement); + } RobustIO.SaveXElement(xml, SettingsFilePath); // Color palette settings are stored in a separate Json file @@ -674,6 +749,46 @@ public void Load() ShowBlorgLanguageQrCode = ReadBoolean(xml, "ShowBlorgLanguageQrCode", true); BadgeQrCodeLabel = ReadString(xml, "BadgeQrCodeLabel", ""); + AiTranslationTargetLanguageTag = ReadString( + xml, + "AiTranslationTargetLanguageTag", + "" + ); + // This is an unreleased experiment, so there is no migration from the old + // single-provider settings: any old AiTranslation* elements are just ignored. + AiTranslationEngines = xml.Descendants("AiTranslationEngine") + .Select(engineElement => + { + bool.TryParse(engineElement.Attribute("enabled")?.Value, out var enabled); + bool.TryParse( + engineElement.Attribute("validationSucceeded")?.Value, + out var validationSucceeded + ); + bool.TryParse( + engineElement.Attribute("validationTargetLanguageNotSupported")?.Value, + out var validationTargetLanguageNotSupported + ); + return new AiTranslationEngineSettings + { + ProviderId = engineElement.Attribute("id")?.Value ?? "", + Enabled = enabled, + LastValidationSucceeded = validationSucceeded, + LastValidationTargetLanguageNotSupported = + validationTargetLanguageNotSupported, + ValidatedConfigurationFingerprint = + engineElement.Attribute("validatedFingerprint")?.Value ?? "", + ApiKey = engineElement.Element("ApiKey")?.Value ?? "", + ServiceAccountEmail = + engineElement.Element("ServiceAccountEmail")?.Value ?? "", + PrivateKey = engineElement.Element("PrivateKey")?.Value ?? "", + SourceLanguageTag = + engineElement.Element("SourceLanguageTag")?.Value ?? "", + LastValidationMessage = + engineElement.Element("ValidationMessage")?.Value ?? "", + }; + }) + .ToList(); + EnsureAiTranslationEngines(); LoadDictionary(xml, "Palette", ColorPalettes); } diff --git a/src/BloomExe/Collection/CollectionSettingsDialog.cs b/src/BloomExe/Collection/CollectionSettingsDialog.cs index 6634a1586b89..e2366bceef92 100644 --- a/src/BloomExe/Collection/CollectionSettingsDialog.cs +++ b/src/BloomExe/Collection/CollectionSettingsDialog.cs @@ -1,7 +1,9 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.Linq; using System.Windows.Forms; +using Bloom.AiTranslation; using Bloom.Book; using Bloom.MiscUI; using Bloom.Properties; @@ -51,6 +53,9 @@ public string PendingDefaultBookshelf internal bool PendingAllowTeamCollection; internal bool PendingAllowAppBuilder; + internal bool PendingAllowAiSourceBubbles; + public string PendingAiTranslationTargetLanguageTag; + public List PendingAiTranslationEngines; internal bool AllowTeamCollectionOptionEnabled = false; // "Internal" so CollectionSettingsApi can update these. @@ -121,6 +126,15 @@ XMatterPackFinder xmatterPackFinder PendingAllowAppBuilder = ExperimentalFeatures.IsFeatureEnabled( ExperimentalFeatures.kAppBuilder ); + PendingAllowAiSourceBubbles = ExperimentalFeatures.IsFeatureEnabled( + ExperimentalFeatures.kAiSourceBubbles + ); + PendingAiTranslationTargetLanguageTag = + _collectionSettings.AiTranslationTargetLanguageTag; + _collectionSettings.EnsureAiTranslationEngines(); + PendingAiTranslationEngines = _collectionSettings + .AiTranslationEngines.Select(engine => engine.Clone()) + .ToList(); if ( !ExperimentalFeatures.IsFeatureEnabled(ExperimentalFeatures.kTeamCollections) @@ -410,6 +424,12 @@ private void _okButton_Click(object sender, EventArgs e) UpdateExperimentalBookSources(); UpdateTeamCollectionAllowed(); UpdateAppBuilderAllowed(); + UpdateAiSourceBubblesAllowed(); + _collectionSettings.AiTranslationTargetLanguageTag = + PendingAiTranslationTargetLanguageTag; + _collectionSettings.AiTranslationEngines = PendingAiTranslationEngines + .Select(engine => engine.Clone()) + .ToList(); _collectionSettings.Country = _countryText.Text.Trim(); _collectionSettings.Province = _provinceText.Text.Trim(); @@ -829,5 +849,14 @@ private void UpdateAppBuilderAllowed() // NB: This change does not require a restart. ExperimentalFeatures.SetValue(ExperimentalFeatures.kAppBuilder, PendingAllowAppBuilder); } + + private void UpdateAiSourceBubblesAllowed() + { + // NB: This change does not require a restart. + ExperimentalFeatures.SetValue( + ExperimentalFeatures.kAiSourceBubbles, + PendingAllowAiSourceBubbles + ); + } } } diff --git a/src/BloomExe/Edit/EditingModel.cs b/src/BloomExe/Edit/EditingModel.cs index 1c70e4d99c69..3e5ef9dbf809 100644 --- a/src/BloomExe/Edit/EditingModel.cs +++ b/src/BloomExe/Edit/EditingModel.cs @@ -9,6 +9,7 @@ using System.Text; using System.Threading.Tasks; using System.Windows.Forms; +using Bloom.AiTranslation; using Bloom.Api; using Bloom.Book; using Bloom.Collection; @@ -55,6 +56,7 @@ public class EditingModel private IPage _previouslySelectedPage; private BloomServer _server; private readonly BloomWebSocketServer _webSocketServer; + private readonly AiTranslationBookUpdater _aiTranslationBookUpdater; internal IPage PageChangingLayout; // used to save the page on which the choose different layout command was invoked while the dialog is active. // This event fires after the EditingModel has finished responding to a PageSelection change. @@ -105,7 +107,8 @@ public EditingModel( CollectionSettings collectionSettings, BloomServer server, BloomWebSocketServer webSocketServer, - ITemplateFinder sourceCollectionsList + ITemplateFinder sourceCollectionsList, + AiTranslationBookUpdater aiTranslationBookUpdater ) { _bookSelection = bookSelection; @@ -114,6 +117,7 @@ ITemplateFinder sourceCollectionsList _server = server; _webSocketServer = webSocketServer; _sourceCollectionsList = sourceCollectionsList; + _aiTranslationBookUpdater = aiTranslationBookUpdater; _stateMachine = new EditingStateMachine( // navigate, @@ -949,6 +953,15 @@ public void OnBecomeVisible() var contentLanguages = GetMultilingualContentLanguages(); CurrentBook.SetMultilingualContentLanguages(contentLanguages); CurrentBook.PrepareForEditing(); + + // Scan the book for AI Source Bubbles translation work (missing/stale translations + // for any active engine, or stale AI content needing cleanup) and, if any is found, + // block here -- showing a modal progress dialog -- until it is translated, applied, + // and saved, or the user cancels. This must happen before the page below is loaded, so + // the user edits pages that already reflect any newly-applied AI translations. When + // nothing needs doing (the normal case once a book is up to date), this returns + // immediately with no dialog and no delay. + _aiTranslationBookUpdater.UpdateBookIfNeeded(CurrentBook); } _currentlyDisplayedBook = CurrentBook; diff --git a/src/BloomExe/ExperimentalFeatures.cs b/src/BloomExe/ExperimentalFeatures.cs index fc8f2c61df57..0e89dfa9eb5d 100644 --- a/src/BloomExe/ExperimentalFeatures.cs +++ b/src/BloomExe/ExperimentalFeatures.cs @@ -11,6 +11,7 @@ public static class ExperimentalFeatures public const string kExperimentalSourceBooks = "experimental-source-books"; public const string kTeamCollections = "team-collections"; public const string kAppBuilder = "app-builder"; + public const string kAiSourceBubbles = "ai-source-bubbles"; public static string TokensOfEnabledFeatures => Settings.Default.EnabledExperimentalFeatures; diff --git a/src/BloomExe/ProjectContext.cs b/src/BloomExe/ProjectContext.cs index 7a681fcbc377..4fd43e56100f 100644 --- a/src/BloomExe/ProjectContext.cs +++ b/src/BloomExe/ProjectContext.cs @@ -6,6 +6,7 @@ using System.Reflection; using System.Windows.Forms; using Autofac; +using Bloom.AiTranslation; using Bloom.Api; using Bloom.Book; using Bloom.Collection; @@ -161,6 +162,8 @@ IContainer parentContainer typeof(RabPublishApi), typeof(AccessibilityCheckApi), typeof(CollectionSettingsApi), + typeof(AiTranslationService), + typeof(AiTranslationBookUpdater), typeof(SubscriptionSettingsEditorApi), typeof(FeatureStatusApi), typeof(CollectionTabView), diff --git a/src/BloomExe/SubscriptionAndFeatures/FeatureRegistry.cs b/src/BloomExe/SubscriptionAndFeatures/FeatureRegistry.cs index d12cc757a6a1..738df5853212 100644 --- a/src/BloomExe/SubscriptionAndFeatures/FeatureRegistry.cs +++ b/src/BloomExe/SubscriptionAndFeatures/FeatureRegistry.cs @@ -29,6 +29,7 @@ public enum FeatureName BulkBloomPub, Bookshelf, AppBuilder, + AiSourceBubbles, } public static class FeatureRegistry @@ -194,6 +195,12 @@ public static class FeatureRegistry SubscriptionTier = SubscriptionTier.Pro, ExperimentalFeatureToken = Bloom.ExperimentalFeatures.kAppBuilder, }, + new FeatureInfo + { + Feature = FeatureName.AiSourceBubbles, + SubscriptionTier = SubscriptionTier.Pro, + ExperimentalFeatureToken = Bloom.ExperimentalFeatures.kAiSourceBubbles, + }, // ---------------------------------------- // Enterprise Tier Features // ---------------------------------------- diff --git a/src/BloomExe/web/controllers/BookCommandsApi.cs b/src/BloomExe/web/controllers/BookCommandsApi.cs index 09b81828c49b..c481ed0a9a19 100644 --- a/src/BloomExe/web/controllers/BookCommandsApi.cs +++ b/src/BloomExe/web/controllers/BookCommandsApi.cs @@ -8,6 +8,7 @@ using System.Text; using System.Threading.Tasks; using System.Windows.Forms; +using Bloom.AiTranslation; using Bloom.Api; using Bloom.Book; using Bloom.Collection; @@ -181,6 +182,19 @@ public void RegisterWithApiHandler(BloomApiHandler apiHandler) }, true ); + apiHandler.RegisterEndpointHandler( + "bookCommand/removeAiSourceTranslations", + (request) => + { + var book = GetBookObjectFromPost(request); + HandleRemoveAiSourceTranslations(book); + request.PostSucceeded(); + }, + // On the UI thread (it shows a modal confirmation dialog) but not sync: the dialog + // makes further API requests that would deadlock if this endpoint held the lock. + true, + false + ); } public void RequestButtonLabelUpdate(string collectionPath, string id) @@ -517,6 +531,35 @@ private void HandleBringBookUpToDate(Book.Book book) } } + /// + /// Removes every AI-generated source translation from the book, saves it, and shows a + /// confirmation dialog. Reached only from the (save-permission-gated) "Remove AI source + /// translations" collection-tab menu item. + /// + private void HandleRemoveAiSourceTranslations(Book.Book book) + { + // RemoveAllAiDivs finds AI content purely by its "-x-ai" language marker, so the + // scanner needs no engine/target-language/source-priority context here. + var scanner = new AiTranslationBookScanner( + book.OurHtmlDom, + targetLanguageTag: null, + enabledEngines: null, + sourceLanguagePriorities: null + ); + var removedCount = scanner.RemoveAllAiDivs(); + book.Save(); + Logger.WriteEvent( + $"AI translation: removed {removedCount} AI source translation div(s) at user request." + ); + + BloomMessageBox.ShowInfo( + LocalizationManager.GetString( + "CollectionTab.BookMenu.RemovedAiSourceTranslations", + "Bloom removed the AI source translations from this book." + ) + ); + } + private BookInfo GetBookInfoFromPost(ApiRequest request) { var bookId = request.RequiredPostString(); diff --git a/src/BloomExe/web/controllers/CollectionSettingsApi.cs b/src/BloomExe/web/controllers/CollectionSettingsApi.cs index 0cc43ad69eb5..960417a24f7f 100644 --- a/src/BloomExe/web/controllers/CollectionSettingsApi.cs +++ b/src/BloomExe/web/controllers/CollectionSettingsApi.cs @@ -2,7 +2,14 @@ using System.Collections.Generic; using System.Dynamic; using System.Globalization; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Security.Cryptography; using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Bloom.AiTranslation; using Bloom.Api; using Bloom.Book; using Bloom.Collection; @@ -80,13 +87,34 @@ public void RegisterWithApiHandler(BloomApiHandler apiHandler) var dialog = DialogBeingEdited; if (dialog != null) { - StoreAdvancedSettingsData(request, dialog); + StoreAdvancedSettingsData( + JObject.Parse(request.RequiredPostJson()), + dialog + ); } request.PostSucceeded(); } }, true ); + apiHandler.RegisterAsyncEndpointHandler( + kApiUrlPart + "validateAiTranslationEngine", + HandleValidateAiTranslationEngineAsync, + false, + true + ); + apiHandler.RegisterAsyncEndpointHandler( + kApiUrlPart + "aiTranslationSupportedLanguages", + HandleGetAiTranslationSupportedLanguagesAsync, + false, + true + ); + apiHandler.RegisterAsyncEndpointHandler( + kApiUrlPart + "aiTranslationAlpha2SourceLanguages", + HandleGetAiTranslationAlpha2SourceLanguagesAsync, + false, + true + ); apiHandler.RegisterBooleanEndpointHandler( kApiUrlPart + "lockedToOneDownloadedBook", request => _collectionSettings.EditingABlorgBook, @@ -318,6 +346,11 @@ private object GetAdvancedSettingsData() ), allowAppBuilder = dialog?.PendingAllowAppBuilder ?? ExperimentalFeatures.IsFeatureEnabled(ExperimentalFeatures.kAppBuilder), + allowAiSourceBubbles = dialog?.PendingAllowAiSourceBubbles + ?? ExperimentalFeatures.IsFeatureEnabled( + ExperimentalFeatures.kAiSourceBubbles + ), + aiTranslation = GetAiTranslationData(dialog), showQrCode = dialog?.PendingShowQrCode ?? _collectionSettings.ShowBlorgLanguageQrCode, qrcodeCaption = dialog?.PendingBadgeQrCodeCaption @@ -330,9 +363,305 @@ private object GetAdvancedSettingsData() }; } - private void StoreAdvancedSettingsData(ApiRequest request, CollectionSettingsDialog dialog) + /// + /// Builds the aiTranslation payload sent to the client: the shared target language and, + /// always in deepl/google/alpha2 order, each engine's configuration plus whether its + /// stored validation is still up to date with that configuration. + /// + private object GetAiTranslationData(CollectionSettingsDialog dialog) + { + var targetLanguageTag = + dialog?.PendingAiTranslationTargetLanguageTag + ?? _collectionSettings.AiTranslationTargetLanguageTag; + if (dialog == null) + _collectionSettings.EnsureAiTranslationEngines(); + var engines = + dialog?.PendingAiTranslationEngines ?? _collectionSettings.AiTranslationEngines; + + return new + { + targetLanguageTag, + engines = engines + .Select(engine => + (object) + new + { + providerId = engine.ProviderId, + enabled = engine.Enabled, + apiKey = engine.ApiKey ?? "", + serviceAccountEmail = engine.ServiceAccountEmail ?? "", + privateKey = engine.PrivateKey ?? "", + sourceLanguageTag = engine.SourceLanguageTag ?? "", + validation = new + { + succeeded = engine.LastValidationSucceeded, + targetLanguageNotSupported = engine.LastValidationTargetLanguageNotSupported, + message = engine.LastValidationMessage ?? "", + upToDate = String.Equals( + engine.ValidatedConfigurationFingerprint, + AiTranslationService.GetEngineFingerprint( + engine, + targetLanguageTag + ), + StringComparison.Ordinal + ), + }, + } + ) + .ToArray(), + }; + } + + private static void InvalidateEngineValidation(AiTranslationEngineSettings engine) + { + engine.ValidatedConfigurationFingerprint = String.Empty; + engine.LastValidationSucceeded = false; + engine.LastValidationTargetLanguageNotSupported = false; + engine.LastValidationMessage = String.Empty; + } + + /// + /// Validates one AI translation engine (identified by providerId in the post body) + /// using the PENDING settings currently being edited in Collection Settings, and stores + /// the result and fingerprint back on that pending engine. + /// + private async Task HandleValidateAiTranslationEngineAsync(ApiRequest request) + { + if (request.HttpMethod != HttpMethods.Post) + { + request.Failed(HttpStatusCode.MethodNotAllowed, "Only POST is supported."); + return; + } + + var dialog = DialogBeingEdited; + if (dialog == null) + { + request.Failed( + HttpStatusCode.BadRequest, + "AI translation can only be validated while Collection Settings is open." + ); + return; + } + + var requestJson = JObject.Parse(request.RequiredPostJson()); + var providerId = AiTranslationService.NormalizeProviderId( + requestJson["providerId"]?.Value() + ); + var engine = dialog.PendingAiTranslationEngines.Single(e => e.ProviderId == providerId); + + var succeeded = false; + var targetLanguageNotSupported = false; + var message = String.Empty; + var tempSettings = new CollectionSettings + { + Subscription = _collectionSettings.Subscription, + AiTranslationTargetLanguageTag = dialog.PendingAiTranslationTargetLanguageTag, + }; + + try + { + var validationResult = await new AiTranslationService( + tempSettings + ).ValidateEngineAsync(engine, CancellationToken.None); + succeeded = validationResult.Succeeded; + targetLanguageNotSupported = validationResult.TargetLanguageNotSupported; + message = validationResult.Message; + engine.ValidatedConfigurationFingerprint = + validationResult.ConfigurationFingerprint; + } + catch (ArgumentException e) + { + message = e.Message; + } + catch (InvalidOperationException e) + { + message = e.Message; + } + catch (HttpRequestException e) + { + message = e.Message; + } + catch (CryptographicException e) + { + message = e.Message; + } + catch (JsonException e) + { + message = e.Message; + } + + engine.LastValidationSucceeded = succeeded; + engine.LastValidationTargetLanguageNotSupported = targetLanguageNotSupported; + engine.LastValidationMessage = message; + // A genuine failure clears the fingerprint so we re-probe next time. The + // "target not supported" outcome, though, is a settled fact for this exact + // configuration: keep the fingerprint (already set from the result above) so the UI + // shows the persisted "will be skipped" note without re-probing on every reopen. + if (!succeeded && !targetLanguageNotSupported) + { + engine.ValidatedConfigurationFingerprint = String.Empty; + } + + request.ReplyWithJson( + new + { + succeeded, + targetLanguageNotSupported, + message, + } + ); + } + + /// + /// Gets the union of provider-backed target languages across the currently-enabled + /// (pending, if Collection Settings is open) AI translation engines. + /// + private async Task HandleGetAiTranslationSupportedLanguagesAsync(ApiRequest request) { - var data = JObject.Parse(request.RequiredPostJson()); + if (request.HttpMethod != HttpMethods.Post) + { + request.Failed(HttpStatusCode.MethodNotAllowed, "Only POST is supported."); + return; + } + + var dialog = DialogBeingEdited; + if (dialog == null) + _collectionSettings.EnsureAiTranslationEngines(); + var tempSettings = new CollectionSettings + { + Subscription = _collectionSettings.Subscription, + AiTranslationTargetLanguageTag = + dialog?.PendingAiTranslationTargetLanguageTag + ?? _collectionSettings.AiTranslationTargetLanguageTag, + AiTranslationEngines = ( + dialog?.PendingAiTranslationEngines ?? _collectionSettings.AiTranslationEngines + ) + .Select(engine => engine.Clone()) + .ToList(), + }; + + try + { + var options = await new AiTranslationService( + tempSettings + ).GetSupportedTargetLanguagesAsync(CancellationToken.None); + var languages = options.Select(option => new + { + tag = option.Value, + name = option.Label, + providerIds = option.ProviderIds, + }); + request.ReplyWithJson(new { languages, message = String.Empty }); + } + catch (ArgumentException e) + { + request.ReplyWithJson( + new { languages = Array.Empty(), message = e.Message } + ); + } + catch (InvalidOperationException e) + { + request.ReplyWithJson( + new { languages = Array.Empty(), message = e.Message } + ); + } + catch (HttpRequestException e) + { + request.ReplyWithJson( + new { languages = Array.Empty(), message = e.Message } + ); + } + catch (CryptographicException e) + { + request.ReplyWithJson( + new { languages = Array.Empty(), message = e.Message } + ); + } + catch (JsonException e) + { + request.ReplyWithJson( + new { languages = Array.Empty(), message = e.Message } + ); + } + } + + /// + /// Gets the (pending, if Collection Settings is open) source languages the Alpha2 engine can + /// translate FROM into the currently-chosen target language, for the Alpha2 source-language + /// chooser. Replies with an empty list when no target is set or Alpha2 has no API key. + /// + private async Task HandleGetAiTranslationAlpha2SourceLanguagesAsync(ApiRequest request) + { + if (request.HttpMethod != HttpMethods.Post) + { + request.Failed(HttpStatusCode.MethodNotAllowed, "Only POST is supported."); + return; + } + + var dialog = DialogBeingEdited; + if (dialog == null) + _collectionSettings.EnsureAiTranslationEngines(); + var targetLanguageTag = + dialog?.PendingAiTranslationTargetLanguageTag + ?? _collectionSettings.AiTranslationTargetLanguageTag; + var engines = + dialog?.PendingAiTranslationEngines ?? _collectionSettings.AiTranslationEngines; + var alpha2Engine = engines.FirstOrDefault(e => + AiTranslationService.NormalizeProviderId(e.ProviderId) == "alpha2" + ); + + if ( + string.IsNullOrWhiteSpace(targetLanguageTag) + || alpha2Engine == null + || string.IsNullOrWhiteSpace(alpha2Engine.ApiKey) + ) + { + request.ReplyWithJson( + new { languages = Array.Empty(), message = String.Empty } + ); + return; + } + + var tempSettings = new CollectionSettings + { + Subscription = _collectionSettings.Subscription, + AiTranslationTargetLanguageTag = targetLanguageTag, + }; + + try + { + var options = await new AiTranslationService( + tempSettings + ).GetSupportedSourceLanguagesAsync( + alpha2Engine.Clone(), + targetLanguageTag, + CancellationToken.None + ); + var languages = options.Select(option => new + { + tag = option.Value, + name = option.Label, + providerIds = option.ProviderIds, + }); + request.ReplyWithJson(new { languages, message = String.Empty }); + } + catch (Exception e) + when (e is ArgumentException + || e is InvalidOperationException + || e is HttpRequestException + || e is CryptographicException + || e is JsonException + ) + { + request.ReplyWithJson( + new { languages = Array.Empty(), message = e.Message } + ); + } + } + + private void StoreAdvancedSettingsData(JObject data, CollectionSettingsDialog dialog) + { + var aiTranslationConfigurationChanged = false; var autoUpdateToken = data["autoUpdate"]; if (autoUpdateToken != null) @@ -360,6 +689,98 @@ private void StoreAdvancedSettingsData(ApiRequest request, CollectionSettingsDia dialog.PendingAllowAppBuilder = allowAppBuilder; } + var allowAiSourceBubblesToken = data["allowAiSourceBubbles"]; + if (allowAiSourceBubblesToken != null) + { + var allowAiSourceBubbles = allowAiSourceBubblesToken.Value(); + dialog.PendingAllowAiSourceBubbles = allowAiSourceBubbles; + } + + var aiTranslationToken = data["aiTranslation"]; + if (aiTranslationToken != null) + { + var targetLanguageTagToken = aiTranslationToken["targetLanguageTag"]; + if (targetLanguageTagToken != null) + { + var targetLanguageTag = targetLanguageTagToken.Value(); + if ( + !String.Equals( + dialog.PendingAiTranslationTargetLanguageTag, + targetLanguageTag, + StringComparison.Ordinal + ) + ) + { + dialog.PendingAiTranslationTargetLanguageTag = targetLanguageTag; + foreach (var engineToInvalidate in dialog.PendingAiTranslationEngines) + InvalidateEngineValidation(engineToInvalidate); + aiTranslationConfigurationChanged = true; + } + } + + if (aiTranslationToken["engines"] is JArray enginesToken) + { + foreach (var engineToken in enginesToken) + { + var providerId = AiTranslationService.NormalizeProviderId( + engineToken["providerId"]?.Value() + ); + var engine = dialog.PendingAiTranslationEngines.SingleOrDefault(e => + e.ProviderId == providerId + ); + if (engine == null) + continue; + + var enabled = engineToken["enabled"]?.Value() ?? engine.Enabled; + var apiKey = engineToken["apiKey"]?.Value() ?? engine.ApiKey; + var serviceAccountEmail = + engineToken["serviceAccountEmail"]?.Value() + ?? engine.ServiceAccountEmail; + var privateKey = + engineToken["privateKey"]?.Value() ?? engine.PrivateKey; + var sourceLanguageTag = + engineToken["sourceLanguageTag"]?.Value() + ?? engine.SourceLanguageTag; + + var engineChanged = + enabled != engine.Enabled + || !String.Equals(apiKey, engine.ApiKey, StringComparison.Ordinal) + || !String.Equals( + serviceAccountEmail, + engine.ServiceAccountEmail, + StringComparison.Ordinal + ) + || !String.Equals( + privateKey, + engine.PrivateKey, + StringComparison.Ordinal + ) + || !String.Equals( + sourceLanguageTag, + engine.SourceLanguageTag, + StringComparison.Ordinal + ); + + engine.Enabled = enabled; + engine.ApiKey = apiKey; + engine.ServiceAccountEmail = serviceAccountEmail; + engine.PrivateKey = privateKey; + engine.SourceLanguageTag = sourceLanguageTag; + + if (engineChanged) + { + InvalidateEngineValidation(engine); + aiTranslationConfigurationChanged = true; + } + } + } + } + + if (aiTranslationConfigurationChanged) + { + dialog.ChangeThatRequiresRestart(); + } + var showQrCodeToken = data["showQrCode"]; if (showQrCodeToken != null) { diff --git a/src/BloomTests/AiTranslation/AiTranslationBookScannerTests.cs b/src/BloomTests/AiTranslation/AiTranslationBookScannerTests.cs new file mode 100644 index 000000000000..c68879a44fda --- /dev/null +++ b/src/BloomTests/AiTranslation/AiTranslationBookScannerTests.cs @@ -0,0 +1,715 @@ +using System.Collections.Generic; +using System.Linq; +using Bloom.AiTranslation; +using Bloom.Book; +using Bloom.Collection; +using Bloom.SafeXml; +using NUnit.Framework; + +namespace BloomTests.AiTranslation +{ + [TestFixture] + public class AiTranslationBookScannerTests + { + private static HtmlDom MakeBookDom(string pagesHtml, string dataDivHtml = "") + { + return new HtmlDom( + $@" +
    {dataDivHtml}
    + {pagesHtml} + " + ); + } + + [Test] + public void Scan_FindsGroupsInDocumentOrder_WithSourceTextChosenFromPriorities() + { + var dom = MakeBookDom( + @" +
    +
    +
    English 1
    +
    French 1
    +
    +
    +
    +
    +
    English 2
    +
    French 2
    +
    +
    " + ); + Assert.That( + dom.RawDom.SafeSelectNodes( + "//div[contains(@class,'bloom-translationGroup')]" + ).Length, + Is.EqualTo(2), + "sanity check: fixture should have 2 translation groups" + ); + + var scanner = new AiTranslationBookScanner( + dom, + "es", + new List(), + new[] { "fr", "en" } + ); + + var scan = scanner.Scan(); + + Assert.That(scan.Groups.Count, Is.EqualTo(2)); + Assert.That(scan.Groups[0].SourceText, Is.EqualTo("French 1")); + Assert.That(scan.Groups[0].SourceLanguageTag, Is.EqualTo("fr")); + Assert.That(scan.Groups[1].SourceText, Is.EqualTo("French 2")); + } + + [Test] + public void Scan_PriorityFallback_UsesSecondPriorityWhenFirstEmpty_ThenAnyNonEmptyIfNoPriorityMatches() + { + var dom = MakeBookDom( + @" +
    +
    +
    +
    English text
    +
    +
    +
    Spanish text
    +
    +
    " + ); + + var scanner = new AiTranslationBookScanner( + dom, + "de", + new List(), + new[] { "fr", "en" } + ); + + var scan = scanner.Scan(); + + Assert.That(scan.Groups.Count, Is.EqualTo(2)); + Assert.That( + scan.Groups[0].SourceText, + Is.EqualTo("English text"), + "first priority (fr) is empty, so it should fall back to the second priority (en)" + ); + Assert.That( + scan.Groups[1].SourceText, + Is.EqualTo("Spanish text"), + "no priority matches (only 'es' is present), so it should fall back to the first non-empty editable" + ); + } + + [Test] + public void ChooseSourceDiv_SkipsZLanguageAndAiLanguageEditables() + { + var dom = MakeBookDom( + @" +
    +
    +
    Z placeholder
    +
    Existing AI text
    +
    English text
    +
    +
    " + ); + + var scanner = new AiTranslationBookScanner( + dom, + "es", + new List(), + new string[0] // no priorities match, so this exercises the fallback path + ); + + var scan = scanner.Scan(); + + Assert.That(scan.Groups.Count, Is.EqualTo(1)); + Assert.That(scan.Groups[0].SourceText, Is.EqualTo("English text")); + } + + [Test] + public void Scan_SkipsGroupsWithEmptyOrWhitespaceOnlySourceText() + { + var dom = MakeBookDom( + @" +
    +
    +
    +
    +
    +
    Real text
    +
    +
    " + ); + + var scanner = new AiTranslationBookScanner( + dom, + "es", + new List(), + new[] { "en" } + ); + + var scan = scanner.Scan(); + + Assert.That(scan.Groups.Count, Is.EqualTo(1)); + Assert.That(scan.Groups[0].SourceText, Is.EqualTo("Real text")); + } + + [Test] + public void Scan_SkipsNoSourceBubbleReadOnlyAndOtherBookDataGroups() + { + var dom = MakeBookDom( + @" +
    +
    +
    Should be skipped 1
    +
    +
    +
    Should be skipped 2
    +
    +
    +
    Should be skipped 3
    +
    +
    +
    Should be included
    +
    +
    " + ); + + var scanner = new AiTranslationBookScanner( + dom, + "es", + new List(), + new[] { "en" } + ); + + var scan = scanner.Scan(); + + Assert.That(scan.Groups.Count, Is.EqualTo(1)); + Assert.That(scan.Groups[0].SourceText, Is.EqualTo("Should be included")); + } + + [Test] + public void Scan_IncludesBookTitleGroup_WithIsBookTitleTrue() + { + var dom = MakeBookDom( + @" +
    +
    +
    My Book
    +
    +
    " + ); + + var scanner = new AiTranslationBookScanner( + dom, + "es", + new List(), + new[] { "en" } + ); + + var scan = scanner.Scan(); + + Assert.That(scan.Groups.Count, Is.EqualTo(1)); + Assert.That(scan.Groups[0].IsBookTitle, Is.True); + Assert.That(scan.Groups[0].SourceText, Is.EqualTo("My Book")); + } + + [Test] + public void GroupsNeedingTranslation_CurrentFingerprintNotIncluded_WrongFingerprintOrEmptyIncluded() + { + var engine = new AiTranslationEngineSettings { ProviderId = "deepl", Enabled = true }; + var aiTag = AiTranslationService.GetAiLanguageTag("es", "deepl"); + Assert.That(aiTag, Is.EqualTo("es-x-ai-deepl"), "sanity check: expected AI tag format"); + var currentFingerprint = AiTranslationBookScanner.ComputeFingerprint( + "en", + "Hello", + aiTag + ); + + var dom = MakeBookDom( + $@" +
    +
    +
    Hello
    +
    Bonjour
    +
    +
    +
    World
    +
    OldTranslation
    +
    +
    +
    Empty div case
    +
    +
    +
    " + ); + + var scanner = new AiTranslationBookScanner( + dom, + "es", + new List { engine }, + new[] { "en" } + ); + + var scan = scanner.Scan(); + Assert.That( + scan.Groups.Count, + Is.EqualTo(3), + "sanity check: all three groups should be eligible" + ); + + var needing = scan.GroupsNeedingTranslation(engine); + + Assert.That(needing.Count, Is.EqualTo(2)); + Assert.That( + needing.Select(g => g.SourceText), + Is.EquivalentTo(new[] { "World", "Empty div case" }) + ); + } + + [Test] + public void ApplyTranslation_WritesDivWithClassLangFingerprint_AndReplacesOnSecondCall() + { + var engine = new AiTranslationEngineSettings { ProviderId = "deepl", Enabled = true }; + var dom = MakeBookDom( + @" +
    +
    +
    Hello
    +
    +
    " + ); + + var scanner = new AiTranslationBookScanner( + dom, + "es", + new List { engine }, + new[] { "en" } + ); + var group = scanner.Scan().Groups.Single(); + Assert.That( + group.SourceText, + Is.EqualTo("Hello"), + "sanity check on chosen source text" + ); + + scanner.ApplyTranslation(group, engine, "Hola"); + + var aiTag = AiTranslationService.GetAiLanguageTag("es", "deepl"); + var aiDivs = group.GroupElement.SafeSelectElements($"div[@lang='{aiTag}']"); + Assert.That( + aiDivs.Length, + Is.EqualTo(1), + "sanity check: exactly one AI div should exist after the first apply" + ); + + var aiDiv = aiDivs[0]; + Assert.That(aiDiv.InnerText, Is.EqualTo("Hola")); + Assert.That(aiDiv.HasClass("bloom-editable"), Is.True); + Assert.That(aiDiv.HasClass("bloom-ai-translation"), Is.True); + var expectedFingerprint = AiTranslationBookScanner.ComputeFingerprint( + "en", + "Hello", + aiTag + ); + Assert.That(aiDiv.GetAttribute("data-ai-fingerprint"), Is.EqualTo(expectedFingerprint)); + + // Applying again should replace the existing div, not duplicate it. + scanner.ApplyTranslation(group, engine, "Hola de nuevo"); + var aiDivsAfterSecondApply = group.GroupElement.SafeSelectElements( + $"div[@lang='{aiTag}']" + ); + Assert.That(aiDivsAfterSecondApply.Length, Is.EqualTo(1)); + Assert.That(aiDivsAfterSecondApply[0].InnerText, Is.EqualTo("Hola de nuevo")); + } + + [Test] + public void ApplyTranslation_ForBookTitleGroup_WritesDataDivEntry() + { + var engine = new AiTranslationEngineSettings { ProviderId = "google", Enabled = true }; + var dom = MakeBookDom( + @" +
    +
    +
    My Book
    +
    +
    " + ); + + var scanner = new AiTranslationBookScanner( + dom, + "fr", + new List { engine }, + new[] { "en" } + ); + var group = scanner.Scan().Groups.Single(); + Assert.That( + group.IsBookTitle, + Is.True, + "sanity check: this group should be recognized as the book title group" + ); + + scanner.ApplyTranslation(group, engine, "Mon Livre"); + + var aiTag = AiTranslationService.GetAiLanguageTag("fr", "google"); + var dataDiv = + dom.RawDom.SelectSingleNode("//div[@id='bloomDataDiv']") as SafeXmlElement; + var titleEntries = dataDiv.SafeSelectElements( + $"div[@data-book='bookTitle' and @lang='{aiTag}']" + ); + Assert.That(titleEntries.Length, Is.EqualTo(1)); + Assert.That(titleEntries[0].InnerText, Is.EqualTo("Mon Livre")); + } + + [Test] + public void ApplyTranslation_BookTitleGroup_NoDataDivPresent_DoesNotThrow() + { + var engine = new AiTranslationEngineSettings { ProviderId = "deepl", Enabled = true }; + var dom = new HtmlDom( + @" +
    +
    +
    My Book
    +
    +
    + " + ); + + var scanner = new AiTranslationBookScanner( + dom, + "es", + new List { engine }, + new[] { "en" } + ); + var group = scanner.Scan().Groups.Single(); + + Assert.DoesNotThrow(() => scanner.ApplyTranslation(group, engine, "Mi Libro")); + } + + [Test] + public void RemoveStaleAiDivs_RemovesDisabledEngineAndStaleFingerprintAndOrphanedDivs_KeepsCurrentOnes_CleansDataDiv() + { + var enabledEngine = new AiTranslationEngineSettings + { + ProviderId = "deepl", + Enabled = true, + }; + const string targetTag = "es"; + var enabledAiTag = AiTranslationService.GetAiLanguageTag(targetTag, "deepl"); + var disabledAiTag = AiTranslationService.GetAiLanguageTag(targetTag, "google"); // google isn't in the enabled-engines list below + var currentFingerprint = AiTranslationBookScanner.ComputeFingerprint( + "en", + "Hello", + enabledAiTag + ); + + var dom = MakeBookDom( + $@" +
    +
    +
    Hello
    +
    Hola
    +
    Should be removed (disabled engine)
    +
    +
    +
    World
    +
    Should be removed (stale fingerprint)
    +
    +
    +
    Excluded group
    +
    Should be removed (orphaned)
    +
    +
    ", + dataDivHtml: $@"
    Stale title translation
    " + ); + + Assert.That( + dom.RawDom.SafeSelectNodes("//div[@lang and contains(@lang,'-x-ai')]").Length, + Is.EqualTo(5), + "sanity check: fixture should start with 5 AI divs total (4 in pages, 1 in the data div)" + ); + + var scanner = new AiTranslationBookScanner( + dom, + targetTag, + new List { enabledEngine }, + new[] { "en" } + ); + + var removedCount = scanner.RemoveStaleAiDivs(); + + Assert.That( + removedCount, + Is.EqualTo(4), + "3 in the pages (disabled-engine, stale-fingerprint, orphaned) plus the data-div bookTitle entry (no matching valid page-level div)" + ); + + var remainingAiDivs = dom.RawDom.SafeSelectElements( + "//div[@lang and contains(@lang,'-x-ai')]" + ); + Assert.That(remainingAiDivs.Length, Is.EqualTo(1)); + Assert.That(remainingAiDivs[0].GetAttribute("lang"), Is.EqualTo(enabledAiTag)); + Assert.That(remainingAiDivs[0].InnerText, Is.EqualTo("Hola")); + } + + [Test] + public void RemoveStaleAiDivs_KeepsCurrentBookTitleDataDivEntry_WhenPageLevelDivIsStillCurrent() + { + var engine = new AiTranslationEngineSettings { ProviderId = "deepl", Enabled = true }; + const string targetTag = "es"; + var aiTag = AiTranslationService.GetAiLanguageTag(targetTag, "deepl"); + var currentFingerprint = AiTranslationBookScanner.ComputeFingerprint( + "en", + "My Book", + aiTag + ); + + var dom = MakeBookDom( + $@" +
    +
    +
    My Book
    +
    Mi Libro
    +
    +
    ", + dataDivHtml: $@"
    Mi Libro
    " + ); + + var scanner = new AiTranslationBookScanner( + dom, + targetTag, + new List { engine }, + new[] { "en" } + ); + + var removedCount = scanner.RemoveStaleAiDivs(); + + Assert.That(removedCount, Is.EqualTo(0)); + var dataDiv = + dom.RawDom.SelectSingleNode("//div[@id='bloomDataDiv']") as SafeXmlElement; + Assert.That( + dataDiv + .SafeSelectElements($"div[@data-book='bookTitle' and @lang='{aiTag}']") + .Length, + Is.EqualTo(1), + "the current data-div bookTitle entry should be kept since the page-level div is still current" + ); + } + + [Test] + public void RemoveAllAiDivs_RemovesEveryAiDivIncludingCurrentOnes_KeepsNonAiDivs_CleansDataDiv() + { + const string targetTag = "es"; + var deepLTag = AiTranslationService.GetAiLanguageTag(targetTag, "deepl"); + var googleTag = AiTranslationService.GetAiLanguageTag(targetTag, "google"); + // A perfectly current translation: RemoveStaleAiDivs would keep this, but RemoveAllAiDivs + // must remove it too. + var currentFingerprint = AiTranslationBookScanner.ComputeFingerprint( + "en", + "Hello", + deepLTag + ); + + var dom = MakeBookDom( + $@" +
    +
    +
    Hello
    +
    Hola
    +
    Hola (google)
    +
    +
    +
    Excluded group
    +
    Orphaned AI text
    +
    +
    ", + dataDivHtml: $@"
    Title translation
    " + ); + + Assert.That( + dom.RawDom.SafeSelectNodes("//div[@lang and contains(@lang,'-x-ai')]").Length, + Is.EqualTo(4), + "sanity check: fixture should start with 4 AI divs total (3 in pages, 1 in the data div)" + ); + + // Constructed with no engine/target/priority context at all, to prove RemoveAllAiDivs + // does not depend on any of them (this is how the menu command calls it). + var scanner = new AiTranslationBookScanner(dom, null, null, null); + + var removedCount = scanner.RemoveAllAiDivs(); + + Assert.That( + removedCount, + Is.EqualTo(4), + "every AI div should be removed: current, disabled-engine, orphaned, and the data-div entry" + ); + Assert.That( + dom.RawDom.SafeSelectNodes("//div[@lang and contains(@lang,'-x-ai')]").Length, + Is.EqualTo(0), + "no AI divs should remain anywhere in the book" + ); + // The non-AI source divs must be untouched. + Assert.That( + dom.RawDom.SafeSelectNodes("//div[@lang='en']").Length, + Is.EqualTo(2), + "both English source divs should be left in place" + ); + } + + [Test] + public void GroupsNeedingTranslation_FixedSourceEngine_UsesConfiguredSourceLanguageText() + { + var alpha2 = new AiTranslationEngineSettings + { + ProviderId = "alpha2", + Enabled = true, + SourceLanguageTag = "fr", + }; + var dom = MakeBookDom( + @" +
    +
    +
    Hello
    +
    Bonjour
    +
    +
    " + ); + var scanner = new AiTranslationBookScanner( + dom, + "es", + new[] { alpha2 }, + new[] { "en" } // priority prefers en, but alpha2's fixed source must win for alpha2. + ); + + var needing = scanner.Scan().GroupsNeedingTranslation(alpha2); + + Assert.That(needing.Count, Is.EqualTo(1)); + Assert.That( + needing[0].SourceLanguageTag, + Is.EqualTo("fr"), + "a fixed-source engine translates from its configured source, not the priority default" + ); + Assert.That(needing[0].SourceText, Is.EqualTo("Bonjour")); + } + + [Test] + public void GroupsNeedingTranslation_FixedSourceEngine_MissingSourceLanguage_ExcludedAndCountedAsSkipped() + { + var alpha2 = new AiTranslationEngineSettings + { + ProviderId = "alpha2", + Enabled = true, + SourceLanguageTag = "fr", + }; + var dom = MakeBookDom( + @" +
    +
    +
    Hello
    +
    +
    " + ); + var scanner = new AiTranslationBookScanner(dom, "es", new[] { alpha2 }, new[] { "en" }); + var scan = scanner.Scan(); + Assert.That(scan.Groups.Count, Is.EqualTo(1), "sanity check: the group is eligible"); + + Assert.That( + scan.GroupsNeedingTranslation(alpha2), + Is.Empty, + "the group has no French text, so the fixed-source alpha2 engine must skip it" + ); + Assert.That(scan.CountGroupsSkippedForEngine(alpha2), Is.EqualTo(1)); + } + + [Test] + public void GroupsNeedingTranslation_SameGroup_Alpha2AndDeeplGetDifferentSources() + { + var deepl = new AiTranslationEngineSettings { ProviderId = "deepl", Enabled = true }; + var alpha2 = new AiTranslationEngineSettings + { + ProviderId = "alpha2", + Enabled = true, + SourceLanguageTag = "fr", + }; + var dom = MakeBookDom( + @" +
    +
    +
    Hello
    +
    Bonjour
    +
    +
    " + ); + var scanner = new AiTranslationBookScanner( + dom, + "es", + new[] { deepl, alpha2 }, + new[] { "en" } + ); + var scan = scanner.Scan(); + + var deeplWork = scan.GroupsNeedingTranslation(deepl).Single(); + var alpha2Work = scan.GroupsNeedingTranslation(alpha2).Single(); + + Assert.That( + deeplWork.SourceLanguageTag, + Is.EqualTo("en"), + "deepl uses the priority-chosen default source" + ); + Assert.That(deeplWork.SourceText, Is.EqualTo("Hello")); + Assert.That( + alpha2Work.SourceLanguageTag, + Is.EqualTo("fr"), + "alpha2 uses its fixed configured source for the same group" + ); + Assert.That(alpha2Work.SourceText, Is.EqualTo("Bonjour")); + } + + [Test] + public void RemoveStaleAiDivs_FixedSourceEngineLostItsSourceText_RemovesDiv() + { + var alpha2 = new AiTranslationEngineSettings + { + ProviderId = "alpha2", + Enabled = true, + SourceLanguageTag = "fr", + }; + const string targetTag = "es"; + var alpha2AiTag = AiTranslationService.GetAiLanguageTag(targetTag, "alpha2"); + // The group has English text but NO French text, yet carries an alpha2 (French-source) + // AI div left over from when French text existed. Since alpha2 can no longer resolve a + // French source for the group, its div is stale and must be removed. + var dom = MakeBookDom( + $@" +
    +
    +
    Hello
    +
    Hola (from French, now orphaned)
    +
    +
    " + ); + Assert.That( + dom.RawDom.SafeSelectNodes("//div[@lang and contains(@lang,'-x-ai')]").Length, + Is.EqualTo(1), + "sanity check: fixture starts with one alpha2 AI div" + ); + + var scanner = new AiTranslationBookScanner( + dom, + targetTag, + new[] { alpha2 }, + new[] { "en" } + ); + + var removedCount = scanner.RemoveStaleAiDivs(); + + Assert.That(removedCount, Is.EqualTo(1)); + Assert.That( + dom.RawDom.SafeSelectNodes("//div[@lang and contains(@lang,'-x-ai')]").Length, + Is.EqualTo(0), + "the fixed-source engine's div is stale because its source language is gone" + ); + } + } +} diff --git a/src/BloomTests/AiTranslation/AiTranslationBookUpdaterTests.cs b/src/BloomTests/AiTranslation/AiTranslationBookUpdaterTests.cs new file mode 100644 index 000000000000..1668fa7a4874 --- /dev/null +++ b/src/BloomTests/AiTranslation/AiTranslationBookUpdaterTests.cs @@ -0,0 +1,541 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Bloom.AiTranslation; +using Bloom.Book; +using Bloom.Collection; +using BloomTests.TeamCollection; +using NUnit.Framework; + +namespace BloomTests.AiTranslation +{ + /// + /// Tests for the dialog-free orchestration core of AiTranslationBookUpdater: RunEnginesAsync (the + /// parallel translate phase) and ApplyOutcomes (the DOM-write phase). The dialog-hosting parts + /// (RunWithProgressDialog, MakeDialog) require a real Form/ShowDialog and are not unit tested here; + /// see the manual test plan for those. + /// + [TestFixture] + public class AiTranslationBookUpdaterTests + { + private static HtmlDom MakeBookDom(string pagesHtml, string dataDivHtml = "") + { + return new HtmlDom( + $@" +
    {dataDivHtml}
    + {pagesHtml} + " + ); + } + + private static AiTranslationEngineSettings MakeEngine(string providerId) => + new AiTranslationEngineSettings { ProviderId = providerId, Enabled = true }; + + [Test] + public async Task RunEnginesAsync_NothingNeeded_MakesNoTranslateCalls() + { + var engine = MakeEngine("deepl"); + var aiTag = AiTranslationService.GetAiLanguageTag("es", "deepl"); + var currentFingerprint = AiTranslationBookScanner.ComputeFingerprint( + "en", + "Hello", + aiTag + ); + var dom = MakeBookDom( + $@" +
    +
    +
    Hello
    +
    Hola
    +
    +
    " + ); + var scanner = new AiTranslationBookScanner(dom, "es", new[] { engine }, new[] { "en" }); + var scan = scanner.Scan(); + Assert.That( + scan.GroupsNeedingTranslation(engine), + Is.Empty, + "sanity check: the existing translation should already be current" + ); + + var callCount = 0; + Task translate( + AiTranslationEngineSettings e, + string[] segments, + string sourceLang, + CancellationToken ct + ) + { + callCount++; + return Task.FromResult(segments); + } + + var outcomes = await AiTranslationBookUpdater.RunEnginesAsync( + scan, + new[] { engine }, + translate, + new ProgressSpy(), + CancellationToken.None + ); + + Assert.That( + callCount, + Is.EqualTo(0), + "no engine has work, so translate must never be called" + ); + Assert.That(outcomes, Is.Empty); + } + + [Test] + public async Task RunEnginesAsync_TwoEngines_GroupsByLanguageAndAppliesCorrectTextsToCorrectGroups() + { + var deepl = MakeEngine("deepl"); + var google = MakeEngine("google"); + var dom = MakeBookDom( + @" +
    +
    +
    Hello
    +
    +
    +
    Bonjour
    +
    +
    " + ); + var scanner = new AiTranslationBookScanner( + dom, + "es", + new[] { deepl, google }, + new[] { "en", "fr" } + ); + var scan = scanner.Scan(); + Assert.That(scan.Groups.Count, Is.EqualTo(2), "sanity check on fixture"); + + var calls = new List<(string Engine, string Lang, string[] Segments)>(); + Task translate( + AiTranslationEngineSettings e, + string[] segments, + string sourceLang, + CancellationToken ct + ) + { + lock (calls) + calls.Add((e.ProviderId, sourceLang, segments)); + return Task.FromResult( + segments.Select(s => $"{e.ProviderId}/{sourceLang}:{s}").ToArray() + ); + } + + var outcomes = await AiTranslationBookUpdater.RunEnginesAsync( + scan, + new[] { deepl, google }, + translate, + new ProgressSpy(), + CancellationToken.None + ); + + Assert.That(outcomes.Count, Is.EqualTo(2), "both engines had work"); + Assert.That(calls.Count, Is.EqualTo(4), "2 engines x 2 source languages (en, fr) each"); + foreach (var outcome in outcomes) + { + Assert.That(outcome.Succeeded, Is.True); + Assert.That(outcome.Translations.Count, Is.EqualTo(2)); + var byText = outcome.Translations.ToDictionary( + t => t.Group.SourceText, + t => t.TranslatedText + ); + Assert.That( + byText["Hello"], + Is.EqualTo($"{outcome.Engine.ProviderId}/en:Hello"), + "the English group's translation should come from the en-language call" + ); + Assert.That( + byText["Bonjour"], + Is.EqualTo($"{outcome.Engine.ProviderId}/fr:Bonjour"), + "the French group's translation should come from the fr-language call" + ); + } + } + + [Test] + public async Task RunEnginesAsync_TwoEngines_RunConcurrentlyNotSequentially() + { + var deepl = MakeEngine("deepl"); + var google = MakeEngine("google"); + var dom = MakeBookDom( + @"
    +
    Hello
    +
    " + ); + var scanner = new AiTranslationBookScanner( + dom, + "es", + new[] { deepl, google }, + new[] { "en" } + ); + var scan = scanner.Scan(); + + var deeplStarted = new TaskCompletionSource(); + var googleStarted = new TaskCompletionSource(); + + async Task translate( + AiTranslationEngineSettings e, + string[] segments, + string sourceLang, + CancellationToken ct + ) + { + if (e.ProviderId == "deepl") + { + deeplStarted.SetResult(true); + await Task.WhenAny(googleStarted.Task, Task.Delay(5000, ct)); + } + else + { + googleStarted.SetResult(true); + await Task.WhenAny(deeplStarted.Task, Task.Delay(5000, ct)); + } + return segments; + } + + await AiTranslationBookUpdater.RunEnginesAsync( + scan, + new[] { deepl, google }, + translate, + new ProgressSpy(), + CancellationToken.None + ); + + Assert.That( + deeplStarted.Task.IsCompletedSuccessfully + && googleStarted.Task.IsCompletedSuccessfully, + Is.True, + "each engine's translate call should have observed the other one having also started, proving they ran concurrently rather than one waiting for the other to fully finish" + ); + } + + [Test] + public async Task RunEnginesAsync_OneEngineThrows_OtherEnginesResultsStillApplied_FailureRecorded() + { + var deepl = MakeEngine("deepl"); + var google = MakeEngine("google"); + var dom = MakeBookDom( + @"
    +
    Hello
    +
    " + ); + var scanner = new AiTranslationBookScanner( + dom, + "es", + new[] { deepl, google }, + new[] { "en" } + ); + var scan = scanner.Scan(); + + Task translate( + AiTranslationEngineSettings e, + string[] segments, + string sourceLang, + CancellationToken ct + ) + { + if (e.ProviderId == "deepl") + throw new InvalidOperationException("DeepL quota exceeded"); + return Task.FromResult(segments.Select(s => $"google:{s}").ToArray()); + } + + var outcomes = await AiTranslationBookUpdater.RunEnginesAsync( + scan, + new[] { deepl, google }, + translate, + new ProgressSpy(), + CancellationToken.None + ); + + var deeplOutcome = outcomes.Single(o => o.Engine.ProviderId == "deepl"); + var googleOutcome = outcomes.Single(o => o.Engine.ProviderId == "google"); + + Assert.That(deeplOutcome.Succeeded, Is.False); + Assert.That(deeplOutcome.ErrorMessage, Does.Contain("DeepL quota exceeded")); + Assert.That(deeplOutcome.Translations, Is.Empty); + + Assert.That(googleOutcome.Succeeded, Is.True); + Assert.That( + googleOutcome.Translations.Single().TranslatedText, + Is.EqualTo("google:Hello") + ); + } + + [Test] + public void RunEnginesAsync_Cancelled_ThrowsAndLeavesDomUnchanged() + { + var engine = MakeEngine("deepl"); + var dom = MakeBookDom( + @"
    +
    Hello
    +
    " + ); + var scanner = new AiTranslationBookScanner(dom, "es", new[] { engine }, new[] { "en" }); + var scan = scanner.Scan(); + var group = scan.Groups.Single(); + + using var cts = new CancellationTokenSource(); + cts.Cancel(); // simulate the user having already clicked Cancel + + Task translate( + AiTranslationEngineSettings e, + string[] segments, + string sourceLang, + CancellationToken ct + ) + { + Assert.Fail( + "translate should never be called once cancellation has already been requested" + ); + return Task.FromResult(segments); + } + + Assert.ThrowsAsync(async () => + await AiTranslationBookUpdater.RunEnginesAsync( + scan, + new[] { engine }, + translate, + new ProgressSpy(), + cts.Token + ) + ); + + // Since RunEnginesAsync threw, the real caller (AiTranslationBookUpdater.RunWithProgressDialog) + // never reaches ApplyOutcomes/RemoveStaleAiDivs/book.Save() -- so nothing should have touched + // the DOM. Confirm no AI div was ever written for the one group that would otherwise have + // been translated. + var aiTag = AiTranslationService.GetAiLanguageTag("es", "deepl"); + Assert.That( + group.GroupElement.SafeSelectElements($"div[@lang='{aiTag}']"), + Is.Empty, + "cancellation must leave the book DOM completely untouched" + ); + } + + [Test] + public async Task ApplyOutcomes_WritesEachEnginesTranslationsToTheDom_AndRemoveStaleAiDivsThenCleansUp() + { + var deepl = MakeEngine("deepl"); + var staleEngineTag = AiTranslationService.GetAiLanguageTag("es", "deepl"); + // A group that already has a stale (wrong-fingerprint) AI div, to prove the success path's + // subsequent RemoveStaleAiDivs() call still cleans it up after ApplyOutcomes runs. + var dom = MakeBookDom( + $@" +
    +
    +
    Hello
    +
    +
    +
    World
    +
    Old
    +
    +
    " + ); + var scanner = new AiTranslationBookScanner(dom, "es", new[] { deepl }, new[] { "en" }); + var scan = scanner.Scan(); + + Task translate( + AiTranslationEngineSettings e, + string[] segments, + string sourceLang, + CancellationToken ct + ) => Task.FromResult(segments.Select(s => $"ES:{s}").ToArray()); + + var outcomes = await AiTranslationBookUpdater.RunEnginesAsync( + scan, + new[] { deepl }, + translate, + new ProgressSpy(), + CancellationToken.None + ); + + AiTranslationBookUpdater.ApplyOutcomes(scanner, outcomes); + + var aiTag = AiTranslationService.GetAiLanguageTag("es", "deepl"); + var helloGroupAiDiv = scan + .Groups.First(g => g.SourceText == "Hello") + .GroupElement.SafeSelectElements($"div[@lang='{aiTag}']") + .Single(); + Assert.That(helloGroupAiDiv.InnerText, Is.EqualTo("ES:Hello")); + + var worldGroupAiDiv = scan + .Groups.First(g => g.SourceText == "World") + .GroupElement.SafeSelectElements($"div[@lang='{aiTag}']") + .Single(); + Assert.That( + worldGroupAiDiv.InnerText, + Is.EqualTo("ES:World"), + "ApplyTranslation should have replaced the previously-stale div with the fresh translation" + ); + + // Only on the success path does the real caller go on to call RemoveStaleAiDivs(); simulate + // that here directly against the same scanner/DOM now that ApplyOutcomes has run. + var removedCount = scanner.RemoveStaleAiDivs(); + Assert.That( + removedCount, + Is.EqualTo(0), + "both AI divs are now current (freshly applied), so there should be nothing left to remove" + ); + } + + [Test] + public void CountStaleAiDivsWithoutMutating_ReportsCountWithoutTouchingTheRealDom() + { + var deepl = MakeEngine("deepl"); + var staleTag = AiTranslationService.GetAiLanguageTag("es", "deepl"); + var dom = MakeBookDom( + $@" +
    +
    +
    Hello
    +
    Old
    +
    +
    " + ); + + var countBefore = dom + .RawDom.SafeSelectNodes("//div[@lang and contains(@lang,'-x-ai')]") + .Length; + Assert.That(countBefore, Is.EqualTo(1), "sanity check on fixture"); + + var staleCount = AiTranslationBookUpdater.CountStaleAiDivsWithoutMutating( + dom, + "es", + new[] { deepl }, + new[] { "en" } + ); + + Assert.That(staleCount, Is.EqualTo(1)); + // The real DOM must be untouched -- the stale div should still be there. + var countAfter = dom + .RawDom.SafeSelectNodes("//div[@lang and contains(@lang,'-x-ai')]") + .Length; + Assert.That( + countAfter, + Is.EqualTo(1), + "CountStaleAiDivsWithoutMutating must not mutate the real book DOM" + ); + } + + [Test] + public async Task RunEnginesAsync_Alpha2FixedSource_DeeplPrioritySource_SameGroup() + { + var deepl = MakeEngine("deepl"); + var alpha2 = new AiTranslationEngineSettings + { + ProviderId = "alpha2", + Enabled = true, + SourceLanguageTag = "fr", + }; + var dom = MakeBookDom( + @" +
    +
    +
    Hello
    +
    Bonjour
    +
    +
    " + ); + var scanner = new AiTranslationBookScanner( + dom, + "es", + new[] { deepl, alpha2 }, + new[] { "en" } + ); + var scan = scanner.Scan(); + + var calls = new List<(string Engine, string Lang, string[] Segments)>(); + Task translate( + AiTranslationEngineSettings e, + string[] segments, + string sourceLang, + CancellationToken ct + ) + { + lock (calls) + calls.Add((e.ProviderId, sourceLang, segments)); + return Task.FromResult(segments.Select(s => $"{e.ProviderId}:{s}").ToArray()); + } + + await AiTranslationBookUpdater.RunEnginesAsync( + scan, + new[] { deepl, alpha2 }, + translate, + new ProgressSpy(), + CancellationToken.None + ); + + var deeplCall = calls.Single(c => c.Engine == "deepl"); + var alpha2Call = calls.Single(c => c.Engine == "alpha2"); + Assert.That(deeplCall.Lang, Is.EqualTo("en"), "deepl uses the priority source"); + Assert.That(deeplCall.Segments, Is.EqualTo(new[] { "Hello" })); + Assert.That( + alpha2Call.Lang, + Is.EqualTo("fr"), + "alpha2 uses its fixed configured source for the same group" + ); + Assert.That(alpha2Call.Segments, Is.EqualTo(new[] { "Bonjour" })); + } + + [Test] + public async Task RunEnginesAsync_Alpha2_SkipsGroupWithNoSourceText_NoTranslateCallForIt() + { + var alpha2 = new AiTranslationEngineSettings + { + ProviderId = "alpha2", + Enabled = true, + SourceLanguageTag = "fr", + }; + var dom = MakeBookDom( + @" +
    +
    +
    Hello
    +
    Bonjour
    +
    +
    +
    World
    +
    +
    " + ); + var scanner = new AiTranslationBookScanner(dom, "es", new[] { alpha2 }, new[] { "en" }); + var scan = scanner.Scan(); + Assert.That(scan.Groups.Count, Is.EqualTo(2), "sanity check on fixture"); + + var translatedSegments = new List(); + Task translate( + AiTranslationEngineSettings e, + string[] segments, + string sourceLang, + CancellationToken ct + ) + { + lock (translatedSegments) + translatedSegments.AddRange(segments); + return Task.FromResult(segments.Select(s => $"ES:{s}").ToArray()); + } + + var outcomes = await AiTranslationBookUpdater.RunEnginesAsync( + scan, + new[] { alpha2 }, + translate, + new ProgressSpy(), + CancellationToken.None + ); + + Assert.That( + translatedSegments, + Is.EqualTo(new[] { "Bonjour" }), + "only the group with French text is translated; the English-only group is skipped" + ); + Assert.That(outcomes.Single().Translations.Count, Is.EqualTo(1)); + } + } +} diff --git a/src/BloomTests/AiTranslation/AiTranslationServiceTests.cs b/src/BloomTests/AiTranslation/AiTranslationServiceTests.cs new file mode 100644 index 000000000000..6fdafb677cc4 --- /dev/null +++ b/src/BloomTests/AiTranslation/AiTranslationServiceTests.cs @@ -0,0 +1,802 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Bloom; +using Bloom.AiTranslation; +using Bloom.Collection; +using Bloom.SubscriptionAndFeatures; +using NUnit.Framework; + +namespace BloomTests.AiTranslation +{ + [TestFixture] + public class AiTranslationServiceTests + { + private bool _previousAiSourceBubblesEnabled; + + [SetUp] + public void Setup() + { + _previousAiSourceBubblesEnabled = ExperimentalFeatures.IsFeatureEnabled( + ExperimentalFeatures.kAiSourceBubbles + ); + ExperimentalFeatures.SetValue(ExperimentalFeatures.kAiSourceBubbles, true); + } + + [TearDown] + public void TearDown() + { + ExperimentalFeatures.SetValue( + ExperimentalFeatures.kAiSourceBubbles, + _previousAiSourceBubblesEnabled + ); + } + + [Test] + public void NormalizeProviderId_GoogleTranslateAlias_ReturnsGoogle() + { + Assert.That( + AiTranslationService.NormalizeProviderId("googleTranslate"), + Is.EqualTo("google") + ); + } + + [Test] + public void GetAiLanguageTag_AppendsNormalizedProviderSuffix() + { + Assert.That( + AiTranslationService.GetAiLanguageTag("fr", "googleTranslate"), + Is.EqualTo("fr-x-ai-google") + ); + } + + [Test] + public void GetGoogleProjectIdFromServiceAccountEmail_ParsesProjectId() + { + Assert.That( + AiTranslationService.GetGoogleProjectIdFromServiceAccountEmail( + "translator@test-project-123.iam.gserviceaccount.com" + ), + Is.EqualTo("test-project-123") + ); + } + + [Test] + public void GetEngineFingerprint_ChangesWhenApiKeyChanges() + { + var engine = new AiTranslationEngineSettings + { + ProviderId = "deepl", + ApiKey = "first-key", + }; + var originalFingerprint = AiTranslationService.GetEngineFingerprint(engine, "fr"); + + engine.ApiKey = "second-key"; + + Assert.That( + AiTranslationService.GetEngineFingerprint(engine, "fr"), + Is.Not.EqualTo(originalFingerprint) + ); + } + + [Test] + public void GetEngineFingerprint_StableForUnchangedSettings() + { + var engine = new AiTranslationEngineSettings + { + ProviderId = "deepl", + ApiKey = "same-key", + }; + + var first = AiTranslationService.GetEngineFingerprint(engine, "fr"); + var second = AiTranslationService.GetEngineFingerprint(engine, "fr"); + + Assert.That(second, Is.EqualTo(first)); + } + + [Test] + public void GetEngineFingerprint_ChangesWhenTargetLanguageChanges() + { + var engine = new AiTranslationEngineSettings + { + ProviderId = "deepl", + ApiKey = "same-key", + }; + + var frenchFingerprint = AiTranslationService.GetEngineFingerprint(engine, "fr"); + var spanishFingerprint = AiTranslationService.GetEngineFingerprint(engine, "es"); + + Assert.That(spanishFingerprint, Is.Not.EqualTo(frenchFingerprint)); + } + + [Test] + public void GetEngineFingerprint_IsIndependentOfOtherEnginesCredentials() + { + // Two independently-configured engines should each fingerprint based only on their + // own provider/target/credentials, regardless of what other engines are configured with. + var deeplEngine = new AiTranslationEngineSettings + { + ProviderId = "deepl", + ApiKey = "deepl-key", + }; + var googleEngine = new AiTranslationEngineSettings + { + ProviderId = "google", + ServiceAccountEmail = "first@example.com", + PrivateKey = "first-private-key", + }; + + var originalFingerprint = AiTranslationService.GetEngineFingerprint(deeplEngine, "fr"); + + googleEngine.ServiceAccountEmail = "second@example.com"; + googleEngine.PrivateKey = "second-private-key"; + + Assert.That( + AiTranslationService.GetEngineFingerprint(deeplEngine, "fr"), + Is.EqualTo(originalFingerprint) + ); + } + + [Test] + public void TranslateSegmentsAsync_WithoutTargetLanguageTag_ThrowsHelpfulError() + { + var collectionSettings = MakeCollectionSettings(); + collectionSettings.AiTranslationTargetLanguageTag = ""; + var service = new AiTranslationService(collectionSettings); + var engine = new AiTranslationEngineSettings { ProviderId = "deepl", ApiKey = "key" }; + + var exception = Assert.ThrowsAsync(async () => + await service.TranslateSegmentsAsync( + engine, + new[] { "Hello world." }, + "en", + CancellationToken.None + ) + ); + + Assert.That(exception.Message, Does.Contain("target language tag")); + } + + [Test] + public async Task TranslateSegmentsAsync_FewerSegmentsThanLimit_MakesOneCall() + { + var fakeProvider = new FakeAiTranslationProvider( + "deepl", + maxSegmentsPerRequest: 50, + maxRequestBytes: 1_000_000 + ); + var service = MakeService(fakeProvider); + var engine = new AiTranslationEngineSettings { ProviderId = "deepl", ApiKey = "key" }; + var segments = Enumerable.Range(0, 50).Select(i => $"segment {i}").ToArray(); + + var results = await service.TranslateSegmentsAsync( + engine, + segments, + "en", + CancellationToken.None + ); + + Assert.That(fakeProvider.Calls.Count, Is.EqualTo(1)); + Assert.That(results, Is.EqualTo(segments.Select(s => "[fr] " + s))); + } + + [Test] + public async Task TranslateSegmentsAsync_MoreSegmentsThanLimit_ChunksBySegmentCount() + { + var fakeProvider = new FakeAiTranslationProvider( + "deepl", + maxSegmentsPerRequest: 50, + maxRequestBytes: 1_000_000 + ); + var service = MakeService(fakeProvider); + var engine = new AiTranslationEngineSettings { ProviderId = "deepl", ApiKey = "key" }; + var segments = Enumerable.Range(0, 51).Select(i => $"segment {i}").ToArray(); + + var results = await service.TranslateSegmentsAsync( + engine, + segments, + "en", + CancellationToken.None + ); + + Assert.That(fakeProvider.Calls.Count, Is.EqualTo(2)); + Assert.That(fakeProvider.Calls[0].Length, Is.EqualTo(50)); + Assert.That(fakeProvider.Calls[1].Length, Is.EqualTo(1)); + Assert.That(results, Is.EqualTo(segments.Select(s => "[fr] " + s))); + } + + [Test] + public async Task TranslateSegmentsAsync_ChunksByByteSize() + { + var fakeProvider = new FakeAiTranslationProvider( + "deepl", + maxSegmentsPerRequest: 1000, + maxRequestBytes: 30 + ); + var service = MakeService(fakeProvider); + var engine = new AiTranslationEngineSettings { ProviderId = "deepl", ApiKey = "key" }; + // Each segment is 9 bytes; a byte cap of 30 should limit each chunk to 3 segments. + var segments = Enumerable.Range(0, 7).Select(i => $"aaaaaaaa{i}").ToArray(); + Assert.That( + segments.All(s => Encoding.UTF8.GetByteCount(s) == 9), + "sanity check: each segment should be 9 bytes" + ); + + var results = await service.TranslateSegmentsAsync( + engine, + segments, + "en", + CancellationToken.None + ); + + Assert.That(fakeProvider.Calls.Count, Is.EqualTo(3)); + Assert.That(fakeProvider.Calls[0].Length, Is.EqualTo(3)); + Assert.That(fakeProvider.Calls[1].Length, Is.EqualTo(3)); + Assert.That(fakeProvider.Calls[2].Length, Is.EqualTo(1)); + Assert.That(results, Is.EqualTo(segments.Select(s => "[fr] " + s))); + } + + [Test] + public async Task TranslateSegmentsAsync_OversizedSingleSegment_GetsItsOwnChunk() + { + var fakeProvider = new FakeAiTranslationProvider( + "deepl", + maxSegmentsPerRequest: 1000, + maxRequestBytes: 5 + ); + var service = MakeService(fakeProvider); + var engine = new AiTranslationEngineSettings { ProviderId = "deepl", ApiKey = "key" }; + var segments = new[] { "short", "this-segment-is-longer-than-the-byte-cap", "short2" }; + + var results = await service.TranslateSegmentsAsync( + engine, + segments, + "en", + CancellationToken.None + ); + + Assert.That(fakeProvider.Calls.Count, Is.EqualTo(3)); + Assert.That(fakeProvider.Calls[1].Single(), Is.EqualTo(segments[1])); + Assert.That(results, Is.EqualTo(segments.Select(s => "[fr] " + s))); + } + + [Test] + public void TranslateSegmentsAsync_WhenProviderReturnsWrongCount_Throws() + { + var fakeProvider = new FakeAiTranslationProvider( + "deepl", + maxSegmentsPerRequest: 50, + maxRequestBytes: 1_000_000, + mismatchCount: true + ); + var service = MakeService(fakeProvider); + var engine = new AiTranslationEngineSettings { ProviderId = "deepl", ApiKey = "key" }; + + Assert.ThrowsAsync(async () => + await service.TranslateSegmentsAsync( + engine, + new[] { "one", "two" }, + "en", + CancellationToken.None + ) + ); + } + + [Test] + public async Task GetSupportedTargetLanguagesAsync_OneEngineFailsToListLanguages_StillReturnsOthers() + { + var service = MakeLanguageListService( + new FakeLanguageListProvider( + "deepl", + throwMessage: "403 Forbidden. Missing required scope(s): languages:read." + ), + new FakeLanguageListProvider("google", languages: new[] { ("es", "Spanish (es)") }) + ); + + var options = await service.GetSupportedTargetLanguagesAsync(CancellationToken.None); + + Assert.That( + options.Count, + Is.EqualTo(1), + "Google's language should still be returned even though DeepL failed to list languages" + ); + Assert.That(options[0].Value, Is.EqualTo("es")); + Assert.That(options[0].ProviderIds, Does.Contain("google")); + Assert.That(options[0].ProviderIds, Does.Not.Contain("deepl")); + } + + [Test] + public void GetSupportedTargetLanguagesAsync_EveryEngineFailsToListLanguages_ThrowsWithCombinedMessage() + { + var service = MakeLanguageListService( + new FakeLanguageListProvider( + "deepl", + throwMessage: "403 Forbidden. Missing required scope(s): languages:read." + ), + new FakeLanguageListProvider("google", throwMessage: "boom") + ); + + var exception = Assert.ThrowsAsync(async () => + await service.GetSupportedTargetLanguagesAsync(CancellationToken.None) + ); + + // Sanity check: with no engine succeeding, the user still gets an informative error + // that names the failing engine and its reason. + Assert.That(exception.Message, Does.Contain("DeepL")); + Assert.That(exception.Message, Does.Contain("languages:read")); + } + + [Test] + public void GetEngineFingerprint_ChangesWhenSourceLanguageTagChanges() + { + var engine = new AiTranslationEngineSettings + { + ProviderId = "alpha2", + ApiKey = "same-key", + SourceLanguageTag = "en", + }; + + var englishSource = AiTranslationService.GetEngineFingerprint(engine, "fr"); + engine.SourceLanguageTag = "es"; + var spanishSource = AiTranslationService.GetEngineFingerprint(engine, "fr"); + + Assert.That(spanishSource, Is.Not.EqualTo(englishSource)); + } + + [Test] + public async Task GetSupportedTargetLanguagesAsync_LooksUpAlpha2TargetsForItsConfiguredSourceOnly() + { + var provider = new FakeLanguageListProvider( + "alpha2", + languages: new[] { ("es", "Spanish") } + ); + var collectionSettings = new CollectionSettings + { + Subscription = Subscription.CreateTempSubscriptionForTier(SubscriptionTier.Pro), + AiTranslationTargetLanguageTag = "de", + Language1Tag = "fr", + AiTranslationEngines = new List + { + new AiTranslationEngineSettings + { + ProviderId = "alpha2", + Enabled = true, + SourceLanguageTag = "es", + }, + }, + }; + var service = new AiTranslationService( + collectionSettings, + new Dictionary { { "alpha2", provider } } + ); + + await service.GetSupportedTargetLanguagesAsync(CancellationToken.None); + + // Only the engine's configured source is used. We must NOT union in English or the + // collection languages: alpha2 unions its target results across every source given, and + // English reaches a near-superset of targets, so including it made the list look the + // same regardless of the chosen source (BL-16549). + Assert.That(provider.LastLikelySourceLanguageTags, Is.EquivalentTo(new[] { "es" })); + } + + [Test] + public async Task ValidateEngineAsync_TargetMissingFromNonEmptyList_ReturnsTargetLanguageNotSupported_WithoutProbing() + { + var provider = new FakeValidatingProvider( + "alpha2", + supportedTargets: new[] { "es", "fr" } + ); + var collectionSettings = new CollectionSettings + { + Subscription = Subscription.CreateTempSubscriptionForTier(SubscriptionTier.Pro), + AiTranslationTargetLanguageTag = "de", // not in the provider's supported list + }; + var service = new AiTranslationService( + collectionSettings, + new Dictionary { { "alpha2", provider } } + ); + var engine = new AiTranslationEngineSettings { ProviderId = "alpha2", ApiKey = "key" }; + + var result = await service.ValidateEngineAsync(engine, CancellationToken.None); + + Assert.That(result.Succeeded, Is.False); + Assert.That(result.TargetLanguageNotSupported, Is.True); + Assert.That( + provider.TranslateCallCount, + Is.EqualTo(0), + "an unsupported target must short-circuit before the probe translation" + ); + } + + [Test] + public async Task ValidateEngineAsync_TargetSupported_ProbesUsingConfiguredSourceLanguage() + { + var provider = new FakeValidatingProvider("alpha2", supportedTargets: new[] { "es" }); + var collectionSettings = new CollectionSettings + { + Subscription = Subscription.CreateTempSubscriptionForTier(SubscriptionTier.Pro), + AiTranslationTargetLanguageTag = "es", + }; + var service = new AiTranslationService( + collectionSettings, + new Dictionary { { "alpha2", provider } } + ); + var engine = new AiTranslationEngineSettings + { + ProviderId = "alpha2", + ApiKey = "key", + SourceLanguageTag = "fr", + }; + + var result = await service.ValidateEngineAsync(engine, CancellationToken.None); + + Assert.That(result.Succeeded, Is.True); + Assert.That(result.TargetLanguageNotSupported, Is.False); + Assert.That( + provider.LastTranslateSourceLanguageTag, + Is.EqualTo("fr"), + "the probe should translate FROM the engine's configured source language" + ); + } + + [Test] + public async Task ValidateEngineAsync_EmptySupportedList_FallsThroughToProbe() + { + var provider = new FakeValidatingProvider("alpha2", supportedTargets: new string[0]); + var collectionSettings = new CollectionSettings + { + Subscription = Subscription.CreateTempSubscriptionForTier(SubscriptionTier.Pro), + AiTranslationTargetLanguageTag = "de", + }; + var service = new AiTranslationService( + collectionSettings, + new Dictionary { { "alpha2", provider } } + ); + var engine = new AiTranslationEngineSettings { ProviderId = "alpha2", ApiKey = "key" }; + + var result = await service.ValidateEngineAsync(engine, CancellationToken.None); + + Assert.That(result.Succeeded, Is.True); + Assert.That(result.TargetLanguageNotSupported, Is.False); + Assert.That( + provider.TranslateCallCount, + Is.EqualTo(1), + "an empty supported-languages list is not proof of non-support, so we still probe" + ); + } + + private static AiTranslationService MakeLanguageListService( + params FakeLanguageListProvider[] providers + ) + { + var collectionSettings = new CollectionSettings + { + Subscription = Subscription.CreateTempSubscriptionForTier(SubscriptionTier.Pro), + AiTranslationTargetLanguageTag = "fr", + AiTranslationEngines = providers + .Select(p => new AiTranslationEngineSettings + { + ProviderId = p.ProviderId, + Enabled = true, + }) + .ToList(), + }; + return new AiTranslationService( + collectionSettings, + providers.ToDictionary(p => p.ProviderId, p => (IAiTranslationProvider)p) + ); + } + + private static AiTranslationService MakeService(IAiTranslationProvider fakeProvider) + { + var collectionSettings = MakeCollectionSettings(); + return new AiTranslationService( + collectionSettings, + new Dictionary + { + { fakeProvider.ProviderId, fakeProvider }, + } + ); + } + + private static CollectionSettings MakeCollectionSettings() + { + return new CollectionSettings + { + Subscription = Subscription.CreateTempSubscriptionForTier(SubscriptionTier.Pro), + AiTranslationTargetLanguageTag = "fr", + }; + } + + private sealed class FakeAiTranslationProvider : IAiTranslationProvider + { + private readonly bool _mismatchCount; + + public FakeAiTranslationProvider( + string providerId, + int maxSegmentsPerRequest, + int maxRequestBytes, + bool mismatchCount = false + ) + { + ProviderId = providerId; + MaxSegmentsPerRequest = maxSegmentsPerRequest; + MaxRequestBytes = maxRequestBytes; + _mismatchCount = mismatchCount; + } + + public string ProviderId { get; } + public int MaxSegmentsPerRequest { get; } + public int MaxRequestBytes { get; } + public List Calls { get; } = new List(); + + public Task> GetSupportedTargetLanguagesAsync( + AiTranslationEngineSettings engine, + IReadOnlyList likelySourceLanguageTags, + HttpClient httpClient, + CancellationToken ct + ) + { + return Task.FromResult(new List()); + } + + public Task TranslateBatchAsync( + AiTranslationEngineSettings engine, + string[] segments, + string sourceLanguageTag, + string targetLanguageTag, + HttpClient httpClient, + CancellationToken ct + ) + { + Calls.Add(segments); + if (_mismatchCount) + { + return Task.FromResult(new[] { "only one translation" }); + } + return Task.FromResult( + segments.Select(s => $"[{targetLanguageTag}] " + s).ToArray() + ); + } + } + + /// + /// A provider used only to exercise GetSupportedTargetLanguagesAsync's aggregation: it + /// either returns a fixed set of (tag, name) languages, or throws the given message to + /// simulate an engine that can't list its languages (e.g. a missing API scope). + /// + private sealed class FakeLanguageListProvider : IAiTranslationProvider + { + private readonly (string Tag, string Name)[] _languages; + private readonly string _throwMessage; + + public FakeLanguageListProvider( + string providerId, + (string Tag, string Name)[] languages = null, + string throwMessage = null + ) + { + ProviderId = providerId; + _languages = languages ?? new (string, string)[0]; + _throwMessage = throwMessage; + } + + public string ProviderId { get; } + public int MaxSegmentsPerRequest => 100; + public int MaxRequestBytes => 100_000; + + /// The likelySourceLanguageTags the service passed on the most recent call. + public IReadOnlyList LastLikelySourceLanguageTags { get; private set; } + + public Task> GetSupportedTargetLanguagesAsync( + AiTranslationEngineSettings engine, + IReadOnlyList likelySourceLanguageTags, + HttpClient httpClient, + CancellationToken ct + ) + { + LastLikelySourceLanguageTags = likelySourceLanguageTags; + if (_throwMessage != null) + throw new HttpRequestException(_throwMessage); + + return Task.FromResult( + _languages + .Select(l => new AiTranslationTargetLanguageOption + { + Value = l.Tag, + Label = l.Name, + ProviderIds = new List { ProviderId }, + }) + .ToList() + ); + } + + public Task TranslateBatchAsync( + AiTranslationEngineSettings engine, + string[] segments, + string sourceLanguageTag, + string targetLanguageTag, + HttpClient httpClient, + CancellationToken ct + ) + { + throw new System.NotSupportedException( + "FakeLanguageListProvider is only for supported-languages tests." + ); + } + } + + /// + /// A provider that both lists a fixed set of supported target languages AND "translates", + /// recording how the probe was called, so ValidateEngineAsync's target-support check and + /// probe-source behavior can be exercised without any network. + /// + private sealed class FakeValidatingProvider : IAiTranslationProvider + { + private readonly string[] _supportedTargets; + + public FakeValidatingProvider(string providerId, string[] supportedTargets) + { + ProviderId = providerId; + _supportedTargets = supportedTargets ?? new string[0]; + } + + public string ProviderId { get; } + public int MaxSegmentsPerRequest => 100; + public int MaxRequestBytes => 100_000; + public int TranslateCallCount { get; private set; } + public string LastTranslateSourceLanguageTag { get; private set; } + + public Task> GetSupportedTargetLanguagesAsync( + AiTranslationEngineSettings engine, + IReadOnlyList likelySourceLanguageTags, + HttpClient httpClient, + CancellationToken ct + ) + { + return Task.FromResult( + _supportedTargets + .Select(tag => new AiTranslationTargetLanguageOption + { + Value = tag, + Label = tag, + ProviderIds = new List { ProviderId }, + }) + .ToList() + ); + } + + public Task TranslateBatchAsync( + AiTranslationEngineSettings engine, + string[] segments, + string sourceLanguageTag, + string targetLanguageTag, + HttpClient httpClient, + CancellationToken ct + ) + { + TranslateCallCount++; + LastTranslateSourceLanguageTag = sourceLanguageTag; + return Task.FromResult(segments.Select(s => "translated:" + s).ToArray()); + } + } + } + + public abstract class AiTranslationLiveTranslationTestsBase + { + private bool _previousAiSourceBubblesEnabled; + + protected abstract string ProviderId { get; } + protected abstract string[] RequiredEnvironmentVariables { get; } + protected abstract AiTranslationEngineSettings MakeEngine(); + + [SetUp] + public void Setup() + { + _previousAiSourceBubblesEnabled = ExperimentalFeatures.IsFeatureEnabled( + ExperimentalFeatures.kAiSourceBubbles + ); + ExperimentalFeatures.SetValue(ExperimentalFeatures.kAiSourceBubbles, true); + } + + [TearDown] + public void TearDown() + { + ExperimentalFeatures.SetValue( + ExperimentalFeatures.kAiSourceBubbles, + _previousAiSourceBubblesEnabled + ); + } + + [Test] + public async Task TranslateSegmentsAsync_ConfiguredProvider_ReturnsTranslationsInOrder() + { + var missingVariables = RequiredEnvironmentVariables + .Where(variableName => + string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable(variableName)) + ) + .ToArray(); + if (missingVariables.Any()) + { + Assert.Ignore( + $"Manual AI translation provider test. Set {string.Join(", ", missingVariables)} to run it." + ); + } + + var collectionSettings = new CollectionSettings + { + Subscription = Subscription.CreateTempSubscriptionForTier(SubscriptionTier.Pro), + AiTranslationTargetLanguageTag = "fr", + }; + var service = new AiTranslationService(collectionSettings); + var engine = MakeEngine(); + var segments = new[] { "Hello world.", "Good morning.", "See you tomorrow." }; + + var results = await service.TranslateSegmentsAsync( + engine, + segments, + "en", + CancellationToken.None + ); + + Assert.That(results.Length, Is.EqualTo(segments.Length)); + for (var i = 0; i < segments.Length; i++) + { + Assert.That(results[i], Is.Not.Null.And.Not.Empty); + Assert.That(results[i], Is.Not.EqualTo(segments[i])); + } + } + } + + [TestFixture] + [Category("SkipOnTeamCity")] + [NonParallelizable] + public class DeepLLiveTranslationTests : AiTranslationLiveTranslationTestsBase + { + protected override string ProviderId => "deepl"; + + protected override string[] RequiredEnvironmentVariables => new[] { "BLOOM_DEEPL_KEY" }; + + protected override AiTranslationEngineSettings MakeEngine() + { + return new AiTranslationEngineSettings + { + ProviderId = "deepl", + ApiKey = Environment.GetEnvironmentVariable("BLOOM_DEEPL_KEY"), + }; + } + } + + [TestFixture] + [Category("SkipOnTeamCity")] + [NonParallelizable] + public class GoogleLiveTranslationTests : AiTranslationLiveTranslationTestsBase + { + protected override string ProviderId => "google"; + + protected override string[] RequiredEnvironmentVariables => + new[] + { + "BLOOM_GOOGLE_TRANSLATION_SERVICE_ACCOUNT_EMAIL", + "BLOOM_GOOGLE_TRANSLATION_SERVICE_PRIVATE_KEY", + }; + + protected override AiTranslationEngineSettings MakeEngine() + { + return new AiTranslationEngineSettings + { + ProviderId = "google", + ServiceAccountEmail = Environment.GetEnvironmentVariable( + "BLOOM_GOOGLE_TRANSLATION_SERVICE_ACCOUNT_EMAIL" + ), + PrivateKey = Environment.GetEnvironmentVariable( + "BLOOM_GOOGLE_TRANSLATION_SERVICE_PRIVATE_KEY" + ), + }; + } + } +} diff --git a/src/BloomTests/AiTranslation/Alpha2TranslationProviderTests.cs b/src/BloomTests/AiTranslation/Alpha2TranslationProviderTests.cs new file mode 100644 index 000000000000..3929dc126bf8 --- /dev/null +++ b/src/BloomTests/AiTranslation/Alpha2TranslationProviderTests.cs @@ -0,0 +1,218 @@ +using System; +using System.Linq; +using System.Threading; +using Bloom.AiTranslation; +using Bloom.Collection; +using NUnit.Framework; + +namespace BloomTests.AiTranslation +{ + [TestFixture] + public class Alpha2TranslationProviderTests + { + [Test] + public void MapToIso6393_TwoLetterEnglish_ReturnsEng() + { + Assert.That(Alpha2TranslationProvider.MapToIso6393("en"), Is.EqualTo("eng")); + } + + [Test] + public void MapToIso6393_TwoLetterFrench_ReturnsFra() + { + Assert.That(Alpha2TranslationProvider.MapToIso6393("fr"), Is.EqualTo("fra")); + } + + [Test] + public void MapToIso6393_AlreadyThreeLetterTag_PassesThrough() + { + Assert.That(Alpha2TranslationProvider.MapToIso6393("eng"), Is.EqualTo("eng")); + } + + [Test] + public void MapToIso6393_RegionQualifiedTag_MapsPrimarySubtag() + { + Assert.That(Alpha2TranslationProvider.MapToIso6393("es-ES"), Is.EqualTo("spa")); + } + + [Test] + public void MapToIso6393_UnmappableTag_Throws() + { + var exception = Assert.Throws(() => + Alpha2TranslationProvider.MapToIso6393("zzzzz") + ); + + Assert.That(exception.Message, Does.Contain("zzzzz")); + } + + [Test] + public void IsTranslationComplete_CompleteStatus_ReturnsTrue() + { + Assert.That(Alpha2TranslationProvider.IsTranslationComplete("complete"), Is.True); + } + + [Test] + public void IsTranslationComplete_IsCaseInsensitive() + { + Assert.That(Alpha2TranslationProvider.IsTranslationComplete("COMPLETE"), Is.True); + } + + // The live Alpha2 API reports a queued segment as "pending"; this must be treated as + // still-in-progress, not a failure (regression guard for the "unexpected status 'pending'" + // bug that aborted otherwise-successful translations). + [TestCase("pending")] + [TestCase("running")] + [TestCase("")] + [TestCase(null)] + public void IsTranslationComplete_InProgressStatus_ReturnsFalse(string status) + { + Assert.That(Alpha2TranslationProvider.IsTranslationComplete(status), Is.False); + } + + [Test] + public void IsTranslationComplete_UnrecognizedStatus_Throws() + { + // Sanity check: a status we've never seen is a hard failure, not silently ignored. + var exception = Assert.Throws(() => + Alpha2TranslationProvider.IsTranslationComplete("exploded") + ); + + Assert.That(exception.Message, Does.Contain("exploded")); + } + + [Test] + public void MapIso6393ToBloomTag_MapsToShortestRegisteredTag() + { + // Alpha2's 3-letter codes should collapse to the familiar 2-letter tags so they dedup + // with DeepL/Google's "fr"/"en" in the settings union. + Assert.That(Alpha2TranslationProvider.MapIso6393ToBloomTag("fra"), Is.EqualTo("fr")); + Assert.That(Alpha2TranslationProvider.MapIso6393ToBloomTag("eng"), Is.EqualTo("en")); + } + + [Test] + public void MapIso6393ToBloomTag_UnmappedCodePassesThrough() + { + // A code with no shorter registered tag is returned unchanged (and lowercased). + Assert.That(Alpha2TranslationProvider.MapIso6393ToBloomTag("zzz"), Is.EqualTo("zzz")); + Assert.That(Alpha2TranslationProvider.MapIso6393ToBloomTag("FRA"), Is.EqualTo("fr")); + } + + [Test] + public void ParseTranslationLanguages_ReverseMapsSortsByLabelAndTagsAlpha2() + { + var json = + @"[ + {""iso"":""fra"",""name"":""French"",""display"":""Français"",""models"":[{""id"":1,""name"":""m1""}]}, + {""iso"":""spa"",""name"":""Spanish"",""models"":[]}, + {""iso"":""zzz"",""name"":""Madeup""} + ]"; + + var options = Alpha2TranslationProvider.ParseTranslationLanguages(json); + + // Sorted by label: French, Madeup, Spanish. + Assert.That( + options.Select(o => o.Label), + Is.EqualTo(new[] { "French", "Madeup", "Spanish" }) + ); + var valueByLabel = options.ToDictionary(o => o.Label, o => o.Value); + Assert.That(valueByLabel["French"], Is.EqualTo("fr")); + Assert.That(valueByLabel["Spanish"], Is.EqualTo("es")); + Assert.That( + valueByLabel["Madeup"], + Is.EqualTo("zzz"), + "an unmapped ISO code should pass through unchanged" + ); + Assert.That( + options.All(o => o.ProviderIds.SequenceEqual(new[] { "alpha2" })), + Is.True, + "every option must be tagged as coming from alpha2" + ); + } + + [Test] + public void ParseTranslationLanguages_DedupsEntriesThatMapToTheSameTag() + { + var json = + @"[ + {""iso"":""eng"",""name"":""English""}, + {""iso"":""eng"",""name"":""English (again)""} + ]"; + + var options = Alpha2TranslationProvider.ParseTranslationLanguages(json); + + Assert.That(options.Count, Is.EqualTo(1)); + Assert.That(options[0].Value, Is.EqualTo("en")); + } + + [Test] + public void ParseTranslationLanguages_SkipsEntriesWithNoIso() + { + var json = + @"[ + {""name"":""No iso here""}, + {""iso"":""fra"",""name"":""French""} + ]"; + + var options = Alpha2TranslationProvider.ParseTranslationLanguages(json); + + Assert.That(options.Count, Is.EqualTo(1)); + Assert.That(options[0].Value, Is.EqualTo("fr")); + } + } + + /// + /// Live round trip against the real Alpha2 service. Gated on BLOOM_ALPHA2_KEY, following the + /// same pattern as the DeepL/Google live tests in AiTranslationServiceTests.cs. + /// + [TestFixture] + [Category("SkipOnTeamCity")] + [NonParallelizable] + public class Alpha2LiveTranslationTests + { + [Test] + public async System.Threading.Tasks.Task TranslateSegmentsAsync_ThreeDistinctSegments_ReturnsTranslationsInOrder() + { + var apiKey = Environment.GetEnvironmentVariable("BLOOM_ALPHA2_KEY"); + if (string.IsNullOrWhiteSpace(apiKey)) + { + Assert.Ignore( + "Manual AI translation provider test. Set BLOOM_ALPHA2_KEY to run it." + ); + } + + var collectionSettings = new CollectionSettings + { + Subscription = + Bloom.SubscriptionAndFeatures.Subscription.CreateTempSubscriptionForTier( + Bloom.SubscriptionAndFeatures.SubscriptionTier.Pro + ), + AiTranslationTargetLanguageTag = "es", + }; + var service = new AiTranslationService(collectionSettings); + var engine = new AiTranslationEngineSettings { ProviderId = "alpha2", ApiKey = apiKey }; + + // Distinct number words per segment so we can verify order is preserved, not just + // that translation happened. + var segments = new[] { "I have one cat.", "I have two dogs.", "I have three birds." }; + + var results = await service.TranslateSegmentsAsync( + engine, + segments, + "en", + CancellationToken.None + ); + + Assert.That(results.Length, Is.EqualTo(segments.Length)); + for (var i = 0; i < segments.Length; i++) + { + Assert.That(results[i], Is.Not.Null.And.Not.Empty); + Assert.That(results[i], Is.Not.EqualTo(segments[i])); + } + + // Spanish number words for one/two/three, verifying each translation landed in the + // slot matching its source segment (not merged/reordered). + Assert.That(results[0].ToLowerInvariant(), Does.Contain("uno").Or.Contain("un ")); + Assert.That(results[1].ToLowerInvariant(), Does.Contain("dos")); + Assert.That(results[2].ToLowerInvariant(), Does.Contain("tres")); + } + } +} diff --git a/src/BloomTests/Book/RuntimeInformationInjectorTests.cs b/src/BloomTests/Book/RuntimeInformationInjectorTests.cs index b1154bf61431..4e108bf35df0 100644 --- a/src/BloomTests/Book/RuntimeInformationInjectorTests.cs +++ b/src/BloomTests/Book/RuntimeInformationInjectorTests.cs @@ -3,8 +3,12 @@ using System.Linq; using System.Text; using System.Threading.Tasks; +using Bloom; +using Bloom.AiTranslation; using Bloom.Book; using Bloom.Collection; +using Bloom.SubscriptionAndFeatures; +using Bloom.web.controllers; using NUnit.Framework; namespace BloomTests.Book @@ -12,6 +16,25 @@ namespace BloomTests.Book public class RuntimeInformationInjectorTests { private HtmlDom _bookDom; + private bool _previousAiSourceBubblesEnabled; + + [SetUp] + public void Setup() + { + _previousAiSourceBubblesEnabled = ExperimentalFeatures.IsFeatureEnabled( + ExperimentalFeatures.kAiSourceBubbles + ); + ExperimentalFeatures.SetValue(ExperimentalFeatures.kAiSourceBubbles, true); + } + + [TearDown] + public void TearDown() + { + ExperimentalFeatures.SetValue( + ExperimentalFeatures.kAiSourceBubbles, + _previousAiSourceBubblesEnabled + ); + } private void SetDom(string bodyContents) { @@ -83,5 +106,78 @@ public void AddLanguagesUsedInPage_InvalidLanguageSubTag_DoesNotThrow(string inv settings ); } + + [Test] + public void AddUISettingsToDom_AiSourceBubblesDisabledWhenNoEngineEnabled() + { + SetDom("
    "); + var collectionSettings = MakeAiSourceBubblesCollectionSettings(); + var bookData = new BookData(_bookDom, collectionSettings, null); + + RuntimeInformationInjector.AddUISettingsToDom(_bookDom, bookData, null); + + var scriptContents = _bookDom + .RawDom.SelectSingleNode("//script[@id='ui-settings']") + .InnerText; + Assert.That(scriptContents, Does.Contain("\"allowAiSourceBubbles\":false")); + } + + [Test] + public void AddUISettingsToDom_AiSourceBubblesDisabledWhenValidationFingerprintStale() + { + SetDom("
    "); + var collectionSettings = MakeAiSourceBubblesCollectionSettings(); + var engine = collectionSettings.AiTranslationEngines.Single(e => + e.ProviderId == "deepl" + ); + engine.Enabled = true; + engine.ApiKey = "sample-key"; + engine.LastValidationSucceeded = true; + engine.ValidatedConfigurationFingerprint = "stale"; + var bookData = new BookData(_bookDom, collectionSettings, null); + + RuntimeInformationInjector.AddUISettingsToDom(_bookDom, bookData, null); + + var scriptContents = _bookDom + .RawDom.SelectSingleNode("//script[@id='ui-settings']") + .InnerText; + Assert.That(scriptContents, Does.Contain("\"allowAiSourceBubbles\":false")); + } + + [Test] + public void AddUISettingsToDom_AiSourceBubblesEnabledWhenAnEngineValidatedAndCurrent() + { + SetDom("
    "); + var collectionSettings = MakeAiSourceBubblesCollectionSettings(); + var engine = collectionSettings.AiTranslationEngines.Single(e => + e.ProviderId == "deepl" + ); + engine.Enabled = true; + engine.ApiKey = "sample-key"; + engine.LastValidationSucceeded = true; + engine.ValidatedConfigurationFingerprint = AiTranslationService.GetEngineFingerprint( + engine, + collectionSettings.AiTranslationTargetLanguageTag + ); + var bookData = new BookData(_bookDom, collectionSettings, null); + + RuntimeInformationInjector.AddUISettingsToDom(_bookDom, bookData, null); + + var scriptContents = _bookDom + .RawDom.SelectSingleNode("//script[@id='ui-settings']") + .InnerText; + Assert.That(scriptContents, Does.Contain("\"allowAiSourceBubbles\":true")); + } + + private static CollectionSettings MakeAiSourceBubblesCollectionSettings() + { + var collectionSettings = new CollectionSettings + { + Subscription = Subscription.CreateTempSubscriptionForTier(SubscriptionTier.Pro), + AiTranslationTargetLanguageTag = "fr", + }; + collectionSettings.EnsureAiTranslationEngines(); + return collectionSettings; + } } } diff --git a/src/BloomTests/ExperimentalFeaturesTests.cs b/src/BloomTests/ExperimentalFeaturesTests.cs index 2303e32ad47c..42610d4eb718 100644 --- a/src/BloomTests/ExperimentalFeaturesTests.cs +++ b/src/BloomTests/ExperimentalFeaturesTests.cs @@ -32,18 +32,25 @@ public void SetValueWorksProperly() ExperimentalFeatures.SetValue(ExperimentalFeatures.kExperimentalSourceBooks, true); ExperimentalFeatures.SetValue(ExperimentalFeatures.kTeamCollections, true); + ExperimentalFeatures.SetValue(ExperimentalFeatures.kAiSourceBubbles, true); ExperimentalFeatures.SetValue(ExperimentalFeatures.kExperimentalSourceBooks, true); ExperimentalFeatures.SetValue(ExperimentalFeatures.kTeamCollections, true); + ExperimentalFeatures.SetValue(ExperimentalFeatures.kAiSourceBubbles, true); Assert.IsTrue( ExperimentalFeatures.IsFeatureEnabled(ExperimentalFeatures.kExperimentalSourceBooks) ); Assert.IsTrue( ExperimentalFeatures.IsFeatureEnabled(ExperimentalFeatures.kTeamCollections) ); + Assert.IsTrue( + ExperimentalFeatures.IsFeatureEnabled(ExperimentalFeatures.kAiSourceBubbles) + ); Assert.AreEqual( ExperimentalFeatures.kExperimentalSourceBooks + "," - + ExperimentalFeatures.kTeamCollections, + + ExperimentalFeatures.kTeamCollections + + "," + + ExperimentalFeatures.kAiSourceBubbles, ExperimentalFeatures.TokensOfEnabledFeatures ); @@ -54,8 +61,11 @@ public void SetValueWorksProperly() Assert.IsTrue( ExperimentalFeatures.IsFeatureEnabled(ExperimentalFeatures.kTeamCollections) ); + Assert.IsTrue( + ExperimentalFeatures.IsFeatureEnabled(ExperimentalFeatures.kAiSourceBubbles) + ); Assert.AreEqual( - ExperimentalFeatures.kTeamCollections, + ExperimentalFeatures.kTeamCollections + "," + ExperimentalFeatures.kAiSourceBubbles, ExperimentalFeatures.TokensOfEnabledFeatures ); @@ -66,6 +76,18 @@ public void SetValueWorksProperly() Assert.IsFalse( ExperimentalFeatures.IsFeatureEnabled(ExperimentalFeatures.kTeamCollections) ); + Assert.IsTrue( + ExperimentalFeatures.IsFeatureEnabled(ExperimentalFeatures.kAiSourceBubbles) + ); + Assert.AreEqual( + ExperimentalFeatures.kAiSourceBubbles, + ExperimentalFeatures.TokensOfEnabledFeatures + ); + + ExperimentalFeatures.SetValue(ExperimentalFeatures.kAiSourceBubbles, false); + Assert.IsFalse( + ExperimentalFeatures.IsFeatureEnabled(ExperimentalFeatures.kAiSourceBubbles) + ); Assert.AreEqual("", ExperimentalFeatures.TokensOfEnabledFeatures); } } diff --git a/src/BloomTests/Subscription/FeatureStatusTests.cs b/src/BloomTests/Subscription/FeatureStatusTests.cs index 0093def14bd3..50f6d9d5d4aa 100644 --- a/src/BloomTests/Subscription/FeatureStatusTests.cs +++ b/src/BloomTests/Subscription/FeatureStatusTests.cs @@ -134,6 +134,52 @@ public void GetFeatureStatus_ExperimentalFeatureStillRequiresSubscription() Assert.That(status.SubscriptionTier, Is.EqualTo(SubscriptionTier.Pro)); } + [Test] + public void GetFeatureStatus_AiSourceBubblesHiddenUnlessEnabled() + { + var subscription = Subscription.CreateTempSubscriptionForTier(SubscriptionTier.Pro); + + ExperimentalFeatures.SetValue(ExperimentalFeatures.kAiSourceBubbles, false); + var hiddenStatus = FeatureStatus.GetFeatureStatus( + subscription, + FeatureName.AiSourceBubbles + ); + + ExperimentalFeatures.SetValue(ExperimentalFeatures.kAiSourceBubbles, true); + var visibleStatus = FeatureStatus.GetFeatureStatus( + subscription, + FeatureName.AiSourceBubbles + ); + + ExperimentalFeatures.SetValue(ExperimentalFeatures.kAiSourceBubbles, false); + + Assert.That(hiddenStatus.Visible, Is.False); + Assert.That(hiddenStatus.Enabled, Is.True); + Assert.That(visibleStatus.Visible, Is.True); + Assert.That(visibleStatus.Enabled, Is.True); + } + + [TestCase(SubscriptionTier.Basic, false)] + [TestCase(SubscriptionTier.Pro, true)] + [TestCase(SubscriptionTier.LocalCommunity, true)] + [TestCase(SubscriptionTier.Enterprise, true)] + public void GetFeatureStatus_AiSourceBubblesRequiresProOrHigher( + SubscriptionTier tier, + bool expectedEnabled + ) + { + var subscription = Subscription.CreateTempSubscriptionForTier(tier); + ExperimentalFeatures.SetValue(ExperimentalFeatures.kAiSourceBubbles, true); + + var status = FeatureStatus.GetFeatureStatus(subscription, FeatureName.AiSourceBubbles); + + ExperimentalFeatures.SetValue(ExperimentalFeatures.kAiSourceBubbles, false); + + Assert.That(status.Visible, Is.True); + Assert.That(status.Enabled, Is.EqualTo(expectedEnabled)); + Assert.That(status.SubscriptionTier, Is.EqualTo(SubscriptionTier.Pro)); + } + [Test] public void ForSerialization_ReturnsValidObject() {