diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/BuildJobOptions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/BuildJobOptions.cs index b6b9da57..c1dac5b0 100644 --- a/src/Machine/src/Serval.Machine.Shared/Configuration/BuildJobOptions.cs +++ b/src/Machine/src/Serval.Machine.Shared/Configuration/BuildJobOptions.cs @@ -7,4 +7,6 @@ public class BuildJobOptions public IList ClearML { get; set; } = new List(); public bool PreserveBuildFiles { get; set; } = false; public int MaxWarnings { get; set; } = 1000; + public int MaxDiagnostics { get; set; } = 1000; + public int MinimumTrainCount { get; set; } = 600; } diff --git a/src/Machine/src/Serval.Machine.Shared/Models/Build.cs b/src/Machine/src/Serval.Machine.Shared/Models/Build.cs index 4a7df99b..d8034a76 100644 --- a/src/Machine/src/Serval.Machine.Shared/Models/Build.cs +++ b/src/Machine/src/Serval.Machine.Shared/Models/Build.cs @@ -30,6 +30,7 @@ public record Build public required BuildStage Stage { get; init; } public DateTimeOffset QueuedAt { get; init; } public string? Options { get; set; } + public string? Model { get; init; } public string? JobData { get; init; } public required BuildExecutionData ExecutionData { get; init; } } diff --git a/src/Machine/src/Serval.Machine.Shared/Models/BuildExecutionData.cs b/src/Machine/src/Serval.Machine.Shared/Models/BuildExecutionData.cs index 004ad374..1f077502 100644 --- a/src/Machine/src/Serval.Machine.Shared/Models/BuildExecutionData.cs +++ b/src/Machine/src/Serval.Machine.Shared/Models/BuildExecutionData.cs @@ -9,6 +9,8 @@ public record BuildExecutionData public IReadOnlyDictionary>? TrainVerseCount { get; init; } public IReadOnlyDictionary>? InferenceVerseCount { get; init; } public IReadOnlyList? Warnings { get; init; } + public IReadOnlyList? Diagnostics { get; init; } + public bool? DiagnosticsTruncated { get; init; } public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } public string? ResolvedSourceLanguage { get; init; } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/BuildJobService.cs b/src/Machine/src/Serval.Machine.Shared/Services/BuildJobService.cs index 526520f8..0ee28ff6 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/BuildJobService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/BuildJobService.cs @@ -68,6 +68,7 @@ public async Task StartBuildJobAsync( BuildStage stage, object? data = null, string? buildOptions = null, + string? model = null, CancellationToken cancellationToken = default ) { @@ -79,6 +80,7 @@ public async Task StartBuildJobAsync( stage, data, buildOptions, + model, cancellationToken ); try @@ -106,6 +108,7 @@ public async Task StartBuildJobAsync( JobState = BuildJobState.Pending, QueuedAt = DateTimeOffset.UtcNow, Options = buildOptions, + Model = model, JobData = jobData, ExecutionData = new BuildExecutionData(), } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ClearMLBuildJobRunner.cs b/src/Machine/src/Serval.Machine.Shared/Services/ClearMLBuildJobRunner.cs index a4211047..825c70fa 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ClearMLBuildJobRunner.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ClearMLBuildJobRunner.cs @@ -40,6 +40,7 @@ public async Task DeleteEngineAsync(string engineId, CancellationToken cancellat BuildStage stage, object? data = null, string? buildOptions = null, + string? model = null, CancellationToken cancellationToken = default ) { @@ -57,6 +58,7 @@ public async Task DeleteEngineAsync(string engineId, CancellationToken cancellat _options[engineType].ModelType, stage, buildOptions, + model, cancellationToken ); string jobId = await _clearMLService.CreateTaskAsync( diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs index 5f2f91d4..3a825302 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs @@ -298,7 +298,7 @@ CancellationToken cancellationToken BuildStage.Postprocess, (corpusSize, confidence), buildOptions, - cancellationToken + cancellationToken: cancellationToken ); } finally diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IBuildJobRunner.cs b/src/Machine/src/Serval.Machine.Shared/Services/IBuildJobRunner.cs index c9b5b37d..afaaf5e3 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/IBuildJobRunner.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/IBuildJobRunner.cs @@ -15,6 +15,7 @@ public interface IBuildJobRunner BuildStage stage, object? data = null, string? buildOptions = null, + string? model = null, CancellationToken cancellationToken = default ); diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IBuildJobService.cs b/src/Machine/src/Serval.Machine.Shared/Services/IBuildJobService.cs index b92d6f90..95b26d56 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/IBuildJobService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/IBuildJobService.cs @@ -16,6 +16,7 @@ Task StartBuildJobAsync( BuildStage stage, object? data = default, string? buildOptions = default, + string? model = null, CancellationToken cancellationToken = default ); diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IClearMLBuildJobFactory.cs b/src/Machine/src/Serval.Machine.Shared/Services/IClearMLBuildJobFactory.cs index 3dbd6e2b..0c4c34a6 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/IClearMLBuildJobFactory.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/IClearMLBuildJobFactory.cs @@ -10,6 +10,7 @@ Task CreateJobScriptAsync( string modelType, BuildStage stage, string? buildOptions = null, + string? model = null, CancellationToken cancellationToken = default ); } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/LocalBuildJobRunner.cs b/src/Machine/src/Serval.Machine.Shared/Services/LocalBuildJobRunner.cs index ead5e954..98322f07 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/LocalBuildJobRunner.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/LocalBuildJobRunner.cs @@ -44,6 +44,7 @@ public Task CreateEngineAsync( BuildStage stage, object? data = null, string? buildOptions = null, + string? model = null, CancellationToken cancellationToken = default ) { diff --git a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs index 3aaa410c..4f98f6b9 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs @@ -8,6 +8,7 @@ public abstract class PreprocessBuildJob( IBuildJobService buildJobService, ISharedFileService sharedFileService, IParallelCorpusService parallelCorpusService, + IBuildDiagnosticService buildDiagnosticService, IOptionsMonitor options ) : BuildJob>( @@ -32,6 +33,7 @@ IOptionsMonitor options protected readonly BuildJobOptions BuildJobOptions = options.CurrentValue; protected readonly ISharedFileService SharedFileService = sharedFileService; protected readonly IParallelCorpusService ParallelCorpusService = parallelCorpusService; + protected readonly IBuildDiagnosticService BuildDiagnosticService = buildDiagnosticService; protected override async Task DoWorkAsync( string engineId, @@ -46,6 +48,7 @@ CancellationToken cancellationToken throw new OperationCanceledException($"Engine {engineId} does not exist. Build canceled."); PreprocessStats stats = await WriteDataFilesAsync(engineId, buildId, data, buildOptions, cancellationToken); + bool isNonPersistedTranslationEngine = engine is IPersistableTrainingEngine { IsModelPersisted: false }; await UpdateBuildExecutionData( engineId, @@ -53,16 +56,17 @@ await UpdateBuildExecutionData( stats, engine.SourceLanguage, engine.TargetLanguage, + isNonPersistedTranslationEngine, data, cancellationToken ); await UpdateTargetQuoteConventionAsync(engineId, buildId, data, cancellationToken); - if (stats.InferenceCount == 0 && engine is IPersistableTrainingEngine { IsModelPersisted: false }) + if (stats.InferenceCount == 0 && isNonPersistedTranslationEngine) { throw new InvalidOperationException( - $"There was no data specified for inferencing in build {buildId}. Build canceled." + $"There was no data specified for inferencing in build {buildId} and the model is not persisted. Build canceled." ); } @@ -87,6 +91,7 @@ protected abstract Task UpdateBuildExecutionData( PreprocessStats stats, string sourceLanguageTag, string targetLanguageTag, + bool isNonPersistedTranslationEngine, IReadOnlyList parallelCorpora, CancellationToken cancellationToken ); @@ -121,53 +126,150 @@ protected override async Task CleanupAsync(string engineId, string buildId, JobC } } - protected virtual IReadOnlyList GetWarnings( + protected virtual IReadOnlyList GetDiagnostics( int trainCount, int inferenceCount, string sourceLanguageTag, string targetLanguageTag, + bool sourceLanguageHasNativeSupport, + bool targetLanguageHasNativeSupport, + bool isNonPersistedTranslationEngine, + string modelName, IReadOnlyList parallelCorpora ) { - List warnings = []; - HashSet versifications = []; + List diagnostics = []; + Dictionary projectVersifications = []; foreach ( ( string parallelCorpusId, string monolingualCorpusId, string projectName, + string projectGuid, string versificationName, - IReadOnlyList diagnostics + IReadOnlyList usfmDiagnostics ) in ParallelCorpusService.AnalyzeUsfmVersification(parallelCorpora) ) { - versifications.Add(versificationName); - foreach (UsfmVersificationDiagnosticContract diagnostic in diagnostics) + projectVersifications[projectGuid] = versificationName; + foreach (UsfmVersificationDiagnosticContract usfmDiagnostic in usfmDiagnostics) { - string diagnosticDetails = - $"in project {projectName} at {diagnostic.Filename} " - + (diagnostic.LineNumbers.Count == 1 ? "line " : "lines ") - + $"{string.Join(", ", diagnostic.LineNumbers)}, " - + (diagnostic.NumAffectedVerses == 1 ? "verse " : "verses ") - + $"{string.Join(", ", diagnostic.References)} " - + $"(parallel corpus {parallelCorpusId}, monolingual corpus {monolingualCorpusId})."; - warnings.Add( - diagnostic.Type switch + diagnostics.Add( + usfmDiagnostic.Type switch { Serval.Shared.Contracts.UsfmVersificationDiagnosticType.InvalidChapter => - $"Invalid chapter number {diagnosticDetails}", + BuildDiagnosticService.CreateDiagnostic( + "USFM-0001", + new Dictionary + { + { "projectName", projectName }, + { "projectGuid", projectGuid }, + { "usfmFilename", usfmDiagnostic.Filename }, + { + "lineNumber", + usfmDiagnostic.LineNumbers.Count > 0 ? usfmDiagnostic.LineNumbers[0] : -1 + }, + { + "verseReference", + usfmDiagnostic.References.Count > 0 ? usfmDiagnostic.References[0] : "" + }, + { "parallelCorpusId", parallelCorpusId }, + { "monolingualCorpusId", monolingualCorpusId }, + } + ), Serval.Shared.Contracts.UsfmVersificationDiagnosticType.InvalidVerse => - $"Invalid verse number {diagnosticDetails}", + BuildDiagnosticService.CreateDiagnostic( + "USFM-0002", + new Dictionary + { + { "projectName", projectName }, + { "projectGuid", projectGuid }, + { "usfmFilename", usfmDiagnostic.Filename }, + { + "lineNumber", + usfmDiagnostic.LineNumbers.Count > 0 ? usfmDiagnostic.LineNumbers[0] : -1 + }, + { + "verseReference", + usfmDiagnostic.References.Count > 0 ? usfmDiagnostic.References[0] : "" + }, + { "parallelCorpusId", parallelCorpusId }, + { "monolingualCorpusId", monolingualCorpusId }, + } + ), + Serval.Shared.Contracts.UsfmVersificationDiagnosticType.Extra => - $"{diagnostic.NumAffectedVerses} extra verses {diagnosticDetails}", + BuildDiagnosticService.CreateDiagnostic( + "USFM-0003", + new Dictionary + { + { "numberOfVerses", usfmDiagnostic.NumAffectedVerses }, + { "projectName", projectName }, + { "projectGuid", projectGuid }, + { "usfmFilename", usfmDiagnostic.Filename }, + { "lineNumbers", usfmDiagnostic.LineNumbers.ToList() }, + { "verseReferences", usfmDiagnostic.References.ToList() }, + { "parallelCorpusId", parallelCorpusId }, + { "monolingualCorpusId", monolingualCorpusId }, + } + ), Serval.Shared.Contracts.UsfmVersificationDiagnosticType.Missing => - $"Missing {diagnostic.NumAffectedVerses} verses {diagnosticDetails}", + BuildDiagnosticService.CreateDiagnostic( + "USFM-0004", + new Dictionary + { + { "numberOfVerses", usfmDiagnostic.NumAffectedVerses }, + { "projectName", projectName }, + { "projectGuid", projectGuid }, + { "usfmFilename", usfmDiagnostic.Filename }, + { "lineNumbers", usfmDiagnostic.LineNumbers.ToList() }, + { "verseReferences", usfmDiagnostic.References.ToList() }, + { "parallelCorpusId", parallelCorpusId }, + { "monolingualCorpusId", monolingualCorpusId }, + } + ), Serval.Shared.Contracts.UsfmVersificationDiagnosticType.IncorrectVerseSegment => - $"Incorrect verse segment {diagnosticDetails}", + BuildDiagnosticService.CreateDiagnostic( + "USFM-0005", + new Dictionary + { + { "projectName", projectName }, + { "projectGuid", projectGuid }, + { "usfmFilename", usfmDiagnostic.Filename }, + { + "lineNumber", + usfmDiagnostic.LineNumbers.Count > 0 ? usfmDiagnostic.LineNumbers[0] : -1 + }, + { + "verseReference", + usfmDiagnostic.References.Count > 0 ? usfmDiagnostic.References[0] : "" + }, + { "parallelCorpusId", parallelCorpusId }, + { "monolingualCorpusId", monolingualCorpusId }, + } + ), Serval.Shared.Contracts.UsfmVersificationDiagnosticType.UnsupportedVerseRange => - $"Unsupported verse range {diagnosticDetails}", - _ => $"USFM versification issue {diagnosticDetails}", + BuildDiagnosticService.CreateDiagnostic( + "USFM-0006", + new Dictionary + { + { "projectName", projectName }, + { "projectGuid", projectGuid }, + { "usfmFilename", usfmDiagnostic.Filename }, + { + "lineNumber", + usfmDiagnostic.LineNumbers.Count > 0 ? usfmDiagnostic.LineNumbers[0] : -1 + }, + { + "verseReference", + usfmDiagnostic.References.Count > 0 ? usfmDiagnostic.References[0] : "" + }, + { "parallelCorpusId", parallelCorpusId }, + { "monolingualCorpusId", monolingualCorpusId }, + } + ), + _ => throw new InvalidEnumArgumentException(nameof(usfmDiagnostic.Type)), } ); } @@ -181,19 +283,37 @@ MissingParentProjectErrorContract error ) in ParallelCorpusService.FindMissingParentProjects(parallelCorpora) ) { - warnings.Add( - $"Unable to locate parent project {error.ParentProjectName} of daughter project {error.ProjectName} (parallel corpus {parallelCorpusId}, monolingual corpus {monolingualCorpusId})" + diagnostics.Add( + BuildDiagnosticService.CreateDiagnostic( + "CONFIG-0001", + new Dictionary + { + { "parentProjectName", error.ParentProjectName }, + { "parentProjectGuid", error.ParentProjectGuid }, + { "daughterProjectName", error.ProjectName }, + { "daughterProjectGuid", error.ProjectGuid }, + { "parallelCorpusId", parallelCorpusId }, + { "monolingualCorpusId", monolingualCorpusId }, + } + ) ); } - if (versifications.Count > 1) + if (projectVersifications.Values.Distinct().Count() > 1) { - warnings.Add( - $"Multiple versifications represented among Paratext projects selected for training or inferencing: {string.Join(", ", versifications)}" + diagnostics.Add( + BuildDiagnosticService.CreateDiagnostic( + "CONFIG-0002", + new Dictionary { { "projectVersifications", projectVersifications } } + ) ); } - return warnings; + if (inferenceCount == 0 && isNonPersistedTranslationEngine) + { + diagnostics.Add(BuildDiagnosticService.CreateDiagnostic("CONFIG-0004", [])); + } + return diagnostics; } protected static (bool IsTrainFilteredByChapter, bool IsInferenceFilteredByChapter) CheckChapterFilters( diff --git a/src/Machine/src/Serval.Machine.Translation/Models/BaseModels.cs b/src/Machine/src/Serval.Machine.Translation/Models/BaseModels.cs new file mode 100644 index 00000000..22b1fb67 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Translation/Models/BaseModels.cs @@ -0,0 +1,8 @@ +namespace Serval.Machine.Translation.Models; + +public static class Models +{ + public const string Nllb = "NLLB"; + public const string Nllb600m = "NLLB600m"; + public const string NllbTesting = "NLLBTesting"; +} diff --git a/src/Machine/src/Serval.Machine.Translation/Services/EchoPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Translation/Services/EchoPreprocessBuildJob.cs index 9d35d5a1..d64995b6 100644 --- a/src/Machine/src/Serval.Machine.Translation/Services/EchoPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Translation/Services/EchoPreprocessBuildJob.cs @@ -8,6 +8,7 @@ public class EchoPreprocessBuildJob( IBuildJobService buildJobService, ISharedFileService sharedFileService, IParallelCorpusService parallelCorpusService, + IBuildDiagnosticService buildDiagnosticService, ITranslationPlatformService translationPlatformService, IOptionsMonitor options ) @@ -19,6 +20,7 @@ IOptionsMonitor options buildJobService, sharedFileService, parallelCorpusService, + buildDiagnosticService, options ) { @@ -30,18 +32,36 @@ protected override async Task UpdateBuildExecutionData( PreprocessStats stats, string sourceLanguageTag, string targetLanguageTag, + bool isNonPersistedTranslationEngine, IReadOnlyList parallelCorpora, CancellationToken cancellationToken ) { - IReadOnlyList warnings = GetWarnings( + string modelName = + (await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken))?.CurrentBuild?.Model?.ToString() + ?? "Unknown"; + IReadOnlyList diagnostics = GetDiagnostics( stats.TrainCount, stats.InferenceCount, sourceLanguageTag, targetLanguageTag, + sourceLanguageHasNativeSupport: true, + targetLanguageHasNativeSupport: true, + isNonPersistedTranslationEngine, + modelName, parallelCorpora ); + IReadOnlyList warnings = diagnostics.Select(d => d.Message).ToList(); + + int maxDiagnostics = BuildJobOptions.MaxDiagnostics; + bool diagnosticsTruncated = false; + if (diagnostics.Count > maxDiagnostics) + { + diagnosticsTruncated = true; + diagnostics = diagnostics.OrderByDescending(d => d.Severity).Take(maxDiagnostics).ToList(); + } + int maxWarnings = BuildJobOptions.MaxWarnings; if (warnings.Count > maxWarnings) { @@ -74,6 +94,8 @@ CancellationToken cancellationToken TrainVerseCount = stats.TrainVerseCount, InferenceVerseCount = stats.InferenceVerseCount, Warnings = warnings, + Diagnostics = diagnostics, + DiagnosticsTruncated = diagnosticsTruncated, EngineSourceLanguageTag = sourceLanguageTag, EngineTargetLanguageTag = targetLanguageTag, ResolvedSourceLanguage = sourceLanguageTag, diff --git a/src/Machine/src/Serval.Machine.Translation/Services/EchoTranslationEngineService.cs b/src/Machine/src/Serval.Machine.Translation/Services/EchoTranslationEngineService.cs index 568b3ee7..2e334ee8 100644 --- a/src/Machine/src/Serval.Machine.Translation/Services/EchoTranslationEngineService.cs +++ b/src/Machine/src/Serval.Machine.Translation/Services/EchoTranslationEngineService.cs @@ -72,6 +72,7 @@ public async Task StartBuildAsync( string buildId, IReadOnlyList corpora, string? options = null, + string? model = null, CancellationToken cancellationToken = default ) { @@ -83,7 +84,8 @@ public async Task StartBuildAsync( BuildStage.Preprocess, corpora, options, - cancellationToken + model, + cancellationToken: cancellationToken ); // If there is a pending/running build, then no need to start a new one. if (building) diff --git a/src/Machine/src/Serval.Machine.Translation/Services/NmtClearMLBuildJobFactory.cs b/src/Machine/src/Serval.Machine.Translation/Services/NmtClearMLBuildJobFactory.cs index 0aece6c6..4b0d29fe 100644 --- a/src/Machine/src/Serval.Machine.Translation/Services/NmtClearMLBuildJobFactory.cs +++ b/src/Machine/src/Serval.Machine.Translation/Services/NmtClearMLBuildJobFactory.cs @@ -18,6 +18,7 @@ public async Task CreateJobScriptAsync( string modelType, BuildStage stage, string? buildOptions = null, + string? model = null, CancellationToken cancellationToken = default ) { @@ -32,6 +33,19 @@ public async Task CreateJobScriptAsync( string folder = sharedFileUri.GetComponents(UriComponents.Path, UriFormat.Unescaped); _languageTagService.ConvertToFlores200Code(engine.SourceLanguage, out string srcLang); _languageTagService.ConvertToFlores200Code(engine.TargetLanguage, out string trgLang); + if (buildOptions != null && model != null) + { + try + { + JsonNode? buildOptionsJsonNode = JsonNode.Parse(buildOptions); + if (buildOptionsJsonNode != null && buildOptionsJsonNode is JsonObject buildOptionsJsonObject) + buildOptionsJsonObject["parent_model_name"] = GetFullModelName(model); + } + catch (Exception e) + { + throw new InvalidOperationException($"Unable to parse field build options : {e.Message}", e); + } + } return "from machine.jobs.build_nmt_engine import run\n" + "args = {\n" + $" 'model_type': '{modelType}',\n" @@ -54,4 +68,15 @@ public async Task CreateJobScriptAsync( throw new ArgumentException("Unknown build stage.", nameof(stage)); } } + + private static string GetFullModelName(string model) + { + return model switch + { + Models.Models.Nllb => "facebook/nllb-200-distilled-1.3B", + Models.Models.Nllb600m => "facebook/nllb-200-distilled-600M", + Models.Models.NllbTesting => "hf-internal-testing/tiny-random-nllb", + _ => throw new ArgumentException($"Unknown base model {model}."), + }; + } } diff --git a/src/Machine/src/Serval.Machine.Translation/Services/NmtEngineService.cs b/src/Machine/src/Serval.Machine.Translation/Services/NmtEngineService.cs index 5ce1e36f..586d8fce 100644 --- a/src/Machine/src/Serval.Machine.Translation/Services/NmtEngineService.cs +++ b/src/Machine/src/Serval.Machine.Translation/Services/NmtEngineService.cs @@ -85,6 +85,7 @@ public async Task StartBuildAsync( string buildId, IReadOnlyList corpora, string? options = null, + string? model = null, CancellationToken cancellationToken = default ) { @@ -96,6 +97,7 @@ public async Task StartBuildAsync( BuildStage.Preprocess, corpora, options, + model, cancellationToken ); // If there is a pending/running build, then no need to start a new one. diff --git a/src/Machine/src/Serval.Machine.Translation/Services/NmtPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Translation/Services/NmtPreprocessBuildJob.cs index 2ab39cfc..b2ce81aa 100644 --- a/src/Machine/src/Serval.Machine.Translation/Services/NmtPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Translation/Services/NmtPreprocessBuildJob.cs @@ -9,6 +9,7 @@ public class NmtPreprocessBuildJob( ISharedFileService sharedFileService, ILanguageTagService languageTagService, IParallelCorpusService parallelCorpusService, + IBuildDiagnosticService buildDiagnosticService, IOptionsMonitor options ) : TranslationPreprocessBuildJob( @@ -19,6 +20,7 @@ IOptionsMonitor options buildJobService, sharedFileService, parallelCorpusService, + buildDiagnosticService, options ) { @@ -55,6 +57,7 @@ protected override async Task UpdateBuildExecutionData( PreprocessStats stats, string sourceLanguageTag, string targetLanguageTag, + bool isNonPersistedTranslationEngine, IReadOnlyList parallelCorpora, CancellationToken cancellationToken ) @@ -62,21 +65,31 @@ CancellationToken cancellationToken bool sourceLanguageHasNativeSupport = ResolveLanguageCode(sourceLanguageTag, out string resolvedSourceLanguage); bool targetLanguageHasNativeSupport = ResolveLanguageCode(targetLanguageTag, out string resolvedTargetLanguage); - if (stats.TrainCount == 0 && (!sourceLanguageHasNativeSupport || !targetLanguageHasNativeSupport)) - { - throw new InvalidOperationException( - $"At least one language code in build {buildId} is unknown to the base model, and the data specified for training was empty. Build canceled." - ); - } - - IReadOnlyList warnings = GetWarnings( + string modelName = + (await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken))?.CurrentBuild?.Model?.ToString() + ?? "Unknown"; + IReadOnlyList diagnostics = GetDiagnostics( stats.TrainCount, stats.InferenceCount, sourceLanguageTag, targetLanguageTag, + sourceLanguageHasNativeSupport, + targetLanguageHasNativeSupport, + isNonPersistedTranslationEngine, + modelName, parallelCorpora ); + IReadOnlyList warnings = diagnostics.Select(d => d.Message).ToList(); + + int maxDiagnostics = BuildJobOptions.MaxDiagnostics; + bool diagnosticsTruncated = false; + if (diagnostics.Count > maxDiagnostics) + { + diagnosticsTruncated = true; + diagnostics = diagnostics.OrderByDescending(d => d.Severity).Take(maxDiagnostics).ToList(); + } + int maxWarnings = BuildJobOptions.MaxWarnings; if (warnings.Count > maxWarnings) { @@ -109,31 +122,50 @@ CancellationToken cancellationToken IsTrainFilteredByChapter = stats.IsTrainFilteredByChapter, IsInferenceFilteredByChapter = stats.IsInferenceFilteredByChapter, Warnings = warnings, + Diagnostics = diagnostics, + DiagnosticsTruncated = diagnosticsTruncated, EngineSourceLanguageTag = sourceLanguageTag, EngineTargetLanguageTag = targetLanguageTag, ResolvedSourceLanguage = resolvedSourceLanguage, ResolvedTargetLanguage = resolvedTargetLanguage, }; await PlatformService.UpdateBuildExecutionDataAsync(engineId, buildId, executionData, cancellationToken); + + if (stats.TrainCount == 0 && (!sourceLanguageHasNativeSupport || !targetLanguageHasNativeSupport)) + { + throw new InvalidOperationException( + $"At least one language code in build {buildId} is unknown to the base model {modelName}, and no data was specified for training. Build canceled." + ); + } } - protected override IReadOnlyList GetWarnings( + protected override IReadOnlyList GetDiagnostics( int trainCount, int inferenceCount, string sourceLanguageTag, string targetLanguageTag, + bool sourceLanguageHasNativeSupport, + bool targetLanguageHasNativeSupport, + bool isNonPersistedTranslationEngine, + string modelName, IReadOnlyList parallelCorpora ) { - List warnings = - [ - .. base.GetWarnings(trainCount, inferenceCount, sourceLanguageTag, targetLanguageTag, parallelCorpora), - ]; + List diagnostics = []; // Has at least a Gospel of Mark amount of data and not the special case of no data which will be caught elsewhere if (trainCount < 600 && trainCount != 0) { - warnings.Add($"Only {trainCount} segments were selected for training."); + diagnostics.Add( + BuildDiagnosticService.CreateDiagnostic( + "CONFIG-0003", + new Dictionary + { + { "trainCount", trainCount }, + { "minimumTrainCount", BuildJobOptions.MinimumTrainCount }, + } + ) + ); } if ( @@ -141,14 +173,59 @@ .. base.GetWarnings(trainCount, inferenceCount, sourceLanguageTag, targetLanguag == Flores200Support.None ) { - warnings.Add($"The script for the source language '{resolvedCode}' is not in Flores-200"); + diagnostics.Add( + BuildDiagnosticService.CreateDiagnostic( + "MODEL-0001", + new Dictionary { { "resolvedCode", resolvedCode }, { "modelName", modelName } } + ) + ); } if (_languageTagService.ConvertToFlores200Code(targetLanguageTag, out resolvedCode) == Flores200Support.None) { - warnings.Add($"The script for the target language '{resolvedCode}' is not in Flores-200"); + diagnostics.Add( + BuildDiagnosticService.CreateDiagnostic( + "MODEL-0002", + new Dictionary { { "resolvedCode", resolvedCode }, { "modelName", modelName } } + ) + ); } - return warnings; + if (trainCount == 0 && (!sourceLanguageHasNativeSupport || !targetLanguageHasNativeSupport)) + { + List unknownLanguageCodes = new[] + { + !sourceLanguageHasNativeSupport ? sourceLanguageTag : "", + !targetLanguageHasNativeSupport ? targetLanguageTag : "", + } + .Where(s => !string.IsNullOrEmpty(s)) + .ToList(); + diagnostics.Add( + BuildDiagnosticService.CreateDiagnostic( + "MODEL-0004", + new Dictionary + { + { "unknownLanguageCodes", unknownLanguageCodes }, + { "modelName", modelName }, + } + ) + ); + } + + return + [ + .. base.GetDiagnostics( + trainCount, + inferenceCount, + sourceLanguageTag, + targetLanguageTag, + sourceLanguageHasNativeSupport, + targetLanguageHasNativeSupport, + isNonPersistedTranslationEngine, + modelName, + parallelCorpora + ), + .. diagnostics, + ]; } } diff --git a/src/Machine/src/Serval.Machine.Translation/Services/ServalTranslationPlatformService.cs b/src/Machine/src/Serval.Machine.Translation/Services/ServalTranslationPlatformService.cs index 1c7d4fce..d5a7be79 100644 --- a/src/Machine/src/Serval.Machine.Translation/Services/ServalTranslationPlatformService.cs +++ b/src/Machine/src/Serval.Machine.Translation/Services/ServalTranslationPlatformService.cs @@ -105,6 +105,8 @@ public Task UpdateBuildExecutionDataAsync( IsTrainFilteredByChapter = executionData.IsTrainFilteredByChapter, IsPretranslateFilteredByChapter = executionData.IsInferenceFilteredByChapter, Warnings = executionData.Warnings, + Diagnostics = executionData.Diagnostics, + DiagnosticsTruncated = executionData.DiagnosticsTruncated, EngineSourceLanguageTag = executionData.EngineSourceLanguageTag, EngineTargetLanguageTag = executionData.EngineTargetLanguageTag, ResolvedSourceLanguage = executionData.ResolvedSourceLanguage, diff --git a/src/Machine/src/Serval.Machine.Translation/Services/SmtTransferClearMLBuildJobFactory.cs b/src/Machine/src/Serval.Machine.Translation/Services/SmtTransferClearMLBuildJobFactory.cs index 5cfe488f..85c3ec26 100644 --- a/src/Machine/src/Serval.Machine.Translation/Services/SmtTransferClearMLBuildJobFactory.cs +++ b/src/Machine/src/Serval.Machine.Translation/Services/SmtTransferClearMLBuildJobFactory.cs @@ -16,6 +16,7 @@ public async Task CreateJobScriptAsync( string modelType, BuildStage stage, string? buildOptions = null, + string? model = null, CancellationToken cancellationToken = default ) { diff --git a/src/Machine/src/Serval.Machine.Translation/Services/SmtTransferEngineService.cs b/src/Machine/src/Serval.Machine.Translation/Services/SmtTransferEngineService.cs index c1fa8a4b..20793653 100644 --- a/src/Machine/src/Serval.Machine.Translation/Services/SmtTransferEngineService.cs +++ b/src/Machine/src/Serval.Machine.Translation/Services/SmtTransferEngineService.cs @@ -184,6 +184,7 @@ public async Task StartBuildAsync( string buildId, IReadOnlyList corpora, string? options = null, + string? model = null, CancellationToken cancellationToken = default ) { @@ -195,7 +196,8 @@ public async Task StartBuildAsync( BuildStage.Preprocess, corpora, options, - cancellationToken + model, + cancellationToken: cancellationToken ); // If there is a pending/running build, then no need to start a new one. if (building) diff --git a/src/Machine/src/Serval.Machine.Translation/Services/SmtTransferPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Translation/Services/SmtTransferPreprocessBuildJob.cs index 2e72847d..47bcad65 100644 --- a/src/Machine/src/Serval.Machine.Translation/Services/SmtTransferPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Translation/Services/SmtTransferPreprocessBuildJob.cs @@ -10,6 +10,7 @@ public class SmtTransferPreprocessBuildJob( SmtTransferEngineStateService stateService, IRepository trainSegmentPairs, IParallelCorpusService parallelCorpusService, + IBuildDiagnosticService buildDiagnosticService, IOptionsMonitor options ) : TranslationPreprocessBuildJob( @@ -20,6 +21,7 @@ IOptionsMonitor options buildJobService, sharedFileService, parallelCorpusService, + buildDiagnosticService, options ) { diff --git a/src/Machine/src/Serval.Machine.Translation/Services/TranslationPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Translation/Services/TranslationPreprocessBuildJob.cs index 9fce8004..dda2f72a 100644 --- a/src/Machine/src/Serval.Machine.Translation/Services/TranslationPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Translation/Services/TranslationPreprocessBuildJob.cs @@ -8,6 +8,7 @@ public class TranslationPreprocessBuildJob( IBuildJobService buildJobService, ISharedFileService sharedFileService, IParallelCorpusService parallelCorpusService, + IBuildDiagnosticService buildDiagnosticService, IOptionsMonitor options ) : PreprocessBuildJob( @@ -18,6 +19,7 @@ IOptionsMonitor options buildJobService, sharedFileService, parallelCorpusService, + buildDiagnosticService, options ) { @@ -89,18 +91,44 @@ protected override async Task UpdateBuildExecutionData( PreprocessStats stats, string sourceLanguageTag, string targetLanguageTag, + bool isNonPersistedTranslationEngine, IReadOnlyList parallelCorpora, CancellationToken cancellationToken ) { - IReadOnlyList warnings = GetWarnings( + string modelName = + (await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken))?.CurrentBuild?.Model?.ToString() + ?? "Unknown"; + IReadOnlyList diagnostics = GetDiagnostics( stats.TrainCount, stats.InferenceCount, sourceLanguageTag, targetLanguageTag, + true, + true, + isNonPersistedTranslationEngine, + modelName, parallelCorpora ); + IReadOnlyList warnings = diagnostics.Select(d => d.Message).ToList(); + + int maxDiagnostics = BuildJobOptions.MaxDiagnostics; + bool diagnosticsTruncated = false; + if (diagnostics.Count > maxDiagnostics) + { + diagnosticsTruncated = true; + diagnostics = diagnostics.OrderByDescending(d => d.Severity).Take(maxDiagnostics).ToList(); + } + + int maxWarnings = BuildJobOptions.MaxWarnings; + if (warnings.Count > maxWarnings) + { + string tooManyWarningsWarning = + $"There were {warnings.Count} warnings. Only the first {maxWarnings} are shown."; + warnings = [tooManyWarningsWarning, .. warnings.Take(maxWarnings)]; + } + // Log summary of build data JsonObject buildPreprocessSummary = new() { @@ -123,6 +151,8 @@ CancellationToken cancellationToken TrainVerseCount = stats.TrainVerseCount, InferenceVerseCount = stats.InferenceVerseCount, Warnings = warnings, + Diagnostics = diagnostics, + DiagnosticsTruncated = diagnosticsTruncated, EngineSourceLanguageTag = sourceLanguageTag, EngineTargetLanguageTag = targetLanguageTag, }; diff --git a/src/Machine/src/Serval.Machine.WordAlignment/Services/EchoWordAlignmentEngineService.cs b/src/Machine/src/Serval.Machine.WordAlignment/Services/EchoWordAlignmentEngineService.cs index 5482a3ba..8fc1b967 100644 --- a/src/Machine/src/Serval.Machine.WordAlignment/Services/EchoWordAlignmentEngineService.cs +++ b/src/Machine/src/Serval.Machine.WordAlignment/Services/EchoWordAlignmentEngineService.cs @@ -85,7 +85,7 @@ public async Task StartBuildAsync( BuildStage.Preprocess, corpora, options, - cancellationToken + cancellationToken: cancellationToken ); // If there is a pending/running build, then no need to start a new one. if (building) diff --git a/src/Machine/src/Serval.Machine.WordAlignment/Services/EchoWordAlignmentPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.WordAlignment/Services/EchoWordAlignmentPreprocessBuildJob.cs index 8d6ffaab..bd847597 100644 --- a/src/Machine/src/Serval.Machine.WordAlignment/Services/EchoWordAlignmentPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.WordAlignment/Services/EchoWordAlignmentPreprocessBuildJob.cs @@ -8,6 +8,7 @@ public class EchoWordAlignmentPreprocessBuildJob( IBuildJobService buildJobService, ISharedFileService sharedFileService, IParallelCorpusService parallelCorpusService, + IBuildDiagnosticService buildDiagnosticService, IWordAlignmentPlatformService wordAlignmentPlatformService, IOptionsMonitor options ) @@ -19,6 +20,7 @@ IOptionsMonitor options buildJobService, sharedFileService, parallelCorpusService, + buildDiagnosticService, options ) { @@ -90,18 +92,44 @@ protected override async Task UpdateBuildExecutionData( PreprocessStats stats, string sourceLanguageTag, string targetLanguageTag, + bool isNonPersistedTranslationEngine, IReadOnlyList parallelCorpora, CancellationToken cancellationToken ) { - IReadOnlyList warnings = GetWarnings( + string modelName = + (await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken))?.CurrentBuild?.Model?.ToString() + ?? "Unknown"; + IReadOnlyList diagnostics = GetDiagnostics( stats.TrainCount, stats.InferenceCount, sourceLanguageTag, targetLanguageTag, + sourceLanguageHasNativeSupport: true, + targetLanguageHasNativeSupport: true, + isNonPersistedTranslationEngine, + modelName, parallelCorpora ); + IReadOnlyList warnings = diagnostics.Select(d => d.Message).ToList(); + + int maxDiagnostics = BuildJobOptions.MaxDiagnostics; + bool diagnosticsTruncated = false; + if (diagnostics.Count > maxDiagnostics) + { + diagnosticsTruncated = true; + diagnostics = diagnostics.OrderByDescending(d => d.Severity).Take(maxDiagnostics).ToList(); + } + + int maxWarnings = BuildJobOptions.MaxWarnings; + if (warnings.Count > maxWarnings) + { + string tooManyWarningsWarning = + $"There were {warnings.Count} warnings. Only the first {maxWarnings} are shown."; + warnings = [tooManyWarningsWarning, .. warnings.Take(maxWarnings)]; + } + // Log summary of build data var buildPreprocessSummary = new JsonObject { @@ -124,6 +152,8 @@ CancellationToken cancellationToken IsInferenceFilteredByChapter = stats.IsInferenceFilteredByChapter, IsTrainFilteredByChapter = stats.IsTrainFilteredByChapter, Warnings = warnings, + Diagnostics = diagnostics, + DiagnosticsTruncated = diagnosticsTruncated, EngineSourceLanguageTag = sourceLanguageTag, EngineTargetLanguageTag = targetLanguageTag, }; diff --git a/src/Machine/src/Serval.Machine.WordAlignment/Services/ServalWordAlignmentPlatformService.cs b/src/Machine/src/Serval.Machine.WordAlignment/Services/ServalWordAlignmentPlatformService.cs index 42c69019..05b34533 100644 --- a/src/Machine/src/Serval.Machine.WordAlignment/Services/ServalWordAlignmentPlatformService.cs +++ b/src/Machine/src/Serval.Machine.WordAlignment/Services/ServalWordAlignmentPlatformService.cs @@ -104,6 +104,8 @@ public Task UpdateBuildExecutionDataAsync( TrainVerseCount = executionData.TrainVerseCount, WordAlignVerseCount = executionData.InferenceVerseCount, Warnings = executionData.Warnings, + Diagnostics = executionData.Diagnostics, + DiagnosticsTruncated = executionData.DiagnosticsTruncated, EngineSourceLanguageTag = executionData.EngineSourceLanguageTag, EngineTargetLanguageTag = executionData.EngineTargetLanguageTag, }, diff --git a/src/Machine/src/Serval.Machine.WordAlignment/Services/StatisticalClearMLBuildJobFactory.cs b/src/Machine/src/Serval.Machine.WordAlignment/Services/StatisticalClearMLBuildJobFactory.cs index 60a7303c..e593679e 100644 --- a/src/Machine/src/Serval.Machine.WordAlignment/Services/StatisticalClearMLBuildJobFactory.cs +++ b/src/Machine/src/Serval.Machine.WordAlignment/Services/StatisticalClearMLBuildJobFactory.cs @@ -16,6 +16,7 @@ public async Task CreateJobScriptAsync( string modelType, BuildStage stage, string? buildOptions = null, + string? model = null, CancellationToken cancellationToken = default ) { diff --git a/src/Machine/src/Serval.Machine.WordAlignment/Services/StatisticalEngineService.cs b/src/Machine/src/Serval.Machine.WordAlignment/Services/StatisticalEngineService.cs index c036730c..0e0c3742 100644 --- a/src/Machine/src/Serval.Machine.WordAlignment/Services/StatisticalEngineService.cs +++ b/src/Machine/src/Serval.Machine.WordAlignment/Services/StatisticalEngineService.cs @@ -121,7 +121,7 @@ public async Task StartBuildAsync( BuildStage.Preprocess, corpora, options, - cancellationToken + cancellationToken: cancellationToken ); // If there is a pending/running build, then no need to start a new one. if (building) diff --git a/src/Machine/src/Serval.Machine.WordAlignment/Services/WordAlignmentPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.WordAlignment/Services/WordAlignmentPreprocessBuildJob.cs index 55591e5d..3b61b9bf 100644 --- a/src/Machine/src/Serval.Machine.WordAlignment/Services/WordAlignmentPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.WordAlignment/Services/WordAlignmentPreprocessBuildJob.cs @@ -8,6 +8,7 @@ public class WordAlignmentPreprocessBuildJob( IBuildJobService buildJobService, ISharedFileService sharedFileService, IParallelCorpusService parallelCorpusService, + IBuildDiagnosticService buildDiagnosticService, IOptionsMonitor options ) : PreprocessBuildJob( @@ -18,6 +19,7 @@ IOptionsMonitor options buildJobService, sharedFileService, parallelCorpusService, + buildDiagnosticService, options ) { @@ -88,18 +90,44 @@ protected override async Task UpdateBuildExecutionData( PreprocessStats stats, string sourceLanguageTag, string targetLanguageTag, + bool isNonPersistedTranslationEngine, IReadOnlyList parallelCorpora, CancellationToken cancellationToken ) { - IReadOnlyList warnings = GetWarnings( + string modelName = + (await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken))?.CurrentBuild?.Model?.ToString() + ?? "Unknown"; + IReadOnlyList diagnostics = GetDiagnostics( stats.TrainCount, stats.InferenceCount, sourceLanguageTag, targetLanguageTag, + sourceLanguageHasNativeSupport: true, + targetLanguageHasNativeSupport: true, + isNonPersistedTranslationEngine, + modelName, parallelCorpora ); + IReadOnlyList warnings = diagnostics.Select(d => d.Message).ToList(); + + int maxDiagnostics = BuildJobOptions.MaxDiagnostics; + bool diagnosticsTruncated = false; + if (diagnostics.Count > maxDiagnostics) + { + diagnosticsTruncated = true; + diagnostics = diagnostics.OrderByDescending(d => d.Severity).Take(maxDiagnostics).ToList(); + } + + int maxWarnings = BuildJobOptions.MaxWarnings; + if (warnings.Count > maxWarnings) + { + string tooManyWarningsWarning = + $"There were {warnings.Count} warnings. Only the first {maxWarnings} are shown."; + warnings = [tooManyWarningsWarning, .. warnings.Take(maxWarnings)]; + } + // Log summary of build data JsonObject buildPreprocessSummary = new() { @@ -122,6 +150,9 @@ CancellationToken cancellationToken IsInferenceFilteredByChapter = stats.IsInferenceFilteredByChapter, IsTrainFilteredByChapter = stats.IsTrainFilteredByChapter, Warnings = warnings, + Diagnostics = diagnostics, + DiagnosticsTruncated = diagnosticsTruncated, + EngineSourceLanguageTag = sourceLanguageTag, EngineTargetLanguageTag = targetLanguageTag, }; diff --git a/src/Machine/test/Serval.Machine.IntegrationTests/Services/ClearMLMonitorServiceTests.cs b/src/Machine/test/Serval.Machine.IntegrationTests/Services/ClearMLMonitorServiceTests.cs index 624d124d..5a86ae62 100644 --- a/src/Machine/test/Serval.Machine.IntegrationTests/Services/ClearMLMonitorServiceTests.cs +++ b/src/Machine/test/Serval.Machine.IntegrationTests/Services/ClearMLMonitorServiceTests.cs @@ -291,6 +291,7 @@ public async Task MonitorClearMLTasksPerDomain_CompletedStatus_ProperlyHandlesCo BuildStage.Postprocess, Arg.Is<(int, double)>(x => x.Item1 == ExpectedCorpusSize && x.Item2 == ExpectedConfidence), engine.CurrentBuild.Options, + engine.CurrentBuild.Model, Arg.Any() ) .Returns(true); @@ -318,6 +319,7 @@ await _buildJobService BuildStage.Postprocess, Arg.Is<(int, double)>(x => x.Item1 == ExpectedCorpusSize && x.Item2 == ExpectedConfidence), engine.CurrentBuild.Options, + engine.CurrentBuild.Model, Arg.Any() ); } diff --git a/src/Machine/test/Serval.Machine.Translation.Tests/Services/EchoTranslationEngineServiceTests.cs b/src/Machine/test/Serval.Machine.Translation.Tests/Services/EchoTranslationEngineServiceTests.cs index 29f9c498..422e3495 100644 --- a/src/Machine/test/Serval.Machine.Translation.Tests/Services/EchoTranslationEngineServiceTests.cs +++ b/src/Machine/test/Serval.Machine.Translation.Tests/Services/EchoTranslationEngineServiceTests.cs @@ -163,6 +163,7 @@ public TestEnvironment() services.AddScoped(_ => new MemoryDataAccessContext()); services.AddSingleton(Substitute.For()); services.AddSingleton(Substitute.For()); + services.AddSingleton(Substitute.For()); services.AddSingleton(buildJobOptions); services.AddSingleton(Substitute.For()); services.AddLogging(); diff --git a/src/Machine/test/Serval.Machine.Translation.Tests/Services/NmtEngineServiceTests.cs b/src/Machine/test/Serval.Machine.Translation.Tests/Services/NmtEngineServiceTests.cs index 89dbb108..aa81249e 100644 --- a/src/Machine/test/Serval.Machine.Translation.Tests/Services/NmtEngineServiceTests.cs +++ b/src/Machine/test/Serval.Machine.Translation.Tests/Services/NmtEngineServiceTests.cs @@ -171,6 +171,7 @@ public TestEnvironment() services.AddSingleton(SharedFileService); services.AddSingleton(new LanguageTagService()); services.AddSingleton(Substitute.For()); + services.AddSingleton(Substitute.For()); services.AddSingleton(BuildJobOptions); services.AddLogging(); _serviceProvider = services.BuildServiceProvider(); diff --git a/src/Machine/test/Serval.Machine.Translation.Tests/Services/PreprocessBuildJobTests.cs b/src/Machine/test/Serval.Machine.Translation.Tests/Services/PreprocessBuildJobTests.cs index e90670d1..c9952a66 100644 --- a/src/Machine/test/Serval.Machine.Translation.Tests/Services/PreprocessBuildJobTests.cs +++ b/src/Machine/test/Serval.Machine.Translation.Tests/Services/PreprocessBuildJobTests.cs @@ -41,14 +41,47 @@ public async Task RunAsync_BuildWarnings() }, ], }; + + env.ParallelCorpusService.PreprocessAsync( + Arg.Any>(), + Arg.Any>(), + Arg.Any>(), + Arg.Any(), + Arg.Any>() + ) + .Returns(Task.CompletedTask) + .AndDoes(ci => + ci.ArgAt>(1) + .Invoke( + new ParallelRowContract( + "MAT", + [ScriptureRef.Parse("MAT 1:1")], + [ScriptureRef.Parse("MAT 1:1")], + "Source Matthew 1:1", + "Target Matthew 1:1", + 1 + ), + TrainingDataType.Text + ) + ); + env.ParallelCorpusService.AnalyzeUsfmVersification(Arg.Any>()) .Returns([ ( "corpusId1", "src_1", "pt-source1", + "0000", "Original", [ + new() + { + NumAffectedVerses = 2, + Filename = "41MAT.SFM", + References = ["MAT 1:2", "MAT 1:3"], + LineNumbers = [4, 5], + Type = Serval.Shared.Contracts.UsfmVersificationDiagnosticType.Extra, + }, new() { NumAffectedVerses = 1, @@ -57,31 +90,294 @@ public async Task RunAsync_BuildWarnings() LineNumbers = [3], Type = Serval.Shared.Contracts.UsfmVersificationDiagnosticType.Missing, }, + ] + ), + ( + "corpusId1", + "trg_1", + "pt-target1", + "1111", + "English", + [ new() { - NumAffectedVerses = 2, + NumAffectedVerses = 1, Filename = "41MAT.SFM", - References = ["MAT 1:2", "MAT 1:3"], - LineNumbers = [4, 5], - Type = Serval.Shared.Contracts.UsfmVersificationDiagnosticType.Extra, + References = ["MAT 1:4a"], + LineNumbers = [6], + Type = Serval.Shared.Contracts.UsfmVersificationDiagnosticType.IncorrectVerseSegment, + }, + new() + { + NumAffectedVerses = 1, + Filename = "41MAT.SFM", + References = ["MAT :1"], + LineNumbers = [12], + Type = Serval.Shared.Contracts.UsfmVersificationDiagnosticType.InvalidChapter, + }, + new() + { + NumAffectedVerses = 1, + Filename = "41MAT.SFM", + References = ["MAT 2:1$"], + LineNumbers = [13], + Type = Serval.Shared.Contracts.UsfmVersificationDiagnosticType.InvalidVerse, + }, + new() + { + NumAffectedVerses = 15, + Filename = "41MAT.SFM", + References = ["MAT 2:2-16"], + LineNumbers = [20], + Type = Serval.Shared.Contracts.UsfmVersificationDiagnosticType.UnsupportedVerseRange, }, ] ), ]); + env.ParallelCorpusService.FindMissingParentProjects(Arg.Any>()) + .Returns([ + ( + "corpusId1", + "src_1", + new MissingParentProjectErrorContract + { + ProjectName = "pt-source1", + ProjectGuid = "0000", + ParentProjectGuid = "1111", + ParentProjectName = "pt-source1-parent", + } + ), + ]); + + env.LanguageTagService.ConvertToFlores200Code(Arg.Any(), out Arg.Any()) + .Returns(Flores200Support.None); + await env.RunBuildJobAsync(corpus1, engineId: "engine4"); - Assert.That(env.ExecutionData.Warnings, Has.Count.EqualTo(2)); + Assert.That(env.ExecutionData.Warnings, Has.Count.EqualTo(11)); + Assert.That(env.ExecutionData.Diagnostics, Has.Count.EqualTo(11)); + Assert.That(env.ExecutionData.DiagnosticsTruncated, Is.False); + + Assert.That(env.ExecutionData.Diagnostics[0].Code, Is.EqualTo("USFM-0003")); + Dictionary data = env.ExecutionData.Diagnostics[0].Data; + Assert.That(data, Has.Count.EqualTo(8)); + using (Assert.EnterMultipleScope()) + { + Assert.That(data["numberOfVerses"] is int numberOfVerses && numberOfVerses == 2); + Assert.That(data["projectName"] is string projectName && projectName == "pt-source1"); + Assert.That(data["projectGuid"] is string projectGuid && projectGuid == "0000"); + Assert.That(data["usfmFilename"] is string usfmFilename && usfmFilename == "41MAT.SFM"); + Assert.That(data["lineNumbers"] is List lineNumbers && lineNumbers.SequenceEqual([4, 5])); + Assert.That( + data["verseReferences"] is List verseReferences + && verseReferences.SequenceEqual(["MAT 1:2", "MAT 1:3"]) + ); + Assert.That(data["parallelCorpusId"] is string parallelCorpusId && parallelCorpusId == "corpusId1"); + Assert.That(data["monolingualCorpusId"] is string monolingualCorpusId && monolingualCorpusId == "src_1"); + } + + Assert.That(env.ExecutionData.Diagnostics[1].Code, Is.EqualTo("USFM-0004")); + data = env.ExecutionData.Diagnostics[1].Data; + Assert.That(data, Has.Count.EqualTo(8)); + using (Assert.EnterMultipleScope()) + { + Assert.That(data["numberOfVerses"] is int numberOfVerses && numberOfVerses == 1); + Assert.That(data["projectName"] is string projectName && projectName == "pt-source1"); + Assert.That(data["projectGuid"] is string projectGuid && projectGuid == "0000"); + Assert.That(data["usfmFilename"] is string usfmFilename && usfmFilename == "41MAT.SFM"); + Assert.That(data["lineNumbers"] is List lineNumbers && lineNumbers.SequenceEqual([3])); + Assert.That( + data["verseReferences"] is List verseReferences && verseReferences.SequenceEqual(["MAT 1:1"]) + ); + Assert.That(data["parallelCorpusId"] is string parallelCorpusId && parallelCorpusId == "corpusId1"); + Assert.That(data["monolingualCorpusId"] is string monolingualCorpusId && monolingualCorpusId == "src_1"); + } - env.BuildJobOptions.CurrentValue.Returns(new BuildJobOptions() { MaxWarnings = 1 }); + Assert.That(env.ExecutionData.Diagnostics[2].Code, Is.EqualTo("USFM-0005")); + data = env.ExecutionData.Diagnostics[2].Data; + Assert.That(data, Has.Count.EqualTo(7)); + using (Assert.EnterMultipleScope()) + { + Assert.That(data["projectName"] is string projectName && projectName == "pt-target1"); + Assert.That(data["projectGuid"] is string projectGuid && projectGuid == "1111"); + Assert.That(data["usfmFilename"] is string usfmFilename && usfmFilename == "41MAT.SFM"); + Assert.That(data["lineNumber"] is int lineNumber && lineNumber == 6); + Assert.That(data["verseReference"] is string verseReference && verseReference == "MAT 1:4a"); + Assert.That(data["parallelCorpusId"] is string parallelCorpusId && parallelCorpusId == "corpusId1"); + Assert.That(data["monolingualCorpusId"] is string monolingualCorpusId && monolingualCorpusId == "trg_1"); + } + + Assert.That(env.ExecutionData.Diagnostics[3].Code, Is.EqualTo("USFM-0001")); + data = env.ExecutionData.Diagnostics[3].Data; + Assert.That(data, Has.Count.EqualTo(7)); + using (Assert.EnterMultipleScope()) + { + Assert.That(data["projectName"] is string projectName && projectName == "pt-target1"); + Assert.That(data["projectGuid"] is string projectGuid && projectGuid == "1111"); + Assert.That(data["usfmFilename"] is string usfmFilename && usfmFilename == "41MAT.SFM"); + Assert.That(data["lineNumber"] is int lineNumber && lineNumber == 12); + Assert.That(data["verseReference"] is string verseReference && verseReference == "MAT :1"); + Assert.That(data["parallelCorpusId"] is string parallelCorpusId && parallelCorpusId == "corpusId1"); + Assert.That(data["monolingualCorpusId"] is string monolingualCorpusId && monolingualCorpusId == "trg_1"); + } + + Assert.That(env.ExecutionData.Diagnostics[4].Code, Is.EqualTo("USFM-0002")); + data = env.ExecutionData.Diagnostics[4].Data; + Assert.That(data, Has.Count.EqualTo(7)); + using (Assert.EnterMultipleScope()) + { + Assert.That(data["projectName"] is string projectName && projectName == "pt-target1"); + Assert.That(data["projectGuid"] is string projectGuid && projectGuid == "1111"); + Assert.That(data["usfmFilename"] is string usfmFilename && usfmFilename == "41MAT.SFM"); + Assert.That(data["lineNumber"] is int lineNumber && lineNumber == 13); + Assert.That(data["verseReference"] is string verseReference && verseReference == "MAT 2:1$"); + Assert.That(data["parallelCorpusId"] is string parallelCorpusId && parallelCorpusId == "corpusId1"); + Assert.That(data["monolingualCorpusId"] is string monolingualCorpusId && monolingualCorpusId == "trg_1"); + } + + Assert.That(env.ExecutionData.Diagnostics[5].Code, Is.EqualTo("USFM-0006")); + data = env.ExecutionData.Diagnostics[5].Data; + Assert.That(data, Has.Count.EqualTo(7)); + using (Assert.EnterMultipleScope()) + { + Assert.That(data["projectName"] is string projectName && projectName == "pt-target1"); + Assert.That(data["projectGuid"] is string projectGuid && projectGuid == "1111"); + Assert.That(data["usfmFilename"] is string usfmFilename && usfmFilename == "41MAT.SFM"); + Assert.That(data["lineNumber"] is int lineNumber && lineNumber == 20); + Assert.That(data["verseReference"] is string verseReference && verseReference == "MAT 2:2-16"); + Assert.That(data["parallelCorpusId"] is string parallelCorpusId && parallelCorpusId == "corpusId1"); + Assert.That(data["monolingualCorpusId"] is string monolingualCorpusId && monolingualCorpusId == "trg_1"); + } + + Assert.That(env.ExecutionData.Diagnostics[6].Code, Is.EqualTo("CONFIG-0001")); + data = env.ExecutionData.Diagnostics[6].Data; + Assert.That(data, Has.Count.EqualTo(6)); + using (Assert.EnterMultipleScope()) + { + Assert.That( + data["parentProjectName"] is string parentProjectName && parentProjectName == "pt-source1-parent" + ); + Assert.That(data["parentProjectGuid"] is string parentProjectGuid && parentProjectGuid == "1111"); + Assert.That( + data["daughterProjectName"] is string daughterProjectName && daughterProjectName == "pt-source1" + ); + Assert.That(data["daughterProjectGuid"] is string daughterProjectGuid && daughterProjectGuid == "0000"); + Assert.That(data["parallelCorpusId"] is string parallelCorpusId && parallelCorpusId == "corpusId1"); + Assert.That(data["monolingualCorpusId"] is string monolingualCorpusId && monolingualCorpusId == "src_1"); + } + + Assert.That(env.ExecutionData.Diagnostics[7].Code, Is.EqualTo("CONFIG-0002")); + data = env.ExecutionData.Diagnostics[7].Data; + Assert.That(data, Has.Count.EqualTo(1)); + Assert.That( + data["projectVersifications"] is Dictionary projectVersifications + && projectVersifications.Count == 2 + && projectVersifications["0000"] == "Original" + && projectVersifications["1111"] == "English" + ); + + Assert.That(env.ExecutionData.Diagnostics[8].Code, Is.EqualTo("CONFIG-0003")); + data = env.ExecutionData.Diagnostics[8].Data; + Assert.That(data, Has.Count.EqualTo(2)); + using (Assert.EnterMultipleScope()) + { + Assert.That(data["trainCount"] is int trainCount && trainCount == 1); + Assert.That(data["minimumTrainCount"] is int minimumTrainCount && minimumTrainCount == 600); + } + + Assert.That(env.ExecutionData.Diagnostics[9].Code, Is.EqualTo("MODEL-0001")); + data = env.ExecutionData.Diagnostics[9].Data; + Assert.That(data, Has.Count.EqualTo(2)); + using (Assert.EnterMultipleScope()) + { + Assert.That(data["resolvedCode"], Is.Null); + Assert.That(data["modelName"] is string modelName && modelName == "NLLB"); + } + + Assert.That(env.ExecutionData.Diagnostics[10].Code, Is.EqualTo("MODEL-0002")); + data = env.ExecutionData.Diagnostics[10].Data; + Assert.That(data, Has.Count.EqualTo(2)); + using (Assert.EnterMultipleScope()) + { + Assert.That(data["resolvedCode"], Is.Null); + Assert.That(data["modelName"] is string modelName && modelName == "NLLB"); + } + + env.BuildJobOptions.CurrentValue.Returns(new BuildJobOptions() { MaxWarnings = 1, MaxDiagnostics = 1 }); await env.RunBuildJobAsync(corpus1, engineId: "engine4"); // Two warnings after truncation + one warning mentioning that warnings were truncated Assert.That(env.ExecutionData.Warnings, Has.Count.EqualTo(2)); + Assert.That(env.ExecutionData.Diagnostics, Has.Count.EqualTo(1)); + Assert.That(env.ExecutionData.DiagnosticsTruncated, Is.True); + + Assert.That(env.ExecutionData.Diagnostics[0].Code, Is.EqualTo("USFM-0004")); + + env.ParallelCorpusService.ClearSubstitute(); + env.ParallelCorpusService.AnalyzeUsfmVersification(Arg.Any>()).Returns([]); + env.ParallelCorpusService.FindMissingParentProjects(Arg.Any>()).Returns([]); + + env.ParallelCorpusService.PreprocessAsync( + Arg.Any>(), + Arg.Any>(), + Arg.Any>(), + Arg.Any(), + Arg.Any>() + ) + .Returns(Task.CompletedTask); + + env.BuildJobOptions.ClearSubstitute(); + env.BuildJobOptions.CurrentValue.Returns(new BuildJobOptions() { MaxWarnings = 1_000, MaxDiagnostics = 1_000 }); + + Assert.ThrowsAsync(async () => + { + await env.RunBuildJobAsync(corpus1, engineId: "engine1"); + }); + + Assert.That(env.ExecutionData.DiagnosticsTruncated, Is.False); + Assert.That(env.ExecutionData.Diagnostics, Has.Count.EqualTo(4)); + + Assert.That(env.ExecutionData.Diagnostics[0].Code, Is.EqualTo("CONFIG-0004")); + data = env.ExecutionData.Diagnostics[0].Data; + Assert.That(data, Is.Empty); + + Assert.That(env.ExecutionData.Diagnostics[1].Code, Is.EqualTo("MODEL-0001")); + data = env.ExecutionData.Diagnostics[1].Data; + Assert.That(data, Has.Count.EqualTo(2)); + using (Assert.EnterMultipleScope()) + { + Assert.That(data["resolvedCode"], Is.Null); + Assert.That(data["modelName"] is string modelName && modelName == "NLLB"); + } + + Assert.That(env.ExecutionData.Diagnostics[2].Code, Is.EqualTo("MODEL-0002")); + data = env.ExecutionData.Diagnostics[2].Data; + Assert.That(data, Has.Count.EqualTo(2)); + using (Assert.EnterMultipleScope()) + { + Assert.That(data["resolvedCode"], Is.Null); + Assert.That(data["modelName"] is string modelName && modelName == "NLLB"); + } + + Assert.That(env.ExecutionData.Diagnostics[3].Code, Is.EqualTo("MODEL-0004")); + data = env.ExecutionData.Diagnostics[3].Data; + Assert.That(data, Has.Count.EqualTo(2)); + using (Assert.EnterMultipleScope()) + { + Assert.That(data["modelName"] is string modelName && modelName == "NLLB"); + Assert.That( + data["unknownLanguageCodes"] is List unknownLanguageCodes + && unknownLanguageCodes.SequenceEqual(["es", "en"]) + ); + } } [Test] public void RunAsync_UnknownLanguageTagsNoData() { TestEnvironment env = new(); + env.LanguageTagService.ConvertToFlores200Code("xxx", out Arg.Any()) + .Returns(Flores200Support.OnlyScript); + env.LanguageTagService.ConvertToFlores200Code("zzz", out Arg.Any()).Returns(Flores200Support.None); ParallelCorpusContract corpus1 = TestEnvironment.TextFileCorpus(sourceLanguage: "xxx", targetLanguage: "zzz"); Assert.ThrowsAsync(async () => @@ -94,6 +390,9 @@ public void RunAsync_UnknownLanguageTagsNoData() public async Task RunAsync_UnknownLanguageTagsNoDataSmtTransfer() { TestEnvironment env = new(); + env.LanguageTagService.ConvertToFlores200Code("xxx", out Arg.Any()) + .Returns(Flores200Support.OnlyScript); + env.LanguageTagService.ConvertToFlores200Code("zzz", out Arg.Any()).Returns(Flores200Support.None); ParallelCorpusContract corpus1 = TestEnvironment.TextFileCorpus(sourceLanguage: "xxx", targetLanguage: "zzz"); await env.RunBuildJobAsync(corpus1, engineId: "engine3", engineType: EngineType.SmtTransfer); @@ -296,7 +595,9 @@ private class TestEnvironment public IBuildJobService BuildJobService { get; } public IClearMLService ClearMLService { get; } public IOptionsMonitor BuildJobOptions { get; } + public ILanguageTagService LanguageTagService { get; } public IParallelCorpusService ParallelCorpusService { get; } + public IBuildDiagnosticService BuildDiagnosticService { get; } public SmtTransferEngineStateService StateService { get; private set; } public BuildExecutionData ExecutionData { get; private set; } = new BuildExecutionData(); @@ -325,6 +626,7 @@ public TestEnvironment() BuildJobRunner = BuildJobRunnerType.Local, Stage = BuildStage.Preprocess, ExecutionData = new BuildExecutionData(), + Model = Models.Models.Nllb, }, } ); @@ -388,6 +690,7 @@ public TestEnvironment() BuildJobRunner = BuildJobRunnerType.Local, Stage = BuildStage.Preprocess, ExecutionData = new BuildExecutionData(), + Model = Models.Models.Nllb, }, } ); @@ -443,6 +746,7 @@ public TestEnvironment() ) .Returns(Task.FromResult("job1")); SharedFileService = new SharedFileService(Substitute.For()); + LanguageTagService = Substitute.For(); BuildJobService = new BuildJobService( [ new TranslationEngineLocalBuildJobRunner( @@ -453,11 +757,7 @@ public TestEnvironment() new ClearMLBuildJobRunner( ClearMLService, [ - new NmtClearMLBuildJobFactory( - SharedFileService, - Substitute.For(), - Engines - ), + new NmtClearMLBuildJobFactory(SharedFileService, LanguageTagService, Engines), new SmtTransferClearMLBuildJobFactory(SharedFileService, Engines), ], BuildJobOptions @@ -466,6 +766,22 @@ public TestEnvironment() Engines ); ParallelCorpusService = Substitute.For(); + BuildDiagnosticService = Substitute.For(); + BuildDiagnosticService + .CreateDiagnostic(Arg.Any(), Arg.Any>()) + .Returns(ci => + { + string code = ci.ArgAt(0); + return new DiagnosticContract + { + Code = code, + Message = "", + Category = "", + Data = ci.ArgAt>(1), + //So that we can confirm that higher severity diagnostics are preserved when truncating + Severity = code == "USFM-0003" ? DiagnosticSeverity.Info : DiagnosticSeverity.Warn, + }; + }); StateService = CreateStateService(); } @@ -482,8 +798,9 @@ public PreprocessBuildJob GetBuildJob(EngineType engineType) Substitute.For>(), BuildJobService, SharedFileService, - new LanguageTagService(), + LanguageTagService, ParallelCorpusService, + BuildDiagnosticService, BuildJobOptions ); } @@ -499,6 +816,7 @@ public PreprocessBuildJob GetBuildJob(EngineType engineType) StateService, TrainSegmentPairs, ParallelCorpusService, + BuildDiagnosticService, BuildJobOptions ); } diff --git a/src/Machine/test/Serval.Machine.Translation.Tests/Services/SmtTransferEngineServiceTests.cs b/src/Machine/test/Serval.Machine.Translation.Tests/Services/SmtTransferEngineServiceTests.cs index 737470bb..ee59dd71 100644 --- a/src/Machine/test/Serval.Machine.Translation.Tests/Services/SmtTransferEngineServiceTests.cs +++ b/src/Machine/test/Serval.Machine.Translation.Tests/Services/SmtTransferEngineServiceTests.cs @@ -317,6 +317,7 @@ [new SmtTransferClearMLBuildJobFactory(SharedFileService, Engines)], services.AddScoped(_ => new MemoryDataAccessContext()); services.AddSingleton(SharedFileService); services.AddSingleton(Substitute.For()); + services.AddSingleton(Substitute.For()); services.AddSingleton(BuildJobOptions); services.AddSingleton(_truecaserFactory); services.AddSingleton(SmtModelFactory); diff --git a/src/Machine/test/Serval.Machine.Translation.Tests/Usings.cs b/src/Machine/test/Serval.Machine.Translation.Tests/Usings.cs index fcde2ede..e708fe95 100644 --- a/src/Machine/test/Serval.Machine.Translation.Tests/Usings.cs +++ b/src/Machine/test/Serval.Machine.Translation.Tests/Usings.cs @@ -4,6 +4,7 @@ global using Microsoft.Extensions.Logging; global using Microsoft.Extensions.Options; global using NSubstitute; +global using NSubstitute.ClearExtensions; global using NUnit.Framework; global using Serval.Machine.Shared.Configuration; global using Serval.Machine.Shared.Models; diff --git a/src/Machine/test/Serval.Machine.WordAlignment.Tests/Services/EchoWordAlignmentEngineServiceTests.cs b/src/Machine/test/Serval.Machine.WordAlignment.Tests/Services/EchoWordAlignmentEngineServiceTests.cs index a7aa218c..75916edb 100644 --- a/src/Machine/test/Serval.Machine.WordAlignment.Tests/Services/EchoWordAlignmentEngineServiceTests.cs +++ b/src/Machine/test/Serval.Machine.WordAlignment.Tests/Services/EchoWordAlignmentEngineServiceTests.cs @@ -120,6 +120,7 @@ public TestEnvironment() services.AddScoped(_ => new MemoryDataAccessContext()); services.AddSingleton(Substitute.For()); services.AddSingleton(Substitute.For()); + services.AddSingleton(Substitute.For()); services.AddSingleton(buildJobOptions); services.AddSingleton(Substitute.For()); services.AddLogging(); diff --git a/src/Machine/test/Serval.Machine.WordAlignment.Tests/Services/StatisticalEngineServiceTests.cs b/src/Machine/test/Serval.Machine.WordAlignment.Tests/Services/StatisticalEngineServiceTests.cs index 56a5f0de..303d227c 100644 --- a/src/Machine/test/Serval.Machine.WordAlignment.Tests/Services/StatisticalEngineServiceTests.cs +++ b/src/Machine/test/Serval.Machine.WordAlignment.Tests/Services/StatisticalEngineServiceTests.cs @@ -223,6 +223,7 @@ [new StatisticalClearMLBuildJobFactory(SharedFileService, Engines)], services.AddScoped(_ => new MemoryDataAccessContext()); services.AddSingleton(SharedFileService); services.AddSingleton(Substitute.For()); + services.AddSingleton(Substitute.For()); services.AddSingleton(BuildJobOptions); services.AddSingleton(WordAlignmentModelFactory); services.AddSingleton(statisticalEngineOptions); diff --git a/src/Serval/src/Serval.Client/Client.g.cs b/src/Serval/src/Serval.Client/Client.g.cs index 1177fae2..92dbf5ec 100644 --- a/src/Serval/src/Serval.Client/Client.g.cs +++ b/src/Serval/src/Serval.Client/Client.g.cs @@ -3518,6 +3518,8 @@ public partial interface ITranslationEnginesClient ///
Note that when using a parallel corpus: ///
* If, within a single parallel corpus, multiple source corpora have data for the same text ids (for text files or Paratext Projects) or books (for Paratext Projects only using the scripture range), those sources will be mixed where they overlap by randomly choosing from each source per line/verse. ///
* If, within a single parallel corpus, multiple target corpora have data for the same text ids (for text files or Paratext Projects) or books (for Paratext Projects only using the scripture range), only the first of the targets that includes that text id/book will be used for that text id/book. + ///
+ ///
See [here](https://github.com/sillsdev/serval/wiki/Build-Diagnostics) for information about diagnostics emitted during the build process. /// /// The translation engine id /// The build config (see remarks) @@ -5824,6 +5826,8 @@ public string BaseUrl ///
Note that when using a parallel corpus: ///
* If, within a single parallel corpus, multiple source corpora have data for the same text ids (for text files or Paratext Projects) or books (for Paratext Projects only using the scripture range), those sources will be mixed where they overlap by randomly choosing from each source per line/verse. ///
* If, within a single parallel corpus, multiple target corpora have data for the same text ids (for text files or Paratext Projects) or books (for Paratext Projects only using the scripture range), only the first of the targets that includes that text id/book will be used for that text id/book. + ///
+ ///
See [here](https://github.com/sillsdev/serval/wiki/Build-Diagnostics) for information about diagnostics emitted during the build process. /// /// The translation engine id /// The build config (see remarks) @@ -11639,8 +11643,15 @@ public partial class ExecutionData [Newtonsoft.Json.JsonProperty("warnings", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required] + [System.Obsolete] public System.Collections.Generic.IList Warnings { get; set; } = new System.Collections.ObjectModel.Collection(); + [Newtonsoft.Json.JsonProperty("diagnostics", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? Diagnostics { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("diagnosticsTruncated", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public bool? DiagnosticsTruncated { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("engineSourceLanguageTag", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public string? EngineSourceLanguageTag { get; set; } = default!; @@ -11658,6 +11669,48 @@ public partial class ExecutionData } + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.7.1.0 (NJsonSchema v11.6.1.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class Diagnostic + { + + [Newtonsoft.Json.JsonProperty("code", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Code { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("category", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Category { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("message", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string Message { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("severity", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + [Newtonsoft.Json.JsonConverter(typeof(Newtonsoft.Json.Converters.StringEnumConverter))] + public DiagnosticSeverity Severity { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("data", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required] + public System.Collections.Generic.IDictionary Data { get; set; } = new System.Collections.Generic.Dictionary(); + + } + + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.7.1.0 (NJsonSchema v11.6.1.0 (Newtonsoft.Json v13.0.0.0))")] + public enum DiagnosticSeverity + { + + [System.Runtime.Serialization.EnumMember(Value = @"Info")] + Info = 0, + + [System.Runtime.Serialization.EnumMember(Value = @"Warn")] + Warn = 1, + + [System.Runtime.Serialization.EnumMember(Value = @"Error")] + Error = 2, + + } + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.7.1.0 (NJsonSchema v11.6.1.0 (Newtonsoft.Json v13.0.0.0))")] public partial class Phase { @@ -12024,6 +12077,9 @@ public partial class TranslationBuildConfig [Newtonsoft.Json.JsonProperty("options", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public object? Options { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("model", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string? Model { get; set; } = default!; + } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.7.1.0 (NJsonSchema v11.6.1.0 (Newtonsoft.Json v13.0.0.0))")] @@ -12391,8 +12447,15 @@ public partial class WordAlignmentExecutionData [Newtonsoft.Json.JsonProperty("warnings", Required = Newtonsoft.Json.Required.Always)] [System.ComponentModel.DataAnnotations.Required] + [System.Obsolete] public System.Collections.Generic.IList Warnings { get; set; } = new System.Collections.ObjectModel.Collection(); + [Newtonsoft.Json.JsonProperty("diagnostics", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? Diagnostics { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("diagnosticsTruncated", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public bool? DiagnosticsTruncated { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("engineSourceLanguageTag", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public string? EngineSourceLanguageTag { get; set; } = default!; diff --git a/src/Serval/src/Serval.Shared.Contracts/DiagnosticContract.cs b/src/Serval/src/Serval.Shared.Contracts/DiagnosticContract.cs new file mode 100644 index 00000000..5ac0b036 --- /dev/null +++ b/src/Serval/src/Serval.Shared.Contracts/DiagnosticContract.cs @@ -0,0 +1,17 @@ +namespace Serval.Shared.Contracts; + +public record DiagnosticContract +{ + public required string Code { get; init; } + public required string Category { get; init; } + public required string Message { get; init; } + public required DiagnosticSeverity Severity { get; init; } + public required Dictionary Data { get; init; } +} + +public enum DiagnosticSeverity +{ + Info, + Warn, + Error, +} diff --git a/src/Serval/src/Serval.Shared.Contracts/IBuildDiagnosticService.cs b/src/Serval/src/Serval.Shared.Contracts/IBuildDiagnosticService.cs new file mode 100644 index 00000000..9a36c9a7 --- /dev/null +++ b/src/Serval/src/Serval.Shared.Contracts/IBuildDiagnosticService.cs @@ -0,0 +1,6 @@ +namespace Serval.Shared.Contracts; + +public interface IBuildDiagnosticService +{ + DiagnosticContract CreateDiagnostic(string code, Dictionary data); +} diff --git a/src/Serval/src/Serval.Shared.Contracts/IParallelCorpusService.cs b/src/Serval/src/Serval.Shared.Contracts/IParallelCorpusService.cs index 6f1071fe..59c94bf1 100644 --- a/src/Serval/src/Serval.Shared.Contracts/IParallelCorpusService.cs +++ b/src/Serval/src/Serval.Shared.Contracts/IParallelCorpusService.cs @@ -8,6 +8,7 @@ public interface IParallelCorpusService string ParallelCorpusId, string MonolingualCorpusId, string ProjectName, + string ProjectGuid, string VersificationName, IReadOnlyList Diagnostics )> AnalyzeUsfmVersification(IEnumerable parallelCorpora); diff --git a/src/Serval/src/Serval.Shared.Contracts/MissingParentProjectErrorContract.cs b/src/Serval/src/Serval.Shared.Contracts/MissingParentProjectErrorContract.cs index bb382db1..52d843ef 100644 --- a/src/Serval/src/Serval.Shared.Contracts/MissingParentProjectErrorContract.cs +++ b/src/Serval/src/Serval.Shared.Contracts/MissingParentProjectErrorContract.cs @@ -3,5 +3,7 @@ namespace Serval.Shared.Contracts; public record MissingParentProjectErrorContract { public required string ProjectName { get; init; } + public required string ProjectGuid { get; init; } public required string ParentProjectName { get; init; } + public required string ParentProjectGuid { get; init; } } diff --git a/src/Serval/src/Serval.Shared/Configuration/IServiceCollectionExtensions.cs b/src/Serval/src/Serval.Shared/Configuration/IServiceCollectionExtensions.cs index 85e5ceea..a1817e1a 100644 --- a/src/Serval/src/Serval.Shared/Configuration/IServiceCollectionExtensions.cs +++ b/src/Serval/src/Serval.Shared/Configuration/IServiceCollectionExtensions.cs @@ -10,6 +10,7 @@ Action configure { services.AddTransient(); services.AddSingleton(); + services.AddSingleton(); services.AddScoped(); services.Configure(configuration.GetSection(DataFileOptions.Key)); diff --git a/src/Serval/src/Serval.Shared/Dtos/DiagnosticDto.cs b/src/Serval/src/Serval.Shared/Dtos/DiagnosticDto.cs new file mode 100644 index 00000000..34ec8d4e --- /dev/null +++ b/src/Serval/src/Serval.Shared/Dtos/DiagnosticDto.cs @@ -0,0 +1,17 @@ +namespace Serval.Shared.Dtos; + +public record DiagnosticDto +{ + public required string Code { get; init; } + public required string Category { get; init; } + public required string Message { get; init; } + public required DiagnosticSeverity Severity { get; init; } + public required Dictionary Data { get; init; } +} + +public enum DiagnosticSeverity +{ + Info, + Warn, + Error, +} diff --git a/src/Serval/src/Serval.Shared/Models/Diagnostic.cs b/src/Serval/src/Serval.Shared/Models/Diagnostic.cs new file mode 100644 index 00000000..6285efda --- /dev/null +++ b/src/Serval/src/Serval.Shared/Models/Diagnostic.cs @@ -0,0 +1,17 @@ +namespace Serval.Shared.Models; + +public record Diagnostic +{ + public required string Code { get; init; } + public required string Category { get; init; } + public required string Message { get; init; } + public required DiagnosticSeverity Severity { get; init; } + public required Dictionary Data { get; init; } +} + +public enum DiagnosticSeverity +{ + Info, + Warn, + Error, +} diff --git a/src/Serval/src/Serval.Shared/Services/BuildDiagnosticService.cs b/src/Serval/src/Serval.Shared/Services/BuildDiagnosticService.cs new file mode 100644 index 00000000..6a2ae62c --- /dev/null +++ b/src/Serval/src/Serval.Shared/Services/BuildDiagnosticService.cs @@ -0,0 +1,285 @@ +using System.Globalization; + +namespace Serval.Shared.Services; + +public class BuildDiagnosticService : IBuildDiagnosticService +{ + private record DiagnosticInfo + { + public required string MessageFormat { get; init; } + public Dictionary> DataFormatters { get; init; } = []; + public required string Category { get; init; } + public required Contracts.DiagnosticSeverity Severity { get; init; } + public required Dictionary DataTypes { get; init; } + + public string FormatMessage(Dictionary data) + { + string?[] formattedParameters = DataTypes + .Select(kvp => DataFormatters.GetValueOrDefault(kvp.Key, obj => obj.ToString())(data[kvp.Key])) + .ToArray(); + + return string.Format(CultureInfo.InvariantCulture, MessageFormat, formattedParameters); + } + } + + private static readonly Dictionary Diagnostics = new() + { + ["MODEL-0001"] = new DiagnosticInfo + { + MessageFormat = "The script for the source language ‘{0}’ is not known to the base model {1}.", + Category = "MODEL", + Severity = Contracts.DiagnosticSeverity.Warn, + DataTypes = new Dictionary + { + ["resolvedCode"] = typeof(string), + ["modelName"] = typeof(string), + }, + }, + ["MODEL-0002"] = new DiagnosticInfo + { + MessageFormat = "The script for the target language ‘{0}’ is not known to the base model {1}.", + Category = "MODEL", + Severity = Contracts.DiagnosticSeverity.Warn, + DataTypes = new Dictionary + { + ["resolvedCode"] = typeof(string), + ["modelName"] = typeof(string), + }, + }, + ["MODEL-0003"] = new DiagnosticInfo + { + MessageFormat = + "The average pretranslation model confidence {0} in book {1} is unusually low for the base model {2}.", + Category = "MODEL", + Severity = Contracts.DiagnosticSeverity.Warn, + DataTypes = new Dictionary + { + ["averagePretranslationConfidence"] = typeof(double), + ["bookId"] = typeof(string), + ["modelName"] = typeof(string), + }, + }, + ["MODEL-0004"] = new DiagnosticInfo + { + MessageFormat = + "The following language codes are unknown to the base model {0}: {1}; and no language data was selected for training.", + Category = "MODEL", + Severity = Contracts.DiagnosticSeverity.Error, + DataTypes = new Dictionary + { + ["modelName"] = typeof(string), + ["unknownLanguageCodes"] = typeof(List), + }, + DataFormatters = new Dictionary> + { + ["unknownLanguageCodes"] = obj => string.Join(", ", (List)obj), + }, + }, + ["CONFIG-0001"] = new DiagnosticInfo + { + MessageFormat = + "Unable to locate parent project {0} {1} of daughter project {2} {3} (parallel corpus {4}, monolingual corpus {5}).", + Category = "CONFIG", + Severity = Contracts.DiagnosticSeverity.Warn, + DataTypes = new Dictionary + { + ["parentProjectName"] = typeof(string), + ["parentProjectGuid"] = typeof(string), + ["daughterProjectName"] = typeof(string), + ["daughterProjectGuid"] = typeof(string), + ["parallelCorpusId"] = typeof(string), + ["monolingualCorpusId"] = typeof(string), + }, + }, + ["CONFIG-0002"] = new DiagnosticInfo + { + MessageFormat = + "There are multiple versifications represented among Paratext projects selected for training or inferencing: {0}.", + Category = "CONFIG", + Severity = Contracts.DiagnosticSeverity.Info, + DataTypes = new Dictionary { ["projectVersifications"] = typeof(Dictionary) }, + DataFormatters = new Dictionary> + { + ["projectVersifications"] = obj => + { + var projectVersifications = (Dictionary)obj; + return $"{{{string.Join( + ", ", + projectVersifications.Select(kvp => $"{kvp.Key}: {kvp.Value}") + )}}}"; + }, + }, + }, + ["CONFIG-0003"] = new DiagnosticInfo + { + MessageFormat = + "Only {0} segments were selected for training. Training on fewer than {1} segments is not recommended.", + Category = "CONFIG", + Severity = Contracts.DiagnosticSeverity.Warn, + DataTypes = new Dictionary + { + ["trainCount"] = typeof(int), + ["minimumTrainCount"] = typeof(int), + }, + }, + ["CONFIG-0004"] = new DiagnosticInfo + { + MessageFormat = "There was no data specified for inferencing and the model is not persisted.", + Category = "CONFIG", + Severity = Contracts.DiagnosticSeverity.Error, + DataTypes = [], + }, + ["USFM-0001"] = new DiagnosticInfo + { + MessageFormat = + "Invalid chapter number in project {0} {1} at {2} line {3}, verse {4} (parallel corpus {5}, monolingual corpus {6}).", + Category = "USFM", + Severity = Contracts.DiagnosticSeverity.Warn, + DataTypes = new Dictionary + { + ["projectName"] = typeof(string), + ["projectGuid"] = typeof(string), + ["usfmFileName"] = typeof(string), + ["lineNumber"] = typeof(int), + ["verseReference"] = typeof(string), + ["parallelCorpusId"] = typeof(string), + ["monolingualCorpusId"] = typeof(string), + }, + }, + ["USFM-0002"] = new DiagnosticInfo + { + MessageFormat = + "Invalid verse number in project {0} {1} at {2} line {3}, verse {4} (parallel corpus {5}, monolingual corpus {6}).", + Category = "USFM", + Severity = Contracts.DiagnosticSeverity.Warn, + DataTypes = new Dictionary + { + ["projectName"] = typeof(string), + ["projectGuid"] = typeof(string), + ["usfmFileName"] = typeof(string), + ["lineNumber"] = typeof(int), + ["verseReference"] = typeof(string), + ["parallelCorpusId"] = typeof(string), + ["monolingualCorpusId"] = typeof(string), + }, + }, + ["USFM-0003"] = new DiagnosticInfo + { + MessageFormat = + "{0} extra verses in project {1} {2} at {3} lines {4}, verses {5} (parallel corpus {6}, monolingual corpus {7}).", + Category = "USFM", + Severity = Contracts.DiagnosticSeverity.Info, + DataTypes = new Dictionary + { + ["numberOfVerses"] = typeof(int), + ["projectName"] = typeof(string), + ["projectGuid"] = typeof(string), + ["usfmFileName"] = typeof(string), + ["lineNumbers"] = typeof(List), + ["verseReferences"] = typeof(List), + ["parallelCorpusId"] = typeof(string), + ["monolingualCorpusId"] = typeof(string), + }, + DataFormatters = new Dictionary> + { + ["lineNumbers"] = obj => string.Join(", ", (List)obj), + ["verseReferences"] = obj => string.Join(", ", (List)obj), + }, + }, + ["USFM-0004"] = new DiagnosticInfo + { + MessageFormat = + "Missing {0} verses in project {1} {2} at {3} lines {4}, verse {5} (parallel corpus {6}, monolingual corpus {7}).", + Category = "USFM", + Severity = Contracts.DiagnosticSeverity.Warn, + DataTypes = new Dictionary + { + ["numberOfVerses"] = typeof(int), + ["projectName"] = typeof(string), + ["projectGuid"] = typeof(string), + ["usfmFileName"] = typeof(string), + ["lineNumbers"] = typeof(List), + ["verseReferences"] = typeof(List), + ["parallelCorpusId"] = typeof(string), + ["monolingualCorpusId"] = typeof(string), + }, + DataFormatters = new Dictionary> + { + ["lineNumbers"] = obj => string.Join(", ", (List)obj), + ["verseReferences"] = obj => string.Join(", ", (List)obj), + }, + }, + ["USFM-0005"] = new DiagnosticInfo + { + MessageFormat = + "Incorrect verse segment in project {0} {1} at {2} line {3}, verse {4} (parallel corpus {5}, monolingual corpus {6}).", + Category = "USFM", + Severity = Contracts.DiagnosticSeverity.Info, + DataTypes = new Dictionary + { + ["projectName"] = typeof(string), + ["projectGuid"] = typeof(string), + ["usfmFileName"] = typeof(string), + ["lineNumber"] = typeof(int), + ["verseReference"] = typeof(string), + ["parallelCorpusId"] = typeof(string), + ["monolingualCorpusId"] = typeof(string), + }, + }, + ["USFM-0006"] = new DiagnosticInfo + { + MessageFormat = + "Unsupported verse range in project {0} {1} at {2} line {3}, verse {4} (parallel corpus {5}, monolingual corpus {6}).", + Category = "USFM", + Severity = Contracts.DiagnosticSeverity.Info, + DataTypes = new Dictionary + { + ["projectName"] = typeof(string), + ["projectGuid"] = typeof(string), + ["usfmFileName"] = typeof(string), + ["lineNumber"] = typeof(int), + ["verseReference"] = typeof(string), + ["parallelCorpusId"] = typeof(string), + ["monolingualCorpusId"] = typeof(string), + }, + }, + }; + + public DiagnosticContract CreateDiagnostic(string code, Dictionary data) + { + DiagnosticInfo diagnosticInfo = GetDiagnosticInfo(code, data); + return new DiagnosticContract + { + Code = code, + Message = diagnosticInfo.FormatMessage(data), + Severity = diagnosticInfo.Severity, + Category = diagnosticInfo.Category, + Data = data, + }; + } + + private static DiagnosticInfo GetDiagnosticInfo(string code, Dictionary data) + { + if (!Diagnostics.TryGetValue(code, out DiagnosticInfo? diagnosticInfo)) + { + throw new ArgumentException($"Unknown diagnostic code ‘{code}’."); + } + + foreach (var (key, type) in diagnosticInfo.DataTypes) + { + if (!data.TryGetValue(key, out object? value)) + { + throw new ArgumentException($"Missing required data for diagnostic code {code}: {key}."); + } + + if (value is null || type != value.GetType()) + { + throw new ArgumentException( + $"Invalid data type for diagnostic code {code}: {key} must be of type {type.Name}." + ); + } + } + + return diagnosticInfo; + } +} diff --git a/src/Serval/src/Serval.Shared/Services/ParallelCorpusService.cs b/src/Serval/src/Serval.Shared/Services/ParallelCorpusService.cs index d0d4f131..e782473f 100644 --- a/src/Serval/src/Serval.Shared/Services/ParallelCorpusService.cs +++ b/src/Serval/src/Serval.Shared/Services/ParallelCorpusService.cs @@ -11,6 +11,7 @@ public class ParallelCorpusService : IParallelCorpusService string ParallelCorpusId, string MonolingualCorpusId, string ProjectName, + string ProjectGuid, string VersificationName, IReadOnlyList Diagnostics )> AnalyzeUsfmVersification(IEnumerable parallelCorpora) @@ -20,6 +21,7 @@ IReadOnlyList Diagnostics string ParallelCorpusId, string MonolingualCorpusId, string ProjectName, + string ProjectGuid, string VersificationName, IReadOnlyList Diagnostics )> diagnosticsPerCorpus = []; @@ -44,6 +46,7 @@ IReadOnlyList Diagnostics parallelCorpus.Id, monolingualCorpus.Id, analysis.ProjectSettings.Name, + analysis.ProjectSettings.Guid, analysis.ProjectSettings.Versification.Name, [ .. analysis.Diagnostics.Select(d => new UsfmVersificationDiagnosticContract @@ -167,7 +170,13 @@ MissingParentProjectErrorContract Error ( parallelCorpus.Id, monolingualCorpus.Id, - new() { ProjectName = settings.Name, ParentProjectName = settings.ParentName } + new() + { + ProjectName = settings.Name, + ProjectGuid = settings.Guid, + ParentProjectName = settings.ParentName, + ParentProjectGuid = settings.ParentGuid, + } ) ); } diff --git a/src/Serval/src/Serval.Translation.Contracts/ExecutionDataContract.cs b/src/Serval/src/Serval.Translation.Contracts/ExecutionDataContract.cs index d7fd7422..6c2e94d4 100644 --- a/src/Serval/src/Serval.Translation.Contracts/ExecutionDataContract.cs +++ b/src/Serval/src/Serval.Translation.Contracts/ExecutionDataContract.cs @@ -9,6 +9,8 @@ public record ExecutionDataContract public IReadOnlyDictionary>? TrainVerseCount { get; init; } public IReadOnlyDictionary>? PretranslateVerseCount { get; init; } public IReadOnlyList? Warnings { get; init; } + public IReadOnlyList? Diagnostics { get; init; } + public bool? DiagnosticsTruncated { get; init; } public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } public string? ResolvedSourceLanguage { get; init; } diff --git a/src/Serval/src/Serval.Translation.Contracts/ITranslationEngineService.cs b/src/Serval/src/Serval.Translation.Contracts/ITranslationEngineService.cs index f70d063d..eadcf8a4 100644 --- a/src/Serval/src/Serval.Translation.Contracts/ITranslationEngineService.cs +++ b/src/Serval/src/Serval.Translation.Contracts/ITranslationEngineService.cs @@ -46,6 +46,7 @@ Task StartBuildAsync( string buildId, IReadOnlyList corpora, string? options = null, + string? model = null, CancellationToken cancellationToken = default ); diff --git a/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs b/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs index ce427261..3ff38726 100644 --- a/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs +++ b/src/Serval/src/Serval.Translation/Dtos/ExecutionDataDto.cs @@ -8,7 +8,11 @@ public record ExecutionDataDto public bool? IsPretranslateFilteredByChapter { get; init; } public IReadOnlyDictionary>? TrainVerseCount { get; init; } public IReadOnlyDictionary>? PretranslateVerseCount { get; init; } + + [Obsolete] public IReadOnlyList Warnings { get; init; } = []; + public IReadOnlyList? Diagnostics { get; init; } + public bool? DiagnosticsTruncated { get; init; } public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } public string? ResolvedSourceLanguage { get; init; } diff --git a/src/Serval/src/Serval.Translation/Features/Engines/StartBuild.cs b/src/Serval/src/Serval.Translation/Features/Engines/StartBuild.cs index 91f94468..275bd648 100644 --- a/src/Serval/src/Serval.Translation/Features/Engines/StartBuild.cs +++ b/src/Serval/src/Serval.Translation/Features/Engines/StartBuild.cs @@ -12,6 +12,7 @@ public record TranslationBuildConfigDto /// } /// public object? Options { get; init; } + public string? Model { get; init; } } public record PretranslateCorpusConfigDto @@ -79,6 +80,7 @@ await builds.ExistsAsync( Options = MapOptions(request.BuildConfig.Options), DeploymentVersion = configuration.GetValue("deploymentVersion") ?? "Unknown", DateCreated = DateTime.UtcNow, + Model = request.BuildConfig.Model, }; await builds.InsertAsync(build, ct); @@ -119,7 +121,7 @@ await builds.ExistsAsync( await engineFactory .GetEngineService(engine.Type) - .StartBuildAsync(engine.Id, build.Id, corpora, buildOptions, ct); + .StartBuildAsync(engine.Id, build.Id, corpora, buildOptions, build.Model, ct); return new StartBuildResponse(dtoMapper.Map(build)); }, cancellationToken @@ -405,6 +407,8 @@ public partial class TranslationEnginesController /// Note that when using a parallel corpus: /// * If, within a single parallel corpus, multiple source corpora have data for the same text ids (for text files or Paratext Projects) or books (for Paratext Projects only using the scripture range), those sources will be mixed where they overlap by randomly choosing from each source per line/verse. /// * If, within a single parallel corpus, multiple target corpora have data for the same text ids (for text files or Paratext Projects) or books (for Paratext Projects only using the scripture range), only the first of the targets that includes that text id/book will be used for that text id/book. + /// + /// See [here](https://github.com/sillsdev/serval/wiki/Build-Diagnostics) for information about diagnostics emitted during the build process. /// /// The translation engine id /// The build config (see remarks) diff --git a/src/Serval/src/Serval.Translation/Models/Build.cs b/src/Serval/src/Serval.Translation/Models/Build.cs index 4363b5a5..1433d3b3 100644 --- a/src/Serval/src/Serval.Translation/Models/Build.cs +++ b/src/Serval/src/Serval.Translation/Models/Build.cs @@ -16,6 +16,7 @@ public record Build : IOwnedEntity public JobState State { get; init; } = JobState.Pending; public DateTime? DateFinished { get; init; } public IReadOnlyDictionary? Options { get; init; } + public string? Model { get; init; } public string? DeploymentVersion { get; init; } public ExecutionData ExecutionData { get; init; } = new ExecutionData(); public DateTime? DateCreated { get; set; } diff --git a/src/Serval/src/Serval.Translation/Models/ExecutionData.cs b/src/Serval/src/Serval.Translation/Models/ExecutionData.cs index feb9a2af..38530114 100644 --- a/src/Serval/src/Serval.Translation/Models/ExecutionData.cs +++ b/src/Serval/src/Serval.Translation/Models/ExecutionData.cs @@ -9,6 +9,8 @@ public record ExecutionData public IReadOnlyDictionary>? TrainVerseCount { get; init; } public IReadOnlyDictionary>? PretranslateVerseCount { get; init; } public IReadOnlyList? Warnings { get; init; } + public IReadOnlyList? Diagnostics { get; init; } + public bool? DiagnosticsTruncated { get; init; } public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } public string? ResolvedSourceLanguage { get; init; } diff --git a/src/Serval/src/Serval.Translation/Services/DtoMapper.cs b/src/Serval/src/Serval.Translation/Services/DtoMapper.cs index ad09d355..bfe7832f 100644 --- a/src/Serval/src/Serval.Translation/Services/DtoMapper.cs +++ b/src/Serval/src/Serval.Translation/Services/DtoMapper.cs @@ -236,12 +236,25 @@ private static ExecutionDataDto Map(ExecutionData source) => IsPretranslateFilteredByChapter = source.IsPretranslateFilteredByChapter ?? false, IsTrainFilteredByChapter = source.IsTrainFilteredByChapter ?? false, Warnings = source.Warnings ?? [], + Diagnostics = source.Diagnostics?.Select(Map).ToList() ?? [], EngineSourceLanguageTag = source.EngineSourceLanguageTag, EngineTargetLanguageTag = source.EngineTargetLanguageTag, ResolvedSourceLanguage = source.ResolvedSourceLanguage, ResolvedTargetLanguage = source.ResolvedTargetLanguage, AveragePretranslationConfidence = source.AveragePretranslationConfidence, }; + + private static DiagnosticDto Map(Diagnostic source) + { + return new DiagnosticDto + { + Code = source.Code, + Category = source.Category, + Message = source.Message, + Severity = (Shared.Dtos.DiagnosticSeverity)source.Severity, + Data = source.Data, + }; + } } #pragma warning restore CS0612 // Type or member is obsolete diff --git a/src/Serval/src/Serval.Translation/Services/PlatformService.cs b/src/Serval/src/Serval.Translation/Services/PlatformService.cs index 88b6e694..f42c4b1c 100644 --- a/src/Serval/src/Serval.Translation/Services/PlatformService.cs +++ b/src/Serval/src/Serval.Translation/Services/PlatformService.cs @@ -1,3 +1,5 @@ +using SIL.Machine.Corpora; + namespace Serval.Translation.Services; public class PlatformService( @@ -5,16 +7,17 @@ public class PlatformService( IRepository engines, IRepository pretranslations, IDataAccessContext dataAccessContext, - IEventRouter eventRouter + IEventRouter eventRouter, + IBuildDiagnosticService buildDiagnosticService ) : ITranslationPlatformService { private const int PretranslationInsertBatchSize = 128; - private readonly IRepository _builds = builds; private readonly IRepository _engines = engines; private readonly IRepository _pretranslations = pretranslations; private readonly IDataAccessContext _dataAccessContext = dataAccessContext; private readonly IEventRouter _eventRouter = eventRouter; + private readonly IBuildDiagnosticService _buildDiagnosticService = buildDiagnosticService; public async Task BuildStartedAsync(string buildId, CancellationToken cancellationToken = default) { @@ -316,6 +319,17 @@ await _builds.UpdateAsync( IsTrainFilteredByChapter = executionData.IsTrainFilteredByChapter, IsPretranslateFilteredByChapter = executionData.IsPretranslateFilteredByChapter, Warnings = executionData.Warnings?.ToList() ?? [], + Diagnostics = executionData + .Diagnostics?.Select(d => new Diagnostic + { + Code = d.Code, + Category = d.Category, + Message = d.Message, + Severity = (Shared.Models.DiagnosticSeverity)d.Severity, + Data = d.Data, + }) + .ToList(), + DiagnosticsTruncated = executionData.DiagnosticsTruncated, EngineSourceLanguageTag = executionData.EngineSourceLanguageTag, EngineTargetLanguageTag = executionData.EngineTargetLanguageTag, ResolvedSourceLanguage = executionData.ResolvedSourceLanguage, @@ -383,6 +397,8 @@ public async Task InsertPretranslationsAsync( double logConfidenceTotal = 0.0; int confidenceCount = 0; int numPretranslations = 0; + Dictionary logConfidenceTotalPerBook = []; + Dictionary confidenceCountPerBook = []; await foreach (PretranslationContract item in pretranslations.WithCancellation(cancellationToken)) { batch.Add( @@ -399,7 +415,7 @@ public async Task InsertPretranslationsAsync( SourceTokens = item.SourceTokens, TranslationTokens = item.TranslationTokens, Alignment = item - .Alignment?.Select(a => new AlignedWordPair + .Alignment?.Select(a => new Shared.Models.AlignedWordPair { SourceIndex = a.SourceIndex, TargetIndex = a.TargetIndex, @@ -412,8 +428,25 @@ public async Task InsertPretranslationsAsync( double? confidence = item.Confidence; if (confidence != null && confidence > 0.0) { - logConfidenceTotal += Math.Log((double)confidence); + double logConfidence = Math.Log((double)confidence); + logConfidenceTotal += logConfidence; confidenceCount++; + + if ( + item.TargetRefs.Count > 0 + && ScriptureRef.TryParse(item.TargetRefs[0], out ScriptureRef scriptureRef) + ) + { + string bookId = scriptureRef.Book; + + if (!logConfidenceTotalPerBook.ContainsKey(bookId)) + logConfidenceTotalPerBook[bookId] = 0.0; + logConfidenceTotalPerBook[bookId] += logConfidence; + + if (!confidenceCountPerBook.ContainsKey(bookId)) + confidenceCountPerBook[bookId] = 0; + confidenceCountPerBook[bookId]++; + } } numPretranslations++; @@ -426,16 +459,81 @@ public async Task InsertPretranslationsAsync( if (batch.Count > 0) await _pretranslations.InsertAllAsync(batch, CancellationToken.None); + string? model = (await _builds.GetAsync(b => b.Id == buildId, cancellationToken))?.Model; + + List badBookConfidences = logConfidenceTotalPerBook + .Select(kvp => + { + string bookId = kvp.Key; + double logTotal = kvp.Value; + int count = confidenceCountPerBook[bookId]; + double averageConfidence = count > 0 ? Math.Exp(logTotal / count) : 0.0; + return (bookId, averageConfidence); + }) + .Where(b => + PretranslationConfidenceEvaluator.IsBookPretranslationConfidenceUnusuallyLow( + b.averageConfidence, + b.bookId, + model + ) + ) + .Select(b => + _buildDiagnosticService.CreateDiagnostic( + "MODEL-0003", + new Dictionary + { + { "bookId", b.bookId }, + { "averagePretranslationConfidence", b.averageConfidence }, + { "modelName", model ?? "Unknown" }, + } + ) + ) + .Select(d => new Diagnostic + { + Code = d.Code, + Category = d.Category, + Message = d.Message, + Severity = (Shared.Models.DiagnosticSeverity)d.Severity, + Data = d.Data, + }) + .ToList(); + + Build? currentBuild = null; + if (badBookConfidences.Count > 0) + { + currentBuild = await _builds.GetAsync(b => b.Id == buildId, cancellationToken); + + await _builds.UpdateAsync( + b => b.Id == buildId, + u => + u.Set( + b => b.ExecutionData.Diagnostics, + currentBuild?.ExecutionData.Diagnostics is null + ? [.. badBookConfidences] + : [.. currentBuild.ExecutionData.Diagnostics, .. badBookConfidences] + ), + cancellationToken: cancellationToken + ); + } + await _builds.UpdateAsync( b => b.Id == buildId, u => + { u.Set( b => b.ExecutionData.AveragePretranslationConfidence, // Calculate the geometric mean of the pretranslation confidences confidenceCount > 0 ? Math.Exp(logConfidenceTotal / confidenceCount) : 0.0 - ), + ); + u.Set( + b => b.ExecutionData.Diagnostics, + currentBuild?.ExecutionData.Diagnostics is null + ? [.. badBookConfidences] + : [.. currentBuild.ExecutionData.Diagnostics, .. badBookConfidences] + ); + }, cancellationToken: cancellationToken ); } diff --git a/src/Serval/src/Serval.Translation/Usings.cs b/src/Serval/src/Serval.Translation/Usings.cs index a13902bd..c2edbc34 100644 --- a/src/Serval/src/Serval.Translation/Usings.cs +++ b/src/Serval/src/Serval.Translation/Usings.cs @@ -33,4 +33,5 @@ global using Serval.Translation.Dtos; global using Serval.Translation.Models; global using Serval.Translation.Services; +global using Serval.Translation.Utils; global using SIL.DataAccess; diff --git a/src/Serval/src/Serval.Translation/Utils/PretranslationConfidenceEvaluator.cs b/src/Serval/src/Serval.Translation/Utils/PretranslationConfidenceEvaluator.cs new file mode 100644 index 00000000..776055c9 --- /dev/null +++ b/src/Serval/src/Serval.Translation/Utils/PretranslationConfidenceEvaluator.cs @@ -0,0 +1,11 @@ +namespace Serval.Translation.Utils; + +public class PretranslationConfidenceEvaluator +{ + private const double LowConfidenceThreshold = 0.25; + + public static bool IsBookPretranslationConfidenceUnusuallyLow(double confidence, string bookId, string? model) + { + return confidence < LowConfidenceThreshold; + } +} diff --git a/src/Serval/src/Serval.WordAlignment.Contracts/ExecutionDataContract.cs b/src/Serval/src/Serval.WordAlignment.Contracts/ExecutionDataContract.cs index ac0d1dc0..017514a5 100644 --- a/src/Serval/src/Serval.WordAlignment.Contracts/ExecutionDataContract.cs +++ b/src/Serval/src/Serval.WordAlignment.Contracts/ExecutionDataContract.cs @@ -9,6 +9,8 @@ public record ExecutionDataContract public IReadOnlyDictionary>? TrainVerseCount { get; init; } public IReadOnlyDictionary>? WordAlignVerseCount { get; init; } public IReadOnlyList? Warnings { get; init; } + public IReadOnlyList? Diagnostics { get; init; } + public bool? DiagnosticsTruncated { get; init; } public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } } diff --git a/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentExecutionDataDto.cs b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentExecutionDataDto.cs index 9ed2a182..7d1534d4 100644 --- a/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentExecutionDataDto.cs +++ b/src/Serval/src/Serval.WordAlignment/Dtos/WordAlignmentExecutionDataDto.cs @@ -8,7 +8,11 @@ public record WordAlignmentExecutionDataDto public bool? IsWordAlignFilteredByChapter { get; init; } public IReadOnlyDictionary>? TrainVerseCount { get; init; } public IReadOnlyDictionary>? WordAlignVerseCount { get; init; } + + [Obsolete] public IReadOnlyList Warnings { get; init; } = []; + public IReadOnlyList? Diagnostics { get; init; } + public bool? DiagnosticsTruncated { get; init; } public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } } diff --git a/src/Serval/src/Serval.WordAlignment/Models/ExecutionData.cs b/src/Serval/src/Serval.WordAlignment/Models/ExecutionData.cs index 824e80b2..1480ae4e 100644 --- a/src/Serval/src/Serval.WordAlignment/Models/ExecutionData.cs +++ b/src/Serval/src/Serval.WordAlignment/Models/ExecutionData.cs @@ -9,6 +9,8 @@ public record ExecutionData public IReadOnlyDictionary>? TrainVerseCount { get; init; } public IReadOnlyDictionary>? WordAlignVerseCount { get; init; } public IReadOnlyList? Warnings { get; init; } + public IReadOnlyList? Diagnostics { get; init; } + public bool? DiagnosticsTruncated { get; init; } public string? EngineSourceLanguageTag { get; init; } public string? EngineTargetLanguageTag { get; init; } } diff --git a/src/Serval/src/Serval.WordAlignment/Services/DtoMapper.cs b/src/Serval/src/Serval.WordAlignment/Services/DtoMapper.cs index 1b368ac2..b21dc884 100644 --- a/src/Serval/src/Serval.WordAlignment/Services/DtoMapper.cs +++ b/src/Serval/src/Serval.WordAlignment/Services/DtoMapper.cs @@ -195,8 +195,22 @@ private static WordAlignmentExecutionDataDto Map(ExecutionData source) IsTrainFilteredByChapter = source.IsTrainFilteredByChapter, IsWordAlignFilteredByChapter = source.IsWordAlignFilteredByChapter, Warnings = source.Warnings ?? [], + Diagnostics = source.Diagnostics?.Select(Map).ToList() ?? [], + DiagnosticsTruncated = source.DiagnosticsTruncated, EngineSourceLanguageTag = source.EngineSourceLanguageTag, EngineTargetLanguageTag = source.EngineTargetLanguageTag, }; } + + private static DiagnosticDto Map(Diagnostic source) + { + return new DiagnosticDto + { + Code = source.Code, + Category = source.Category, + Message = source.Message, + Severity = (Shared.Dtos.DiagnosticSeverity)source.Severity, + Data = source.Data, + }; + } } diff --git a/src/Serval/src/Serval.WordAlignment/Services/PlatformService.cs b/src/Serval/src/Serval.WordAlignment/Services/PlatformService.cs index be4812a5..40a2c51c 100644 --- a/src/Serval/src/Serval.WordAlignment/Services/PlatformService.cs +++ b/src/Serval/src/Serval.WordAlignment/Services/PlatformService.cs @@ -368,6 +368,7 @@ await _builds.UpdateAsync( IsTrainFilteredByChapter = executionData.IsTrainFilteredByChapter, IsWordAlignFilteredByChapter = executionData.IsWordAlignFilteredByChapter, Warnings = executionData.Warnings?.ToList() ?? [], + DiagnosticsTruncated = executionData.DiagnosticsTruncated, EngineSourceLanguageTag = executionData.EngineSourceLanguageTag, EngineTargetLanguageTag = executionData.EngineTargetLanguageTag, } diff --git a/src/Serval/test/Serval.Shared.Tests/Services/BuildDiagnosticServiceTests.cs b/src/Serval/test/Serval.Shared.Tests/Services/BuildDiagnosticServiceTests.cs new file mode 100644 index 00000000..8589d430 --- /dev/null +++ b/src/Serval/test/Serval.Shared.Tests/Services/BuildDiagnosticServiceTests.cs @@ -0,0 +1,456 @@ +namespace Serval.Shared.Services; + +[TestFixture] +public class BuildDiagnosticServiceTests +{ + private IBuildDiagnosticService _service; + + [SetUp] + public void SetUp() + { + _service = new BuildDiagnosticService(); + } + + [Test] + public void CreateDiagnostic_UnknownCode() + { + var code = "ASDF-1234"; + var data = new Dictionary { { "parentProjectName", "Tes" } }; + + ArgumentException? ex = Assert.Throws(() => _service.CreateDiagnostic(code, data)); + Assert.That(ex.Message, Is.EqualTo("Unknown diagnostic code ‘ASDF-1234’.")); + } + + [Test] + public void CreateDiagnostic_Config0001_MissingData() + { + var code = "CONFIG-0001"; + var data = new Dictionary + { + { "parentProjectName", "Tes" }, + { "parentProjectGuid", "parent-guid" }, + { "daughterProjectName", "TesBT" }, + { "daughterProjectGuid", "daughter-guid" }, + { "monolingualCorpusId", "monolingual-corpus-id" }, + }; + + ArgumentException? ex = Assert.Throws(() => _service.CreateDiagnostic(code, data)); + Assert.That(ex.Message, Is.EqualTo("Missing required data for diagnostic code CONFIG-0001: parallelCorpusId.")); + } + + [Test] + public void CreateDiagnostic_Config0001_IncorrectDataType() + { + var code = "CONFIG-0001"; + var data = new Dictionary + { + { "parentProjectName", "Tes" }, + { "parentProjectGuid", 1234 }, + { "daughterProjectName", "TesBT" }, + { "daughterProjectGuid", "daughter-guid" }, + { "monolingualCorpusId", "monolingual-corpus-id" }, + }; + + ArgumentException? ex = Assert.Throws(() => _service.CreateDiagnostic(code, data)); + Assert.That( + ex.Message, + Is.EqualTo("Invalid data type for diagnostic code CONFIG-0001: parentProjectGuid must be of type String.") + ); + } + + [Test] + public void CreateDiagnostic_Config0001_DataOutOfOrder() + { + var code = "CONFIG-0001"; + var data = new Dictionary + { + { "daughterProjectGuid", "daughter-guid" }, + { "parallelCorpusId", "parallel-corpus-id" }, + { "parentProjectName", "Tes" }, + { "parentProjectGuid", "parent-guid" }, + { "monolingualCorpusId", "monolingual-corpus-id" }, + { "daughterProjectName", "TesBT" }, + }; + + DiagnosticContract diagnostic = _service.CreateDiagnostic(code, data); + + Assert.That(diagnostic.Code, Is.EqualTo(code)); + Assert.That(diagnostic.Category, Is.EqualTo("CONFIG")); + Assert.That(diagnostic.Severity, Is.EqualTo(Contracts.DiagnosticSeverity.Warn)); + Assert.That( + diagnostic.Message, + Is.EqualTo( + "Unable to locate parent project Tes parent-guid of daughter project TesBT daughter-guid (parallel corpus parallel-corpus-id, monolingual corpus monolingual-corpus-id)." + ) + ); + Assert.That(diagnostic.Data, Is.EqualTo(data)); + } + + [Test] + public void CreateDiagnostic_Config0001() + { + var code = "CONFIG-0001"; + var data = new Dictionary + { + { "parentProjectName", "Tes" }, + { "parentProjectGuid", "parent-guid" }, + { "daughterProjectName", "TesBT" }, + { "daughterProjectGuid", "daughter-guid" }, + { "parallelCorpusId", "parallel-corpus-id" }, + { "monolingualCorpusId", "monolingual-corpus-id" }, + }; + + DiagnosticContract diagnostic = _service.CreateDiagnostic(code, data); + + Assert.That(diagnostic.Code, Is.EqualTo(code)); + Assert.That(diagnostic.Category, Is.EqualTo("CONFIG")); + Assert.That(diagnostic.Severity, Is.EqualTo(Contracts.DiagnosticSeverity.Warn)); + Assert.That( + diagnostic.Message, + Is.EqualTo( + "Unable to locate parent project Tes parent-guid of daughter project TesBT daughter-guid (parallel corpus parallel-corpus-id, monolingual corpus monolingual-corpus-id)." + ) + ); + Assert.That(diagnostic.Data, Is.EqualTo(data)); + } + + [Test] + public void CreateDiagnostic_Config0002() + { + var code = "CONFIG-0002"; + var data = new Dictionary + { + { + "projectVersifications", + new Dictionary { { "project-guid-1", "Original" }, { "project-guid-2", "English" } } + }, + }; + + DiagnosticContract diagnostic = _service.CreateDiagnostic(code, data); + + Assert.That(diagnostic.Code, Is.EqualTo(code)); + Assert.That(diagnostic.Category, Is.EqualTo("CONFIG")); + Assert.That(diagnostic.Severity, Is.EqualTo(Contracts.DiagnosticSeverity.Info)); + Assert.That( + diagnostic.Message, + Is.EqualTo( + "There are multiple versifications represented among Paratext projects selected for training or inferencing: {project-guid-1: Original, project-guid-2: English}." + ) + ); + Assert.That(diagnostic.Data, Is.EqualTo(data)); + } + + [Test] + public void CreateDiagnostic_Config0003() + { + var code = "CONFIG-0003"; + var data = new Dictionary { { "trainCount", 10 }, { "minimumTrainCount", 600 } }; + + DiagnosticContract diagnostic = _service.CreateDiagnostic(code, data); + + Assert.That(diagnostic.Code, Is.EqualTo(code)); + Assert.That(diagnostic.Category, Is.EqualTo("CONFIG")); + Assert.That(diagnostic.Severity, Is.EqualTo(Contracts.DiagnosticSeverity.Warn)); + Assert.That( + diagnostic.Message, + Is.EqualTo( + "Only 10 segments were selected for training. Training on fewer than 600 segments is not recommended." + ) + ); + Assert.That(diagnostic.Data, Is.EqualTo(data)); + } + + [Test] + public void CreateDiagnostic_Config0004() + { + var code = "CONFIG-0004"; + var data = new Dictionary(); + + DiagnosticContract diagnostic = _service.CreateDiagnostic(code, data); + + Assert.That(diagnostic.Code, Is.EqualTo(code)); + Assert.That(diagnostic.Category, Is.EqualTo("CONFIG")); + Assert.That(diagnostic.Severity, Is.EqualTo(Contracts.DiagnosticSeverity.Error)); + Assert.That( + diagnostic.Message, + Is.EqualTo("There was no data specified for inferencing and the model is not persisted.") + ); + Assert.That(diagnostic.Data, Is.EqualTo(data)); + } + + [Test] + public void CreateDiagnostic_Model0001() + { + var code = "MODEL-0001"; + var data = new Dictionary { { "resolvedCode", "eng_Latn" }, { "modelName", "test-model" } }; + + DiagnosticContract diagnostic = _service.CreateDiagnostic(code, data); + + Assert.That(diagnostic.Code, Is.EqualTo(code)); + Assert.That(diagnostic.Category, Is.EqualTo("MODEL")); + Assert.That(diagnostic.Severity, Is.EqualTo(Contracts.DiagnosticSeverity.Warn)); + Assert.That( + diagnostic.Message, + Is.EqualTo("The script for the source language ‘eng_Latn’ is not known to the base model test-model.") + ); + Assert.That(diagnostic.Data, Is.EqualTo(data)); + } + + [Test] + public void CreateDiagnostic_Model0002() + { + var code = "MODEL-0002"; + var data = new Dictionary { { "resolvedCode", "eng_Latn" }, { "modelName", "test-model" } }; + + DiagnosticContract diagnostic = _service.CreateDiagnostic(code, data); + + Assert.That(diagnostic.Code, Is.EqualTo(code)); + Assert.That(diagnostic.Category, Is.EqualTo("MODEL")); + Assert.That(diagnostic.Severity, Is.EqualTo(Contracts.DiagnosticSeverity.Warn)); + Assert.That( + diagnostic.Message, + Is.EqualTo("The script for the target language ‘eng_Latn’ is not known to the base model test-model.") + ); + Assert.That(diagnostic.Data, Is.EqualTo(data)); + } + + [Test] + public void CreateDiagnostic_Model0003() + { + var code = "MODEL-0003"; + var data = new Dictionary + { + { "averagePretranslationConfidence", 0.37 }, + { "bookId", "MAT" }, + { "modelName", "test-model" }, + }; + + DiagnosticContract diagnostic = _service.CreateDiagnostic(code, data); + + Assert.That(diagnostic.Code, Is.EqualTo(code)); + Assert.That(diagnostic.Category, Is.EqualTo("MODEL")); + Assert.That(diagnostic.Severity, Is.EqualTo(Contracts.DiagnosticSeverity.Warn)); + Assert.That( + diagnostic.Message, + Is.EqualTo( + "The average pretranslation model confidence 0.37 in book MAT is unusually low for the base model test-model." + ) + ); + Assert.That(diagnostic.Data, Is.EqualTo(data)); + } + + [Test] + public void CreateDiagnostic_Model0004() + { + var code = "MODEL-0004"; + var data = new Dictionary + { + { "modelName", "test-model" }, + { + "unknownLanguageCodes", + new List { "spa_Latn", "eng_Latn" } + }, + }; + + DiagnosticContract diagnostic = _service.CreateDiagnostic(code, data); + + Assert.That(diagnostic.Code, Is.EqualTo(code)); + Assert.That(diagnostic.Category, Is.EqualTo("MODEL")); + Assert.That(diagnostic.Severity, Is.EqualTo(Contracts.DiagnosticSeverity.Error)); + Assert.That( + diagnostic.Message, + Is.EqualTo( + "The following language codes are unknown to the base model test-model: spa_Latn, eng_Latn; and no language data was selected for training." + ) + ); + Assert.That(diagnostic.Data, Is.EqualTo(data)); + } + + [Test] + public void CreateDiagnostic_Usfm0001() + { + var code = "USFM-0001"; + var data = new Dictionary + { + { "projectName", "Tes" }, + { "projectGuid", "project-guid" }, + { "usfmFileName", "MAT.USFM" }, + { "lineNumber", 12 }, + { "verseReference", "1" }, + { "parallelCorpusId", "parallel-corpus-id" }, + { "monolingualCorpusId", "monolingual-corpus-id" }, + }; + + DiagnosticContract diagnostic = _service.CreateDiagnostic(code, data); + + Assert.That(diagnostic.Code, Is.EqualTo(code)); + Assert.That(diagnostic.Category, Is.EqualTo("USFM")); + Assert.That(diagnostic.Severity, Is.EqualTo(Contracts.DiagnosticSeverity.Warn)); + Assert.That( + diagnostic.Message, + Is.EqualTo( + "Invalid chapter number in project Tes project-guid at MAT.USFM line 12, verse 1 (parallel corpus parallel-corpus-id, monolingual corpus monolingual-corpus-id)." + ) + ); + Assert.That(diagnostic.Data, Is.EqualTo(data)); + } + + [Test] + public void CreateDiagnostic_Usfm0002() + { + var code = "USFM-0002"; + var data = new Dictionary + { + { "projectName", "Tes" }, + { "projectGuid", "project-guid" }, + { "usfmFileName", "MAT.USFM" }, + { "lineNumber", 12 }, + { "verseReference", "1" }, + { "parallelCorpusId", "parallel-corpus-id" }, + { "monolingualCorpusId", "monolingual-corpus-id" }, + }; + + DiagnosticContract diagnostic = _service.CreateDiagnostic(code, data); + + Assert.That(diagnostic.Code, Is.EqualTo(code)); + Assert.That(diagnostic.Category, Is.EqualTo("USFM")); + Assert.That(diagnostic.Severity, Is.EqualTo(Contracts.DiagnosticSeverity.Warn)); + Assert.That( + diagnostic.Message, + Is.EqualTo( + "Invalid verse number in project Tes project-guid at MAT.USFM line 12, verse 1 (parallel corpus parallel-corpus-id, monolingual corpus monolingual-corpus-id)." + ) + ); + Assert.That(diagnostic.Data, Is.EqualTo(data)); + } + + [Test] + public void CreateDiagnostic_Usfm0003() + { + var code = "USFM-0003"; + var data = new Dictionary + { + { "numberOfVerses", 2 }, + { "projectName", "Tes" }, + { "projectGuid", "project-guid" }, + { "usfmFileName", "MAT.USFM" }, + { + "lineNumbers", + new List { 3, 4 } + }, + { + "verseReferences", + new List { "MAT 1:1", "MAT 1:2" } + }, + { "parallelCorpusId", "parallel-corpus-id" }, + { "monolingualCorpusId", "monolingual-corpus-id" }, + }; + + DiagnosticContract diagnostic = _service.CreateDiagnostic(code, data); + + Assert.That(diagnostic.Code, Is.EqualTo(code)); + Assert.That(diagnostic.Category, Is.EqualTo("USFM")); + Assert.That(diagnostic.Severity, Is.EqualTo(Contracts.DiagnosticSeverity.Info)); + Assert.That( + diagnostic.Message, + Is.EqualTo( + "2 extra verses in project Tes project-guid at MAT.USFM lines 3, 4, verses MAT 1:1, MAT 1:2 (parallel corpus parallel-corpus-id, monolingual corpus monolingual-corpus-id)." + ) + ); + Assert.That(diagnostic.Data, Is.EqualTo(data)); + } + + [Test] + public void CreateDiagnostic_Usfm0004() + { + var code = "USFM-0004"; + var data = new Dictionary + { + { "numberOfVerses", 2 }, + { "projectName", "Tes" }, + { "projectGuid", "project-guid" }, + { "usfmFileName", "MAT.USFM" }, + { + "lineNumbers", + new List { 3, 4 } + }, + { + "verseReferences", + new List { "MAT 1:1", "MAT 1:2" } + }, + { "parallelCorpusId", "parallel-corpus-id" }, + { "monolingualCorpusId", "monolingual-corpus-id" }, + }; + + DiagnosticContract diagnostic = _service.CreateDiagnostic(code, data); + + Assert.That(diagnostic.Code, Is.EqualTo(code)); + Assert.That(diagnostic.Category, Is.EqualTo("USFM")); + Assert.That(diagnostic.Severity, Is.EqualTo(Contracts.DiagnosticSeverity.Warn)); + Assert.That( + diagnostic.Message, + Is.EqualTo( + "Missing 2 verses in project Tes project-guid at MAT.USFM lines 3, 4, verse MAT 1:1, MAT 1:2 (parallel corpus parallel-corpus-id, monolingual corpus monolingual-corpus-id)." + ) + ); + Assert.That(diagnostic.Data, Is.EqualTo(data)); + } + + [Test] + public void CreateDiagnostic_Usfm0005() + { + var code = "USFM-0005"; + var data = new Dictionary + { + { "projectName", "Tes" }, + { "projectGuid", "project-guid" }, + { "usfmFileName", "MAT.USFM" }, + { "lineNumber", 12 }, + { "verseReference", "MAT 1:1a" }, + { "parallelCorpusId", "parallel-corpus-id" }, + { "monolingualCorpusId", "monolingual-corpus-id" }, + }; + + DiagnosticContract diagnostic = _service.CreateDiagnostic(code, data); + + Assert.That(diagnostic.Code, Is.EqualTo(code)); + Assert.That(diagnostic.Category, Is.EqualTo("USFM")); + Assert.That(diagnostic.Severity, Is.EqualTo(Contracts.DiagnosticSeverity.Info)); + Assert.That( + diagnostic.Message, + Is.EqualTo( + "Incorrect verse segment in project Tes project-guid at MAT.USFM line 12, verse MAT 1:1a (parallel corpus parallel-corpus-id, monolingual corpus monolingual-corpus-id)." + ) + ); + Assert.That(diagnostic.Data, Is.EqualTo(data)); + } + + [Test] + public void CreateDiagnostic_Usfm0006() + { + var code = "USFM-0006"; + var data = new Dictionary + { + { "projectName", "Tes" }, + { "projectGuid", "project-guid" }, + { "usfmFileName", "MAT.USFM" }, + { "lineNumber", 12 }, + { "verseReference", "MAT 1:1-12" }, + { "parallelCorpusId", "parallel-corpus-id" }, + { "monolingualCorpusId", "monolingual-corpus-id" }, + }; + + DiagnosticContract diagnostic = _service.CreateDiagnostic(code, data); + + Assert.That(diagnostic.Code, Is.EqualTo(code)); + Assert.That(diagnostic.Category, Is.EqualTo("USFM")); + Assert.That(diagnostic.Severity, Is.EqualTo(Contracts.DiagnosticSeverity.Info)); + Assert.That( + diagnostic.Message, + Is.EqualTo( + "Unsupported verse range in project Tes project-guid at MAT.USFM line 12, verse MAT 1:1-12 (parallel corpus parallel-corpus-id, monolingual corpus monolingual-corpus-id)." + ) + ); + Assert.That(diagnostic.Data, Is.EqualTo(data)); + } +} diff --git a/src/Serval/test/Serval.Shared.Tests/Services/CorpusBundleTests.cs b/src/Serval/test/Serval.Shared.Tests/Services/CorpusBundleTests.cs index 16ab950c..6a6b6e0f 100644 --- a/src/Serval/test/Serval.Shared.Tests/Services/CorpusBundleTests.cs +++ b/src/Serval/test/Serval.Shared.Tests/Services/CorpusBundleTests.cs @@ -1,5 +1,6 @@ namespace Serval.Shared.Services; +[TestFixture] public class CorpusBundleTests { [Test] diff --git a/src/Serval/test/Serval.Shared.Tests/Services/ParallelCorpusServiceTests.cs b/src/Serval/test/Serval.Shared.Tests/Services/ParallelCorpusServiceTests.cs index 98780e33..53f9ed04 100644 --- a/src/Serval/test/Serval.Shared.Tests/Services/ParallelCorpusServiceTests.cs +++ b/src/Serval/test/Serval.Shared.Tests/Services/ParallelCorpusServiceTests.cs @@ -196,7 +196,7 @@ corpus with expected = new Dictionary> { { "MAT", [] } }; Assert.That(actual, Is.EqualTo(expected)); - // Test merging hapters without numbers specified + // Test merging chapters without numbers specified actual = ParallelCorpusService.GetBookIdsAndChapters( corpus with { @@ -218,6 +218,7 @@ public void AnalyzeUsfmVersification() string ParallelCorpusId, string MonolingualCorpusId, string ProjectName, + string ProjectGuid, string VersificationName, IReadOnlyList Diagnostics )> analysis = env.Processor.AnalyzeUsfmVersification([parallelCorpus]); diff --git a/src/Serval/test/Serval.Translation.Tests/Features/Engines/EnginesHandlersTests.cs b/src/Serval/test/Serval.Translation.Tests/Features/Engines/EnginesHandlersTests.cs index 6642e6d3..9fbfdbdc 100644 --- a/src/Serval/test/Serval.Translation.Tests/Features/Engines/EnginesHandlersTests.cs +++ b/src/Serval/test/Serval.Translation.Tests/Features/Engines/EnginesHandlersTests.cs @@ -225,6 +225,7 @@ await env }, ]), null, + null, Arg.Any() ); } @@ -302,6 +303,7 @@ await env }, ]), null, + null, Arg.Any() ); } @@ -379,6 +381,7 @@ await env }, ]), null, + null, Arg.Any() ); } @@ -451,6 +454,7 @@ await env }, ]), null, + null, Arg.Any() ); } @@ -527,6 +531,7 @@ await env }, ]), null, + null, Arg.Any() ); } @@ -645,6 +650,7 @@ await env }, ]), null, + null, Arg.Any() ); } @@ -751,6 +757,7 @@ await env }, ]), null, + null, Arg.Any() ); } @@ -828,6 +835,7 @@ await env }, ]), null, + null, Arg.Any() ); } @@ -943,6 +951,7 @@ await env }, ]), null, + null, Arg.Any() ); } @@ -1029,6 +1038,7 @@ await env }, ]), null, + null, Arg.Any() ); } @@ -1157,6 +1167,7 @@ await env }, ]), null, + null, Arg.Any() ); } @@ -1272,6 +1283,7 @@ await env }, ]), null, + null, Arg.Any() ); } @@ -1403,6 +1415,7 @@ await env }, ]), null, + null, Arg.Any() ); } @@ -1526,6 +1539,7 @@ await env }, ]), null, + null, Arg.Any() ); } @@ -1682,6 +1696,7 @@ await env }, ]), null, + null, Arg.Any() ); } @@ -1778,6 +1793,7 @@ await env }, ]), null, + null, Arg.Any() ); } @@ -1893,6 +1909,7 @@ await env }, ]), null, + null, Arg.Any() ); } @@ -2758,6 +2775,7 @@ public TestEnvironment() Arg.Any(), Arg.Any>(), Arg.Any(), + Arg.Any(), Arg.Any() ) .Returns(Task.CompletedTask); diff --git a/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs b/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs index 23c032f4..3123a1a1 100644 --- a/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs +++ b/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs @@ -97,6 +97,7 @@ await env.Builds.InsertAsync( Id = "b0", EngineRef = "e0", Owner = "owner1", + Model = "NLLB", } ); @@ -104,14 +105,25 @@ await env.Builds.InsertAsync( await env.PlatformService.InsertPretranslationsAsync( "e0", "b0", - GetTestPretranslationsWithConfidences([0.25, 0.5, 1.0]) + GetTestScripturePretranslationsWithConfidences([0.25, 0.49, 0.25, 0.125]) ); await env.PlatformService.BuildCompletedAsync("b0", 0, 0.0); - + ExecutionData? executionData = (await env.Builds.GetAsync(b => b.Id == "b0"))?.ExecutionData; + Assert.That(executionData, Is.Not.Null); + Assert.That(executionData.AveragePretranslationConfidence, Is.EqualTo(0.2487).Within(0.0001)); + Assert.That(executionData.Diagnostics, Has.Count.EqualTo(1)); + Assert.That(executionData.Diagnostics[0].Code, Is.EqualTo("MODEL-0003")); Assert.That( - (await env.Builds.GetAsync(b => b.Id == "b0"))?.ExecutionData.AveragePretranslationConfidence, - Is.EqualTo(0.5).Within(0.0001) + executionData + .Diagnostics[0] + .Data.Keys.SequenceEqual(["bookId", "averagePretranslationConfidence", "modelName"]) + ); + Assert.That(executionData.Diagnostics[0].Data["bookId"], Is.EqualTo("MAT")); + Assert.That( + executionData.Diagnostics[0].Data["averagePretranslationConfidence"], + Is.EqualTo(0.2487).Within(0.0001) ); + Assert.That(executionData.Diagnostics[0].Data["modelName"], Is.EqualTo("NLLB")); } [Test] @@ -381,7 +393,7 @@ private static async IAsyncEnumerable GetTestPretranslat await Task.CompletedTask; } - private static async IAsyncEnumerable GetTestPretranslationsWithConfidences( + private static async IAsyncEnumerable GetTestScripturePretranslationsWithConfidences( IReadOnlyList confidences ) { @@ -390,9 +402,9 @@ IReadOnlyList confidences yield return new PretranslationContract { CorpusId = "e0", - TextId = $"text{index}", - SourceRefs = [$"ref{index}"], - TargetRefs = [$"ref{index}"], + TextId = "MAT", + SourceRefs = [$"MAT 1:{index + 1}"], + TargetRefs = [$"MAT 1:{index + 1}"], Translation = "test", SourceTokens = [], TranslationTokens = [], @@ -426,12 +438,25 @@ public TestEnvironment() return ((Func)x[0])((CancellationToken)x[1]); }); + var buildDiagnosticService = Substitute.For(); + buildDiagnosticService + .CreateDiagnostic(Arg.Any(), Arg.Any>()) + .Returns(ci => new DiagnosticContract + { + Code = ci.ArgAt(0), + Category = "", + Severity = Shared.Contracts.DiagnosticSeverity.Info, + Message = "", + Data = ci.ArgAt>(1), + }); + PlatformService = new PlatformService( Builds, Engines, Pretranslations, DataAccessContext, - Substitute.For() + Substitute.For(), + buildDiagnosticService ); }