Structured build diagnostics

src/Machine/src/Serval.Machine.Shared/Configuration/BuildJobOptions.cs

@@ -7,4 +7,6 @@ public class BuildJobOptions
     public IList<ClearMLBuildQueue> ClearML { get; set; } = new List<ClearMLBuildQueue>();
     public bool PreserveBuildFiles { get; set; } = false;
     public int MaxWarnings { get; set; } = 1000;
+    public int MaxDiagnostics { get; set; } = 1000;
+    public int MinimumTrainCount { get; set; } = 600;
 }

src/Machine/src/Serval.Machine.Shared/Models/Build.cs

@@ -30,6 +30,7 @@ public record Build
     public required BuildStage Stage { get; init; }
     public DateTimeOffset QueuedAt { get; init; }
     public string? Options { get; set; }
+    public BaseModelContract? BaseModel { get; init; }
     public string? JobData { get; init; }
     public required BuildExecutionData ExecutionData { get; init; }
 }

src/Machine/src/Serval.Machine.Shared/Models/BuildExecutionData.cs

@@ -9,6 +9,8 @@ public record BuildExecutionData
     public IReadOnlyDictionary<string, Dictionary<string, int>>? TrainVerseCount { get; init; }
     public IReadOnlyDictionary<string, Dictionary<string, int>>? InferenceVerseCount { get; init; }
     public IReadOnlyList<string>? Warnings { get; init; }
+    public IReadOnlyList<DiagnosticContract>? Diagnostics { get; init; }
+    public bool? DiagnosticsTruncated { get; init; }
     public string? EngineSourceLanguageTag { get; init; }
     public string? EngineTargetLanguageTag { get; init; }
     public string? ResolvedSourceLanguage { get; init; }

src/Machine/src/Serval.Machine.Shared/Services/BuildJobService.cs

@@ -68,6 +68,7 @@ public async Task<bool> StartBuildJobAsync(
         BuildStage stage,
         object? data = null,
         string? buildOptions = null,
+        BaseModelContract? baseModel = null,
         CancellationToken cancellationToken = default
     )
     {
@@ -106,6 +107,7 @@ public async Task<bool> StartBuildJobAsync(
                             JobState = BuildJobState.Pending,
                             QueuedAt = DateTimeOffset.UtcNow,
                             Options = buildOptions,
+                            BaseModel = baseModel,
                             JobData = jobData,
                             ExecutionData = new BuildExecutionData(),
                         }

src/Machine/src/Serval.Machine.Shared/Services/ClearMLMonitorService.cs

@@ -298,7 +298,7 @@ CancellationToken cancellationToken
                 BuildStage.Postprocess,
                 (corpusSize, confidence),
                 buildOptions,
-                cancellationToken
+                cancellationToken: cancellationToken
             );
         }
         finally

src/Machine/src/Serval.Machine.Shared/Services/IBuildJobService.cs

@@ -16,6 +16,7 @@ Task<bool> StartBuildJobAsync(
         BuildStage stage,
         object? data = default,
         string? buildOptions = default,
+        BaseModelContract? baseModel = null,
         CancellationToken cancellationToken = default
     );
 

src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs

@@ -8,6 +8,7 @@ public abstract class PreprocessBuildJob<TEngine>(
     IBuildJobService<TEngine> buildJobService,
     ISharedFileService sharedFileService,
     IParallelCorpusService parallelCorpusService,
+    IBuildDiagnosticService buildDiagnosticService,
     IOptionsMonitor<BuildJobOptions> options
 )
     : BuildJob<TEngine, IReadOnlyList<ParallelCorpusContract>>(
@@ -32,6 +33,7 @@ IOptionsMonitor<BuildJobOptions> options
     protected readonly BuildJobOptions BuildJobOptions = options.CurrentValue;
     protected readonly ISharedFileService SharedFileService = sharedFileService;
     protected readonly IParallelCorpusService ParallelCorpusService = parallelCorpusService;
+    protected readonly IBuildDiagnosticService BuildDiagnosticService = buildDiagnosticService;
 
     protected override async Task DoWorkAsync(
         string engineId,
@@ -46,23 +48,25 @@ CancellationToken cancellationToken
             throw new OperationCanceledException($"Engine {engineId} does not exist.  Build canceled.");
 
         PreprocessStats stats = await WriteDataFilesAsync(engineId, buildId, data, buildOptions, cancellationToken);
+        bool isNonPersistedTranslationEngine = engine is IPersistableTrainingEngine { IsModelPersisted: false };
 
         await UpdateBuildExecutionData(
             engineId,
             buildId,
             stats,
             engine.SourceLanguage,
             engine.TargetLanguage,
+            isNonPersistedTranslationEngine,
             data,
             cancellationToken
         );
 
         await UpdateTargetQuoteConventionAsync(engineId, buildId, data, cancellationToken);
 
-        if (stats.InferenceCount == 0 && engine is IPersistableTrainingEngine { IsModelPersisted: false })
+        if (stats.InferenceCount == 0 && isNonPersistedTranslationEngine)
         {
             throw new InvalidOperationException(
-                $"There was no data specified for inferencing in build {buildId}. Build canceled."
+                $"There was no data specified for inferencing in build {buildId} and the model is not persisted. Build canceled."
             );
         }
 
@@ -87,6 +91,7 @@ protected abstract Task UpdateBuildExecutionData(
         PreprocessStats stats,
         string sourceLanguageTag,
         string targetLanguageTag,
+        bool isNonPersistedTranslationEngine,
         IReadOnlyList<ParallelCorpusContract> parallelCorpora,
         CancellationToken cancellationToken
     );
@@ -121,53 +126,150 @@ protected override async Task CleanupAsync(string engineId, string buildId, JobC
         }
     }
 
-    protected virtual IReadOnlyList<string> GetWarnings(
+    protected virtual IReadOnlyList<DiagnosticContract> GetDiagnostics(
         int trainCount,
         int inferenceCount,
         string sourceLanguageTag,
         string targetLanguageTag,
+        bool sourceLanguageHasNativeSupport,
+        bool targetLanguageHasNativeSupport,
+        bool isNonPersistedTranslationEngine,
+        string modelName,
         IReadOnlyList<ParallelCorpusContract> parallelCorpora
     )
     {
-        List<string> warnings = [];
-        HashSet<string> versifications = [];
+        List<DiagnosticContract> diagnostics = [];
+        Dictionary<string, string> projectVersifications = [];
 
         foreach (
             (
                 string parallelCorpusId,
                 string monolingualCorpusId,
                 string projectName,
+                string projectGuid,
                 string versificationName,
-                IReadOnlyList<UsfmVersificationDiagnosticContract> diagnostics
+                IReadOnlyList<UsfmVersificationDiagnosticContract> usfmDiagnostics
             ) in ParallelCorpusService.AnalyzeUsfmVersification(parallelCorpora)
         )
         {
-            versifications.Add(versificationName);
-            foreach (UsfmVersificationDiagnosticContract diagnostic in diagnostics)
+            projectVersifications[projectGuid] = versificationName;
+            foreach (UsfmVersificationDiagnosticContract usfmDiagnostic in usfmDiagnostics)
             {
-                string diagnosticDetails =
-                    $"in project {projectName} at {diagnostic.Filename} "
-                    + (diagnostic.LineNumbers.Count == 1 ? "line " : "lines ")
-                    + $"{string.Join(", ", diagnostic.LineNumbers)}, "
-                    + (diagnostic.NumAffectedVerses == 1 ? "verse " : "verses ")
-                    + $"{string.Join(", ", diagnostic.References)} "
-                    + $"(parallel corpus {parallelCorpusId}, monolingual corpus {monolingualCorpusId}).";
-                warnings.Add(
-                    diagnostic.Type switch
+                diagnostics.Add(
+                    usfmDiagnostic.Type switch
                     {
                         Serval.Shared.Contracts.UsfmVersificationDiagnosticType.InvalidChapter =>
-                            $"Invalid chapter number {diagnosticDetails}",
+                            BuildDiagnosticService.CreateDiagnostic(
+                                "USFM-0001",
+                                new Dictionary<string, object>
+                                {
+                                    { "projectName", projectName },
+                                    { "projectGuid", projectGuid },
+                                    { "usfmFilename", usfmDiagnostic.Filename },
+                                    {
+                                        "lineNumber",
+                                        usfmDiagnostic.LineNumbers.Count > 0 ? usfmDiagnostic.LineNumbers[0] : -1
+                                    },
+                                    {
+                                        "verseReference",
+                                        usfmDiagnostic.References.Count > 0 ? usfmDiagnostic.References[0] : ""
+                                    },
+                                    { "parallelCorpusId", parallelCorpusId },
+                                    { "monolingualCorpusId", monolingualCorpusId },
+                                }
+                            ),
                         Serval.Shared.Contracts.UsfmVersificationDiagnosticType.InvalidVerse =>
-                            $"Invalid verse number {diagnosticDetails}",
+                            BuildDiagnosticService.CreateDiagnostic(
+                                "USFM-0002",
+                                new Dictionary<string, object>
+                                {
+                                    { "projectName", projectName },
+                                    { "projectGuid", projectGuid },
+                                    { "usfmFilename", usfmDiagnostic.Filename },
+                                    {
+                                        "lineNumber",
+                                        usfmDiagnostic.LineNumbers.Count > 0 ? usfmDiagnostic.LineNumbers[0] : -1
+                                    },
+                                    {
+                                        "verseReference",
+                                        usfmDiagnostic.References.Count > 0 ? usfmDiagnostic.References[0] : ""
+                                    },
+                                    { "parallelCorpusId", parallelCorpusId },
+                                    { "monolingualCorpusId", monolingualCorpusId },
+                                }
+                            ),
+
                         Serval.Shared.Contracts.UsfmVersificationDiagnosticType.Extra =>
-                            $"{diagnostic.NumAffectedVerses} extra verses {diagnosticDetails}",
+                            BuildDiagnosticService.CreateDiagnostic(
+                                "USFM-0003",
+                                new Dictionary<string, object>
+                                {
+                                    { "numberOfVerses", usfmDiagnostic.NumAffectedVerses },
+                                    { "projectName", projectName },
+                                    { "projectGuid", projectGuid },
+                                    { "usfmFilename", usfmDiagnostic.Filename },
+                                    { "lineNumbers", usfmDiagnostic.LineNumbers.ToList() },
+                                    { "verseReferences", usfmDiagnostic.References.ToList() },
+                                    { "parallelCorpusId", parallelCorpusId },
+                                    { "monolingualCorpusId", monolingualCorpusId },
+                                }
+                            ),
                         Serval.Shared.Contracts.UsfmVersificationDiagnosticType.Missing =>
-                            $"Missing {diagnostic.NumAffectedVerses} verses {diagnosticDetails}",
+                            BuildDiagnosticService.CreateDiagnostic(
+                                "USFM-0004",
+                                new Dictionary<string, object>
+                                {
+                                    { "numberOfVerses", usfmDiagnostic.NumAffectedVerses },
+                                    { "projectName", projectName },
+                                    { "projectGuid", projectGuid },
+                                    { "usfmFilename", usfmDiagnostic.Filename },
+                                    { "lineNumbers", usfmDiagnostic.LineNumbers.ToList() },
+                                    { "verseReferences", usfmDiagnostic.References.ToList() },
+                                    { "parallelCorpusId", parallelCorpusId },
+                                    { "monolingualCorpusId", monolingualCorpusId },
+                                }
+                            ),
                         Serval.Shared.Contracts.UsfmVersificationDiagnosticType.IncorrectVerseSegment =>
-                            $"Incorrect verse segment {diagnosticDetails}",
+                            BuildDiagnosticService.CreateDiagnostic(
+                                "USFM-0005",
+                                new Dictionary<string, object>
+                                {
+                                    { "projectName", projectName },
+                                    { "projectGuid", projectGuid },
+                                    { "usfmFilename", usfmDiagnostic.Filename },
+                                    {
+                                        "lineNumber",
+                                        usfmDiagnostic.LineNumbers.Count > 0 ? usfmDiagnostic.LineNumbers[0] : -1
+                                    },
+                                    {
+                                        "verseReference",
+                                        usfmDiagnostic.References.Count > 0 ? usfmDiagnostic.References[0] : ""
+                                    },
+                                    { "parallelCorpusId", parallelCorpusId },
+                                    { "monolingualCorpusId", monolingualCorpusId },
+                                }
+                            ),
                         Serval.Shared.Contracts.UsfmVersificationDiagnosticType.UnsupportedVerseRange =>
-                            $"Unsupported verse range {diagnosticDetails}",
-                        _ => $"USFM versification issue {diagnosticDetails}",
+                            BuildDiagnosticService.CreateDiagnostic(
+                                "USFM-0006",
+                                new Dictionary<string, object>
+                                {
+                                    { "projectName", projectName },
+                                    { "projectGuid", projectGuid },
+                                    { "usfmFilename", usfmDiagnostic.Filename },
+                                    {
+                                        "lineNumber",
+                                        usfmDiagnostic.LineNumbers.Count > 0 ? usfmDiagnostic.LineNumbers[0] : -1
+                                    },
+                                    {
+                                        "verseReference",
+                                        usfmDiagnostic.References.Count > 0 ? usfmDiagnostic.References[0] : ""
+                                    },
+                                    { "parallelCorpusId", parallelCorpusId },
+                                    { "monolingualCorpusId", monolingualCorpusId },
+                                }
+                            ),
+                        _ => throw new InvalidEnumArgumentException(nameof(usfmDiagnostic.Type)),
                     }
                 );
             }
@@ -181,19 +283,37 @@ MissingParentProjectErrorContract error
             ) in ParallelCorpusService.FindMissingParentProjects(parallelCorpora)
         )
         {
-            warnings.Add(
-                $"Unable to locate parent project {error.ParentProjectName} of daughter project {error.ProjectName} (parallel corpus {parallelCorpusId}, monolingual corpus {monolingualCorpusId})"
+            diagnostics.Add(
+                BuildDiagnosticService.CreateDiagnostic(
+                    "CONFIG-0001",
+                    new Dictionary<string, object>
+                    {
+                        { "parentProjectName", error.ParentProjectName },
+                        { "parentProjectGuid", error.ParentProjectGuid },
+                        { "daughterProjectName", error.ProjectName },
+                        { "daughterProjectGuid", error.ProjectGuid },
+                        { "parallelCorpusId", parallelCorpusId },
+                        { "monolingualCorpusId", monolingualCorpusId },
+                    }
+                )
             );
         }
 
-        if (versifications.Count > 1)
+        if (projectVersifications.Values.Distinct().Count() > 1)
         {
-            warnings.Add(
-                $"Multiple versifications represented among Paratext projects selected for training or inferencing: {string.Join(", ", versifications)}"
+            diagnostics.Add(
+                BuildDiagnosticService.CreateDiagnostic(
+                    "CONFIG-0002",
+                    new Dictionary<string, object> { { "projectVersifications", projectVersifications } }
+                )
             );
         }
 
-        return warnings;
+        if (inferenceCount == 0 && isNonPersistedTranslationEngine)
+        {
+            diagnostics.Add(BuildDiagnosticService.CreateDiagnostic("CONFIG-0004", []));
+        }
+        return diagnostics;
     }
 
     protected static (bool IsTrainFilteredByChapter, bool IsInferenceFilteredByChapter) CheckChapterFilters(

src/Machine/src/Serval.Machine.Translation/Services/EchoPreprocessBuildJob.cs

@@ -8,6 +8,7 @@ public class EchoPreprocessBuildJob(
     IBuildJobService<TranslationEngine> buildJobService,
     ISharedFileService sharedFileService,
     IParallelCorpusService parallelCorpusService,
+    IBuildDiagnosticService buildDiagnosticService,
     ITranslationPlatformService translationPlatformService,
     IOptionsMonitor<BuildJobOptions> options
 )
@@ -19,6 +20,7 @@ IOptionsMonitor<BuildJobOptions> options
         buildJobService,
         sharedFileService,
         parallelCorpusService,
+        buildDiagnosticService,
         options
     )
 {
@@ -30,18 +32,36 @@ protected override async Task UpdateBuildExecutionData(
         PreprocessStats stats,
         string sourceLanguageTag,
         string targetLanguageTag,
+        bool isNonPersistedTranslationEngine,
         IReadOnlyList<ParallelCorpusContract> parallelCorpora,
         CancellationToken cancellationToken
     )
     {
-        IReadOnlyList<string> warnings = GetWarnings(
+        string modelName =
+            (await Engines.GetAsync(e => e.EngineId == engineId, cancellationToken))?.CurrentBuild?.BaseModel.ToString()
+            ?? "Unknown";
+        IReadOnlyList<DiagnosticContract> diagnostics = GetDiagnostics(
             stats.TrainCount,
             stats.InferenceCount,
             sourceLanguageTag,
             targetLanguageTag,
+            sourceLanguageHasNativeSupport: true,
+            targetLanguageHasNativeSupport: true,
+            isNonPersistedTranslationEngine,
+            modelName,
             parallelCorpora
         );
 
+        IReadOnlyList<string> warnings = diagnostics.Select(d => d.Message).ToList();
+
+        int maxDiagnostics = BuildJobOptions.MaxDiagnostics;
+        bool diagnosticsTruncated = false;
+        if (diagnostics.Count > maxDiagnostics)
+        {
+            diagnosticsTruncated = true;
+            diagnostics = diagnostics.OrderByDescending(d => d.Severity).Take(maxDiagnostics).ToList();
+        }
+
         int maxWarnings = BuildJobOptions.MaxWarnings;
         if (warnings.Count > maxWarnings)
         {
@@ -74,6 +94,8 @@ CancellationToken cancellationToken
             TrainVerseCount = stats.TrainVerseCount,
             InferenceVerseCount = stats.InferenceVerseCount,
             Warnings = warnings,
+            Diagnostics = diagnostics,
+            DiagnosticsTruncated = diagnosticsTruncated,
             EngineSourceLanguageTag = sourceLanguageTag,
             EngineTargetLanguageTag = targetLanguageTag,
             ResolvedSourceLanguage = sourceLanguageTag,