Multi-Computer Estimation and Runs

README.md

@@ -68,6 +68,23 @@ In XTMF2, open **Settings**, open **RunServers**, and add a remote endpoint. Ent
 
 The GUI pins the server certificate to this fingerprint and authenticates with the token before creating the RunServer bus. A mismatched certificate or token is rejected. Do not expose the TCP port to untrusted networks; use firewall rules or a private network as appropriate.
 
+## Distributed estimation
+
+Estimation runs can evaluate candidate parameter vectors concurrently across multiple connected RunServers. Normal model runs and calibration runs continue to use one RunServer.
+
+To start a distributed estimation run:
+
+1. Connect the required RunServers from **Settings** > **RunServers**.
+2. Open the model system and choose **Run Estimation**.
+3. Select two or more RunServers in the run configuration dialog and choose the orchestrator RunServer. The other selected servers are workers.
+4. If the model has an estimation `InputDirectory`, review its worker-local value for each selected worker. These values are saved in the GUI settings by model node ID and are reused on later estimation runs.
+
+The orchestrator RunServer owns the estimation algorithm and connects directly to the workers. Each worker constructs and validates its own local copy of the model. Input-directory overrides are applied only to that worker's copy; the saved model system is not modified. The orchestrator must be able to reach every worker's configured TCP endpoint.
+
+After submission, the GUI is not required for the optimization to continue. The orchestrator persists `estimation-completion.json` in the run directory when the job finishes, including the best parameter values and completion status.
+
+RunServers that receive path overrides must be running the current shared-estimation worker build. Runs without path overrides retain the version-1 shared-estimation protocol and remain compatible with workers that support the original distributed-estimation protocol.
+
 ## Main Branches
 
 There are 4 major branches for XTMF 2 intended for different purposes:

XTMF2.sln

@@ -33,6 +33,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "XTMF2.Run", "src\XTMF2.Run\
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "XTMF2.GUI", "src\XTMF2.GUI\XTMF2.GUI.csproj", "{A16D11CD-0D70-4558-8CEA-52F0601C215A}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "XTMF2.RunServer.Updater", "src\XTMF2.RunServer.Updater\XTMF2.RunServer.Updater.csproj", "{E3239050-BF93-44A7-BC91-6F865087EBB7}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -139,6 +141,18 @@ Global
 		{A16D11CD-0D70-4558-8CEA-52F0601C215A}.Release|x64.Build.0 = Release|Any CPU
 		{A16D11CD-0D70-4558-8CEA-52F0601C215A}.Release|x86.ActiveCfg = Release|Any CPU
 		{A16D11CD-0D70-4558-8CEA-52F0601C215A}.Release|x86.Build.0 = Release|Any CPU
+		{E3239050-BF93-44A7-BC91-6F865087EBB7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E3239050-BF93-44A7-BC91-6F865087EBB7}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E3239050-BF93-44A7-BC91-6F865087EBB7}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{E3239050-BF93-44A7-BC91-6F865087EBB7}.Debug|x64.Build.0 = Debug|Any CPU
+		{E3239050-BF93-44A7-BC91-6F865087EBB7}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{E3239050-BF93-44A7-BC91-6F865087EBB7}.Debug|x86.Build.0 = Debug|Any CPU
+		{E3239050-BF93-44A7-BC91-6F865087EBB7}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{E3239050-BF93-44A7-BC91-6F865087EBB7}.Release|Any CPU.Build.0 = Release|Any CPU
+		{E3239050-BF93-44A7-BC91-6F865087EBB7}.Release|x64.ActiveCfg = Release|Any CPU
+		{E3239050-BF93-44A7-BC91-6F865087EBB7}.Release|x64.Build.0 = Release|Any CPU
+		{E3239050-BF93-44A7-BC91-6F865087EBB7}.Release|x86.ActiveCfg = Release|Any CPU
+		{E3239050-BF93-44A7-BC91-6F865087EBB7}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -152,6 +166,7 @@ Global
 		{A1B2C3D4-E5F6-7890-ABCD-EF1234567890} = {77789778-93F5-46A6-9114-CCE855E5919C}
 		{F6999FCE-4C4F-4CF6-9B70-0DEA31FB5E6D} = {9040CC77-712E-414D-8453-391F9F348218}
 		{A16D11CD-0D70-4558-8CEA-52F0601C215A} = {9040CC77-712E-414D-8453-391F9F348218}
+		{E3239050-BF93-44A7-BC91-6F865087EBB7} = {9040CC77-712E-414D-8453-391F9F348218}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {15025AE3-C5AA-4CD4-BC38-D1C1BA8904AF}

src/XTMF2.Client/Properties/launchSettings.json

@@ -3,6 +3,10 @@
     "XTMF.Run": {
       "commandName": "Project",
       "commandLineArgs": "-loadDLL \"../../../../../tests/XTMF2.UnitTests/bin/Debug/netcoreapp3.1/XTMF2.UnitTests.dll\" -namedPipe 123"
+    },
+    "XTMF.RunServer": {
+      "commandName": "Project",
+      "commandLineArgs": "-tcp 0.0.0.0 1447 -security .\\runserver-security\\"
     }
   }
 }

src/XTMF2.Client/XTMF2.RunServer.csproj

@@ -26,6 +26,7 @@
     <ProjectReference Include="..\XTMF2.Run\XTMF2.Run.csproj" />
     <ProjectReference Include="..\XTMF2\XTMF2.csproj" />
     <ProjectReference Include="..\XTMF2.Interfaces\XTMF2.Interfaces.csproj" />
+    <ProjectReference Include="..\XTMF2.RunServer.Updater\XTMF2.RunServer.Updater.csproj" ReferenceOutputAssembly="false" />
   </ItemGroup>
 
 </Project>

src/XTMF2.GUI/App.axaml

@@ -332,6 +332,12 @@
         <Style Selector="Window.neon-dialog TextBlock">
             <Setter Property="Foreground" Value="{DynamicResource DlgFg}"/>
         </Style>
+        <Style Selector="Window.neon-dialog Grid.activity-row">
+            <Setter Property="Background" Value="Transparent"/>
+        </Style>
+        <Style Selector="Window.neon-dialog Grid.activity-row:pointerover">
+            <Setter Property="Background" Value="{DynamicResource DlgSelected}"/>
+        </Style>
         <Style Selector="Window.neon-dialog TextBox">
             <Setter Property="Background" Value="{DynamicResource DlgInputBg}"/>
             <Setter Property="Foreground" Value="{DynamicResource DlgFg}"/>

src/XTMF2.GUI/MainWindow.axaml

@@ -39,6 +39,7 @@
                           InputGesture="Ctrl+B"/>
             </MenuItem>
             <MenuItem Header="{res:Localize MainWindow_MenuTools}">
+                <MenuItem Header="RunServer Activity" Click="RunServerActivity_Click"/>
                 <MenuItem Header="{res:Localize MainWindow_MenuSettings}" Click="Settings_Click"/>
             </MenuItem>
             <MenuItem Header="{res:Localize MainWindow_MenuHelp}">

src/XTMF2.GUI/MainWindow.axaml.cs

@@ -38,12 +38,15 @@ You should have received a copy of the GNU General Public License
 using System.Collections.ObjectModel;
 using System.Collections.Specialized;
 using System.ComponentModel;
+using System.IO;
 using System.Linq;
 using System.Net.Http;
 using System.Threading;
 using System.Threading.Tasks;
 using XTMF2;
 using XTMF2.AI;
+using XTMF2.Bus;
+using XTMF2.Bus.Optimization;
 using XTMF2.Editing;
 using XTMF2.GUI.AI;
 using XTMF2.GUI.Controls;
@@ -63,6 +66,7 @@ public partial class MainWindow : Window
     private bool _documentCloseInProgress;
     private SettingsWindow? _settingsWindow;
     private RunServersWindow? _runServersWindow;
+    private RunServerActivityWindow? _runServerActivityWindow;
     private readonly HttpClient _aiHttpClient = new();
     private readonly AiProviderRegistry _aiProviders = new();
 
@@ -312,7 +316,10 @@ public void InitializeWithRuntime(XTMFRuntime runtime, RunController? runControl
 
         // Add the Runs tab so it is always visible.
         if (_runController is not null)
+        {
             Documents.Add(_runController.RunsViewModel);
+            _runController.EnableAutomaticRecoveredRunBinding(user);
+        }
 
         // Add the Projects tab as the permanent first document
         Documents.Add(new ProjectsViewModel(runtime, user));
@@ -421,6 +428,43 @@ public void FocusEditorTab(ModelSystemEditorViewModel editorVm)
         ActivateDocument(editorVm);
     }
 
+    public bool BindRecoveredRunToActiveModelSystem(string runId, out string? error)
+    {
+        if (_activeEditorVm is null || _currentUser is null || _runController is null)
+        {
+            error = "This run could not be matched automatically. Open its model system, make it active, and bind it manually.";
+            return false;
+        }
+        return _runController.BindRecoveredRemoteRun(runId, _activeEditorVm.Session,
+            _activeEditorVm.User, out error);
+    }
+
+    public bool RetryRemoteRunReceipt(string runId, out string? error)
+    {
+        if (_runController is null)
+        {
+            error = "The run controller is unavailable.";
+            return false;
+        }
+        return _runController.RetryRemoteRunReceipt(runId, out error);
+    }
+
+    public bool RequestRemoteRunOutputTransfer(string runId, out string? error)
+    {
+        if (_runController is null)
+        {
+            error = "The run controller is unavailable.";
+            return false;
+        }
+        return _runController.RequestRemoteRunOutputTransfer(runId, out error);
+    }
+
+    public Task<RemoteRunDeletionResponse> DeleteRemoteRunAsync(string runId)
+        => _runController is null
+            ? Task.FromResult(new RemoteRunDeletionResponse(string.Empty, runId, false,
+                "The run controller is unavailable."))
+            : _runController.DeleteRemoteRunAsync(runId);
+
     /// <summary>
     /// Switches the active document to the Runs tab.
     /// </summary>
@@ -479,6 +523,19 @@ private void Settings_Click(object? sender, RoutedEventArgs e)
         }
     }
 
+    private void RunServerActivity_Click(object? sender, RoutedEventArgs e)
+    {
+        if (_runServerActivityWindow is not null && _runServerActivityWindow.IsVisible)
+        {
+            _runServerActivityWindow.Activate();
+            return;
+        }
+
+        _runServerActivityWindow = new RunServerActivityWindow(_runController);
+        _runServerActivityWindow.Closed += (_, _) => _runServerActivityWindow = null;
+        _runServerActivityWindow.Show(this);
+    }
+
     private void ShowRunServersWindow()
     {
         var settingsOwner = _settingsWindow;
@@ -643,6 +700,43 @@ protected override async void OnClosing(WindowClosingEventArgs e)
                     return;
             }
 
+            if (_runController is not null)
+            {
+                IReadOnlyList<RunServerActivity> localActivities;
+                try
+                {
+                    localActivities = await _runController.QueryLocalRunServerActivityAsync();
+                }
+                catch (Exception exception) when (exception is IOException or OperationCanceledException or
+                    ObjectDisposedException or InvalidOperationException)
+                {
+                    var confirmation = new ConfirmDialog("Local RunServer Status Unavailable",
+                        $"The local RunServer's workload could not be checked ({exception.Message}). " +
+                        "Closing may terminate a running model system. Close the program anyway?");
+                    await confirmation.ShowDialog(this);
+                    if (!confirmation.Result)
+                        return;
+                    localActivities = Array.Empty<RunServerActivity>();
+                }
+
+                if (localActivities.Count > 0)
+                {
+                    var runningCount = localActivities.Count(activity => activity.State == RunServerActivityState.Running);
+                    var queuedCount = localActivities.Count(activity => activity.State == RunServerActivityState.Queued);
+                    var workload = new List<string>();
+                    if (runningCount > 0)
+                        workload.Add($"{runningCount} running");
+                    if (queuedCount > 0)
+                        workload.Add($"{queuedCount} queued");
+                    var confirmation = new ConfirmDialog("Local RunServer Is Busy",
+                        $"The local RunServer has {string.Join(" and ", workload)} model system(s). " +
+                        "Closing the program will stop the Local RunServer and terminate this work. Close anyway?");
+                    await confirmation.ShowDialog(this);
+                    if (!confirmation.Result)
+                        return;
+                }
+            }
+
             _allowDocumentClose = true;
             _allowClose = true;
             Close();

src/XTMF2.GUI/Properties/RunServerEndpoint.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 
 namespace XTMF2.GUI.Properties;
 
@@ -15,6 +16,7 @@ public sealed class RunServerEndpoint
     public bool IsLocal { get; set; }
     public string Token { get; set; } = string.Empty;
     public string CertificateFingerprint { get; set; } = string.Empty;
+    public Dictionary<string, string> BasicParameterOverrides { get; set; } = new(StringComparer.OrdinalIgnoreCase);
 
     public static RunServerEndpoint CreateLocal()
         => new()
@@ -26,7 +28,8 @@ public static RunServerEndpoint CreateLocal()
             Port = 0,
             IsLocal = true,
             Token = string.Empty,
-            CertificateFingerprint = string.Empty
+            CertificateFingerprint = string.Empty,
+            BasicParameterOverrides = new(StringComparer.OrdinalIgnoreCase)
         };
 
     public RunServerEndpoint Clone()
@@ -39,6 +42,8 @@ public RunServerEndpoint Clone()
             Port = Port,
             IsLocal = IsLocal,
             Token = Token,
-            CertificateFingerprint = CertificateFingerprint
+            CertificateFingerprint = CertificateFingerprint,
+            BasicParameterOverrides = new Dictionary<string, string>(BasicParameterOverrides ?? new Dictionary<string, string>(),
+                StringComparer.OrdinalIgnoreCase)
         };
 }

src/XTMF2.GUI/Properties/Settings.cs

@@ -149,6 +149,8 @@ private static List<RunServerEndpoint> NormalizeRunServers(IEnumerable<RunServer
                 if (endpoint is null || string.IsNullOrWhiteSpace(endpoint.Address) || endpoint.Port is < 0 or > 65535)
                     continue;
 
+                endpoint.BasicParameterOverrides ??= new(StringComparer.OrdinalIgnoreCase);
+
                 if (normalized.Any(existing => existing.Id == endpoint.Id))
                     continue;
 

src/XTMF2.GUI/RemoteRunOutputPaths.cs

@@ -0,0 +1,16 @@
+using System;
+using System.IO;
+using System.Security.Cryptography;
+using System.Text;
+
+namespace XTMF2.GUI;
+
+internal static class RemoteRunOutputPaths
+{
+    public static string GetLocalDirectory(string runId)
+    {
+        var key = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(runId)));
+        return Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
+            "XTMF2", "GUI", "RemoteRunOutputs", key);
+    }
+}