skyline: Port the native waters_connect/UNIFI readers to IdentityModel 7

Jamroot.jam

@@ -951,6 +951,14 @@ else
     .installer-vendor-files-location = $(PWIZ_BUILD_PATH)/without-cxt/$(PLATFORM) ;
 }
 
+# Assemblies that every installer already declares for itself. They reach the <assembly>
+# enumeration below because a vendor API references them, but listing them here as well would
+# declare the same file twice and fail the WiX link with LGHT0091 (duplicate symbol).
+# pwiz.CommonUtil.dll is ours rather than a vendor redistributable, and is named directly by
+# scripts/wix/pwiz-setup.wxs.template, pwiz_tools/Bumbershoot/idpicker/Deploy/Deploy.wxs.template
+# and pwiz_tools/Skyline/Executables/Installer/FileList64-template.txt.
+.installer-vendor-files-exclusions = pwiz.CommonUtil.dll ;
+
 rule make_INSTALLER_VENDOR_FILES ( targets * : sources * : properties * )
 {
     # enumerate .NET assemblies and their native DLL dependencies
@@ -961,7 +969,8 @@ rule make_INSTALLER_VENDOR_FILES ( targets * : sources * : properties * )
         for local assembly in $(assemblies)
         {
             local assembly-path = [ path.basename [ $(assembly).name ] ] ;
-            if ! $(assembly-path:L) in $(.unique-vendor-files:L)
+            if ! $(assembly-path:L) in $(.unique-vendor-files:L) &&
+               ! $(assembly-path:L) in $(.installer-vendor-files-exclusions:L)
             {
                 .unique-vendor-files += $(assembly-path) ;
             }

libraries/boost-build/src/tools/msvc.jam

@@ -861,6 +861,11 @@ rule set-assemblies ( target : properties * )
     {
         for local assembly in $(assemblies)
         {
+            # an assembly can be generated by this build rather than vendored as a prebuilt file,
+            # in which case the compile referencing it with /FU has to wait for it to be built;
+            # copy-assemblies already declares the same edge for the copy it makes
+            DEPENDS $(target) : [ $(assembly).actualize ] ;
+
             local assembly-path = [ $(assembly).name ] ;
             if $(assembly-path:S) != ".pdb"
             {
@@ -883,6 +888,8 @@ rule set-assemblies ( target : properties * )
         local assembly-include-paths ;
         for local dependency in $(assembly-dependencies)
         {
+            DEPENDS $(target) : [ $(dependency).actualize ] ;
+
             local dependency-path = [ $(dependency).name ] ;
             if ! [ path.is-rooted $(dependency-path) ]
             {

pwiz_aux/msrc/utility/vendor_api/UNIFI/Jamfile.jam

@@ -45,9 +45,54 @@ if [ modules.peek : NT ]
                    System.Memory.dll
                    System.Numerics.Vectors.dll
                    System.Runtime.CompilerServices.Unsafe.dll
+                   System.Text.Json.dll
                    System.Threading.Tasks.Extensions.dll
                    System.ValueTuple.dll ;
 
+# pwiz.CommonUtil.dll supplies OAuthPasswordGrantClient - the IdentityModel-7-compatible OAuth
+# password-grant request shared with the managed WatersConnect/Unifi account classes, so this
+# native reader does not duplicate that request logic in C++/CLI. Unlike everything in
+# .shared-assemblies it is built from this repo rather than vendored, so it is declared as a
+# target here and referenced by target ID below: <assembly> is a dependency feature, so bjam
+# builds this before compiling anything that references it, and msvc.jam's copy-assemblies
+# stages it next to each consuming binary.
+#
+# MSBuild, not "dotnet build": the .NET SDK's MSBuild resolves cultures through ICU, which does
+# not know the legacy zh-CHS of CommonUtil's satellite .resx files, so it gives them the same
+# manifest name as the neutral resources and fails with MSB3577. Framework/VS MSBuild uses NLS
+# and still recognizes zh-CHS.
+rule build-common-util-properties ( targets + : sources * : properties * )
+{
+    local .msvcSetupScript = [ get-current-msvc-setup-script $(properties) ] ;
+    JAM_SEMAPHORE on $(targets) = "dotNetSemaphore" ;
+    MSVC_CURRENT_SETUP_SCRIPT on $(targets) = $(.msvcSetupScript) ;
+}
+
+rule do_build_common_util ( targets + : sources * : properties * )
+{
+    return [ build-common-util-properties $(targets) : $(sources) : $(properties) ] ;
+}
+
+actions do_build_common_util
+{
+    $(MSVC_CURRENT_SETUP_SCRIPT)
+    echo Building pwiz.CommonUtil.dll for pwiz_vendor_api_unifi
+    msbuild "$(PWIZ_ROOT_PATH)\pwiz_tools\Shared\CommonUtil\CommonUtil.csproj" /p:Configuration=Release;Platform=x64;OutDir=$(<:D)\ /nologo /verbosity:minimal
+    set status=%ERRORLEVEL%
+    exit %status%
+}
+
+make pwiz.CommonUtil.dll
+    : # sources
+    : # actions
+        @do_build_common_util
+    : # requirements
+        <conditional>@no-express-requirement
+        <conditional>@msvc-dotnet-requirement
+    ;
+
+explicit pwiz.CommonUtil.dll ;
+
 rule vendor-api-requirements ( properties * )
 {
     local result ;
@@ -67,6 +112,7 @@ rule vendor-api-requirements ( properties * )
         #result += <assembly>$(dll_location)/unifi-protobuf-net.dll ;
         result += <assembly>$(PWIZ_ROOT_PATH)/pwiz_aux/msrc/utility/vendor_api/ABI/protobuf-net.dll ;
         result += <assembly>$(PWIZ_ROOT_PATH)/pwiz_tools/Shared/Lib/$(.shared-assemblies) ;
+        result += <assembly>$(PWIZ_ROOT_PATH)/pwiz_aux/msrc/utility/vendor_api/UNIFI//pwiz.CommonUtil.dll ;
         result += <assembly>$(dll_location)/System.Runtime.Caching.Generic.dll ;
         result += <assembly>$(dll_location)/ParallelExtensionsExtras.dll ;
         result += <include>$(dll_location) ;

pwiz_aux/msrc/utility/vendor_api/UNIFI/UnifiData.cpp

@@ -63,8 +63,8 @@ using System::Threading::Tasks::Task;
 using System::Threading::Tasks::TaskScheduler;
 using System::Threading::Tasks::Schedulers::QueuedTaskScheduler;
 using System::Uri;
-using IdentityModel::Client::TokenClient;
 using IdentityModel::Client::TokenResponse;
+using pwiz::Common::SystemUtil::OAuthPasswordGrantClient;
 using std::size_t;
 
 
@@ -1002,14 +1002,8 @@ class UnifiData::Impl
             password = userPassPair[1];
         }
 
-        auto fields = gcnew System::Collections::Generic::Dictionary<System::String^, System::String^>();
-        fields->Add(IdentityModel::OidcConstants::TokenRequest::GrantType, IdentityModel::OidcConstants::GrantTypes::Password);
-        fields->Add(IdentityModel::OidcConstants::TokenRequest::UserName, username);
-        fields->Add(IdentityModel::OidcConstants::TokenRequest::Password, password);
-        fields->Add(IdentityModel::OidcConstants::TokenRequest::Scope, _clientScope);
-
-        auto tokenClient = gcnew TokenClient(tokenEndpoint(), _clientId, _clientSecret, nullptr, IdentityModel::Client::AuthenticationStyle::BasicAuthentication);
-        TokenResponse^ response = tokenClient->RequestAsync(fields, System::Threading::CancellationToken::None)->Result;
+        TokenResponse^ response = OAuthPasswordGrantClient::RequestToken(gcnew Uri(tokenEndpoint()), _clientId, _clientSecret,
+            OAuthPasswordGrantClient::PasswordGrantForm(username, password, _clientScope));
         if (response->IsError)
             throw user_error("authentication error: incorrect hostname, username or password? (" + ToStdString(response->Error) + ")");
 

pwiz_aux/msrc/utility/vendor_api/UNIFI/WatersConnectData.ipp

@@ -61,8 +61,8 @@ using System::Threading::Tasks::TaskScheduler;
 using System::Threading::Tasks::Schedulers::QueuedTaskScheduler;
 using System::Net::Http::HttpClient;
 using System::Uri;
-using IdentityModel::Client::TokenClient;
 using IdentityModel::Client::TokenResponse;
+using pwiz::Common::SystemUtil::OAuthPasswordGrantClient;
 using std::size_t;
 #include "WatersConnectProtobuf.hpp"
 
@@ -475,14 +475,8 @@ private:
 
     static Object^ getAccessTokenResult(String^ uri, AccessTokenRequest^ request)
     {
-        auto fields = gcnew System::Collections::Generic::Dictionary<System::String^, System::String^>();
-        fields->Add(IdentityModel::OidcConstants::TokenRequest::GrantType, IdentityModel::OidcConstants::GrantTypes::Password);
-        fields->Add(IdentityModel::OidcConstants::TokenRequest::UserName, request->Username);
-        fields->Add(IdentityModel::OidcConstants::TokenRequest::Password, request->Password);
-        fields->Add(IdentityModel::OidcConstants::TokenRequest::Scope, request->Scope);
-
-        auto tokenClient = gcnew TokenClient(request->Uri, request->ClientId, request->Secret, nullptr);
-        TokenResponse^ response = tokenClient->RequestAsync(fields, System::Threading::CancellationToken::None)->Result;
+        TokenResponse^ response = OAuthPasswordGrantClient::RequestToken(gcnew Uri(request->Uri), request->ClientId, request->Secret,
+            OAuthPasswordGrantClient::PasswordGrantForm(request->Username, request->Password, request->Scope));
         if (response->IsError)
             throw user_error("authentication error: incorrect hostname, username or password? (" + ToStdString(response->Error) + ")");
         return gcnew KeyValuePair<String^, DateTime>(response->AccessToken, DateTime::UtcNow.AddSeconds(response->ExpiresIn));

pwiz_tools/Shared/CommonMsData/RemoteApi/Unifi/UnifiAccount.cs

@@ -16,6 +16,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Net.Http;
@@ -127,15 +128,19 @@ public string GetFoldersUrl()
 
         public TokenResponse Authenticate()
         {
-            var tokenClient = new TokenClient(IdentityServer + IdentityConnectEndpoint, ClientId,
-                ClientSecret, new HttpClientHandler());
-            return tokenClient.RequestResourceOwnerPasswordAsync(Username, Password, ClientScope).Result;
+            // Shared with WatersConnectAccount.RequestToken, which authenticates against a
+            // sibling Waters-hosted identity server the same way - see OAuthPasswordGrantClient
+            // for the POST, response parsing, and why both needed to stop constructing
+            // TokenResponse directly once IdentityModel 7 removed its constructors.
+            return OAuthPasswordGrantClient.RequestToken(new Uri(IdentityServer + IdentityConnectEndpoint), ClientId, ClientSecret,
+                OAuthPasswordGrantClient.PasswordGrantForm(Username, Password, ClientScope));
         }
 
         public IEnumerable<UnifiFolderObject> GetFolders()
         {
-            var httpClient = GetAuthenticatedHttpClient();
-            var response = httpClient.GetAsync(GetFoldersUrl()).Result;
+            using var httpClient = GetAuthenticatedHttpClient();
+            using var request = new HttpRequestMessage(HttpMethod.Get, GetFoldersUrl());
+            using var response = httpClient.SendRequest(request);
             string responseBody = response.Content.ReadAsStringAsync().Result;
             var jsonObject = JObject.Parse(responseBody);
 
@@ -149,9 +154,10 @@ public IEnumerable<UnifiFolderObject> GetFolders()
 
         public IEnumerable<UnifiFileObject> GetFiles(UnifiFolderObject folder)
         {
-            var httpClient = GetAuthenticatedHttpClient();
+            using var httpClient = GetAuthenticatedHttpClient();
             string url = string.Format(@"/unifi/v1/folders({0})/items", folder.Id);
-            var response = httpClient.GetAsync(ServerUrl + url).Result;
+            using var request = new HttpRequestMessage(HttpMethod.Get, ServerUrl + url);
+            using var response = httpClient.SendRequest(request);
             string responseBody = response.Content.ReadAsStringAsync().Result;
             var jsonObject = JObject.Parse(responseBody);
             var itemsValue = jsonObject[@"value"] as JArray;
@@ -162,13 +168,12 @@ public IEnumerable<UnifiFileObject> GetFiles(UnifiFolderObject folder)
             return itemsValue.OfType<JObject>().Select(f => new UnifiFileObject(f));
         }
 
-        public HttpClient GetAuthenticatedHttpClient()
+        public HttpClientWithProgress GetAuthenticatedHttpClient()
         {
             var tokenResponse = Authenticate();
-            var httpClient = new HttpClient();
-            httpClient.SetBearerToken(tokenResponse.AccessToken);
-            httpClient.DefaultRequestHeaders.Remove(@"Accept");
-            httpClient.DefaultRequestHeaders.Add(@"Accept", @"application/json;odata.metadata=minimal");
+            var httpClient = new HttpClientWithProgress();
+            httpClient.AddAuthorizationHeader(@"Bearer " + tokenResponse.AccessToken);
+            httpClient.AddHeader(@"Accept", @"application/json;odata.metadata=minimal");
             return httpClient;
         }
 

pwiz_tools/Shared/CommonMsData/RemoteApi/Unifi/UnifiSession.cs

@@ -19,6 +19,7 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Net.Http;
 using Newtonsoft.Json.Linq;
 using pwiz.Common.Collections;
 
@@ -44,9 +45,11 @@ public override bool AsyncFetchContents(RemoteUrl remoteUrl, out RemoteServerExc
 
         private ImmutableList<UnifiFolderObject> GetFolders(Uri requestUri)
         {
-            var httpClient = UnifiAccount.GetAuthenticatedHttpClient();
-            var response = httpClient.GetAsync(requestUri).Result;
-            response.EnsureSuccessStatusCode();
+            using var httpClient = UnifiAccount.GetAuthenticatedHttpClient();
+            // SendRequest already throws NetworkRequestException on a non-2xx response, so there
+            // is nothing left for an EnsureSuccessStatusCode() to catch here.
+            using var request = new HttpRequestMessage(HttpMethod.Get, requestUri);
+            using var response = httpClient.SendRequest(request);
             string responseBody = response.Content.ReadAsStringAsync().Result;
             var jsonObject = JObject.Parse(responseBody);
 
@@ -60,9 +63,9 @@ private ImmutableList<UnifiFolderObject> GetFolders(Uri requestUri)
 
         private ImmutableList<UnifiFileObject> GetFiles(Uri requestUri)
         {
-            var httpClient = UnifiAccount.GetAuthenticatedHttpClient();
-            var response = httpClient.GetAsync(requestUri).Result;
-            response.EnsureSuccessStatusCode();
+            using var httpClient = UnifiAccount.GetAuthenticatedHttpClient();
+            using var request = new HttpRequestMessage(HttpMethod.Get, requestUri);
+            using var response = httpClient.SendRequest(request);
             string responseBody = response.Content.ReadAsStringAsync().Result;
             var jsonObject = JObject.Parse(responseBody);
             var itemsValue = jsonObject[@"value"] as JArray;

pwiz_tools/Shared/CommonMsData/RemoteApi/WatersConnect/WatersConnectAccount.cs

@@ -21,9 +21,7 @@
 using System.Collections.Generic;
 using System.Collections.Specialized;
 using System.Globalization;
-using System.Net;
 using System.Security.Authentication;
-using System.Text;
 using System.Xml;
 using System.Xml.Linq;
 using System.Xml.Serialization;
@@ -226,13 +224,7 @@ public TokenResponse Authenticate()
                 }
             }
             // Otherwise, request a new token using the username and password
-            var newToken = RequestToken(new NameValueCollection
-            {
-                [@"grant_type"] = @"password",
-                [@"username"] = Username,
-                [@"password"] = Password,
-                [@"scope"] = ClientScope
-            });
+            var newToken = RequestToken(OAuthPasswordGrantClient.PasswordGrantForm(Username, Password, ClientScope));
             if (newToken.IsError)
             {
                 AuthenticationException ex;
@@ -263,35 +255,11 @@ public TokenResponse Authenticate()
         /// </summary>
         private TokenResponse RequestToken(NameValueCollection form)
         {
-            try
-            {
-                var requestUri = new Uri(IdentityServer + IdentityConnectEndpoint);
-                using var httpClient = new HttpClientWithProgress();
-                httpClient.AddAuthorizationHeader(@"Basic " + Convert.ToBase64String(Encoding.UTF8.GetBytes(
-                    EscapeClientCredential(ClientId) + @":" + EscapeClientCredential(ClientSecret))));
-                httpClient.AddHeader(@"Accept", @"application/json");
-                var raw = Encoding.UTF8.GetString(httpClient.UploadValues(requestUri, @"POST", form));
-                return new TokenResponse(raw);
-            }
-            catch (NetworkRequestException ex)
-            {
-                if (ex.StatusCode == HttpStatusCode.BadRequest && !string.IsNullOrEmpty(ex.ResponseBody))
-                    return new TokenResponse(ex.ResponseBody);
-                return new TokenResponse(ex.StatusCode ?? HttpStatusCode.ServiceUnavailable, ex.Message, ex.ResponseBody);
-            }
-            catch (Exception ex)
-            {
-                return new TokenResponse(ex);
-            }
-        }
-
-        /// <summary>
-        /// RFC 6749 section 2.3.1: client_id and client_secret are form-urlencoded before being
-        /// combined into the Basic authorization credential.
-        /// </summary>
-        private static string EscapeClientCredential(string value)
-        {
-            return Uri.EscapeDataString(value ?? string.Empty).Replace(@"%20", @"+");
+            // Shared with UnifiAccount.Authenticate, which authenticates against a sibling
+            // Waters-hosted identity server the same way - see OAuthPasswordGrantClient for the
+            // POST, response parsing, and why both needed to stop constructing TokenResponse
+            // directly once IdentityModel 7 removed its constructors.
+            return OAuthPasswordGrantClient.RequestToken(new Uri(IdentityServer + IdentityConnectEndpoint), ClientId, ClientSecret, form);
         }
 
         public static AuthenticationErrorType HandleAuthenticationException(AuthenticationException ex, out string message)
@@ -305,18 +273,26 @@ public static AuthenticationErrorType HandleAuthenticationException(Authenticati
             try
             {
                 var tokenResponse = JObject.Parse((string)ex.Data[TOKEN_DATA]);
+                // error_description is frequently empty (e.g. Waters' invalid_scope response is just
+                // {"error":"invalid_scope"}), so fall back to the bare error code rather than leaving
+                // the caller with nothing to show - every classified branch below sets message for the
+                // same reason. Only EditRemoteAccountDlg's InvalidClientSecret case overrides this with
+                // a friendlier string; the others show this raw (deliberately non-L10N) server text.
                 string error = (tokenResponse[@"error_description"] ?? tokenResponse[@"error"] ?? "").ToString();
                 var errorType = (tokenResponse[@"error"] ?? "").ToString();
                 if (errorType == @"invalid_scope")
                 {
+                    message = error;
                     return AuthenticationErrorType.InvalidClientScope;
                 }
                 else if (errorType == @"invalid_client")
                 {
+                    message = error;
                     return AuthenticationErrorType.InvalidClientSecret;
                 }
                 else if (errorType == @"invalid_grant")
                 {
+                    message = error;
                     return AuthenticationErrorType.InvalidPassword;
                 }
                 else if (!string.IsNullOrEmpty(error))
@@ -326,6 +302,7 @@ public static AuthenticationErrorType HandleAuthenticationException(Authenticati
                 }
                 else
                 {
+                    message = ex.Message;
                     return AuthenticationErrorType.InvalidIdentityServer;
                 }
             }

pwiz_tools/Shared/CommonUtil/CommonUtil.csproj

@@ -90,6 +90,9 @@
     <Prefer32Bit>false</Prefer32Bit>
   </PropertyGroup>
   <ItemGroup>
+    <Reference Include="IdentityModel">
+      <HintPath>..\Lib\IdentityModel.dll</HintPath>
+    </Reference>
     <Reference Include="JetBrains.Annotations">
       <HintPath>..\Lib\JetBrains.Annotations.dll</HintPath>
     </Reference>
@@ -245,6 +248,7 @@
     <Compile Include="SystemUtil\ConcurrencyVisualizer.cs" />
     <Compile Include="SystemUtil\FormUtil.cs" />
     <Compile Include="SystemUtil\HttpClientWithProgress.cs" />
+    <Compile Include="SystemUtil\OAuthPasswordGrantClient.cs" />
     <Compile Include="SystemUtil\Immutable.cs" />
     <Compile Include="SystemUtil\IProgressMonitor.cs" />
     <Compile Include="SystemUtil\LocalizationHelper.cs" />