Conformant libc++21 formatter refactor (boundary widening) [WIP]

packages.config

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <packages>
   <package id="CommunityToolkit.Mvvm" version="8.4.0" />
   <package id="CommunityToolkit.WinUI.Animations" version="8.2.250402" />
@@ -21,7 +21,7 @@
   <package id="Microsoft.WSL.Dependencies.arm64fre" version="10.0.27820.1000-250318-1700.rs-base2-hyp" targetFramework="native" />
   <package id="Microsoft.WSL.DeviceHost" version="1.2.48-0" />
   <package id="Microsoft.WSL.Kernel" version="6.18.35.2-1" targetFramework="native" />
-  <package id="Microsoft.WSL.LinuxSdk" version="1.20.0" targetFramework="native" />
+  <package id="Microsoft.WSL.LinuxSdk" version="1.23.0" targetFramework="native" />
   <package id="Microsoft.WSL.TestData" version="0.5.0" />
   <package id="Microsoft.WSL.TestDistro" version="2.7.1-1" />
   <package id="Microsoft.WSLg" version="1.0.79" />

src/shared/inc/stringshared.h

@@ -16,6 +16,9 @@ Module Name:
 #include <set>
 #include <vector>
 #include <string>
+#include <string_view>
+#include <type_traits>
+#include <utility>
 #include <sstream>
 #include <fstream>
 #include <gsl/gsl>
@@ -659,6 +662,40 @@ inline std::wstring MultiByteToWide(const std::string& string)
     return MultiByteToWide(string.c_str());
 }
 
+//
+// Boundary helper used when narrow string arguments must be fed into a wide (wchar_t) std::format
+// context (e.g. the generated Localization::Message*() formatters, and Windows-only test helpers).
+//
+// Any narrow string argument (const char*, char[N], std::string, std::string_view) must be widened
+// first: C++20/23 explicitly disables formatter<char*, wchar_t> and friends ([format.formatter.spec]),
+// and libc++ 20+ ships them as deleted specializations. Converting here, at the single point where
+// arguments enter the wide format context, lets callers keep passing narrow strings without any
+// non-conformant formatter specialization and without per-call-site conversions. Non-string and
+// already-wide arguments are forwarded unchanged (and the widening branches are compiled out on
+// non-Windows builds, where the format context is already narrow).
+//
+template <typename T>
+inline decltype(auto) WideFormatArg([[maybe_unused]] T&& value)
+{
+#ifdef WIN32
+    using Decayed = std::decay_t<T>;
+    if constexpr (std::is_same_v<std::remove_cvref_t<T>, std::string> || std::is_same_v<std::remove_cvref_t<T>, std::string_view>)
+    {
+        return MultiByteToWide(std::string{value});
+    }
+    else if constexpr (std::is_same_v<Decayed, char*> || std::is_same_v<Decayed, const char*>)
+    {
+        return MultiByteToWide(value);
+    }
+    else
+    {
+        return std::forward<T>(value);
+    }
+#else
+    return std::forward<T>(value);
+#endif
+}
+
 inline std::string WideToMultiByte(const wchar_t* string)
 {
 
@@ -927,71 +964,12 @@ struct std::formatter<std::source_location, wchar_t>
     template <typename TCtx>
     auto format(const std::source_location& location, TCtx& ctx) const
     {
-        return std::format_to(ctx.out(), L"{}[{}:{}]", location.function_name(), location.file_name(), location.line());
-    }
-};
-
-template <>
-struct std::formatter<char*, wchar_t>
-{
-    template <typename TCtx>
-    static constexpr auto parse(TCtx& ctx)
-    {
-        return ctx.begin();
-    }
-
-    template <typename TCtx>
-    auto format(const char* str, TCtx& ctx) const
-    {
-        return std::format_to(ctx.out(), "{}", wsl::shared::string::MultiByteToWide(str));
-    }
-};
-
-template <>
-struct std::formatter<const char*, wchar_t>
-{
-    template <typename TCtx>
-    static constexpr auto parse(TCtx& ctx)
-    {
-        return ctx.begin();
-    }
-
-    template <typename TCtx>
-    auto format(const char* str, TCtx& ctx) const
-    {
-        return std::format_to(ctx.out(), "{}", wsl::shared::string::MultiByteToWide(str));
-    }
-};
-
-template <std::size_t N>
-struct std::formatter<char[N], wchar_t>
-{
-    template <typename TCtx>
-    static constexpr auto parse(TCtx& ctx)
-    {
-        return ctx.begin();
-    }
-
-    template <typename TCtx>
-    auto format(const char str[N], TCtx& ctx) const
-    {
-        return std::format_to(ctx.out(), "{}", wsl::shared::string::MultiByteToWide(str));
-    }
-};
-
-template <class Traits, class Allocator>
-struct std::formatter<std::basic_string<char, Traits, Allocator>, wchar_t>
-{
-    template <typename TCtx>
-    static constexpr auto parse(TCtx& ctx)
-    {
-        return ctx.begin();
-    }
-
-    template <typename TCtx>
-    auto format(const std::basic_string<char, Traits, Allocator>& str, TCtx& ctx) const
-    {
-        return std::format_to(ctx.out(), "{}", wsl::shared::string::MultiByteToWide(str));
+        return std::format_to(
+            ctx.out(),
+            L"{}[{}:{}]",
+            wsl::shared::string::MultiByteToWide(location.function_name()),
+            wsl::shared::string::MultiByteToWide(location.file_name()),
+            location.line());
     }
 };
 

src/windows/common/ConsommeNetworking.cpp

@@ -164,7 +164,8 @@ uint16_t ConsommeNetworking::ModifyOpenPorts(
 
     const auto hostAddressStr = wsl::windows::common::string::SockAddrInetToString(hostAddress);
 
-    std::wstring portString = std::format(L"tag={};guest_port={};listen_addr={}", tag, GuestPort, hostAddressStr.c_str());
+    std::wstring portString =
+        std::format(L"tag={};guest_port={};listen_addr={}", tag, GuestPort, wsl::shared::string::MultiByteToWide(hostAddressStr));
 
     if (HostPort != WSLC_EPHEMERAL_PORT)
     {

src/windows/common/ExecutionContext.h

@@ -6,10 +6,22 @@
 
 namespace wsl::windows::common {
 
+// User-facing error/warning messages may be produced as wide strings (Localization on Windows)
+// or narrow strings (e.g. std::string from lower layers). Normalize to wide for reporting.
+inline std::wstring ToUserErrorMessage(std::wstring_view message)
+{
+    return std::wstring{message};
+}
+
+inline std::wstring ToUserErrorMessage(std::string_view message)
+{
+    return wsl::shared::string::MultiByteToWide(std::string{message});
+}
+
 #define THROW_HR_WITH_USER_ERROR(Result, Message) \
     do \
     { \
-        auto _messageWide = std::format(L"{}", Message); \
+        auto _messageWide = wsl::windows::common::ToUserErrorMessage(Message); \
         if (wsl::windows::common::ExecutionContext::ShouldCollectErrorMessage()) \
         { \
             ::wsl::windows::common::SetErrorMessage(std::wstring(_messageWide)); \
@@ -20,7 +32,7 @@ namespace wsl::windows::common {
 #define THROW_HR_WITH_USER_ERROR_MSG(Result, Message, Format, ...) \
     do \
     { \
-        auto _messageWide = std::format(L"{}", Message); \
+        auto _messageWide = wsl::windows::common::ToUserErrorMessage(Message); \
         if (wsl::windows::common::ExecutionContext::ShouldCollectErrorMessage()) \
         { \
             ::wsl::windows::common::SetErrorMessage(std::wstring(_messageWide)); \

src/windows/common/WslClient.cpp

@@ -1293,13 +1293,22 @@ int Version()
     const auto windowsVersion = wsl::windows::common::helpers::GetWindowsVersionString();
     wsl::windows::common::wslutil::PrintMessage(
         Localization::MessagePackageVersions(
-            WSL_PACKAGE_VERSION, KERNEL_VERSION, WSLG_VERSION, MSRDC_VERSION, DIRECT3D_VERSION, DXCORE_VERSION, windowsVersion),
+            STRING_TO_WIDE_STRING(WSL_PACKAGE_VERSION),
+            STRING_TO_WIDE_STRING(KERNEL_VERSION),
+            STRING_TO_WIDE_STRING(WSLG_VERSION),
+            STRING_TO_WIDE_STRING(MSRDC_VERSION),
+            STRING_TO_WIDE_STRING(DIRECT3D_VERSION),
+            STRING_TO_WIDE_STRING(DXCORE_VERSION),
+            windowsVersion),
         stdout);
 
     if constexpr (!wsl::shared::OfficialBuild)
     {
         // Print additional information if running a debug build.
-        wsl::windows::common::wslutil::PrintMessage(Localization::MessageBuildInfo(_MSC_VER, COMMIT_HASH, __TIME__ " " __DATE__), stdout);
+        wsl::windows::common::wslutil::PrintMessage(
+            Localization::MessageBuildInfo(
+                _MSC_VER, STRING_TO_WIDE_STRING(COMMIT_HASH), STRING_TO_WIDE_STRING(__TIME__) L" " STRING_TO_WIDE_STRING(__DATE__)),
+            stdout);
     }
 
     return 0;

src/windows/wslc/commands/VersionCommand.cpp

@@ -29,7 +29,7 @@ std::wstring VersionCommand::LongDescription() const
 
 void VersionCommand::PrintVersion()
 {
-    wsl::windows::common::wslutil::PrintMessage(std::format(L"{} {}", s_ExecutableName, WSL_PACKAGE_VERSION));
+    wsl::windows::common::wslutil::PrintMessage(std::format(L"{} {}", s_ExecutableName, STRING_TO_WIDE_STRING(WSL_PACKAGE_VERSION)));
 }
 
 void VersionCommand::ExecuteInternal(CLIExecutionContext& context) const

src/windows/wslc/services/ImageProgressCallback.cpp

@@ -53,7 +53,7 @@ HRESULT ImageProgressCallback::OnProgress(LPCSTR status, LPCSTR id, ULONGLONG cu
 
         if (id == nullptr || *id == '\0') // Print all 'global' statuses on their own line
         {
-            WriteTerminal(std::format(L"{}\n", status));
+            WriteTerminal(std::format(L"{}\n", wsl::shared::string::MultiByteToWide(status)));
             m_currentLine++;
             return S_OK;
         }
@@ -121,15 +121,19 @@ std::wstring ImageProgressCallback::GenerateStatusLine(LPCSTR status, LPCSTR id,
             progress += std::format(L"/{}", wsl::shared::string::FormatBytes(total));
         }
 
-        line = std::format(L"{}: {} [{}] {}", id, status, bar, progress);
+        line = std::format(L"{}: {} [{}] {}", wsl::shared::string::MultiByteToWide(id), wsl::shared::string::MultiByteToWide(status), bar, progress);
     }
     else if (current != 0)
     {
-        line = std::format(L"{}: {} {}", id, status, wsl::shared::string::FormatBytes(current));
+        line = std::format(
+            L"{}: {} {}",
+            wsl::shared::string::MultiByteToWide(id),
+            wsl::shared::string::MultiByteToWide(status),
+            wsl::shared::string::FormatBytes(current));
     }
     else
     {
-        line = std::format(L"{}: {}", id, status);
+        line = std::format(L"{}: {}", wsl::shared::string::MultiByteToWide(id), wsl::shared::string::MultiByteToWide(status));
     }
 
     // Use the visible window width (not the buffer width) to prevent wrapping.

src/windows/wslc/tasks/ImageTasks.cpp

@@ -162,7 +162,7 @@ void ListImages(CLIExecutionContext& context)
         bool trunc = !context.Args.Contains(ArgType::NoTrunc);
         for (const auto& image : images)
         {
-            context.Reporter.Output(L"{}\n", trunc ? TruncateId(image.Id, true) : image.Id);
+            context.Reporter.Output(L"{}\n", MultiByteToWide(trunc ? TruncateId(image.Id, true) : image.Id));
         }
 
         return;

test/windows/Common.cpp

@@ -2748,7 +2748,8 @@ void VerifyPatternMatch(const std::string& Content, const std::string& Pattern)
 {
     if (!PathMatchSpecA(Content.c_str(), Pattern.c_str()))
     {
-        std::wstring message = std::format(L"Output: '{}' didn't match pattern: '{}'", Content, Pattern);
+        std::wstring message = std::format(
+            L"Output: '{}' didn't match pattern: '{}'", wsl::shared::string::MultiByteToWide(Content), wsl::shared::string::MultiByteToWide(Pattern));
         VERIFY_FAIL(message.c_str());
     }
 }

test/windows/Common.h

@@ -679,15 +679,16 @@ void VerifyAreEqualUnordered(const std::vector<T>& expected, const std::vector<T
     {
         if (actualCounts[value] != count)
         {
-            error += std::format(L"Value '{}' expected {} times but was found {} times.\n", value, count, actualCounts[value]);
+            error += std::format(
+                L"Value '{}' expected {} times but was found {} times.\n", wsl::shared::string::WideFormatArg(value), count, actualCounts[value]);
         }
     }
 
     for (const auto& [value, count] : actualCounts)
     {
         if (expectedCounts.find(value) == expectedCounts.end())
         {
-            error += std::format(L"Unexpected value found: '{}'", value);
+            error += std::format(L"Unexpected value found: '{}'", wsl::shared::string::WideFormatArg(value));
         }
     }
 
@@ -696,14 +697,14 @@ void VerifyAreEqualUnordered(const std::vector<T>& expected, const std::vector<T
         error += std::format(L"Expected ({} elements):\n", expected.size());
         for (const auto& e : expected)
         {
-            error += std::format(L"- {}\n", e);
+            error += std::format(L"- {}\n", wsl::shared::string::WideFormatArg(e));
         }
 
         error += std::format(L"Actual ({} elements):\n", actual.size());
 
         for (const auto& e : actual)
         {
-            error += std::format(L"- {}\n", e);
+            error += std::format(L"- {}\n", wsl::shared::string::WideFormatArg(e));
         }
 
         error += std::format(L"Called from: {}", source);

test/windows/DrvFsTests.cpp

@@ -445,7 +445,7 @@ class DrvFsTests
 
             // Mount the share.
             VERIFY_ARE_EQUAL(LxsstuLaunchWsl(std::format(L"mkdir -p '{}'", mountPoint)), 0);
-            VERIFY_ARE_EQUAL(LxsstuLaunchWsl(std::format(L"mount -t drvfs '{}' '{}'", sourceDir.string(), mountPoint)), 0);
+            VERIFY_ARE_EQUAL(LxsstuLaunchWsl(std::format(L"mount -t drvfs '{}' '{}'", sourceDir.wstring(), mountPoint)), 0);
 
             // Validate that it can be accessed.
             {

test/windows/SimpleTests.cpp

@@ -101,7 +101,9 @@ class SimpleTests
             CATCH_LOG()
         });
 
-        VERIFY_ARE_EQUAL(LxsstuLaunchWsl(std::format(L"{} {} {}", WSL_EXPORT_ARG, LXSS_DISTRO_NAME_TEST, tar.wstring()).c_str()), (DWORD)0);
+        VERIFY_ARE_EQUAL(
+            LxsstuLaunchWsl(std::format(L"{} {} {}", WSL_EXPORT_ARG, STRING_TO_WIDE_STRING(LXSS_DISTRO_NAME_TEST), tar.wstring()).c_str()),
+            (DWORD)0);
         LxsstuLaunchWsl(std::format(L"{} {}", WSL_UNREGISTER_ARG, tempDistro).c_str());
         ValidateOutput(
             std::format(L"{} {} {} {}", WSL_IMPORT_ARG, tempDistro, vhdDir.wstring(), tar.wstring()).c_str(),

test/windows/UnitTests.cpp

@@ -770,7 +770,7 @@ class UnitTests
             DistroFileChange groups(L"/etc/group", true);
             CommandLine = std::format(L"-- for i in $(seq 1 64); do groupadd group$i; usermod -a -G group$i {}; done", LXSST_TEST_USERNAME);
             VERIFY_ARE_EQUAL(LxsstuLaunchWsl(CommandLine), (DWORD)0);
-            VERIFY_ARE_EQUAL(LxsstuLaunchWsl(std::format(L"{} {} {}", WSL_USER_ARG_LONG, LXSST_TEST_USERNAME, "echo success")), (DWORD)0);
+            VERIFY_ARE_EQUAL(LxsstuLaunchWsl(std::format(L"{} {} {}", WSL_USER_ARG_LONG, LXSST_TEST_USERNAME, L"echo success")), (DWORD)0);
         }
 
         //
@@ -962,14 +962,14 @@ class UnitTests
 
         {
             auto [out, err] = LxsstuLaunchWslAndCaptureOutput(L"wslinfo --version");
-            VERIFY_ARE_EQUAL(out, std::format(L"{}\n", WSL_PACKAGE_VERSION));
+            VERIFY_ARE_EQUAL(out, std::format(L"{}\n", STRING_TO_WIDE_STRING(WSL_PACKAGE_VERSION)));
             VERIFY_ARE_EQUAL(err, L"");
         }
 
         {
             // Ensure the old version query command still works.
             const auto [out, err] = LxsstuLaunchWslAndCaptureOutput(L"wslinfo --wsl-version");
-            VERIFY_ARE_EQUAL(out, std::format(L"{}\n", WSL_PACKAGE_VERSION));
+            VERIFY_ARE_EQUAL(out, std::format(L"{}\n", STRING_TO_WIDE_STRING(WSL_PACKAGE_VERSION)));
             VERIFY_ARE_EQUAL(err, L"");
         }
 
@@ -4636,7 +4636,7 @@ VERSION_ID="Invalid|Format"
 
                     VERIFY_ARE_EQUAL(
                         LxsstuLaunchWsl(
-                            std::format(L"--import {} \"{}\" {}", distroName, location, "distro-default-name-icon.tar")),
+                            std::format(L"--import {} \"{}\" {}", distroName, location, L"distro-default-name-icon.tar")),
                         0L);
 
                     auto [json, profile_path] = ValidateDistributionTerminalProfile(distroName, false);
@@ -6443,7 +6443,13 @@ Error code: Wsl/InstallDistro/WSL_E_INVALID_JSON\r\n",
             VERIFY_ARE_EQUAL(lines[1], L"# [network]");
             VERIFY_ARE_EQUAL(lines[2], L"# generateHosts = false");
             VERIFY_ARE_EQUAL(lines[3], L"127.0.0.1\tlocalhost");
-            VERIFY_ARE_EQUAL(lines[4], std::format(L"127.0.1.1\t{}.{}\t{}", hostname, domain, hostname));
+            VERIFY_ARE_EQUAL(
+                lines[4],
+                std::format(
+                    L"127.0.1.1\t{}.{}\t{}",
+                    wsl::shared::string::MultiByteToWide(hostname),
+                    wsl::shared::string::MultiByteToWide(domain),
+                    wsl::shared::string::MultiByteToWide(hostname)));
         }
     }
 
@@ -6809,7 +6815,8 @@ Error code: Wsl/InstallDistro/WSL_E_INVALID_JSON\r\n",
     WSL2_TEST_METHOD(CGroupv1)
     {
         auto expectedMount = [](const char* path, const wchar_t* expected) {
-            auto [out, _] = LxsstuLaunchWslAndCaptureOutput(std::format(L"findmnt -ln '{}' || true", path));
+            auto [out, _] =
+                LxsstuLaunchWslAndCaptureOutput(std::format(L"findmnt -ln '{}' || true", wsl::shared::string::MultiByteToWide(path)));
 
             VERIFY_ARE_EQUAL(out, expected);
         };