Skip to content

cleanup: refactor svccommio class #13843

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
benhillis merged 7 commits into from
Jan 14, 2026
+155 −214
Merged

cleanup: refactor svccommio class #13843

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
addd4ee
cleanup: refactor svccommio class
Dec 6, 2025
c77fded
add logging if setting code page fails
Dec 6, 2025
fb985df
pr feedback
Dec 17, 2025
2fd46f5
remove unneeded if
Dec 19, 2025
b367e20
remove unneeded if
Dec 19, 2025
a1f0386
Merge branch 'master' into user/benhill/svccommio_refactor
benhillis Jan 5, 2026
c8ff079
pr feedbacK'
Jan 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
304 changes: 112 additions & 192 deletions src/windows/common/svccommio.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,14 @@ Module Name:
#pragma hdrstop

namespace {
void ChangeConsoleMode(_In_ HANDLE File, _In_ DWORD ConsoleMode)

bool IsConsoleHandle(_In_ HANDLE Handle)
{
DWORD Mode;
return GetFileType(Handle) == FILE_TYPE_CHAR && GetConsoleMode(Handle, &Mode);
}

void ChangeConsoleMode(_In_ HANDLE Handle, _In_ DWORD Mode)
{
//
// Use the invalid parameter error code to detect the v1 console that does
Expand All @@ -29,8 +36,17 @@ void ChangeConsoleMode(_In_ HANDLE File, _In_ DWORD ConsoleMode)
// disconnected.
//

if (!SetConsoleMode(File, ConsoleMode))
if (!SetConsoleMode(Handle, Mode))
{
// DISABLE_NEWLINE_AUTO_RETURN is not supported everywhere, if the flag was present fall back and try again.
if (WI_IsFlagSet(Mode, DISABLE_NEWLINE_AUTO_RETURN))
{
if (SetConsoleMode(Handle, WI_ClearFlag(Mode, DISABLE_NEWLINE_AUTO_RETURN)))
{
return;
}
}

switch (GetLastError())
{
case ERROR_INVALID_PARAMETER:
Expand All @@ -45,238 +61,142 @@ void ChangeConsoleMode(_In_ HANDLE File, _In_ DWORD ConsoleMode)
}
}

void ConfigureStdHandles(_Inout_ LXSS_STD_HANDLES_INFO& StdHandlesInfo)
void TrySetConsoleMode(_In_ HANDLE Handle, _In_ DWORD Mode)
try
{
//
// Check stdin to see if it is a console or another device. If it is
// a console, configure it to raw processing mode and VT-100 support. If the
// force console I/O is requested, ignore stdin and get active console input
// handle instead.
//
ChangeConsoleMode(Handle, Mode);
}
CATCH_LOG()

UINT NewConsoleInputCP = 0;
DWORD NewConsoleInputMode = 0;
BOOLEAN IsConsoleInput = StdHandlesInfo.IsConsoleInput;
BOOLEAN IsConsoleOutput = StdHandlesInfo.IsConsoleOutput;
BOOLEAN IsConsoleError = StdHandlesInfo.IsConsoleError;
DWORD SavedInputMode = StdHandlesInfo.SavedInputMode;
DWORD SavedOutputMode = StdHandlesInfo.SavedOutputMode;
UINT SavedInputCP = StdHandlesInfo.SavedInputCP;
UINT SavedOutputCP = StdHandlesInfo.SavedOutputCP;
CONSOLE_SCREEN_BUFFER_INFO ScreenBufferInfo;
auto RestoreInputHandle = wil::scope_exit_log(WI_DIAGNOSTICS_INFO, [&] {
if (NewConsoleInputCP != 0)
{
SetConsoleCP(SavedInputCP);
}
} // namespace

if (NewConsoleInputMode != 0)
{
ChangeConsoleMode(StdHandlesInfo.InputHandle, SavedInputMode);
}
});
namespace wsl::windows::common {

IsConsoleInput = FALSE;
if ((GetFileType(StdHandlesInfo.InputHandle) == FILE_TYPE_CHAR) && (GetConsoleMode(StdHandlesInfo.InputHandle, &SavedInputMode)))
std::optional<ConsoleInput> ConsoleInput::Create(HANDLE Handle)
{
DWORD Mode;
if (GetFileType(Handle) == FILE_TYPE_CHAR && GetConsoleMode(Handle, &Mode))
{
IsConsoleInput = TRUE;
NewConsoleInputMode = SavedInputMode;
WI_SetAllFlags(NewConsoleInputMode, (ENABLE_WINDOW_INPUT | ENABLE_VIRTUAL_TERMINAL_INPUT));
WI_ClearAllFlags(NewConsoleInputMode, (ENABLE_ECHO_INPUT | ENABLE_INSERT_MODE | ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT));
ChangeConsoleMode(StdHandlesInfo.InputHandle, NewConsoleInputMode);

//
// Set the console input to the UTF-8 code page.
//

SavedInputCP = GetConsoleCP();
NewConsoleInputCP = CP_UTF8;
THROW_LAST_ERROR_IF(!::SetConsoleCP(NewConsoleInputCP));
return ConsoleInput(Handle, Mode);
}

bool RestoreMode = false;
bool RestoreCp = false;
auto RestoreOutput = wil::scope_exit([&] {
if (RestoreMode)
{
SetConsoleMode(StdHandlesInfo.ConsoleOutputHandle.get(), SavedOutputMode);
}

if (RestoreCp)
{
SetConsoleOutputCP(SavedOutputCP);
}
});

//
// If there is a console output handle, save the output mode and codepage so
// it can be restored.
//

if (StdHandlesInfo.ConsoleOutputHandle)
{
THROW_LAST_ERROR_IF(!::GetConsoleMode(StdHandlesInfo.ConsoleOutputHandle.get(), &SavedOutputMode));

//
// Temporarily try both with and without the custom flag to disable newline
// auto return.
//

DWORD NewConsoleOutputMode = SavedOutputMode | ENABLE_PROCESSED_OUTPUT | ENABLE_VIRTUAL_TERMINAL_PROCESSING | DISABLE_NEWLINE_AUTO_RETURN;
if (SetConsoleMode(StdHandlesInfo.ConsoleOutputHandle.get(), NewConsoleOutputMode) == FALSE)
{
WI_ClearFlag(NewConsoleOutputMode, DISABLE_NEWLINE_AUTO_RETURN);
ChangeConsoleMode(StdHandlesInfo.ConsoleOutputHandle.get(), NewConsoleOutputMode);
}

RestoreMode = true;

//
// Set the console output to the UTF-8 code page.
//

SavedOutputCP = GetConsoleOutputCP();
THROW_LAST_ERROR_IF(!::SetConsoleOutputCP(CP_UTF8));
return std::nullopt;
}

RestoreCp = true;
}
ConsoleInput::ConsoleInput(HANDLE Handle, DWORD SavedMode) : m_Handle(Handle), m_SavedMode(SavedMode)
{
// Save code page.
m_SavedCodePage = GetConsoleCP();

//
// If the force console I/O is requested, ignore stdout and treat the
// console as the output handle.
//
// Configure for raw input with VT support.
DWORD NewMode = m_SavedMode;
WI_SetAllFlags(NewMode, ENABLE_WINDOW_INPUT | ENABLE_VIRTUAL_TERMINAL_INPUT);
WI_ClearAllFlags(NewMode, ENABLE_ECHO_INPUT | ENABLE_INSERT_MODE | ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT);
ChangeConsoleMode(Handle, NewMode);

IsConsoleOutput = FALSE;
if ((GetFileType(StdHandlesInfo.OutputHandle) == FILE_TYPE_CHAR) &&
(GetConsoleScreenBufferInfo(StdHandlesInfo.OutputHandle, &ScreenBufferInfo)))
{
IsConsoleOutput = TRUE;
}
// Set UTF-8 code page.
LOG_IF_WIN32_BOOL_FALSE(SetConsoleCP(CP_UTF8));
}

IsConsoleError = FALSE;
if ((GetFileType(StdHandlesInfo.ErrorHandle) == FILE_TYPE_CHAR) &&
(GetConsoleScreenBufferInfo(StdHandlesInfo.ErrorHandle, &ScreenBufferInfo)))
ConsoleInput::~ConsoleInput()
{
if (m_Handle)
{
IsConsoleError = TRUE;
TrySetConsoleMode(m_Handle, m_SavedMode);
LOG_IF_WIN32_BOOL_FALSE(SetConsoleCP(m_SavedCodePage));
}

RestoreInputHandle.release();
RestoreOutput.release();
StdHandlesInfo.IsConsoleInput = IsConsoleInput;
StdHandlesInfo.IsConsoleOutput = IsConsoleOutput;
StdHandlesInfo.IsConsoleError = IsConsoleError;
StdHandlesInfo.SavedInputMode = SavedInputMode;
StdHandlesInfo.SavedOutputMode = SavedOutputMode;
StdHandlesInfo.SavedInputCP = SavedInputCP;
StdHandlesInfo.SavedOutputCP = SavedOutputCP;
}
} // namespace

wsl::windows::common::SvcCommIo::SvcCommIo()
std::optional<ConsoleOutput> ConsoleOutput::Create()
{
_stdHandlesInfo.InputHandle = GetStdHandle(STD_INPUT_HANDLE);
_stdHandlesInfo.OutputHandle = GetStdHandle(STD_OUTPUT_HANDLE);
_stdHandlesInfo.ErrorHandle = GetStdHandle(STD_ERROR_HANDLE);
_stdHandlesInfo.ConsoleOutputHandle.reset(
wil::unique_hfile ConsoleHandle(
CreateFileW(L"CONOUT$", GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING, 0, nullptr));

ConfigureStdHandles(_stdHandlesInfo);
_stdHandles.StdIn.HandleType = LxssHandleInput;
_stdHandles.StdIn.Handle = HandleToUlong(_stdHandlesInfo.InputHandle);
_stdHandles.StdOut.HandleType = LxssHandleOutput;
_stdHandles.StdOut.Handle = HandleToUlong(_stdHandlesInfo.OutputHandle);
_stdHandles.StdErr.HandleType = LxssHandleOutput;
_stdHandles.StdErr.Handle = HandleToUlong(_stdHandlesInfo.ErrorHandle);

//
// N.B.: The console handle is not supposed to be closed, it is just copied
// from PEB.
//

if (_stdHandlesInfo.IsConsoleInput)
{
_stdHandles.StdIn.Handle = LXSS_HANDLE_USE_CONSOLE;
_stdHandles.StdIn.HandleType = LxssHandleConsole;
}

if (_stdHandlesInfo.IsConsoleOutput)
if (ConsoleHandle)
{
_stdHandles.StdOut.Handle = LXSS_HANDLE_USE_CONSOLE;
_stdHandles.StdOut.HandleType = LxssHandleConsole;
DWORD Mode;
if (GetConsoleMode(ConsoleHandle.get(), &Mode))
{
return ConsoleOutput(std::move(ConsoleHandle), Mode);
}
}

if (_stdHandlesInfo.IsConsoleError)
{
_stdHandles.StdErr.Handle = LXSS_HANDLE_USE_CONSOLE;
_stdHandles.StdErr.HandleType = LxssHandleConsole;
}
return std::nullopt;
}

wsl::windows::common::SvcCommIo::~SvcCommIo()
ConsoleOutput::ConsoleOutput(wil::unique_hfile ConsoleHandle, DWORD SavedMode) :
Copy link
Collaborator

@OneBlue OneBlue Jan 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm surprised that this works since unique_hfile isn't copyable. (I'd expect something like wil::unique_hfile&& ConsoleHandle) here

m_ConsoleHandle(std::move(ConsoleHandle)), m_SavedMode(SavedMode)
{
try
{
RestoreConsoleMode();
}
CATCH_LOG()
}
// Save code page.
m_SavedCodePage = GetConsoleOutputCP();

PLXSS_STD_HANDLES
wsl::windows::common::SvcCommIo::GetStdHandles()
{
return &_stdHandles;
// Configure for VT output.
DWORD NewMode = m_SavedMode;
WI_SetAllFlags(NewMode, ENABLE_PROCESSED_OUTPUT | ENABLE_VIRTUAL_TERMINAL_PROCESSING | DISABLE_NEWLINE_AUTO_RETURN);
ChangeConsoleMode(m_ConsoleHandle.get(), NewMode);

// Set UTF-8 code page.
LOG_IF_WIN32_BOOL_FALSE(SetConsoleOutputCP(CP_UTF8));
}

COORD
wsl::windows::common::SvcCommIo::GetWindowSize() const
ConsoleOutput::~ConsoleOutput()
{
CONSOLE_SCREEN_BUFFER_INFOEX Info{};
Info.cbSize = sizeof(Info);
if (_stdHandlesInfo.IsConsoleOutput)
if (m_ConsoleHandle)
{
THROW_IF_WIN32_BOOL_FALSE(::GetConsoleScreenBufferInfoEx(_stdHandlesInfo.OutputHandle, &Info));
TrySetConsoleMode(m_ConsoleHandle.get(), m_SavedMode);
LOG_IF_WIN32_BOOL_FALSE(SetConsoleOutputCP(m_SavedCodePage));
}
else if (_stdHandlesInfo.IsConsoleError)
{
THROW_IF_WIN32_BOOL_FALSE(::GetConsoleScreenBufferInfoEx(_stdHandlesInfo.ErrorHandle, &Info));
}

return {
static_cast<short>(Info.srWindow.Right - Info.srWindow.Left + 1), static_cast<short>(Info.srWindow.Bottom - Info.srWindow.Top + 1)};
}

void wsl::windows::common::SvcCommIo::RestoreConsoleMode() const

/*++

Routine Description:
SvcCommIo::SvcCommIo()
{
const HANDLE InputHandle = GetStdHandle(STD_INPUT_HANDLE);
const HANDLE OutputHandle = GetStdHandle(STD_OUTPUT_HANDLE);
const HANDLE ErrorHandle = GetStdHandle(STD_ERROR_HANDLE);

Restores the saved input/output console mode.
// Configure input console
m_ConsoleInput = ConsoleInput::Create(InputHandle);

Arguments:
// Configure output console
m_ConsoleOutput = ConsoleOutput::Create();

None.
// Initialize the standard handles structure
const bool IsConsoleInput = m_ConsoleInput.has_value();
m_StdHandles.StdIn.HandleType = IsConsoleInput ? LxssHandleConsole : LxssHandleInput;
m_StdHandles.StdIn.Handle = IsConsoleInput ? LXSS_HANDLE_USE_CONSOLE : HandleToUlong(InputHandle);

Return Value:
const bool IsConsoleOutput = IsConsoleHandle(OutputHandle);
m_StdHandles.StdOut.HandleType = IsConsoleOutput ? LxssHandleConsole : LxssHandleOutput;
m_StdHandles.StdOut.Handle = IsConsoleOutput ? LXSS_HANDLE_USE_CONSOLE : HandleToUlong(OutputHandle);

None.
const bool IsConsoleError = IsConsoleHandle(ErrorHandle);
m_StdHandles.StdErr.HandleType = IsConsoleError ? LxssHandleConsole : LxssHandleOutput;
m_StdHandles.StdErr.Handle = IsConsoleError ? LXSS_HANDLE_USE_CONSOLE : HandleToUlong(ErrorHandle);

--*/
// Cache a console handle for GetWindowSize
m_WindowSizeHandle = IsConsoleOutput ? OutputHandle : (IsConsoleError ? ErrorHandle : nullptr);
}

PLXSS_STD_HANDLES
SvcCommIo::GetStdHandles()
{
//
// Restore the console input and output modes.
//
return &m_StdHandles;
}

if (_stdHandlesInfo.ConsoleOutputHandle)
COORD
SvcCommIo::GetWindowSize() const
{
if (m_WindowSizeHandle)
{
ChangeConsoleMode(_stdHandlesInfo.ConsoleOutputHandle.get(), _stdHandlesInfo.SavedOutputMode);
SetConsoleOutputCP(_stdHandlesInfo.SavedOutputCP);
CONSOLE_SCREEN_BUFFER_INFOEX Info{};
Info.cbSize = sizeof(Info);
THROW_IF_WIN32_BOOL_FALSE(GetConsoleScreenBufferInfoEx(m_WindowSizeHandle, &Info));
return {
static_cast<short>(Info.srWindow.Right - Info.srWindow.Left + 1),
static_cast<short>(Info.srWindow.Bottom - Info.srWindow.Top + 1)};
}

if (_stdHandlesInfo.IsConsoleInput != FALSE)
{
ChangeConsoleMode(_stdHandlesInfo.InputHandle, _stdHandlesInfo.SavedInputMode);
SetConsoleCP(_stdHandlesInfo.SavedInputCP);
}
return {80, 24}; // Default size if no console
}

} // namespace wsl::windows::common
Loading