Feature: Improved Windows console text handling

impl/console.cxx

@@ -1,9 +1,6 @@
 // SPDX-License-Identifier: BSD-3-Clause
 #include <cerrno>
-#if defined(_MSC_VER) && !defined(_WINDOWS)
-#	define _WINDOWS 1
-#endif
-#ifndef _WINDOWS
+#ifndef _WIN32
 #	include <unistd.h>
 #else
 #	ifndef NOMINMAX
@@ -21,13 +18,17 @@
 
 using substrate::operator ""_s;
 using charTraits = std::char_traits<char>;
+#ifdef _WIN32
+using wcharTraits = std::char_traits<wchar_t>;
+#endif
+using char16Traits = std::char_traits<char16_t>;
 
 static const std::string errorPrefix{"[ERR]"_s};
 static const std::string warningPrefix{"[WRN]"_s};
 static const std::string infoPrefix{"[INF]"_s};
 static const std::string debugPrefix{"[DBG]"_s};
 
-#ifndef _WINDOWS
+#ifndef _WIN32
 static const std::string colourRed{"\033[1;31m"_s};
 static const std::string colourYellow{"\033[1;33m"_s};
 static const std::string colourCyan{"\033[36m"_s};
@@ -41,44 +42,90 @@
 
 namespace substrate
 {
+	// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
 	console_t console;
 
 	void consoleStream_t::checkTTY() noexcept { _tty = isatty(fd); }
 
 	void consoleStream_t::write(const void *const buffer, const size_t bufferLen) const noexcept
 	{
 		// We don't actually care if this succeeds. We just try if at all possible.
-#ifndef _WINDOWS
+#ifndef _WIN32
 		SUBSTRATE_NOWARN_UNUSED(const auto result) = ::write(fd, buffer, bufferLen);
 #else
 		SUBSTRATE_NOWARN_UNUSED(const auto result) = ::write(fd, buffer, uint32_t(bufferLen));
 #endif
 		errno = 0; // extra insurance.
 	}
 
+	void consoleStream_t::write(const char *const value) const noexcept
+		{ write(value, value ? charTraits::length(value) : 0U); }
+
 	// WARNING: This assumes you're giving it a TEXT stream so no non-printable stuff you want to preserve.
 	// It will (if necessary) automatically UTF-8 => 16 convert whatever passes through for the sake of windows
-	void consoleStream_t::write(const char *const value) const noexcept
+	void consoleStream_t::write(const char *const value, const size_t valueLen) const noexcept
 	{
 		if (value)
 		{
-#ifdef _WINDOWS
-			const auto valueLen{charTraits::length(value)};
+#ifdef _WIN32
 			// If there's nothing to convert (0-length string), fast-exit doing nothing.
 			if (!valueLen)
 				return;
-			const auto consoleMode{setmode(fd, _O_U8TEXT)};
+			const auto consoleMode{_setmode(fd, _O_U8TEXT)};
 			const auto stringLen{static_cast<size_t>(MultiByteToWideChar(CP_UTF8, MB_PRECOMPOSED | MB_USEGLYPHCHARS,
 				value, int(valueLen), nullptr, 0))};
+			// NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays)
 			auto string{make_unique_nothrow<wchar_t []>(stringLen)};
 			if (!string)
 				return;
 			MultiByteToWideChar(CP_UTF8, MB_PRECOMPOSED | MB_USEGLYPHCHARS, value, int(valueLen),
 				string.get(), int(stringLen));
-			write(string.get(), sizeof(wchar_t) * stringLen);
-			setmode(fd, consoleMode);
+			write(static_cast<const void *>(string.get()), sizeof(wchar_t) * stringLen);
+			_setmode(fd, consoleMode);
+#else
+			write(static_cast<const void *>(value), valueLen);
+#endif
+		}
+		else
+			write(nullString);
+	}
+
+#ifdef _WIN32
+	void consoleStream_t::write(const wchar_t *const value) const noexcept
+		{ write(value, value ? wcharTraits::length(value) : 0U); }
+
+	void consoleStream_t::write(const wchar_t *const value, const size_t valueLen) const noexcept
+	{
+		if (value)
+		{
+			// If there's nothing to convert (0-length string), fast-exit doing nothing.
+			if (!valueLen)
+				return;
+			const auto consoleMode{_setmode(fd, _O_WTEXT)};
+			write(static_cast<const void *>(value), sizeof(wchar_t) * valueLen);
+			_setmode(fd, consoleMode);
+		}
+		else
+			write(nullString);
+	}
+#endif
+
+	void consoleStream_t::write(const char16_t *const value) const noexcept
+		{ write(value, value ? char16Traits::length(value) : 0U); }
+
+	void consoleStream_t::write(const char16_t *const value, const size_t valueLen) const noexcept
+	{
+		if (value)
+		{
+			// If there's nothing to convert (0-length string), fast-exit doing nothing.
+			if (!valueLen)
+				return;
+#ifdef _WIN32
+			const auto consoleMode{_setmode(fd, _O_U16TEXT)};
+			write(static_cast<const void *>(value), sizeof(char16_t) * valueLen);
+			_setmode(fd, consoleMode);
 #else
-			write(value, charTraits::length(value));
+			convertingWrite(value, valueLen);
 #endif
 		}
 		else
@@ -92,7 +139,7 @@
 		outputStream{fileno(outStream)}, errorStream{fileno(errStream)},
 		valid_{outputStream.valid() && errorStream.valid()} { }
 
-#ifndef _WINDOWS
+#ifndef _WIN32
 	inline void red(const consoleStream_t &stream) noexcept
 		{ stream.write(colourRed); }
 	inline void yellow(const consoleStream_t &stream) noexcept
@@ -163,5 +210,62 @@
 			defaults(outputStream);
 		outputStream.write(' ');
 	}
+
+	// These functions implement streamed UTF-16 to UTF-8 conversion for display
+#ifndef _WIN32
+	// Performs safe indexing into the string array, returning the invalid value 0xffff if outside the bounds of the array
+	static inline uint16_t safeIndex(const char16_t *const string, const size_t length, const size_t index) noexcept
+	{
+		if (index >= length)
+			return UINT16_MAX;
+		uint16_t result{};
+		// NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic)
+		memcpy(&result, string + index, sizeof(char16_t));
+		return result;
+	}
+
+	// Convert and stream out the converted code points given by string to the fd
+	void consoleStream_t::convertingWrite(const char16_t *string, const size_t stringLen) const noexcept
+	{
+		for (size_t offset = 0; offset < stringLen; ++offset)
+		{
+			const auto unitA{safeIndex(string, stringLen, offset)};
+			// Check if this is a high-half surrogate pair
+			if ((unitA & 0xfe00U) == 0xd800U)
+			{
+				// Recover the upper 10 (11) bits from the first surrogate of the pair.
+				const auto upper{(unitA & 0x03ffU) + 0x0040U};
+				// Recover the lower 10 bits from the second surrogate of the pair.
+				const auto lower{safeIndex(string, stringLen, ++offset) & 0x03fffU};
+
+				// Encode the 4 code units and write them out
+				write(static_cast<char>(0xf0U | (uint8_t(upper >> 8U) & 0x07U)));
+				write(static_cast<char>(0x80U | (uint8_t(upper >> 2U) & 0x3fU)));
+				write(static_cast<char>(0x80U | (uint8_t(upper << 4U) & 0x30U) | (uint8_t(lower >> 6U) & 0x0fU)));
+				write(static_cast<char>(0x80U | (lower & 0x3fU)));
+			}
+			else
+			{
+				// Something from the Basic Multilingual Plane, figure out how to encode it.
+				// If it's able to be represented as a single byte, write it out as one
+				if (unitA <= 0x007fU)
+					write(static_cast<char>(unitA));
+				// If it's two byte, encode and write out as a code unit pair
+				else if (unitA <= 0x07ffU)
+				{
+					write(static_cast<char>(0xc0U | (uint8_t(unitA >> 6U) & 0x1fU)));
+					write(static_cast<char>(0x80U | uint8_t(unitA & 0x3fU)));
+				}
+				// Otherwise it's 3 byte, encode and write out as a code unit tripple
+				else
+				{
+					write(static_cast<char>(0xe0U | (uint8_t(unitA >> 12U) & 0x0fU)));
+					write(static_cast<char>(0x80U | (uint8_t(unitA >> 6U) & 0x3fU)));
+					write(static_cast<char>(0x80U | (unitA & 0x3fU)));
+				}
+			}
+		}
+	}
+#endif
 } // namespace substrate
 /* vim: set ft=cpp ts=4 sw=4 noexpandtab: */

impl/meson.build

@@ -19,63 +19,99 @@ endif
 
 deps += dependency('threads')
 
-if cxxVersion.version_compare('>=201703')
-	friendTypenameTest = '''
+friendTypenameTest = '''
+	template <typename U>
+	auto foo(U u)
+	{
+		u.v;
+	}
+
+	template <typename T>
+	class X
+	{
+		T v;
+
+	public:
 		template <typename U>
-		auto foo(U u)
-		{
-			u.v;
-		}
-
-		template <typename T>
-		class X
-		{
-			T v;
-
-		public:
-			template <typename U>
-			friend auto foo(U);
-		};
-
-		int main()
-		{
-			::foo(X<int>{});
-		}
-	'''
+		friend auto foo(U);
+	};
+
+	int main()
+	{
+		::foo(X<int>{});
+	}
+'''
+
+stdVariantGCCTest = '''
+	#include <tuple>
+	#include <variant>
+	int main() {
+		using variant_t = std::variant<short, int, long>;
+		constexpr auto variant_v = variant_t{std::in_place_index_t<0>{}, short{}};
+		constexpr auto tuple = std::make_tuple(variant_v);
+		constexpr std::tuple tuple_v{variant_v};
+	}
+'''
+
+stdFilesystemPathTest = '''
+	#include <filesystem>
+	#include <iostream>
+
+	using namespace std::literals::string_view_literals;
+
+	int main() {
+		std::filesystem::path p{"/bin/bash"sv};
+		std::cout << p << std::endl;
+		return 0;
+	}
+'''
+
+initializerListTest = '''
+	#include <array>
+	#include <cstdint>
+
+	using size_t = std::size_t;
+
+	namespace internal
+	{
+	template<std::size_t... seq> using indexSequence_t = std::index_sequence<seq...>;
+	template<std::size_t N> using makeIndexSequence = std::make_index_sequence<N>;
+
+	template<typename T, size_t N, size_t... index> constexpr std::array<T, N>
+		makeArray(T (&&elems)[N], indexSequence_t<index...>)
+	{
+		return {{elems[index]...}};
+	}
+	} // namespace internal
+
+	template<typename T, size_t N> constexpr std::array<T, N>
+		make_array(T (&&elems)[N]) // NOLINT(cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays)
+	{
+		return internal::makeArray(std::move(elems), internal::makeIndexSequence<N>{});
+	}
+
+	int main()
+	{
+		auto test = make_array<const char *>({
+			"program",
+			"choiceC",
+			nullptr,
+		});
+	}
+'''
+
+if cxxVersion.version_compare('>=201703')
 	friendTypenameTemplate = cxx.compiles(
 		friendTypenameTest,
 		name: 'accepts friend declarations for auto functions (LLVM #31852, #33222)'
 	)
 
-	stdVariantGCCTest = '''
-		#include <tuple>
-		#include <variant>
-		int main() {
-			using variant_t = std::variant<short, int, long>;
-			constexpr auto variant_v = variant_t{std::in_place_index_t<0>{}, short{}};
-			constexpr auto tuple = std::make_tuple(variant_v);
-			constexpr std::tuple tuple_v{variant_v};
-		}
-	'''
 
 	stdVariantGCC = cxx.compiles(
 		stdVariantGCCTest,
 		name: 'has a working std::variant implementation (GCC #80165)'
 	)
 
-	stdFilesystemPathTest = '''
-		#include <filesystem>
-		#include <iostream>
-
-		using namespace std::literals::string_view_literals;
-
-		int main() {
-			std::filesystem::path p{"/bin/bash"sv};
-			std::cout << p << std::endl;
-			return 0;
-		}
-	'''
-
 	# GCC < 9.1 splits the filesystem module into a separate library
 	libstdcppFS = cxx.find_library('stdc++fs', required: false)
 
@@ -89,40 +125,6 @@ if cxxVersion.version_compare('>=201703')
 		libSubstrateArgs += ['-DHAVE_FILESYSTEM_PATH']
 	endif
 
-	initializerListTest = '''
-		#include <array>
-		#include <cstdint>
-
-		using size_t = std::size_t;
-
-		namespace internal
-		{
-		template<std::size_t... seq> using indexSequence_t = std::index_sequence<seq...>;
-		template<std::size_t N> using makeIndexSequence = std::make_index_sequence<N>;
-
-		template<typename T, size_t N, size_t... index> constexpr std::array<T, N>
-			makeArray(T (&&elems)[N], indexSequence_t<index...>)
-		{
-			return {{elems[index]...}};
-		}
-		} // namespace internal
-
-		template<typename T, size_t N> constexpr std::array<T, N>
-			make_array(T (&&elems)[N]) // NOLINT(cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays)
-		{
-			return internal::makeArray(std::move(elems), internal::makeIndexSequence<N>{});
-		}
-
-		int main()
-		{
-			auto test = make_array<const char *>({
-				"program",
-				"choiceC",
-				nullptr,
-			});
-		}
-	'''
-
 	initializerList = cxx.compiles(
 		initializerListTest,
 		name: 'accepts nullptr as part of CTAD with pointer type values (GCC #85977)'
@@ -267,7 +269,7 @@ if meson.is_cross_build()
 		gnu_symbol_visibility: 'inlineshidden',
 		implicit_include_directories: false,
 		pic: true,
-		install: (not meson.is_subproject()),
+		install: false,
 		native: true,
 	)
 endif