commit 54a3eb82482a106f50906b5af35bf7f3efb6672c
parent 48dbb90d9cec8c73aeaed01ae0687586752c9319
Author: yao-msft <50888816+yao-msft@users.noreply.github.com>
Date: Mon, 14 Sep 2020 17:02:41 -0700
Fix alignment of table output with extended characters (#562)
Diffstat:
4 files changed, 121 insertions(+), 18 deletions(-)
diff --git a/src/AppInstallerCLICore/TableOutput.h b/src/AppInstallerCLICore/TableOutput.h
@@ -42,7 +42,7 @@ namespace AppInstaller::CLI::Execution
for (size_t i = 0; i < FieldCount; ++i)
{
m_columns[i].Name = std::move(header[i]);
- m_columns[i].MinLength = Utility::UTF8Length(m_columns[i].Name.get());
+ m_columns[i].MinLength = Utility::UTF8ColumnWidth(m_columns[i].Name.get());
m_columns[i].MaxLength = 0;
}
}
@@ -93,7 +93,7 @@ namespace AppInstaller::CLI::Execution
{
for (size_t i = 0; i < FieldCount; ++i)
{
- m_columns[i].MaxLength = std::max(m_columns[i].MaxLength, Utility::UTF8Length(line[i]));
+ m_columns[i].MaxLength = std::max(m_columns[i].MaxLength, Utility::UTF8ColumnWidth(line[i]));
}
}
@@ -189,12 +189,18 @@ namespace AppInstaller::CLI::Execution
if (col.MaxLength)
{
- size_t valueLength = Utility::UTF8Length(line[i]);
+ size_t valueLength = Utility::UTF8ColumnWidth(line[i]);
if (valueLength > col.MaxLength)
{
- out << Utility::UTF8Substring(line[i], 0, col.MaxLength - 1);
- out << "\xE2\x80\xA6"; // UTF8 encoding of ellipsis (…) character
+ size_t actualWidth;
+ out << Utility::UTF8TrimRightToColumnWidth(line[i], col.MaxLength - 1, actualWidth) << "\xE2\x80\xA6"; // UTF8 encoding of ellipsis (…) character
+
+ // Some characters take 2 unit space, the trimmed string length might be 1 less than the expected length.
+ if (actualWidth != col.MaxLength - 1)
+ {
+ out << ' ';
+ }
if (col.SpaceAfter)
{
diff --git a/src/AppInstallerCLITests/Strings.cpp b/src/AppInstallerCLITests/Strings.cpp
@@ -37,6 +37,37 @@ TEST_CASE("UTF8Substring", "[strings]")
REQUIRE(UTF8Substring(s, 1, 8) == "s like \xf0\x9f\x8c\x8a");
}
+TEST_CASE("UTF8ColumnWidth", "[strings]")
+{
+ REQUIRE(UTF8ColumnWidth("") == 0);
+ REQUIRE(UTF8ColumnWidth("a") == 1);
+ REQUIRE(UTF8ColumnWidth(" a b c ") == 7);
+ REQUIRE(UTF8ColumnWidth("K\xC3\xA4se") == 4); // "Käse"
+ REQUIRE(UTF8ColumnWidth("bye\xE2\x80\xA6") == 4); // "bye…"
+ REQUIRE(UTF8ColumnWidth("fi\xEF\xAC\x81") == 3); // "fi[fi]" [fi] is not decoupled
+ REQUIRE(UTF8ColumnWidth("\xf0\x9f\xa6\x86") == 2); // [duck emoji]
+ REQUIRE(UTF8ColumnWidth("\xf0\x9d\x85\xa0\xf0\x9d\x85\xa0") == 2); // [8th note][8th note]
+ REQUIRE(UTF8ColumnWidth("\xe6\xb5\x8b\xe8\xaf\x95") == 4); // 测试
+ REQUIRE(UTF8ColumnWidth("te\xe6\xb5\x8bs\xe8\xaf\x95t") == 8); // te测s试t
+}
+
+TEST_CASE("UTF8TrimRightToColumnWidth", "[strings]")
+{
+ size_t actualWidth;
+ REQUIRE((UTF8TrimRightToColumnWidth("", 0, actualWidth) == "" && actualWidth == 0));
+ REQUIRE((UTF8TrimRightToColumnWidth("abcd", 4, actualWidth) == "abcd" && actualWidth == 4));
+ REQUIRE((UTF8TrimRightToColumnWidth("abcd", 5, actualWidth) == "abcd" && actualWidth == 4));
+ REQUIRE((UTF8TrimRightToColumnWidth("abcd", 2, actualWidth) == "ab" && actualWidth == 2));
+
+ NormalizedString s{ "te\xe6\xb5\x8bs\xe8\xaf\x95t" }; // // te测s试t
+ REQUIRE((UTF8TrimRightToColumnWidth(s, 0, actualWidth) == "" && actualWidth == 0));
+ REQUIRE((UTF8TrimRightToColumnWidth(s, 2, actualWidth) == "te" && actualWidth == 2));
+ REQUIRE((UTF8TrimRightToColumnWidth(s, 3, actualWidth) == "te" && actualWidth == 2));
+ REQUIRE((UTF8TrimRightToColumnWidth(s, 4, actualWidth) == "te\xe6\xb5\x8b" && actualWidth == 4));
+ REQUIRE((UTF8TrimRightToColumnWidth(s, 8, actualWidth) == "te\xe6\xb5\x8bs\xe8\xaf\x95t" && actualWidth == 8));
+ REQUIRE((UTF8TrimRightToColumnWidth(s, 10, actualWidth) == "te\xe6\xb5\x8bs\xe8\xaf\x95t" && actualWidth == 8));
+}
+
TEST_CASE("Normalize", "[strings]")
{
REQUIRE(Normalize("test") == "test");
diff --git a/src/AppInstallerCommonCore/AppInstallerStrings.cpp b/src/AppInstallerCommonCore/AppInstallerStrings.cpp
@@ -79,6 +79,12 @@ namespace AppInstaller::Utility
return m_currentBrk;
}
+ // Returns code point of the chracter at m_currentBrk, or U_SENTINEL if m_currentBrk points to the end.
+ UChar32 CurrentCodePoint()
+ {
+ return utext_char32At(m_text.get(), m_currentBrk);
+ }
+
private:
wil::unique_any<UText*, decltype(utext_close), &utext_close> m_text;
wil::unique_any<UBreakIterator*, decltype(ubrk_close), &ubrk_close> m_brk;
@@ -151,6 +157,25 @@ namespace AppInstaller::Utility
return numGraphemeClusters;
}
+ size_t UTF8ColumnWidth(const NormalizedUTF8<NormalizationC>& input)
+ {
+ ICUBreakIterator itr{ input, UBRK_CHARACTER };
+
+ size_t columnWidth = 0;
+ UChar32 currentCP = 0;
+
+ currentCP = itr.CurrentCodePoint();
+ while (itr.Next() != UBRK_DONE && currentCP != U_SENTINEL)
+ {
+ int32_t width = u_getIntPropertyValue(currentCP, UCHAR_EAST_ASIAN_WIDTH);
+ columnWidth += width == U_EA_FULLWIDTH || width == U_EA_WIDE ? 2 : 1;
+
+ currentCP = itr.CurrentCodePoint();
+ }
+
+ return columnWidth;
+ }
+
std::string_view UTF8Substring(std::string_view input, size_t offset, size_t count)
{
ICUBreakIterator itr{ input, UBRK_CHARACTER };
@@ -177,6 +202,40 @@ namespace AppInstaller::Utility
return input.substr(utf8Offset, utf8Count);
}
+ std::string UTF8TrimRightToColumnWidth(const NormalizedUTF8<NormalizationC>& input, size_t expectedWidth, size_t& actualWidth)
+ {
+ ICUBreakIterator itr{ input, UBRK_CHARACTER };
+
+ size_t columnWidth = 0;
+ UChar32 currentCP = 0;
+ int32_t currentBrk = 0;
+ int32_t nextBrk = 0;
+
+ currentCP = itr.CurrentCodePoint();
+ currentBrk = itr.CurrentBreak();
+ nextBrk = itr.Next();
+ while (nextBrk != UBRK_DONE && currentCP != U_SENTINEL)
+ {
+ int32_t width = u_getIntPropertyValue(currentCP, UCHAR_EAST_ASIAN_WIDTH);
+ int charWidth = width == U_EA_FULLWIDTH || width == U_EA_WIDE ? 2 : 1;
+ columnWidth += charWidth;
+
+ if (columnWidth > expectedWidth)
+ {
+ columnWidth -= charWidth;
+ break;
+ }
+
+ currentCP = itr.CurrentCodePoint();
+ currentBrk = nextBrk;
+ nextBrk = itr.Next();
+ }
+
+ actualWidth = columnWidth;
+
+ return input.substr(0, currentBrk);
+ }
+
std::string Normalize(std::string_view input, NORM_FORM form)
{
if (input.empty())
diff --git a/src/AppInstallerCommonCore/Public/AppInstallerStrings.h b/src/AppInstallerCommonCore/Public/AppInstallerStrings.h
@@ -9,24 +9,12 @@
namespace AppInstaller::Utility
{
- // Compares the two UTF8 strings in a case insensitive manner.
- bool CaseInsensitiveEquals(std::string_view a, std::string_view b);
-
- // Determins if string a starts with string b.
- bool CaseInsensitiveStartsWith(std::string_view a, std::string_view b);
-
// Converts the given UTF16 string to UTF8
std::string ConvertToUTF8(std::wstring_view input);
// Converts the given UTF8 string to UTF16
std::wstring ConvertToUTF16(std::string_view input, UINT codePage = CP_UTF8);
- // Returns the number of grapheme clusters (characters) in an UTF8-encoded string.
- size_t UTF8Length(std::string_view input);
-
- // Returns a substring view in an UTF8-encoded string. Offset and count are measured in grapheme clusters (characters).
- std::string_view UTF8Substring(std::string_view input, size_t offset, size_t count);
-
// Normalizes a UTF8 string to the given form.
std::string Normalize(std::string_view input, NORM_FORM form = NORM_FORM::NormalizationKC);
@@ -40,7 +28,7 @@ namespace AppInstaller::Utility
NormalizedUTF8() = default;
template <size_t Size>
- NormalizedUTF8(const char (&s)[Size]) : std::string(Normalize(std::string_view{ s, (s[Size - 1] == '\0' ? Size - 1 : Size) }, Form)) {}
+ NormalizedUTF8(const char(&s)[Size]) : std::string(Normalize(std::string_view{ s, (s[Size - 1] == '\0' ? Size - 1 : Size) }, Form)) {}
NormalizedUTF8(std::string_view sv) : std::string(Normalize(sv, Form)) {}
@@ -83,6 +71,25 @@ namespace AppInstaller::Utility
using NormalizedString = NormalizedUTF8<>;
+ // Compares the two UTF8 strings in a case insensitive manner.
+ bool CaseInsensitiveEquals(std::string_view a, std::string_view b);
+
+ // Determins if string a starts with string b.
+ bool CaseInsensitiveStartsWith(std::string_view a, std::string_view b);
+
+ // Returns the number of grapheme clusters (characters) in an UTF8-encoded string.
+ size_t UTF8Length(std::string_view input);
+
+ // Returns the number of units the UTF8-encoded string will take in terminal output. Some characters take 2 units in terminal output.
+ size_t UTF8ColumnWidth(const NormalizedUTF8<NormalizationC>& input);
+
+ // Returns a substring view in an UTF8-encoded string. Offset and count are measured in grapheme clusters (characters).
+ std::string_view UTF8Substring(std::string_view input, size_t offset, size_t count);
+
+ // Returns a substring view in an UTF8-encoded string trimmed to be at most expected length. Length is measured as units taken in terminal output.
+ // Note the returned substring view might be less than specified length as some characters might take 2 units in terminal output.
+ std::string UTF8TrimRightToColumnWidth(const NormalizedUTF8<NormalizationC>&, size_t expectedWidth, size_t& actualWidth);
+
// Get the lower case version of the given std::string
std::string ToLower(std::string_view in);
