commit 287325ed82a0443ecf1c39e7d5f233f34a211ae6 parent ca9101a8f9f0638de66fd76dc99cc33278a4c8e0 Author: JohnMcPMS <johnmcp@microsoft.com> Date: Mon, 16 Mar 2020 13:16:43 -0700 Bring in ICU bridge functions and use normalized strings in index (#56) Diffstat:
32 files changed, 912 insertions(+), 97 deletions(-)
diff --git a/src/AppInstallerCLI/AppInstallerCLI.vcxproj b/src/AppInstallerCLI/AppInstallerCLI.vcxproj @@ -133,7 +133,7 @@ <Link> <SubSystem>Console</SubSystem> <GenerateWindowsMetadata>false</GenerateWindowsMetadata> - <AdditionalDependencies Condition="'$(Configuration)'=='Debug'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalDependencies Condition="'$(Configuration)'=='Debug'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;icuuc.lib;icuin.lib;%(AdditionalDependencies)</AdditionalDependencies> </Link> </ItemDefinitionGroup> <ItemDefinitionGroup Condition="'$(Platform)'=='Win32'"> @@ -159,7 +159,7 @@ <EnableCOMDATFolding>true</EnableCOMDATFolding> <OptimizeReferences>true</OptimizeReferences> <GenerateWindowsMetadata>false</GenerateWindowsMetadata> - <AdditionalDependencies Condition="'$(Configuration)'=='Release'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalDependencies Condition="'$(Configuration)'=='Release'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;icuuc.lib;icuin.lib;%(AdditionalDependencies)</AdditionalDependencies> </Link> </ItemDefinitionGroup> <ItemGroup> diff --git a/src/AppInstallerCLICore/Core.cpp b/src/AppInstallerCLICore/Core.cpp @@ -11,11 +11,40 @@ using namespace AppInstaller::CLI; namespace AppInstaller::CLI { + namespace + { + // RAII class to restore the console output codepage. + struct ConsoleOuputCPRestore + { + ConsoleOuputCPRestore(UINT cpToChangeTo) + { + m_previousCP = GetConsoleOutputCP(); + LOG_LAST_ERROR_IF(!SetConsoleOutputCP(cpToChangeTo)); + } + + ~ConsoleOuputCPRestore() + { + SetConsoleOutputCP(m_previousCP); + } + + ConsoleOuputCPRestore(const ConsoleOuputCPRestore&) = delete; + ConsoleOuputCPRestore& operator=(const ConsoleOuputCPRestore&) = delete; + + ConsoleOuputCPRestore(ConsoleOuputCPRestore&&) = delete; + ConsoleOuputCPRestore& operator=(ConsoleOuputCPRestore&&) = delete; + + private: + UINT m_previousCP = 0; + }; + } int CoreMain(int argc, wchar_t const** argv) try { init_apartment(); + // Set output to UTF8 + ConsoleOuputCPRestore utf8CP(CP_UTF8); + // Enable logging (*all* for now, TODO: add common arguments to allow control of logging) Logging::Log().EnableChannel(Logging::Channel::All); Logging::Log().SetLevel(Logging::Level::Verbose); diff --git a/src/AppInstallerCLICore/Workflows/ShellExecuteInstallerHandler.cpp b/src/AppInstallerCLICore/Workflows/ShellExecuteInstallerHandler.cpp @@ -96,7 +96,7 @@ namespace AppInstaller::Workflow std::string ShellExecuteInstallerHandler::GetInstallerArgsTemplate() { std::string installerArgs = ""; - const std::map<ManifestInstaller::InstallerSwitchType, std::string>& installerSwitches = m_manifestInstallerRef.Switches; + const std::map<ManifestInstaller::InstallerSwitchType, Utility::NormalizedString>& installerSwitches = m_manifestInstallerRef.Switches; // Construct install experience arg. if (m_argsRef.Contains(ExecutionArgs::Type::Silent) && installerSwitches.find(ManifestInstaller::InstallerSwitchType::Silent) != installerSwitches.end()) diff --git a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj @@ -114,7 +114,7 @@ <Link> <SubSystem>Console</SubSystem> <GenerateWindowsMetadata>false</GenerateWindowsMetadata> - <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;icuuc.lib;icuin.lib;%(AdditionalDependencies)</AdditionalDependencies> </Link> </ItemDefinitionGroup> <ItemDefinitionGroup Condition="'$(Platform)'=='Win32'"> @@ -124,7 +124,7 @@ <TreatWarningAsError Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</TreatWarningAsError> </ClCompile> <Link> - <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;icuuc.lib;icuin.lib;%(AdditionalDependencies)</AdditionalDependencies> </Link> </ItemDefinitionGroup> <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'"> @@ -141,8 +141,8 @@ <EnableCOMDATFolding>true</EnableCOMDATFolding> <OptimizeReferences>true</OptimizeReferences> <GenerateWindowsMetadata>false</GenerateWindowsMetadata> - <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;%(AdditionalDependencies)</AdditionalDependencies> - <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Release|x64'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;icuuc.lib;icuin.lib;%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Release|x64'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;icuuc.lib;icuin.lib;%(AdditionalDependencies)</AdditionalDependencies> </Link> </ItemDefinitionGroup> <ItemGroup> @@ -156,6 +156,7 @@ <ClCompile Include="MsixInfo.cpp" /> <ClCompile Include="PreIndexedPackageSource.cpp" /> <ClCompile Include="SQLiteIndexSource.cpp" /> + <ClCompile Include="Strings.cpp" /> <ClCompile Include="Versions.cpp" /> <ClCompile Include="WorkFlow.cpp" /> <ClCompile Include="LanguageUtilities.cpp" /> diff --git a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters @@ -80,6 +80,9 @@ <ClCompile Include="Versions.cpp"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="Strings.cpp"> + <Filter>Source Files</Filter> + </ClCompile> </ItemGroup> <ItemGroup> <None Include="PropertySheet.props" /> diff --git a/src/AppInstallerCLITests/SQLiteIndex.cpp b/src/AppInstallerCLITests/SQLiteIndex.cpp @@ -51,8 +51,8 @@ struct IndexFields std::string Moniker; std::string Version; std::string Channel; - std::vector<std::string> Tags; - std::vector<std::string> Commands; + std::vector<NormalizedString> Tags; + std::vector<NormalizedString> Commands; std::string Path; }; @@ -916,3 +916,26 @@ TEST_CASE("SQLiteIndex_Search_QueryAndMultipleFilters", "[sqliteindex]") REQUIRE(result.has_value()); REQUIRE(result.value() == "Id3"); } + +TEST_CASE("SQLiteIndex_Search_SimpleICULike", "[sqliteindex]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + // Insert decomposed character: [upper] A + umlaut + SQLiteIndex index = SearchTestSetup(tempFile, { + { u8"\x41\x308wesomeApp", "HasUmlaut", "Moniker", "Version", "Channel", { "foot" }, { "com34" }, "Path1" }, + { u8"AwesomeApp", "Nope", "Moniker", "Version", "Channel", { "foot" }, { "com34" }, "Path2" }, + }); + + SearchRequest request; + // Search for anything containing: [lower] a + umlaut + request.Filters.emplace_back(ApplicationMatchField::Id, MatchType::Substring, u8"\xE4"); + + auto results = index.Search(request); + REQUIRE(results.size() == 1); + + auto result = index.GetNameStringById(results[0].first); + REQUIRE(result.has_value()); + REQUIRE(result.value() == "HasUmlaut"); +} diff --git a/src/AppInstallerCLITests/Strings.cpp b/src/AppInstallerCLITests/Strings.cpp @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "TestCommon.h" +#include <AppInstallerStrings.h> + +using namespace AppInstaller::Utility; + + +TEST_CASE("Normalize", "[strings]") +{ + REQUIRE(Normalize("test") == "test"); + + // A + combining Dieresis => single A with umlaut char + REQUIRE(Normalize(L"\x41\x308") == L"\xC4"); + // This will stop working in C++20, sigh. + REQUIRE(Normalize(u8"\x41\x308") == u8"\xC4"); + + // Ligature fi => f + i + REQUIRE(Normalize(u8"\xFB01") == u8"fi"); +} + +TEST_CASE("NormalizedString", "[strings]") +{ + REQUIRE(NormalizedString("test") == "test"); + std::string input = "test"; + REQUIRE(NormalizedString(input) == input); + + // A + combining Dieresis => single A with umlaut char + REQUIRE(NormalizedString(std::wstring_view(L"\x41\x308")) == u8"\xC4"); + // This will stop working in C++20, sigh. + input = u8"\x41\x308"; + REQUIRE(NormalizedString(input) == u8"\xC4"); + + // Ligature fi => f + i + std::string_view input2 = u8"\xFB01"; + REQUIRE(NormalizedString(input2) == u8"fi"); +} diff --git a/src/AppInstallerCLITests/WorkFlow.cpp b/src/AppInstallerCLITests/WorkFlow.cpp @@ -70,7 +70,7 @@ struct TestSource : public ISource { TestApplication(const Manifest manifest) : m_manifest(manifest) {} - Manifest GetManifest(std::string_view, std::string_view) override + Manifest GetManifest(const NormalizedString&, const NormalizedString&) override { return m_manifest; } diff --git a/src/AppInstallerCLITests/YamlManifest.cpp b/src/AppInstallerCLITests/YamlManifest.cpp @@ -9,7 +9,7 @@ using namespace TestCommon; using namespace AppInstaller::Manifest; using namespace AppInstaller::Utility; -using MultiValue = std::vector<std::string>; +using MultiValue = std::vector<NormalizedString>; bool operator==(const MultiValue& a, const MultiValue& b) { if (a.size() != b.size()) diff --git a/src/AppInstallerCommonCore/AppInstallerStrings.cpp b/src/AppInstallerCommonCore/AppInstallerStrings.cpp @@ -13,6 +13,11 @@ namespace AppInstaller::Utility std::string ConvertToUTF8(std::wstring_view input) { + if (input.empty()) + { + return {}; + } + int utf8ByteCount = WideCharToMultiByte(CP_UTF8, 0, input.data(), wil::safe_cast<int>(input.length()), nullptr, 0, nullptr, nullptr); THROW_LAST_ERROR_IF(utf8ByteCount == 0); @@ -28,6 +33,11 @@ namespace AppInstaller::Utility std::wstring ConvertToUTF16(std::string_view input) { + if (input.empty()) + { + return {}; + } + int utf16CharCount = MultiByteToWideChar(CP_UTF8, 0, input.data(), wil::safe_cast<int>(input.length()), nullptr, 0); THROW_LAST_ERROR_IF(utf16CharCount == 0); @@ -41,6 +51,49 @@ namespace AppInstaller::Utility return result; } + std::string Normalize(std::string_view input, NORM_FORM form) + { + if (input.empty()) + { + return {}; + } + + return ConvertToUTF8(Normalize(ConvertToUTF16(input), form)); + } + + std::wstring Normalize(std::wstring_view input, NORM_FORM form) + { + if (input.empty()) + { + return {}; + } + + std::wstring result; + + int cchEstimate = NormalizeString(form, input.data(), static_cast<int>(input.length()), NULL, 0); + for (;;) + { + result.resize(cchEstimate); + cchEstimate = NormalizeString(form, input.data(), static_cast<int>(input.length()), &result[0], cchEstimate); + + if (cchEstimate > 0) + { + result.resize(cchEstimate); + return result; + } + else + { + DWORD dwError = GetLastError(); + THROW_LAST_ERROR_IF(dwError != ERROR_INSUFFICIENT_BUFFER); + + // New guess is negative of the return value. + cchEstimate = -cchEstimate; + + THROW_HR_IF_MSG(E_UNEXPECTED, static_cast<size_t>(cchEstimate) <= result.size(), "New estimate should never be less than previous value"); + } + } + } + std::string ToLower(std::string_view in) { std::string result(in); @@ -88,7 +141,9 @@ namespace AppInstaller::Utility auto offset = stream.tellg() - currentPos; stream.seekg(currentPos); - std::string result(offset, '\0'); + // Don't allow use of this API for reading very large streams. + THROW_HR_IF(E_OUTOFMEMORY, offset > static_cast<std::streamoff>(std::numeric_limits<uint32_t>::max())); + std::string result(static_cast<size_t>(offset), '\0'); stream.read(&result[0], offset); return result; diff --git a/src/AppInstallerCommonCore/Public/AppInstallerStrings.h b/src/AppInstallerCommonCore/Public/AppInstallerStrings.h @@ -2,6 +2,7 @@ // Licensed under the MIT License. #pragma once +#include <ostream> #include <string> #include <string_view> @@ -16,6 +17,62 @@ namespace AppInstaller::Utility // Converts the given UTF8 string to UTF16 std::wstring ConvertToUTF16(std::string_view input); + // Normalizes a UTF8 string to the given form. + std::string Normalize(std::string_view input, NORM_FORM form = NORM_FORM::NormalizationKC); + + // Normalizes a UTF16 string to the given form. + std::wstring Normalize(std::wstring_view input, NORM_FORM form = NORM_FORM::NormalizationKC); + + // Type to hold and force a normalized UTF8 string. + template <NORM_FORM Form = NORM_FORM::NormalizationKC> + struct NormalizedUTF8 : public std::string + { + NormalizedUTF8() = default; + + template <size_t Size> + NormalizedUTF8(const char (&s)[Size]) : std::string(Normalize(std::string_view{ s, (s[Size - 1] == '\0' ? Size - 1 : Size) }, Form)) {} + + NormalizedUTF8(std::string_view sv) : std::string(Normalize(sv, Form)) {} + + NormalizedUTF8(const std::string& s) : std::string(Normalize(s, Form)) {} + NormalizedUTF8(std::string&& s) : std::string(Normalize(s, Form)) {} + + NormalizedUTF8(std::wstring_view sv) : std::string(ConvertToUTF8(Normalize(sv, Form))) {} + + NormalizedUTF8(const NormalizedUTF8& other) = default; + NormalizedUTF8& operator=(const NormalizedUTF8& other) = default; + + NormalizedUTF8(NormalizedUTF8&& other) = default; + NormalizedUTF8& operator=(NormalizedUTF8&& other) = default; + + template <size_t Size> + NormalizedUTF8& operator=(const char(&s)[Size]) + { + assign(Normalize(std::string_view{ s, (s[Size - 1] == '\0' ? Size - 1 : Size) }, Form)); + return *this; + } + + NormalizedUTF8& operator=(std::string_view sv) + { + assign(Normalize(sv, Form)); + return *this; + } + + NormalizedUTF8& operator=(const std::string& s) + { + assign(Normalize(s, Form)); + return *this; + } + + NormalizedUTF8& operator=(std::string&& s) + { + assign(Normalize(s, Form)); + return *this; + } + }; + + using NormalizedString = NormalizedUTF8<>; + // Get the lower case version of the given std::string std::string ToLower(std::string_view in); diff --git a/src/AppInstallerCommonCore/SHA256.cpp b/src/AppInstallerCommonCore/SHA256.cpp @@ -149,9 +149,9 @@ namespace AppInstaller::Utility { while (!in.eof()) { in.read((char*)(buffer.get()), bufferSize); - hasher.Add(buffer.get(), in.gcount()); + hasher.Add(buffer.get(), static_cast<size_t>(in.gcount())); } - + std::vector<uint8_t> result; hasher.Get(result); diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj @@ -168,6 +168,7 @@ </Link> </ItemDefinitionGroup> <ItemGroup> + <ClInclude Include="ICU\SQLiteICU.h" /> <ClInclude Include="Manifest\Manifest.h" /> <ClInclude Include="Manifest\ManifestInstaller.h" /> <ClInclude Include="Manifest\ManifestLocalization.h" /> @@ -199,6 +200,16 @@ <ClInclude Include="SQLiteWrapper.h" /> </ItemGroup> <ItemGroup> + <ClCompile Include="ICU\SQLiteICU.c"> + <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">NotUsing</PrecompiledHeader> + <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">NotUsing</PrecompiledHeader> + <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">NotUsing</PrecompiledHeader> + <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">NotUsing</PrecompiledHeader> + <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">NotUsing</PrecompiledHeader> + <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">NotUsing</PrecompiledHeader> + <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">NotUsing</PrecompiledHeader> + <PrecompiledHeader Condition="'$(Configuration)|$(Platform)'=='Release|x64'">NotUsing</PrecompiledHeader> + </ClCompile> <ClCompile Include="Manifest\Manifest.cpp" /> <ClCompile Include="Manifest\ManifestInstaller.cpp" /> <ClCompile Include="Manifest\ManifestLocalization.cpp" /> diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters @@ -28,6 +28,9 @@ <Filter Include="Public"> <UniqueIdentifier>{69ce2e35-fe7f-41af-bd47-91a70131d167}</UniqueIdentifier> </Filter> + <Filter Include="ICU"> + <UniqueIdentifier>{dac1a359-45ac-4456-8a83-f7df10058197}</UniqueIdentifier> + </Filter> </ItemGroup> <ItemGroup> <ClInclude Include="pch.h"> @@ -117,6 +120,9 @@ <ClInclude Include="Microsoft\Schema\1_0\SearchResultsTable.h"> <Filter>Microsoft\Schema\1_0</Filter> </ClInclude> + <ClInclude Include="ICU\SQLiteICU.h"> + <Filter>ICU</Filter> + </ClInclude> </ItemGroup> <ItemGroup> <ClCompile Include="pch.cpp"> @@ -176,6 +182,9 @@ <ClCompile Include="Microsoft\Schema\1_0\SearchResultsTable.cpp"> <Filter>Microsoft\Schema\1_0</Filter> </ClCompile> + <ClCompile Include="ICU\SQLiteICU.c"> + <Filter>ICU</Filter> + </ClCompile> </ItemGroup> <ItemGroup> <None Include="PropertySheet.props" /> diff --git a/src/AppInstallerRepositoryCore/ICU/SQLiteICU.c b/src/AppInstallerRepositoryCore/ICU/SQLiteICU.c @@ -0,0 +1,557 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include <winsqlite/winsqlite3.h> + +// Define values to force static linkage version of extension. +// The code below the ICU header include should be changed as little as possible. +#define SQLITE_CORE +#define SQLITE_ENABLE_ICU +#define SQLITE_PRIVATE +#define SQLITE_DIRECTONLY 0x000080000 +#define SQLITE_INNOCUOUS 0x000200000 + +#define STDCALL_FOR_INTEROP __stdcall + +// Adapted from the file icu.c (v 1.7 2007/12/13 21:54:11) to use the built-in Windows SQLite and ICU binaries. + +// This file implements an integration between the ICU library +// ("International Components for Unicode", an open-source library +// for handling unicode data) and SQLite. The integration uses +// ICU to provide the following to SQLite: +// +// * An implementation of the SQL regexp() function (and hence REGEXP +// operator) using the ICU uregex_XX() APIs. +// +// * Implementations of the SQL scalar upper() and lower() functions +// for case mapping. +// +// * Integration of ICU and SQLite collation sequences. +// +// * An implementation of the LIKE operator that uses ICU to +// provide case-independent matching. + +// Include ICU headers +#include <icu.h> + +/* #include <assert.h> */ +#include <assert.h> + +#ifndef SQLITE_CORE +/* #include "sqlite3ext.h" */ +SQLITE_EXTENSION_INIT1 +#else +/* #include "sqlite3.h" */ +#endif + +/* +** This function is called when an ICU function called from within +** the implementation of an SQL scalar function returns an error. +** +** The scalar function context passed as the first argument is +** loaded with an error message based on the following two args. +*/ +static void icuFunctionError( + sqlite3_context* pCtx, /* SQLite scalar function context */ + const char* zName, /* Name of ICU function that failed */ + UErrorCode e /* Error code returned by ICU function */ +) { + char zBuf[128]; + sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e)); + zBuf[127] = '\0'; + sqlite3_result_error(pCtx, zBuf, -1); +} + +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) + +/* +** Maximum length (in bytes) of the pattern in a LIKE or GLOB +** operator. +*/ +#ifndef SQLITE_MAX_LIKE_PATTERN_LENGTH +# define SQLITE_MAX_LIKE_PATTERN_LENGTH 50000 +#endif + +/* +** Version of sqlite3_free() that is always a function, never a macro. +*/ +static void STDCALL_FOR_INTEROP xFree(void* p) { + sqlite3_free(p); +} + +/* +** This lookup table is used to help decode the first byte of +** a multi-byte UTF8 character. It is copied here from SQLite source +** code file utf8.c. +*/ +static const unsigned char icuUtf8Trans1[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, +}; + +#define SQLITE_ICU_READ_UTF8(zIn, c) \ + c = *(zIn++); \ + if( c>=0xc0 ){ \ + c = icuUtf8Trans1[c-0xc0]; \ + while( (*zIn & 0xc0)==0x80 ){ \ + c = (c<<6) + (0x3f & *(zIn++)); \ + } \ + } + +#define SQLITE_ICU_SKIP_UTF8(zIn) \ + assert( *zIn ); \ + if( *(zIn++)>=0xc0 ){ \ + while( (*zIn & 0xc0)==0x80 ){zIn++;} \ + } + + +/* +** Compare two UTF-8 strings for equality where the first string is +** a "LIKE" expression. Return true (1) if they are the same and +** false (0) if they are different. +*/ +static int icuLikeCompare( + const uint8_t* zPattern, /* LIKE pattern */ + const uint8_t* zString, /* The UTF-8 string to compare against */ + const UChar32 uEsc /* The escape character */ +) { + static const uint32_t MATCH_ONE = (uint32_t)'_'; + static const uint32_t MATCH_ALL = (uint32_t)'%'; + + int prevEscape = 0; /* True if the previous character was uEsc */ + + while (1) { + + /* Read (and consume) the next character from the input pattern. */ + uint32_t uPattern; + SQLITE_ICU_READ_UTF8(zPattern, uPattern); + if (uPattern == 0) break; + + /* There are now 4 possibilities: + ** + ** 1. uPattern is an unescaped match-all character "%", + ** 2. uPattern is an unescaped match-one character "_", + ** 3. uPattern is an unescaped escape character, or + ** 4. uPattern is to be handled as an ordinary character + */ + if (!prevEscape && uPattern == MATCH_ALL) { + /* Case 1. */ + uint8_t c; + + /* Skip any MATCH_ALL or MATCH_ONE characters that follow a + ** MATCH_ALL. For each MATCH_ONE, skip one character in the + ** test string. + */ + while ((c = *zPattern) == MATCH_ALL || c == MATCH_ONE) { + if (c == MATCH_ONE) { + if (*zString == 0) return 0; + SQLITE_ICU_SKIP_UTF8(zString); + } + zPattern++; + } + + if (*zPattern == 0) return 1; + + while (*zString) { + if (icuLikeCompare(zPattern, zString, uEsc)) { + return 1; + } + SQLITE_ICU_SKIP_UTF8(zString); + } + return 0; + + } + else if (!prevEscape && uPattern == MATCH_ONE) { + /* Case 2. */ + if (*zString == 0) return 0; + SQLITE_ICU_SKIP_UTF8(zString); + + } + else if (!prevEscape && uPattern == (uint32_t)uEsc) { + /* Case 3. */ + prevEscape = 1; + + } + else { + /* Case 4. */ + uint32_t uString; + SQLITE_ICU_READ_UTF8(zString, uString); + uString = (uint32_t)u_foldCase((UChar32)uString, U_FOLD_CASE_DEFAULT); + uPattern = (uint32_t)u_foldCase((UChar32)uPattern, U_FOLD_CASE_DEFAULT); + if (uString != uPattern) { + return 0; + } + prevEscape = 0; + } + } + + return *zString == 0; +} + +/* +** Implementation of the like() SQL function. This function implements +** the build-in LIKE operator. The first argument to the function is the +** pattern and the second argument is the string. So, the SQL statements: +** +** A LIKE B +** +** is implemented as like(B, A). If there is an escape character E, +** +** A LIKE B ESCAPE E +** +** is mapped to like(B, A, E). +*/ +static void STDCALL_FOR_INTEROP icuLikeFunc( + sqlite3_context* context, + int argc, + sqlite3_value** argv +) { + const unsigned char* zA = sqlite3_value_text(argv[0]); + const unsigned char* zB = sqlite3_value_text(argv[1]); + UChar32 uEsc = 0; + + /* Limit the length of the LIKE or GLOB pattern to avoid problems + ** of deep recursion and N*N behavior in patternCompare(). + */ + if (sqlite3_value_bytes(argv[0]) > SQLITE_MAX_LIKE_PATTERN_LENGTH) { + sqlite3_result_error(context, "LIKE or GLOB pattern too complex", -1); + return; + } + + + if (argc == 3) { + /* The escape character string must consist of a single UTF-8 character. + ** Otherwise, return an error. + */ + int nE = sqlite3_value_bytes(argv[2]); + const unsigned char* zE = sqlite3_value_text(argv[2]); + int i = 0; + if (zE == 0) return; + U8_NEXT(zE, i, nE, uEsc); + if (i != nE) { + sqlite3_result_error(context, + "ESCAPE expression must be a single character", -1); + return; + } + } + + if (zA && zB) { + sqlite3_result_int(context, icuLikeCompare(zA, zB, uEsc)); + } +} + +/* +** Function to delete compiled regexp objects. Registered as +** a destructor function with sqlite3_set_auxdata(). +*/ +static void STDCALL_FOR_INTEROP icuRegexpDelete(void* p) { + URegularExpression* pExpr = (URegularExpression*)p; + uregex_close(pExpr); +} + +/* +** Implementation of SQLite REGEXP operator. This scalar function takes +** two arguments. The first is a regular expression pattern to compile +** the second is a string to match against that pattern. If either +** argument is an SQL NULL, then NULL Is returned. Otherwise, the result +** is 1 if the string matches the pattern, or 0 otherwise. +** +** SQLite maps the regexp() function to the regexp() operator such +** that the following two are equivalent: +** +** zString REGEXP zPattern +** regexp(zPattern, zString) +** +** Uses the following ICU regexp APIs: +** +** uregex_open() +** uregex_matches() +** uregex_close() +*/ +static void STDCALL_FOR_INTEROP icuRegexpFunc(sqlite3_context* p, int nArg, sqlite3_value** apArg) { + UErrorCode status = U_ZERO_ERROR; + URegularExpression* pExpr; + UBool res; + const UChar* zString = sqlite3_value_text16(apArg[1]); + + (void)nArg; /* Unused parameter */ + + /* If the left hand side of the regexp operator is NULL, + ** then the result is also NULL. + */ + if (!zString) { + return; + } + + pExpr = sqlite3_get_auxdata(p, 0); + if (!pExpr) { + const UChar* zPattern = sqlite3_value_text16(apArg[0]); + if (!zPattern) { + return; + } + pExpr = uregex_open(zPattern, -1, 0, 0, &status); + + if (U_SUCCESS(status)) { + sqlite3_set_auxdata(p, 0, pExpr, icuRegexpDelete); + } + else { + assert(!pExpr); + icuFunctionError(p, "uregex_open", status); + return; + } + } + + /* Configure the text that the regular expression operates on. */ + uregex_setText(pExpr, zString, -1, &status); + if (!U_SUCCESS(status)) { + icuFunctionError(p, "uregex_setText", status); + return; + } + + /* Attempt the match */ + res = uregex_matches(pExpr, 0, &status); + if (!U_SUCCESS(status)) { + icuFunctionError(p, "uregex_matches", status); + return; + } + + /* Set the text that the regular expression operates on to a NULL + ** pointer. This is not really necessary, but it is tidier than + ** leaving the regular expression object configured with an invalid + ** pointer after this function returns. + */ + uregex_setText(pExpr, 0, 0, &status); + + /* Return 1 or 0. */ + sqlite3_result_int(p, res ? 1 : 0); +} + +/* +** Implementations of scalar functions for case mapping - upper() and +** lower(). Function upper() converts its input to upper-case (ABC). +** Function lower() converts to lower-case (abc). +** +** ICU provides two types of case mapping, "general" case mapping and +** "language specific". Refer to ICU documentation for the differences +** between the two. +** +** To utilise "general" case mapping, the upper() or lower() scalar +** functions are invoked with one argument: +** +** upper('ABC') -> 'abc' +** lower('abc') -> 'ABC' +** +** To access ICU "language specific" case mapping, upper() or lower() +** should be invoked with two arguments. The second argument is the name +** of the locale to use. Passing an empty string ("") or SQL NULL value +** as the second argument is the same as invoking the 1 argument version +** of upper() or lower(). +** +** lower('I', 'en_us') -> 'i' +** lower('I', 'tr_tr') -> '\u131' (small dotless i) +** +** http://www.icu-project.org/userguide/posix.html#case_mappings +*/ +static void STDCALL_FOR_INTEROP icuCaseFunc16(sqlite3_context* p, int nArg, sqlite3_value** apArg) { + const UChar* zInput; /* Pointer to input string */ + UChar* zOutput = 0; /* Pointer to output buffer */ + int nInput; /* Size of utf-16 input string in bytes */ + int nOut; /* Size of output buffer in bytes */ + int cnt; + int bToUpper; /* True for toupper(), false for tolower() */ + UErrorCode status; + const char* zLocale = 0; + + assert(nArg == 1 || nArg == 2); + bToUpper = (sqlite3_user_data(p) != 0); + if (nArg == 2) { + zLocale = (const char*)sqlite3_value_text(apArg[1]); + } + + zInput = sqlite3_value_text16(apArg[0]); + if (!zInput) { + return; + } + nOut = nInput = sqlite3_value_bytes16(apArg[0]); + if (nOut == 0) { + sqlite3_result_text16(p, "", 0, SQLITE_STATIC); + return; + } + + for (cnt = 0; cnt < 2; cnt++) { + UChar* zNew = sqlite3_realloc(zOutput, nOut); + if (zNew == 0) { + sqlite3_free(zOutput); + sqlite3_result_error_nomem(p); + return; + } + zOutput = zNew; + status = U_ZERO_ERROR; + if (bToUpper) { + nOut = 2 * u_strToUpper(zOutput, nOut / 2, zInput, nInput / 2, zLocale, &status); + } + else { + nOut = 2 * u_strToLower(zOutput, nOut / 2, zInput, nInput / 2, zLocale, &status); + } + + if (U_SUCCESS(status)) { + sqlite3_result_text16(p, zOutput, nOut, xFree); + } + else if (status == U_BUFFER_OVERFLOW_ERROR) { + assert(cnt == 0); + continue; + } + else { + icuFunctionError(p, bToUpper ? "u_strToUpper" : "u_strToLower", status); + } + return; + } + assert(0); /* Unreachable */ +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) */ + +/* +** Collation sequence destructor function. The pCtx argument points to +** a UCollator structure previously allocated using ucol_open(). +*/ +static void STDCALL_FOR_INTEROP icuCollationDel(void* pCtx) { + UCollator* p = (UCollator*)pCtx; + ucol_close(p); +} + +/* +** Collation sequence comparison function. The pCtx argument points to +** a UCollator structure previously allocated using ucol_open(). +*/ +static int STDCALL_FOR_INTEROP icuCollationColl( + void* pCtx, + int nLeft, + const void* zLeft, + int nRight, + const void* zRight +) { + UCollationResult res; + UCollator* p = (UCollator*)pCtx; + res = ucol_strcoll(p, (UChar*)zLeft, nLeft / 2, (UChar*)zRight, nRight / 2); + switch (res) { + case UCOL_LESS: return -1; + case UCOL_GREATER: return +1; + case UCOL_EQUAL: return 0; + } + assert(!"Unexpected return value from ucol_strcoll()"); + return 0; +} + +/* +** Implementation of the scalar function icu_load_collation(). +** +** This scalar function is used to add ICU collation based collation +** types to an SQLite database connection. It is intended to be called +** as follows: +** +** SELECT icu_load_collation(<locale>, <collation-name>); +** +** Where <locale> is a string containing an ICU locale identifier (i.e. +** "en_AU", "tr_TR" etc.) and <collation-name> is the name of the +** collation sequence to create. +*/ +static void STDCALL_FOR_INTEROP icuLoadCollation( + sqlite3_context* p, + int nArg, + sqlite3_value** apArg +) { + sqlite3* db = (sqlite3*)sqlite3_user_data(p); + UErrorCode status = U_ZERO_ERROR; + const char* zLocale; /* Locale identifier - (eg. "jp_JP") */ + const char* zName; /* SQL Collation sequence name (eg. "japanese") */ + UCollator* pUCollator; /* ICU library collation object */ + int rc; /* Return code from sqlite3_create_collation_x() */ + + assert(nArg == 2); + (void)nArg; /* Unused parameter */ + zLocale = (const char*)sqlite3_value_text(apArg[0]); + zName = (const char*)sqlite3_value_text(apArg[1]); + + if (!zLocale || !zName) { + return; + } + + pUCollator = ucol_open(zLocale, &status); + if (!U_SUCCESS(status)) { + icuFunctionError(p, "ucol_open", status); + return; + } + assert(p); + + rc = sqlite3_create_collation_v2(db, zName, SQLITE_UTF16, (void*)pUCollator, + icuCollationColl, icuCollationDel + ); + if (rc != SQLITE_OK) { + ucol_close(pUCollator); + sqlite3_result_error(p, "Error registering collation function", -1); + } +} + +/* +** Register the ICU extension functions with database db. +*/ +SQLITE_PRIVATE int sqlite3IcuInit(sqlite3* db) { +# define SQLITEICU_EXTRAFLAGS (SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS) + static const struct IcuScalar { + const char* zName; /* Function name */ + unsigned char nArg; /* Number of arguments */ + unsigned int enc; /* Optimal text encoding */ + unsigned char iContext; /* sqlite3_user_data() context */ + void (STDCALL_FOR_INTEROP *xFunc)(sqlite3_context*, int, sqlite3_value**); + } scalars[] = { + {"icu_load_collation",2,SQLITE_UTF8 | SQLITE_DIRECTONLY,1, icuLoadCollation}, + #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) + {"regexp", 2, SQLITE_ANY | SQLITEICU_EXTRAFLAGS, 0, icuRegexpFunc}, + {"lower", 1, SQLITE_UTF16 | SQLITEICU_EXTRAFLAGS, 0, icuCaseFunc16}, + {"lower", 2, SQLITE_UTF16 | SQLITEICU_EXTRAFLAGS, 0, icuCaseFunc16}, + {"upper", 1, SQLITE_UTF16 | SQLITEICU_EXTRAFLAGS, 1, icuCaseFunc16}, + {"upper", 2, SQLITE_UTF16 | SQLITEICU_EXTRAFLAGS, 1, icuCaseFunc16}, + {"lower", 1, SQLITE_UTF8 | SQLITEICU_EXTRAFLAGS, 0, icuCaseFunc16}, + {"lower", 2, SQLITE_UTF8 | SQLITEICU_EXTRAFLAGS, 0, icuCaseFunc16}, + {"upper", 1, SQLITE_UTF8 | SQLITEICU_EXTRAFLAGS, 1, icuCaseFunc16}, + {"upper", 2, SQLITE_UTF8 | SQLITEICU_EXTRAFLAGS, 1, icuCaseFunc16}, + {"like", 2, SQLITE_UTF8 | SQLITEICU_EXTRAFLAGS, 0, icuLikeFunc}, + {"like", 3, SQLITE_UTF8 | SQLITEICU_EXTRAFLAGS, 0, icuLikeFunc}, + #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) */ + }; + int rc = SQLITE_OK; + int i; + + for (i = 0; rc == SQLITE_OK && i < (int)(sizeof(scalars) / sizeof(scalars[0])); i++) { + const struct IcuScalar* p = &scalars[i]; + rc = sqlite3_create_function( + db, p->zName, p->nArg, p->enc, + p->iContext ? (void*)db : (void*)0, + p->xFunc, 0, 0 + ); + } + + return rc; +} + +#ifndef SQLITE_CORE +#ifdef _WIN32 +__declspec(dllexport) +#endif +SQLITE_API int sqlite3_icu_init( + sqlite3* db, + char** pzErrMsg, + const sqlite3_api_routines* pApi +) { + SQLITE_EXTENSION_INIT2(pApi) + return sqlite3IcuInit(db); +} +#endif + +/************** End of icu.c *************************************************/+ \ No newline at end of file diff --git a/src/AppInstallerRepositoryCore/ICU/SQLiteICU.h b/src/AppInstallerRepositoryCore/ICU/SQLiteICU.h @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once +#include <winsqlite/winsqlite3.h> + +// Adapted from the file sqliteicu.h to use the built-in Windows SQLite and ICU binaries. + +extern "C" +{ + int sqlite3IcuInit(sqlite3* db); +} diff --git a/src/AppInstallerRepositoryCore/Manifest/Manifest.cpp b/src/AppInstallerRepositoryCore/Manifest/Manifest.cpp @@ -7,14 +7,14 @@ namespace AppInstaller::Manifest { namespace { - std::vector<std::string> SplitMultiValueField(const std::string& input) + std::vector<Manifest::string_t> SplitMultiValueField(const std::string& input) { if (input.empty()) { return {}; } - std::vector<std::string> result; + std::vector<Manifest::string_t> result; size_t currentPos = 0; while (currentPos < input.size()) { diff --git a/src/AppInstallerRepositoryCore/Manifest/Manifest.h b/src/AppInstallerRepositoryCore/Manifest/Manifest.h @@ -1,9 +1,10 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. #pragma once -#include "AppInstallerErrors.h" #include "ManifestInstaller.h" #include "ManifestLocalization.h" +#include <AppInstallerErrors.h> +#include <AppInstallerStrings.h> #include <yaml-cpp/yaml.h> #include <wil/result.h> @@ -23,38 +24,40 @@ namespace AppInstaller::Manifest // Our representation of the parsed manifest file. struct Manifest { + using string_t = Utility::NormalizedString; + // Required - std::string Id; + string_t Id; // Required - std::string Name; + string_t Name; // Required - std::string Version; + string_t Version; - std::string AppMoniker; + string_t AppMoniker; - std::string Publisher; + string_t Publisher; - std::string Channel; + string_t Channel; - std::string Author; + string_t Author; - std::string License; + string_t License; - std::string MinOSVersion; + string_t MinOSVersion; // Comma separated values - std::vector<std::string> Tags; + std::vector<string_t> Tags; // Comma separated values - std::vector<std::string> Commands; + std::vector<string_t> Commands; // Comma separated values - std::vector<std::string> Protocols; + std::vector<string_t> Protocols; // Comma separated values - std::vector<std::string> FileExtensions; + std::vector<string_t> FileExtensions; std::vector<ManifestInstaller> Installers; @@ -62,13 +65,13 @@ namespace AppInstaller::Manifest ManifestInstaller::InstallerTypeEnum InstallerType; - std::map<ManifestInstaller::InstallerSwitchType, std::string> Switches; + std::map<ManifestInstaller::InstallerSwitchType, string_t> Switches; - std::string Description; + string_t Description; - std::string Homepage; + string_t Homepage; - std::string LicenseUrl; + string_t LicenseUrl; void PopulateManifestFields(const YAML::Node& rootNode); diff --git a/src/AppInstallerRepositoryCore/Manifest/ManifestInstaller.cpp b/src/AppInstallerRepositoryCore/Manifest/ManifestInstaller.cpp @@ -26,7 +26,7 @@ namespace AppInstaller::Manifest ConvertToInstallerTypeEnum(installerNode["InstallerType"].as<std::string>()) : defaultInstaller.InstallerType; - std::map<InstallerSwitchType, std::string> defaultKnownSwitches = GetDefaultKnownSwitches(this->InstallerType); + std::map<InstallerSwitchType, string_t> defaultKnownSwitches = GetDefaultKnownSwitches(this->InstallerType); if (installerNode["Switches"]) { @@ -41,9 +41,9 @@ namespace AppInstaller::Manifest void ManifestInstaller::PopulateSwitchesFields( const YAML::Node* switchesNode, - std::map<InstallerSwitchType, std::string>& switches, - const std::map<InstallerSwitchType, std::string>* manifestRootSwitches, - const std::map<InstallerSwitchType, std::string>* defaultKnownSwitches) + std::map<InstallerSwitchType, string_t>& switches, + const std::map<InstallerSwitchType, string_t>* manifestRootSwitches, + const std::map<InstallerSwitchType, string_t>* defaultKnownSwitches) { PopulateOneSwitchField(switchesNode, "Custom", InstallerSwitchType::Custom, switches, manifestRootSwitches, defaultKnownSwitches); PopulateOneSwitchField(switchesNode, "Silent", InstallerSwitchType::Silent, switches, manifestRootSwitches, defaultKnownSwitches); @@ -58,9 +58,9 @@ namespace AppInstaller::Manifest const YAML::Node* switchesNode, const std::string& switchName, InstallerSwitchType switchType, - std::map<InstallerSwitchType, std::string>& switches, - const std::map<InstallerSwitchType, std::string>* manifestRootSwitches, - const std::map<InstallerSwitchType, std::string>* defaultKnownSwitches) + std::map<InstallerSwitchType, string_t>& switches, + const std::map<InstallerSwitchType, string_t>* manifestRootSwitches, + const std::map<InstallerSwitchType, string_t>* defaultKnownSwitches) { if (switchesNode && (*switchesNode)[switchName]) { @@ -76,7 +76,7 @@ namespace AppInstaller::Manifest } } - std::map<ManifestInstaller::InstallerSwitchType, std::string> ManifestInstaller::GetDefaultKnownSwitches(InstallerTypeEnum installerType) + std::map<ManifestInstaller::InstallerSwitchType, ManifestInstaller::string_t> ManifestInstaller::GetDefaultKnownSwitches(InstallerTypeEnum installerType) { switch (installerType) { @@ -85,25 +85,25 @@ namespace AppInstaller::Manifest case ManifestInstaller::InstallerTypeEnum::Msi: return { - {InstallerSwitchType::Silent, "/quiet"}, - {InstallerSwitchType::SilentWithProgress, "/passive"}, - {InstallerSwitchType::Log, "/log \"" + std::string(ARG_TOKEN_LOGPATH) + "\""}, - {InstallerSwitchType::InstallLocation, "TARGETDIR=\"" + std::string(ARG_TOKEN_INSTALLPATH) + "\""} + {InstallerSwitchType::Silent, string_t("/quiet")}, + {InstallerSwitchType::SilentWithProgress, string_t("/passive")}, + {InstallerSwitchType::Log, string_t("/log \"" + std::string(ARG_TOKEN_LOGPATH) + "\"")}, + {InstallerSwitchType::InstallLocation, string_t("TARGETDIR=\"" + std::string(ARG_TOKEN_INSTALLPATH) + "\"")} }; case ManifestInstaller::InstallerTypeEnum::Nullsoft: return { - {InstallerSwitchType::Silent, "/S"}, - {InstallerSwitchType::SilentWithProgress, "/S"}, - {InstallerSwitchType::InstallLocation, "/D=\"" + std::string(ARG_TOKEN_INSTALLPATH) + "\""} + {InstallerSwitchType::Silent, string_t("/S")}, + {InstallerSwitchType::SilentWithProgress, string_t("/S")}, + {InstallerSwitchType::InstallLocation, string_t("/D=\"" + std::string(ARG_TOKEN_INSTALLPATH) + "\"")} }; case ManifestInstaller::InstallerTypeEnum::Inno: return { - {InstallerSwitchType::Silent, "/VERYSILENT"}, - {InstallerSwitchType::SilentWithProgress, "/SILENT"}, - {InstallerSwitchType::Log, "/LOG=\"" + std::string(ARG_TOKEN_LOGPATH) + "\""}, - {InstallerSwitchType::InstallLocation, "/DIR=\"" + std::string(ARG_TOKEN_INSTALLPATH) + "\""} + {InstallerSwitchType::Silent, string_t("/VERYSILENT")}, + {InstallerSwitchType::SilentWithProgress, string_t("/SILENT")}, + {InstallerSwitchType::Log, string_t("/LOG=\"" + std::string(ARG_TOKEN_LOGPATH) + "\"")}, + {InstallerSwitchType::InstallLocation, string_t("/DIR=\"" + std::string(ARG_TOKEN_INSTALLPATH) + "\"")} }; } return {}; diff --git a/src/AppInstallerRepositoryCore/Manifest/ManifestInstaller.h b/src/AppInstallerRepositoryCore/Manifest/ManifestInstaller.h @@ -2,6 +2,7 @@ // Licensed under the MIT License. #pragma once #include <AppInstallerArchitecture.h> +#include <AppInstallerStrings.h> #include <yaml-cpp/yaml.h> #include <string> @@ -18,6 +19,7 @@ namespace AppInstaller::Manifest class ManifestInstaller { public: + using string_t = Utility::NormalizedString; enum class InstallerTypeEnum { @@ -47,7 +49,7 @@ namespace AppInstaller::Manifest AppInstaller::Utility::Architecture Arch; // Required - std::string Url; + string_t Url; // Required std::vector<BYTE> Sha256; @@ -57,28 +59,28 @@ namespace AppInstaller::Manifest std::vector<BYTE> SignatureSha256; // Empty means default - std::string Language; + string_t Language; // Name TBD - std::string Scope; + string_t Scope; // If present, has more precedence than root InstallerTypeEnum InstallerType; // If present, has more precedence than root - std::map<InstallerSwitchType, std::string> Switches; + std::map<InstallerSwitchType, string_t> Switches; static InstallerTypeEnum ConvertToInstallerTypeEnum(const std::string& in); - static std::map<InstallerSwitchType, std::string> GetDefaultKnownSwitches(InstallerTypeEnum installerType); + static std::map<InstallerSwitchType, string_t> GetDefaultKnownSwitches(InstallerTypeEnum installerType); // Populates InstallerSwitches // The value declared in the manifest takes precedence, then value in the manifest root, then default known values. static void PopulateSwitchesFields( const YAML::Node* switchesNode, - std::map<InstallerSwitchType, std::string>& switches, - const std::map<InstallerSwitchType, std::string>* manifestRootSwitches = nullptr, - const std::map<InstallerSwitchType, std::string>* defaultKnownSwitches = nullptr); + std::map<InstallerSwitchType, string_t>& switches, + const std::map<InstallerSwitchType, string_t>* manifestRootSwitches = nullptr, + const std::map<InstallerSwitchType, string_t>* defaultKnownSwitches = nullptr); // Populates one Installer Switch // The value declared in the manifest takes precedence, then value in the manifest root, then default known values. @@ -86,9 +88,9 @@ namespace AppInstaller::Manifest const YAML::Node* switchesNode, const std::string& switchName, InstallerSwitchType switchType, - std::map<InstallerSwitchType, std::string>& switches, - const std::map<InstallerSwitchType, std::string>* manifestRootSwitches, - const std::map<InstallerSwitchType, std::string>* defaultKnownSwitches); + std::map<InstallerSwitchType, string_t>& switches, + const std::map<InstallerSwitchType, string_t>* manifestRootSwitches, + const std::map<InstallerSwitchType, string_t>* defaultKnownSwitches); // Populates ManifestInstaller // defaultInstaller: if an optional field is not found in the YAML node, the field will be populated with value from defaultInstaller. diff --git a/src/AppInstallerRepositoryCore/Manifest/ManifestLocalization.cpp b/src/AppInstallerRepositoryCore/Manifest/ManifestLocalization.cpp @@ -13,15 +13,15 @@ namespace AppInstaller::Manifest // Optional this->Description = localizationNode["Description"] ? - localizationNode["Description"].as<std::string>() : + string_t(localizationNode["Description"].as<std::string>()) : defaultLocalization.Description; this->Homepage = localizationNode["Homepage"] ? - localizationNode["Homepage"].as<std::string>() : + string_t(localizationNode["Homepage"].as<std::string>()) : defaultLocalization.Homepage; this->LicenseUrl = localizationNode["LicenseUrl"] ? - localizationNode["LicenseUrl"].as<std::string>() : + string_t(localizationNode["LicenseUrl"].as<std::string>()) : defaultLocalization.LicenseUrl; } } diff --git a/src/AppInstallerRepositoryCore/Manifest/ManifestLocalization.h b/src/AppInstallerRepositoryCore/Manifest/ManifestLocalization.h @@ -10,14 +10,16 @@ namespace AppInstaller::Manifest class ManifestLocalization { public: + using string_t = Utility::NormalizedString; + // Required - std::string Language; + string_t Language; - std::string Description; + string_t Description; - std::string Homepage; + string_t Homepage; - std::string LicenseUrl; + string_t LicenseUrl; // Populates ManifestLocalization // defaultLocalization: if an optional field is not found in the YAML node, the field will be populated with value from defaultLocalization. diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp @@ -109,6 +109,7 @@ namespace AppInstaller::Repository::Microsoft SQLiteIndex::SQLiteIndex(const std::string& target, SQLite::Connection::OpenDisposition disposition, SQLite::Connection::OpenFlags flags) : m_dbconn(SQLite::Connection::Create(target, disposition, flags)) { + m_dbconn.EnableICU(); m_version = Schema::Version::GetSchemaVersion(m_dbconn); AICLI_LOG(Repo, Info, << "Opened SQLite Index with version [" << m_version << "], last write [" << GetLastWriteTime() << "]"); m_interface = m_version.CreateISQLiteIndex(); @@ -118,6 +119,7 @@ namespace AppInstaller::Repository::Microsoft SQLiteIndex::SQLiteIndex(const std::string& target, Schema::Version version) : m_dbconn(SQLite::Connection::Create(target, SQLite::Connection::OpenDisposition::Create)) { + m_dbconn.EnableICU(); m_interface = version.CreateISQLiteIndex(); m_version = m_interface->GetVersion(); } diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSource.cpp b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSource.cpp @@ -26,7 +26,7 @@ namespace AppInstaller::Repository::Microsoft return GetSource()->GetIndex().GetNameStringById(m_id).value(); } - Manifest::Manifest GetManifest(std::string_view version, std::string_view channel) override + Manifest::Manifest GetManifest(const Utility::NormalizedString& version, const Utility::NormalizedString& channel) override { std::shared_ptr<SQLiteIndexSource> source = GetSource(); std::string relativePath = source->GetIndex().GetPathStringByKey(m_id, version, channel).value(); diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToManyTable.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToManyTable.cpp @@ -126,7 +126,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 void OneToManyTableEnsureExistsAndInsert(SQLite::Connection& connection, std::string_view tableName, std::string_view valueName, - const std::vector<std::string>& values, SQLite::rowid_t manifestId) + const std::vector<Utility::NormalizedString>& values, SQLite::rowid_t manifestId) { SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, std::string{ tableName } + "_ensureandinsert_v1_0"); @@ -149,7 +149,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 bool OneToManyTableUpdateIfNeededByManifestId(SQLite::Connection& connection, std::string_view tableName, std::string_view valueName, - const std::vector<std::string>& values, SQLite::rowid_t manifestId) + const std::vector<Utility::NormalizedString>& values, SQLite::rowid_t manifestId) { std::vector<SQLite::rowid_t> oldValueIds = GetValueIdsByManifestId(connection, tableName, valueName, manifestId); bool modificationNeeded = false; diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToManyTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToManyTable.h @@ -23,12 +23,12 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 // Ensures that the value exists and inserts mapping entries. void OneToManyTableEnsureExistsAndInsert(SQLite::Connection& connection, std::string_view tableName, std::string_view valueName, - const std::vector<std::string>& values, SQLite::rowid_t manifestId); + const std::vector<Utility::NormalizedString>& values, SQLite::rowid_t manifestId); // Updates the mapping table to represent the given values for the manifest. bool OneToManyTableUpdateIfNeededByManifestId(SQLite::Connection& connection, std::string_view tableName, std::string_view valueName, - const std::vector<std::string>& values, SQLite::rowid_t manifestId); + const std::vector<Utility::NormalizedString>& values, SQLite::rowid_t manifestId); // Deletes the mapping rows for the given manifest, then removes any unused data rows. void OneToManyTableDeleteIfNotNeededByManifestId(SQLite::Connection& connection, std::string_view tableName, std::string_view valueName, SQLite::rowid_t manifestId); @@ -69,13 +69,13 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 } // Ensures that all values exist in the data table, and inserts into the mapping table for the given manifest id. - static void EnsureExistsAndInsert(SQLite::Connection& connection, const std::vector<std::string>& values, SQLite::rowid_t manifestId) + static void EnsureExistsAndInsert(SQLite::Connection& connection, const std::vector<Utility::NormalizedString>& values, SQLite::rowid_t manifestId) { details::OneToManyTableEnsureExistsAndInsert(connection, TableInfo::TableName(), TableInfo::ValueName(), values, manifestId); } // Updates the mapping table to represent the given values for the manifest. - static bool UpdateIfNeededByManifestId(SQLite::Connection& connection, const std::vector<std::string>& values, SQLite::rowid_t manifestId) + static bool UpdateIfNeededByManifestId(SQLite::Connection& connection, const std::vector<Utility::NormalizedString>& values, SQLite::rowid_t manifestId) { return details::OneToManyTableUpdateIfNeededByManifestId(connection, TableInfo::TableName(), TableInfo::ValueName(), values, manifestId); } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/Version.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/Version.cpp @@ -39,9 +39,9 @@ namespace AppInstaller::Repository::Microsoft::Schema // We do not have the capacity to operate on this schema version THROW_HR(HRESULT_FROM_WIN32(ERROR_NOT_SUPPORTED)); } +} - std::ostream& operator<<(std::ostream& out, const Version& version) - { - return (out << version.MajorVersion << '.' << version.MinorVersion); - } +std::ostream& operator<<(std::ostream& out, const AppInstaller::Repository::Microsoft::Schema::Version& version) +{ + return (out << version.MajorVersion << '.' << version.MinorVersion); } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/Version.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/Version.h @@ -53,7 +53,7 @@ namespace AppInstaller::Repository::Microsoft::Schema // Creates the interface object for this version. std::unique_ptr<ISQLiteIndex> CreateISQLiteIndex(); }; - - // Output the version - std::ostream& operator<<(std::ostream& out, const Version& version); } + +// Output the version +std::ostream& operator<<(std::ostream& out, const AppInstaller::Repository::Microsoft::Schema::Version& version); diff --git a/src/AppInstallerRepositoryCore/Public/AppInstallerRepositorySearch.h b/src/AppInstallerRepositoryCore/Public/AppInstallerRepositorySearch.h @@ -2,6 +2,7 @@ // Licensed under the MIT License. #pragma once #include <Manifest/Manifest.h> +#include <AppInstallerStrings.h> #include <AppInstallerVersions.h> #include <memory> @@ -37,9 +38,9 @@ namespace AppInstaller::Repository struct RequestMatch { MatchType Type; - std::string Value; + Utility::NormalizedString Value; - RequestMatch(MatchType t, std::string v) : Type(t), Value(std::move(v)) {} + RequestMatch(MatchType t, const std::string& v) : Type(t), Value(v) {} }; // A match on a specific field to be performed during a search. @@ -47,7 +48,7 @@ namespace AppInstaller::Repository { ApplicationMatchField Field; - ApplicationMatchFilter(ApplicationMatchField f, MatchType t, std::string v) : RequestMatch(t, std::move(v)), Field(f) {} + ApplicationMatchFilter(ApplicationMatchField f, MatchType t, const std::string& v) : RequestMatch(t, v), Field(f) {} }; // Container for data used to filter the available manifests in a source. @@ -80,7 +81,7 @@ namespace AppInstaller::Repository // Gets a manifest for this application. // An empty version implies 'latest'. // An empty channel is the 'general audience'. - virtual Manifest::Manifest GetManifest(std::string_view version, std::string_view channel) = 0; + virtual Manifest::Manifest GetManifest(const Utility::NormalizedString& version, const Utility::NormalizedString& channel) = 0; // Gets all versions of this application. // The versions will be returned in sorted, descending order. diff --git a/src/AppInstallerRepositoryCore/SQLiteWrapper.cpp b/src/AppInstallerRepositoryCore/SQLiteWrapper.cpp @@ -2,6 +2,7 @@ // Licensed under the MIT License. #include "pch.h" #include "SQLiteWrapper.h" +#include "ICU/SQLiteICU.h" #include <wil/result_macros.h> @@ -105,6 +106,12 @@ namespace AppInstaller::Repository::SQLite return result; } + void Connection::EnableICU() + { + AICLI_LOG(SQL, Info, << "Enabling ICU"); + THROW_IF_SQLITE_FAILED(sqlite3IcuInit(m_dbconn.get())); + } + rowid_t Connection::GetLastInsertRowID() { return sqlite3_last_insert_rowid(m_dbconn.get()); diff --git a/src/AppInstallerRepositoryCore/SQLiteWrapper.h b/src/AppInstallerRepositoryCore/SQLiteWrapper.h @@ -135,6 +135,9 @@ namespace AppInstaller::Repository::SQLite ~Connection() = default; + // Enables the ICU integrations on this connection. + void EnableICU(); + // Gets the last inerted rowid to the database. rowid_t GetLastInsertRowID(); diff --git a/src/AppInstallerSQLiteIndexUtil/AppInstallerSQLiteIndexUtil.vcxproj b/src/AppInstallerSQLiteIndexUtil/AppInstallerSQLiteIndexUtil.vcxproj @@ -137,9 +137,9 @@ <ModuleDefinitionFile Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">Source.def</ModuleDefinitionFile> <ModuleDefinitionFile Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">Source.def</ModuleDefinitionFile> <ModuleDefinitionFile Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">Source.def</ModuleDefinitionFile> - <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;%(AdditionalDependencies)</AdditionalDependencies> - <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;%(AdditionalDependencies)</AdditionalDependencies> - <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;icuuc.lib;icuin.lib;%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Debug|ARM64'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;icuuc.lib;icuin.lib;%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;icuuc.lib;icuin.lib;%(AdditionalDependencies)</AdditionalDependencies> </Link> </ItemDefinitionGroup> <ItemDefinitionGroup Condition="'$(Platform)'=='Win32'"> @@ -151,7 +151,7 @@ <Link> <SubSystem Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">Windows</SubSystem> <ModuleDefinitionFile Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">Source.def</ModuleDefinitionFile> - <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;icuuc.lib;icuin.lib;%(AdditionalDependencies)</AdditionalDependencies> </Link> </ItemDefinitionGroup> <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'"> @@ -177,10 +177,10 @@ <ModuleDefinitionFile Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">Source.def</ModuleDefinitionFile> <ModuleDefinitionFile Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">Source.def</ModuleDefinitionFile> <ModuleDefinitionFile Condition="'$(Configuration)|$(Platform)'=='Release|x64'">Source.def</ModuleDefinitionFile> - <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;%(AdditionalDependencies)</AdditionalDependencies> - <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;%(AdditionalDependencies)</AdditionalDependencies> - <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;%(AdditionalDependencies)</AdditionalDependencies> - <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Release|x64'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Release|ARM'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;icuuc.lib;icuin.lib;%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Release|ARM64'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;icuuc.lib;icuin.lib;%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;icuuc.lib;icuin.lib;%(AdditionalDependencies)</AdditionalDependencies> + <AdditionalDependencies Condition="'$(Configuration)|$(Platform)'=='Release|x64'">wininet.lib;shell32.lib;winsqlite3.lib;shlwapi.lib;icuuc.lib;icuin.lib;%(AdditionalDependencies)</AdditionalDependencies> </Link> </ItemDefinitionGroup> <ItemGroup>