commit 5fe31fc8710514f0f3f08d18d8043c43352e3f73 parent 0703b6beece5ed85e1dc18b5e1761215233f5078 Author: JohnMcPMS <johnmcp@microsoft.com> Date: Fri, 10 May 2024 12:14:00 -0700 Index V2 consumption (#4462) ## Change This change adds the ability to actually use the V2 index database. This primarily required fixing the ignored search functionality from the previous PR and updating the `ISource` implementation to be aware of V2 and how to retrieve the package version manifest files. Additionally, a few places in the code were updated to avoid requesting all of the package versions when we could instead use only the latest version. This prevents the need to retrieve the package version file at all, which is the goal for all read-only operations (`search`, `list`, `update` [aka `list` but filtered to things with updates]). In reality, we do need to get the package version files when we need to do ARP version mapping, but that constitutes only 5% of the total packages in `winget-pkgs` currently. While the V2 usage code is always enabled, an experimental feature (`indexV2`) is required to attempt to retrieve the V2 index from the CDN. Disabling this feature should naturally transition one back to V1 on the next update attempt. File caching, based on the known SHA256 hash of the file, is added as a general utility. This is then used for V1 manifest files and also for V2 package version data and manifest files. While the files are generally small, it is not uncommon for them to be used a few times over a short period. Thanks to @msftrubengu for pointing out that I forgot the include the exports in the .def file in the previous PR. 🥇 Diffstat:
83 files changed, 2756 insertions(+), 993 deletions(-)
diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt @@ -361,7 +361,8 @@ pseudocode PSHOST psobject ptstr -publickey +publickey +PVD pvk pvm pwabuilder diff --git a/azure-pipelines.yml b/azure-pipelines.yml @@ -245,6 +245,13 @@ jobs: - template: templates/e2e-test.template.yml parameters: + title: "E2E Tests Packaged (Index V2)" + isPackaged: true + filter: "TestCategory!=InProcess&TestCategory!=OutOfProcess" + experimentalFeatures: indexV2 + + - template: templates/e2e-test.template.yml + parameters: title: "Microsoft.Management.Deployment E2E Tests (In-process)" isPackaged: false filter: "TestCategory=InProcess" diff --git a/doc/Settings.md b/doc/Settings.md @@ -345,3 +345,20 @@ You can enable the feature as shown below. "configureExport": true }, ``` + +### indexV2 + +This feature enables the `winget` source to retrieve the V2 index, which is significantly smaller. +Regardless of the state of this feature, if the index on the machine contains a V2 index, it will be used. +If there is a bug with the V2 index stopping the `winget` CLI from working, disable the feature in your settings file and run this command: +``` +> winget uninstall -s msstore Microsoft.Winget.Source_8wekyb3d8bbwe +``` + +You can enable the feature as shown below. + +```json + "experimentalFeatures": { + "indexV2": true + }, +``` diff --git a/schemas/JSON/settings/settings.schema.0.2.json b/schemas/JSON/settings/settings.schema.0.2.json @@ -290,6 +290,11 @@ "description": "Enable support for the configure export command", "type": "boolean", "default": false + }, + "indexV2": { + "description": "Enable retrieving the V2 index for the winget source", + "type": "boolean", + "default": false } } } diff --git a/src/AppInstallerCLIE2ETests/Constants.cs b/src/AppInstallerCLIE2ETests/Constants.cs @@ -29,6 +29,7 @@ namespace AppInstallerCLIE2ETests public const string PackageCertificatePathParameter = "PackageCertificatePath"; public const string PowerShellModulePathParameter = "PowerShellModulePath"; public const string SkipTestSourceParameter = "SkipTestSource"; + public const string ForcedExperimentalFeaturesParameter = "ForcedExperimentalFeatures"; // Test Sources public const string DefaultWingetSourceName = @"winget"; diff --git a/src/AppInstallerCLIE2ETests/FeaturesCommand.cs b/src/AppInstallerCLIE2ETests/FeaturesCommand.cs @@ -1,4 +1,4 @@ -// ----------------------------------------------------------------------------- +// ----------------------------------------------------------------------------- // <copyright file="FeaturesCommand.cs" company="Microsoft Corporation"> // Copyright (c) Microsoft Corporation. Licensed under the MIT License. // </copyright> @@ -41,7 +41,14 @@ namespace AppInstallerCLIE2ETests var result = TestCommon.RunAICLICommand("features", string.Empty); Assert.AreEqual(Constants.ErrorCode.S_OK, result.ExitCode); Assert.True(result.StdOut.Contains("Direct MSI Installation")); - Assert.False(result.StdOut.Contains("Enabled")); + if (WinGetSettingsHelper.ForcedExperimentalFeatures.Length > 0 && WinGetSettingsHelper.ForcedExperimentalFeatures[0] != "none") + { + Assert.True(result.StdOut.Contains("Enabled")); + } + else + { + Assert.False(result.StdOut.Contains("Enabled")); + } } /// <summary> diff --git a/src/AppInstallerCLIE2ETests/Helpers/TestCommon.cs b/src/AppInstallerCLIE2ETests/Helpers/TestCommon.cs @@ -667,6 +667,9 @@ namespace AppInstallerCLIE2ETests.Helpers /// <param name="useGroupPolicyForTestSource">Use group policy.</param> public static void SetupTestSource(bool useGroupPolicyForTestSource = false) { + // Remove the test source so that its package is also removed. + RunAICLICommand("source remove", Constants.TestSourceName); + RunAICLICommand("source reset", "--force"); RunAICLICommand("source remove", Constants.DefaultWingetSourceName); RunAICLICommand("source remove", Constants.DefaultMSStoreSourceName); diff --git a/src/AppInstallerCLIE2ETests/Helpers/TestSetup.cs b/src/AppInstallerCLIE2ETests/Helpers/TestSetup.cs @@ -1,4 +1,4 @@ -// ----------------------------------------------------------------------------- +// ----------------------------------------------------------------------------- // <copyright file="TestSetup.cs" company="Microsoft Corporation"> // Copyright (c) Microsoft Corporation. Licensed under the MIT License. // </copyright> @@ -43,6 +43,8 @@ namespace AppInstallerCLIE2ETests.Helpers this.ExeInstallerPath = this.InitializeFileParam(Constants.ExeInstallerPathParameter); this.MsiInstallerPath = this.InitializeFileParam(Constants.MsiInstallerPathParameter); this.MsixInstallerPath = this.InitializeFileParam(Constants.MsixInstallerPathParameter); + + this.ForcedExperimentalFeatures = this.InitializeStringArrayParam(Constants.ForcedExperimentalFeaturesParameter); } /// <summary> @@ -138,6 +140,11 @@ namespace AppInstallerCLIE2ETests.Helpers } /// <summary> + /// Gets the experimental features that should be forcibly enabled. + /// </summary> + public string[] ForcedExperimentalFeatures { get; } + + /// <summary> /// Gets a value indicating whether is the default parameters. /// </summary> public bool IsDefault { get; } @@ -162,6 +169,16 @@ namespace AppInstallerCLIE2ETests.Helpers return TestContext.Parameters.Get(paramName); } + private string[] InitializeStringArrayParam(string paramName, string[] defaultValue = null) + { + if (this.IsDefault || !TestContext.Parameters.Exists(paramName)) + { + return defaultValue; + } + + return TestContext.Parameters.Get(paramName).Split('|', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + } + private string InitializeFileParam(string paramName, string defaultValue = null) { if (!TestContext.Parameters.Exists(paramName)) diff --git a/src/AppInstallerCLIE2ETests/Helpers/WinGetSettingsHelper.cs b/src/AppInstallerCLIE2ETests/Helpers/WinGetSettingsHelper.cs @@ -1,4 +1,4 @@ -// ----------------------------------------------------------------------------- +// ----------------------------------------------------------------------------- // <copyright file="WinGetSettingsHelper.cs" company="Microsoft Corporation"> // Copyright (c) Microsoft Corporation. Licensed under the MIT License. // </copyright> @@ -17,6 +17,11 @@ namespace AppInstallerCLIE2ETests.Helpers internal static class WinGetSettingsHelper { /// <summary> + /// Gets or sets the experimental features that should be forcibly enabled. + /// </summary> + public static string[] ForcedExperimentalFeatures { get; set; } + + /// <summary> /// Gets the user settings path by calling winget settings export. /// </summary> /// <returns>Expanded path for user settings.</returns> @@ -33,19 +38,19 @@ namespace AppInstallerCLIE2ETests.Helpers /// </summary> public static void InitializeWingetSettings() { + Hashtable experimentalFeatures = new Hashtable(); + + var forcedExperimentalFeatures = ForcedExperimentalFeatures; + foreach (var feature in forcedExperimentalFeatures) + { + experimentalFeatures[feature] = true; + } + var settingsJson = new Hashtable() { { "experimentalFeatures", - new Hashtable() - { - { "experimentalArg", false }, - { "experimentalCmd", false }, - { "directMSI", false }, - { "windowsFeature", false }, - { "resume", false }, - { "reboot", false }, - } + experimentalFeatures }, { "debugging", @@ -67,28 +72,10 @@ namespace AppInstallerCLIE2ETests.Helpers // when running in unpackaged context TestCommon.RunAICLICommand(string.Empty, "-v"); - SetWingetSettings(settingsJson); - } - - /// <summary> - /// Converts a hashtable to json and writes to the settings file. - /// </summary> - /// <param name="settingsJson">Settings to set.</param> - public static void SetWingetSettings(Hashtable settingsJson) - { SetWingetSettings(JsonConvert.SerializeObject(settingsJson, Formatting.Indented)); } /// <summary> - /// Writes string to settings file. - /// </summary> - /// <param name="settings">Settings as string.</param> - public static void SetWingetSettings(string settings) - { - File.WriteAllText(TestSetup.Parameters.SettingsJsonFilePath, settings); - } - - /// <summary> /// Configure experimental features. /// </summary> /// <param name="featureName">Feature name.</param> @@ -96,10 +83,8 @@ namespace AppInstallerCLIE2ETests.Helpers public static void ConfigureFeature(string featureName, bool status) { JObject settingsJson = GetJsonSettingsObject("experimentalFeatures"); - var experimentalFeatures = settingsJson["experimentalFeatures"]; - experimentalFeatures[featureName] = status; - - File.WriteAllText(TestSetup.Parameters.SettingsJsonFilePath, settingsJson.ToString()); + ConfigureFeature(settingsJson, featureName, status); + SetWingetSettings(settingsJson); } /// <summary> @@ -113,7 +98,7 @@ namespace AppInstallerCLIE2ETests.Helpers var installBehavior = settingsJson["installBehavior"]; installBehavior[settingName] = value; - File.WriteAllText(TestSetup.Parameters.SettingsJsonFilePath, settingsJson.ToString()); + SetWingetSettings(settingsJson); } /// <summary> @@ -134,7 +119,7 @@ namespace AppInstallerCLIE2ETests.Helpers var preferences = installBehavior["preferences"]; preferences[settingName] = value; - File.WriteAllText(TestSetup.Parameters.SettingsJsonFilePath, settingsJson.ToString()); + SetWingetSettings(settingsJson); } /// <summary> @@ -155,7 +140,7 @@ namespace AppInstallerCLIE2ETests.Helpers var preferences = installBehavior["preferences"]; preferences[settingName] = new JArray(value); - File.WriteAllText(TestSetup.Parameters.SettingsJsonFilePath, settingsJson.ToString()); + SetWingetSettings(settingsJson); } /// <summary> @@ -176,7 +161,7 @@ namespace AppInstallerCLIE2ETests.Helpers var requirements = installBehavior["requirements"]; requirements[settingName] = value; - File.WriteAllText(TestSetup.Parameters.SettingsJsonFilePath, settingsJson.ToString()); + SetWingetSettings(settingsJson); } /// <summary> @@ -197,7 +182,7 @@ namespace AppInstallerCLIE2ETests.Helpers var requirements = installBehavior["requirements"]; requirements[settingName] = new JArray(value); - File.WriteAllText(TestSetup.Parameters.SettingsJsonFilePath, settingsJson.ToString()); + SetWingetSettings(settingsJson); } /// <summary> @@ -206,12 +191,28 @@ namespace AppInstallerCLIE2ETests.Helpers /// <param name="status">Initialized feature value.</param> public static void InitializeAllFeatures(bool status) { - ConfigureFeature("experimentalArg", status); - ConfigureFeature("experimentalCmd", status); - ConfigureFeature("directMSI", status); - ConfigureFeature("windowsFeature", status); - ConfigureFeature("resume", status); - ConfigureFeature("reboot", status); + JObject settingsJson = GetJsonSettingsObject("experimentalFeatures"); + + ConfigureFeature(settingsJson, "experimentalArg", status); + ConfigureFeature(settingsJson, "experimentalCmd", status); + ConfigureFeature(settingsJson, "directMSI", status); + ConfigureFeature(settingsJson, "windowsFeature", status); + ConfigureFeature(settingsJson, "resume", status); + ConfigureFeature(settingsJson, "reboot", status); + + SetWingetSettings(settingsJson); + } + + /// <summary> + /// Configure experimental features. + /// </summary> + /// <param name="settingsJson">The settings JSON object to modify.</param> + /// <param name="featureName">Feature name.</param> + /// <param name="status">Status.</param> + private static void ConfigureFeature(JObject settingsJson, string featureName, bool status) + { + var experimentalFeatures = settingsJson["experimentalFeatures"]; + experimentalFeatures[featureName] = status; } private static JObject GetJsonSettingsObject(string objectName) @@ -225,5 +226,29 @@ namespace AppInstallerCLIE2ETests.Helpers return settingsJson; } + + /// <summary> + /// Converts a JObject to a string and writes to the settings file. + /// </summary> + /// <param name="settingsJson">Settings to set.</param> + private static void SetWingetSettings(JObject settingsJson) + { + var forcedExperimentalFeatures = ForcedExperimentalFeatures; + foreach (var feature in forcedExperimentalFeatures) + { + ConfigureFeature(settingsJson, feature, true); + } + + SetWingetSettings(settingsJson.ToString()); + } + + /// <summary> + /// Writes string to settings file. + /// </summary> + /// <param name="settings">Settings as string.</param> + private static void SetWingetSettings(string settings) + { + File.WriteAllText(TestSetup.Parameters.SettingsJsonFilePath, settings); + } } } diff --git a/src/AppInstallerCLIE2ETests/README.md b/src/AppInstallerCLIE2ETests/README.md @@ -1,4 +1,4 @@ -# How to Run End-To-End Tests for Windows Package Manager Client +# How to Run End-To-End Tests for Windows Package Manager Client Most of the tests require having the local test source added into winget. The local test source must be hosted in a localhost web server. @@ -67,7 +67,8 @@ The localhost web server needs to be running for the duration of the tests. The | **CertPassword** | Mandatory | HTTPS Developer Certificate Password | | **Port** | Optional | Port number [Default Port Number: 5001] | | **OutCertFile** | Optional | The exported certificate used | -| **LocalSourceJson** | Optional | The local source definition. If set generates the source. | +| **LocalSourceJson** | Optional | The local source definition. If set generates the source. | +| **ForcedExperimentalFeatures** | Optional | Experimental features that should be forcibly enabled always. | ### How to create and trust an ASP.NET Core HTTPS Development Certificate Windows Package Manager Client (WinGet.exe) requires new sources added to the WinGet repositories be securely accessed through HTTPS. Therefore, in order to verify the LocalhostWebServer, you will need to create a self-signed development certificate to verify the localhost address. diff --git a/src/AppInstallerCLIE2ETests/SetUpFixture.cs b/src/AppInstallerCLIE2ETests/SetUpFixture.cs @@ -1,4 +1,4 @@ -// ----------------------------------------------------------------------------- +// ----------------------------------------------------------------------------- // <copyright file="SetUpFixture.cs" company="Microsoft Corporation"> // Copyright (c) Microsoft Corporation. Licensed under the MIT License. // </copyright> @@ -61,6 +61,7 @@ namespace AppInstallerCLIE2ETests TestIndex.GenerateE2ESource(); } + WinGetSettingsHelper.ForcedExperimentalFeatures = testParams.ForcedExperimentalFeatures; WinGetSettingsHelper.InitializeWingetSettings(); } diff --git a/src/AppInstallerCLIE2ETests/UpgradeCommand.cs b/src/AppInstallerCLIE2ETests/UpgradeCommand.cs @@ -1,4 +1,4 @@ -// ----------------------------------------------------------------------------- +// ----------------------------------------------------------------------------- // <copyright file="UpgradeCommand.cs" company="Microsoft Corporation"> // Copyright (c) Microsoft Corporation. Licensed under the MIT License. // </copyright> @@ -15,6 +15,18 @@ namespace AppInstallerCLIE2ETests /// </summary> public class UpgradeCommand : BaseCommand { + private static readonly string DenyUpgradePackage = "AppInstallerTest.TestUpgradeDeny"; + + /// <summary> + /// Tear down. + /// </summary> + [TearDown] + public void TearDown() + { + // Due to its properties, this being present is problematic. + TestCommon.RunAICLICommand("uninstall", DenyUpgradePackage); + } + /// <summary> /// Test upgrade portable package. /// </summary> @@ -117,7 +129,7 @@ namespace AppInstallerCLIE2ETests [Test] public void UpgradeBehaviorDeny() { - string packageId = "AppInstallerTest.TestUpgradeDeny"; + string packageId = DenyUpgradePackage; var result = TestCommon.RunAICLICommand("install", $"{packageId} -v 1.0.0.0"); Assert.AreEqual(Constants.ErrorCode.S_OK, result.ExitCode); diff --git a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj @@ -102,7 +102,8 @@ <VcpkgEnableManifest>true</VcpkgEnableManifest> <VcpkgApplocalDeps>false</VcpkgApplocalDeps> <VcpkgInstalledDir>$(ProjectDir)..\vcpkg_installed</VcpkgInstalledDir> - <VcpkgUseStatic></VcpkgUseStatic> + <VcpkgUseStatic> + </VcpkgUseStatic> </PropertyGroup> <PropertyGroup Label="Vcpkg" Condition="'$(Configuration)|$(Platform)'=='Debug|ARM'"> <VcpkgTriplet>arm</VcpkgTriplet> @@ -251,6 +252,7 @@ <ClCompile Include="Errors.cpp" /> <ClCompile Include="ExperimentalFeature.cpp" /> <ClCompile Include="ExportFlow.cpp" /> + <ClCompile Include="FileCache.cpp" /> <ClCompile Include="Filesystem.cpp" /> <ClCompile Include="FolderFileWatcher.cpp" /> <ClCompile Include="GroupPolicy.cpp" /> @@ -1006,4 +1008,4 @@ <Error Condition="!Exists('$(SolutionDir)\packages\Microsoft.Windows.CppWinRT.2.0.230706.1\build\native\Microsoft.Windows.CppWinRT.props')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\packages\Microsoft.Windows.CppWinRT.2.0.230706.1\build\native\Microsoft.Windows.CppWinRT.props'))" /> <Error Condition="!Exists('$(SolutionDir)\packages\Microsoft.Windows.CppWinRT.2.0.230706.1\build\native\Microsoft.Windows.CppWinRT.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\packages\Microsoft.Windows.CppWinRT.2.0.230706.1\build\native\Microsoft.Windows.CppWinRT.targets'))" /> </Target> -</Project> +</Project>+ \ No newline at end of file diff --git a/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters b/src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters @@ -353,6 +353,9 @@ <ClCompile Include="Yaml.cpp"> <Filter>Source Files\Common</Filter> </ClCompile> + <ClCompile Include="FileCache.cpp"> + <Filter>Source Files\Common</Filter> + </ClCompile> </ItemGroup> <ItemGroup> <None Include="PropertySheet.props" /> diff --git a/src/AppInstallerCLITests/FileCache.cpp b/src/AppInstallerCLITests/FileCache.cpp @@ -0,0 +1,240 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "TestCommon.h" +#include <AppInstallerStrings.h> +#include <winget/FileCache.h> + +using namespace AppInstaller::Caching; +using namespace AppInstaller::Utility; +using namespace TestCommon; + +struct TestFileCache +{ + struct UpstreamFileInfo + { + TestDataFile OriginalFile; + std::filesystem::path Offset; + std::filesystem::path UpstreamPath; + std::vector<uint8_t> Contents; + SHA256::HashBuffer ContentHash; + }; + + TestFileCache(std::string identifier = {}, size_t upstreamCount = 1) + { + if (identifier.empty()) + { + identifier = ConvertToUTF8(CreateNewGuidNameWString()); + } + + std::vector<std::string> upstreamStrings; + + for (size_t i = 0; i < upstreamCount; ++i) + { + UpstreamSources.emplace_back("TestFileCache"); + upstreamStrings.emplace_back(UpstreamSources.back().GetPath().u8string()); + } + + CachePtr = std::make_unique<FileCache>(FileCache::Type::Tests, std::move(identifier), std::move(upstreamStrings)); + } + + FileCache& Cache() { return *CachePtr; } + FileCache* operator->() { return CachePtr.get(); } + + UpstreamFileInfo PrepareUpstreamFile(const std::filesystem::path& testDataFile, const std::filesystem::path& offset = {}, size_t index = 0) + { + UpstreamFileInfo result{ testDataFile }; + + auto dataFilePath = result.OriginalFile.GetPath(); + + result.Offset = offset.empty() ? dataFilePath.filename() : offset; + result.UpstreamPath = UpstreamSources[index].GetPath() / result.Offset; + + std::filesystem::copy_file(dataFilePath, result.UpstreamPath); + + std::ifstream fileStream{ dataFilePath, std::ios_base::in | std::ios_base::binary }; + result.Contents = ReadEntireStreamAsByteArray(fileStream); + + result.ContentHash = SHA256::ComputeHash(result.Contents); + + return result; + } + + std::filesystem::path GetCacheFilePath(const UpstreamFileInfo& upstreamFileInfo) + { + std::filesystem::path result = CachePtr->GetDetails().GetCachePath() / upstreamFileInfo.Offset; + std::filesystem::create_directories(result.parent_path()); + return result; + } + + std::unique_ptr<std::istream> GetFile(const UpstreamFileInfo& upstreamFileInfo) + { + return CachePtr->GetFile(upstreamFileInfo.Offset, upstreamFileInfo.ContentHash); + } + + void RequireCachedFile(const UpstreamFileInfo& upstreamFileInfo) + { + std::filesystem::path cachedFilePath = GetCacheFilePath(upstreamFileInfo); + REQUIRE(std::filesystem::is_regular_file(cachedFilePath)); + REQUIRE(SHA256::AreEqual(upstreamFileInfo.ContentHash, SHA256::ComputeHashFromFile(cachedFilePath))); + } + + std::unique_ptr<FileCache> CachePtr; + std::vector<TempDirectory> UpstreamSources; +}; + +TEST_CASE("FileCache_TypeLocationsDiffer", "[file_cache]") +{ + std::string identifier = "identifier"; + std::string identifier2 = "identifier2"; + + REQUIRE(FileCache(FileCache::Type::IndexV1_Manifest, identifier, {}).GetDetails().GetCachePath() != FileCache(FileCache::Type::IndexV2_Manifest, identifier, {}).GetDetails().GetCachePath()); + REQUIRE(FileCache(FileCache::Type::IndexV1_Manifest, identifier, {}).GetDetails().GetCachePath() != FileCache(FileCache::Type::IndexV2_PackageVersionData, identifier, {}).GetDetails().GetCachePath()); + REQUIRE(FileCache(FileCache::Type::IndexV2_Manifest, identifier, {}).GetDetails().GetCachePath() != FileCache(FileCache::Type::IndexV2_PackageVersionData, identifier, {}).GetDetails().GetCachePath()); + + REQUIRE(FileCache(FileCache::Type::IndexV1_Manifest, identifier, {}).GetDetails().GetCachePath() != FileCache(FileCache::Type::IndexV1_Manifest, identifier2, {}).GetDetails().GetCachePath()); + REQUIRE(FileCache(FileCache::Type::IndexV2_Manifest, identifier, {}).GetDetails().GetCachePath() != FileCache(FileCache::Type::IndexV2_Manifest, identifier2, {}).GetDetails().GetCachePath()); + REQUIRE(FileCache(FileCache::Type::IndexV2_PackageVersionData, identifier, {}).GetDetails().GetCachePath() != FileCache(FileCache::Type::IndexV2_PackageVersionData, identifier2, {}).GetDetails().GetCachePath()); +} + +TEST_CASE("FileCache_TypeLocationsSame", "[file_cache]") +{ + std::string identifier = "identifier"; + std::string source = "source"; + + REQUIRE(FileCache(FileCache::Type::IndexV1_Manifest, identifier, {}).GetDetails().GetCachePath() == FileCache(FileCache::Type::IndexV1_Manifest, identifier, { source }).GetDetails().GetCachePath()); + REQUIRE(FileCache(FileCache::Type::IndexV2_Manifest, identifier, {}).GetDetails().GetCachePath() == FileCache(FileCache::Type::IndexV2_Manifest, identifier, { source }).GetDetails().GetCachePath()); + REQUIRE(FileCache(FileCache::Type::IndexV2_PackageVersionData, identifier, {}).GetDetails().GetCachePath() == FileCache(FileCache::Type::IndexV2_PackageVersionData, identifier, { source }).GetDetails().GetCachePath()); +} + +TEST_CASE("FileCache_NoCachedFile", "[file_cache]") +{ + TestFileCache testFileCache; + INFO("Cache location: " << testFileCache->GetDetails().GetCachePath().u8string()); + + auto sourceFile = testFileCache.PrepareUpstreamFile("Manifest-Good-SystemReferenceComplex.yaml"); + + auto cachedStream = testFileCache.GetFile(sourceFile); + + REQUIRE(cachedStream); + REQUIRE(SHA256::AreEqual(sourceFile.ContentHash, SHA256::ComputeHash(ReadEntireStreamAsByteArray(*cachedStream)))); + + testFileCache.RequireCachedFile(sourceFile); +} + +TEST_CASE("FileCache_CachedFileIsDirectory", "[file_cache]") +{ + TestFileCache testFileCache; + INFO("Cache location: " << testFileCache->GetDetails().GetCachePath().u8string()); + + auto sourceFile = testFileCache.PrepareUpstreamFile("Manifest-Good.yaml"); + std::filesystem::create_directories(testFileCache.GetCacheFilePath(sourceFile)); + + auto cachedStream = testFileCache.GetFile(sourceFile); + + REQUIRE(cachedStream); + REQUIRE(SHA256::AreEqual(sourceFile.ContentHash, SHA256::ComputeHash(ReadEntireStreamAsByteArray(*cachedStream)))); + + testFileCache.RequireCachedFile(sourceFile); +} + +TEST_CASE("FileCache_CachedFileGoodHash", "[file_cache]") +{ + TestFileCache testFileCache; + INFO("Cache location: " << testFileCache->GetDetails().GetCachePath().u8string()); + + auto sourceFile = testFileCache.PrepareUpstreamFile("InstallFlowTest_MSStore.yaml"); + std::filesystem::copy_file(sourceFile.OriginalFile, testFileCache.GetCacheFilePath(sourceFile)); + + auto cachedStream = testFileCache.GetFile(sourceFile); + + REQUIRE(cachedStream); + REQUIRE(SHA256::AreEqual(sourceFile.ContentHash, SHA256::ComputeHash(ReadEntireStreamAsByteArray(*cachedStream)))); + + testFileCache.RequireCachedFile(sourceFile); +} + +TEST_CASE("FileCache_CachedFileBadHash", "[file_cache]") +{ + TestFileCache testFileCache; + INFO("Cache location: " << testFileCache->GetDetails().GetCachePath().u8string()); + + auto sourceFile = testFileCache.PrepareUpstreamFile("ManifestV1-MultiFile-Version.yaml"); + std::filesystem::copy_file(TestDataFile("Manifest-Bad-ProductCodeOnMSIX.yaml"), testFileCache.GetCacheFilePath(sourceFile)); + + auto cachedStream = testFileCache.GetFile(sourceFile); + + REQUIRE(cachedStream); + REQUIRE(SHA256::AreEqual(sourceFile.ContentHash, SHA256::ComputeHash(ReadEntireStreamAsByteArray(*cachedStream)))); + + testFileCache.RequireCachedFile(sourceFile); +} + +TEST_CASE("FileCache_CachedFileLockedExclusive", "[file_cache]") +{ + TestFileCache testFileCache; + INFO("Cache location: " << testFileCache->GetDetails().GetCachePath().u8string()); + + auto sourceFile = testFileCache.PrepareUpstreamFile("ManifestV1-MultiFile-Installer.yaml"); + TestDataFile wrongFileOriginal = TestDataFile("Manifest-Bad-InvalidLocale.yaml"); + std::filesystem::path wrongFilePath = testFileCache.GetCacheFilePath(sourceFile); + std::filesystem::copy_file(wrongFileOriginal, wrongFilePath); + wil::unique_handle exclusiveFileHandle{ CreateFileW(wrongFilePath.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL) }; + + auto cachedStream = testFileCache.GetFile(sourceFile); + + REQUIRE(cachedStream); + REQUIRE(SHA256::AreEqual(sourceFile.ContentHash, SHA256::ComputeHash(ReadEntireStreamAsByteArray(*cachedStream)))); +} + +TEST_CASE("FileCache_FirstUpstreamDoesNotHaveFile", "[file_cache]") +{ + TestFileCache testFileCache({}, 2); + INFO("Cache location: " << testFileCache->GetDetails().GetCachePath().u8string()); + + auto sourceFile = testFileCache.PrepareUpstreamFile("Manifest-Good-MultiLocale.yaml", {}, 1); + + auto cachedStream = testFileCache.GetFile(sourceFile); + + REQUIRE(cachedStream); + REQUIRE(SHA256::AreEqual(sourceFile.ContentHash, SHA256::ComputeHash(ReadEntireStreamAsByteArray(*cachedStream)))); + + testFileCache.RequireCachedFile(sourceFile); +} + +TEST_CASE("FileCache_FirstUpstreamHasBadHash", "[file_cache]") +{ + TestFileCache testFileCache({}, 2); + INFO("Cache location: " << testFileCache->GetDetails().GetCachePath().u8string()); + + auto badFile = testFileCache.PrepareUpstreamFile("Manifest-Bad-VersionMissing.yaml", {}, 0); + auto sourceFile = testFileCache.PrepareUpstreamFile("Manifest-Good-MultiLocale.yaml", {}, 1); + + auto cachedStream = testFileCache.GetFile(sourceFile); + + REQUIRE(cachedStream); + REQUIRE(SHA256::AreEqual(sourceFile.ContentHash, SHA256::ComputeHash(ReadEntireStreamAsByteArray(*cachedStream)))); + + testFileCache.RequireCachedFile(sourceFile); +} + +TEST_CASE("FileCache_NoUpstreamSources", "[file_cache]") +{ + TestFileCache testFileCache("", 0); + INFO("Cache location: " << testFileCache->GetDetails().GetCachePath().u8string()); + + REQUIRE_THROWS_HR(testFileCache->GetFile("any_file", SHA256::ComputeHash("garbage")), E_NOT_SET); +} + +TEST_CASE("FileCache_PathTooLong", "[file_cache]") +{ + TestFileCache testFileCache(std::string(260, 'a')); + INFO("Cache location: " << testFileCache->GetDetails().GetCachePath().u8string()); + + auto sourceFile = testFileCache.PrepareUpstreamFile("Manifest-Good-SystemReferenceComplex.yaml"); + + auto cachedStream = testFileCache.GetFile(sourceFile); + + REQUIRE(cachedStream); + REQUIRE(SHA256::AreEqual(sourceFile.ContentHash, SHA256::ComputeHash(ReadEntireStreamAsByteArray(*cachedStream)))); +} diff --git a/src/AppInstallerCLITests/PredefinedInstalledSource.cpp b/src/AppInstallerCLITests/PredefinedInstalledSource.cpp @@ -117,11 +117,11 @@ void VerifyMetadataString(const SQLiteIndex::MetadataResult& metadata, PackageVe void VerifyEntryAgainstIndex(const SQLiteIndex& index, SQLiteIndex::IdType manifestId, const ARPEntry& entry) { - REQUIRE(index.GetPropertyByManifestId(manifestId, PackageVersionProperty::Name) == entry.DisplayName); - REQUIRE(index.GetPropertyByManifestId(manifestId, PackageVersionProperty::Version) == entry.DisplayVersion); + REQUIRE(index.GetPropertyByPrimaryId(manifestId, PackageVersionProperty::Name) == entry.DisplayName); + REQUIRE(index.GetPropertyByPrimaryId(manifestId, PackageVersionProperty::Version) == entry.DisplayVersion); - REQUIRE(index.GetMultiPropertyByManifestId(manifestId, PackageVersionMultiProperty::PackageFamilyName).empty()); - auto productCodes = index.GetMultiPropertyByManifestId(manifestId, PackageVersionMultiProperty::ProductCode); + REQUIRE(index.GetMultiPropertyByPrimaryId(manifestId, PackageVersionMultiProperty::PackageFamilyName).empty()); + auto productCodes = index.GetMultiPropertyByPrimaryId(manifestId, PackageVersionMultiProperty::ProductCode); REQUIRE(productCodes.size() == 1); REQUIRE(productCodes[0] == FoldCase(static_cast<std::string_view>(entry.EntryName))); diff --git a/src/AppInstallerCLITests/SQLiteIndex.cpp b/src/AppInstallerCLITests/SQLiteIndex.cpp @@ -58,6 +58,8 @@ SQLiteVersion TestPrepareForRead(SQLiteIndex& index) SQLiteVersion versionMinus1 = SQLiteVersion{ 1, 7 }; SQLiteVersion versionMinus2 = SQLiteVersion{ 1, 6 }; + index.PrepareForPackaging(); + if (index.GetVersion() == versionMinus2) { // Degenerate case where we don't need to do anything @@ -74,13 +76,7 @@ SQLiteVersion TestPrepareForRead(SQLiteIndex& index) } else if (index.GetVersion() == latestVersion) { - SQLiteVersion version = GENERATE_COPY(SQLiteVersion{ versionMinus2 }, SQLiteVersion{ versionMinus1 }, SQLiteVersion{ latestVersion }); - - if (version != latestVersion) - { - index.ForceVersion(version); - return version; - } + // This crosses major versions, so leave it at 2.0 always } return index.GetVersion(); @@ -311,7 +307,7 @@ bool IsManifestMetadataSupported(const SQLiteIndex& index, const SQLiteVersion& bool AreManifestHashesSupported(const SQLiteIndex& index, const SQLiteVersion& testVersion) { UNSCOPED_INFO("Index " << index.GetVersion() << " | Test " << testVersion); - return (index.GetVersion() >= SQLiteVersion{ 1, 3 } && testVersion >= SQLiteVersion{ 1, 3 }); + return (index.GetVersion() >= SQLiteVersion{ 1, 3 } && testVersion >= SQLiteVersion{ 1, 3 } && index.GetVersion() < SQLiteVersion{ 2, 0 }); } bool AreArpVersionsSupported(const SQLiteIndex& index, const SQLiteVersion& testVersion) @@ -320,6 +316,12 @@ bool AreArpVersionsSupported(const SQLiteIndex& index, const SQLiteVersion& test return (index.GetVersion() >= SQLiteVersion{ 1, 5 } && testVersion >= SQLiteVersion{ 1, 5 }); } +bool AreArpVersionsNullable(const SQLiteIndex& index) +{ + UNSCOPED_INFO("Index " << index.GetVersion()); + return (index.GetVersion() >= SQLiteVersion{ 2, 0 }); +} + bool IsMapDataFoldingSupported(const SQLiteIndex& index, const SQLiteVersion& testVersion) { UNSCOPED_INFO("Index " << index.GetVersion() << " | Test " << testVersion); @@ -328,30 +330,63 @@ bool IsMapDataFoldingSupported(const SQLiteIndex& index, const SQLiteVersion& te bool IsMapDataFolded(const SQLiteIndex& index) { + UNSCOPED_INFO("Index " << index.GetVersion()); return (index.GetVersion() >= SQLiteVersion{ 1, 7 }); } -std::string GetPropertyStringByKey(const SQLiteIndex& index, rowid_t manifestId, PackageVersionProperty property) +bool AreVersionKeysSupported(const SQLiteIndex& index) { - auto result = index.GetPropertyByManifestId(manifestId, property); + UNSCOPED_INFO("Index " << index.GetVersion()); + return (index.GetVersion() < SQLiteVersion{ 2, 0 }); +} + +bool AreChannelsSupported(const SQLiteIndex& index) +{ + UNSCOPED_INFO("Index " << index.GetVersion()); + return (index.GetVersion() < SQLiteVersion{ 2, 0 }); +} + +bool AreManifestPathsSupported(const SQLiteIndex& index) +{ + UNSCOPED_INFO("Index " << index.GetVersion()); + return (index.GetVersion() < SQLiteVersion{ 2, 0 }); +} + +std::string GetPropertyStringByKey(const SQLiteIndex& index, rowid_t primaryId, PackageVersionProperty property) +{ + auto result = index.GetPropertyByPrimaryId(primaryId, property); REQUIRE(result); return result.value(); } std::string GetPropertyStringByKey(const SQLiteIndex& index, rowid_t id, PackageVersionProperty property, std::string_view version, std::string_view channel) { - auto manifestId = index.GetManifestIdByKey(id, version, channel); - REQUIRE(manifestId); - auto result = index.GetPropertyByManifestId(manifestId.value(), property); - REQUIRE(result); - return result.value(); + if (AreVersionKeysSupported(index)) + { + auto manifestId = index.GetManifestIdByKey(id, version, channel); + REQUIRE(manifestId); + auto result = index.GetPropertyByPrimaryId(manifestId.value(), property); + REQUIRE(result); + return result.value(); + } + else + { + return GetPropertyStringByKey(index, id, property); + } } std::string GetPropertyStringById(const SQLiteIndex& index, rowid_t id, PackageVersionProperty property) { - auto versions = index.GetVersionKeysById(id); - REQUIRE(!versions.empty()); - return GetPropertyStringByKey(index, versions[0].ManifestId, property); + if (AreVersionKeysSupported(index)) + { + auto versions = index.GetVersionKeysById(id); + REQUIRE(!versions.empty()); + return GetPropertyStringByKey(index, versions[0].ManifestId, property); + } + else + { + return GetPropertyStringByKey(index, id, property); + } } std::string GetIdStringById(const SQLiteIndex& index, rowid_t id) @@ -992,7 +1027,7 @@ TEST_CASE("SQLiteIndex_ValidateManifestWhenManifestIsDependency_StructureBroken_ APPINSTALLER_CLI_ERROR_DEPENDENCIES_VALIDATION_FAILED); } -TEST_CASE("SQLiteIndex_RemoveManifest_EnsureConsistentRowId", "[sqliteindex]") +TEST_CASE("SQLiteIndex_RemoveManifest_EnsureConsistentRowId", "[sqliteindex][V1_7]") { TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; INFO("Using temporary file named: " << tempFile.GetPath()); @@ -1052,12 +1087,12 @@ TEST_CASE("SQLiteIndex_RemoveManifest_EnsureConsistentRowId", "[sqliteindex]") REQUIRE(rowId); REQUIRE(rowId.value() == manifest2RowId); - REQUIRE(manifest2.Id == index.GetPropertyByManifestId(manifest2RowId, PackageVersionProperty::Id)); - REQUIRE(manifest2.DefaultLocalization.Get<Localization::PackageName>() == index.GetPropertyByManifestId(manifest2RowId, PackageVersionProperty::Name)); - REQUIRE(manifest2.Moniker == index.GetPropertyByManifestId(manifest2RowId, PackageVersionProperty::Moniker)); - REQUIRE(manifest2.Version == index.GetPropertyByManifestId(manifest2RowId, PackageVersionProperty::Version)); - REQUIRE(manifest2.Channel == index.GetPropertyByManifestId(manifest2RowId, PackageVersionProperty::Channel)); - REQUIRE(manifest2Path == index.GetPropertyByManifestId(manifest2RowId, PackageVersionProperty::RelativePath)); + REQUIRE(manifest2.Id == index.GetPropertyByPrimaryId(manifest2RowId, PackageVersionProperty::Id)); + REQUIRE(manifest2.DefaultLocalization.Get<Localization::PackageName>() == index.GetPropertyByPrimaryId(manifest2RowId, PackageVersionProperty::Name)); + REQUIRE(manifest2.Moniker == index.GetPropertyByPrimaryId(manifest2RowId, PackageVersionProperty::Moniker)); + REQUIRE(manifest2.Version == index.GetPropertyByPrimaryId(manifest2RowId, PackageVersionProperty::Version)); + REQUIRE(manifest2.Channel == index.GetPropertyByPrimaryId(manifest2RowId, PackageVersionProperty::Channel)); + REQUIRE(manifest2Path == index.GetPropertyByPrimaryId(manifest2RowId, PackageVersionProperty::RelativePath)); } TEST_CASE("SQLiteIndex_RemoveManifestFile", "[sqliteindex][V1_0]") @@ -1708,8 +1743,15 @@ TEST_CASE("SQLiteIndex_Search_MultipleMatch", "[sqliteindex]") auto results = index.Search(request); REQUIRE(results.Matches.size() == 1); - auto result = index.GetVersionKeysById(results.Matches[0].first); - REQUIRE(result.size() == 2); + if (AreVersionKeysSupported(index)) + { + auto result = index.GetVersionKeysById(results.Matches[0].first); + REQUIRE(result.size() == 2); + } + else + { + REQUIRE_THROWS_HR(index.GetVersionKeysById(results.Matches[0].first), E_NOT_VALID_STATE); + } } TEST_CASE("SQLiteIndex_Search_NoMatch", "[sqliteindex]") @@ -1780,6 +1822,11 @@ TEST_CASE("SQLiteIndex_PathString", "[sqliteindex]") TestPrepareForRead(index); + if (!AreManifestPathsSupported(index)) + { + return; + } + SearchRequest request; request.Query = RequestMatch(MatchType::Exact, manifest.Id); @@ -1807,6 +1854,11 @@ TEST_CASE("SQLiteIndex_PathlessString", "[sqliteindex]") TestPrepareForRead(index); + if (!AreManifestPathsSupported(index)) + { + return; + } + SearchRequest request; request.Query = RequestMatch(MatchType::Exact, manifest.Id); @@ -1837,10 +1889,17 @@ TEST_CASE("SQLiteIndex_Versions", "[sqliteindex]") auto results = index.Search(request); REQUIRE(results.Matches.size() == 1); - auto result = index.GetVersionKeysById(results.Matches[0].first); - REQUIRE(result.size() == 1); - REQUIRE(result[0].VersionAndChannel.GetVersion().ToString() == manifest.Version); - REQUIRE(result[0].VersionAndChannel.GetChannel().ToString() == manifest.Channel); + if (AreVersionKeysSupported(index)) + { + auto result = index.GetVersionKeysById(results.Matches[0].first); + REQUIRE(result.size() == 1); + REQUIRE(result[0].VersionAndChannel.GetVersion().ToString() == manifest.Version); + REQUIRE(result[0].VersionAndChannel.GetChannel().ToString() == manifest.Channel); + } + else + { + REQUIRE_THROWS_HR(index.GetVersionKeysById(results.Matches[0].first), E_NOT_VALID_STATE); + } } TEST_CASE("SQLiteIndex_Search_VersionSorting", "[sqliteindex]") @@ -1873,6 +1932,11 @@ TEST_CASE("SQLiteIndex_Search_VersionSorting", "[sqliteindex]") TestPrepareForRead(index); + if (!AreChannelsSupported(index)) + { + return; + } + SearchRequest request; request.Filters.emplace_back(PackageMatchField::Id, MatchType::Exact, "Id"); @@ -1923,6 +1987,11 @@ TEST_CASE("SQLiteIndex_PathString_VersionSorting", "[sqliteindex]") TestPrepareForRead(index); + if (!AreChannelsSupported(index)) + { + return; + } + SearchRequest request; request.Filters.emplace_back(PackageMatchField::Id, MatchType::Exact, "Id"); @@ -1960,6 +2029,11 @@ TEST_CASE("SQLiteIndex_PathString_CaseInsensitive", "[sqliteindex]") TestPrepareForRead(index); + if (!AreChannelsSupported(index)) + { + return; + } + SearchRequest request; request.Filters.emplace_back(PackageMatchField::Id, MatchType::Exact, "Id"); @@ -2688,7 +2762,7 @@ TEST_CASE("SQLiteIndex_GetMultiProperty_PackageFamilyName", "[sqliteindex]") auto results = index.Search(request); REQUIRE(results.Matches.size() == 1); - auto props = index.GetMultiPropertyByManifestId(results.Matches[0].first, PackageVersionMultiProperty::PackageFamilyName); + auto props = index.GetMultiPropertyByPrimaryId(results.Matches[0].first, PackageVersionMultiProperty::PackageFamilyName); if (ArePackageFamilyNameAndProductCodeSupported(index, testVersion)) { @@ -2718,7 +2792,7 @@ TEST_CASE("SQLiteIndex_GetMultiProperty_ProductCode", "[sqliteindex]") auto results = index.Search(request); REQUIRE(results.Matches.size() == 1); - auto props = index.GetMultiPropertyByManifestId(results.Matches[0].first, PackageVersionMultiProperty::ProductCode); + auto props = index.GetMultiPropertyByPrimaryId(results.Matches[0].first, PackageVersionMultiProperty::ProductCode); if (ArePackageFamilyNameAndProductCodeSupported(index, testVersion)) { @@ -2748,7 +2822,7 @@ TEST_CASE("SQLiteIndex_GetMultiProperty_Tag", "[sqliteindex]") auto results = index.Search(request); REQUIRE(results.Matches.size() == 1); - auto props = index.GetMultiPropertyByManifestId(results.Matches[0].first, PackageVersionMultiProperty::Tag); + auto props = index.GetMultiPropertyByPrimaryId(results.Matches[0].first, PackageVersionMultiProperty::Tag); REQUIRE(props.size() == 2); REQUIRE(std::find(props.begin(), props.end(), "Tag1") != props.end()); @@ -2771,7 +2845,7 @@ TEST_CASE("SQLiteIndex_GetMultiProperty_Command", "[sqliteindex]") auto results = index.Search(request); REQUIRE(results.Matches.size() == 1); - auto props = index.GetMultiPropertyByManifestId(results.Matches[0].first, PackageVersionMultiProperty::Command); + auto props = index.GetMultiPropertyByPrimaryId(results.Matches[0].first, PackageVersionMultiProperty::Command); REQUIRE(props.size() == 1); REQUIRE(props[0] == "Command"); @@ -2961,19 +3035,17 @@ TEST_CASE("SQLiteIndex_ManifestHash_Present", "[sqliteindex]") auto results = index.Search({}); REQUIRE(results.Matches.size() == 1); - auto hashResult = index.GetPropertyByManifestId(results.Matches[0].first, PackageVersionProperty::ManifestSHA256Hash); + auto hashResult = index.GetPropertyByPrimaryId(results.Matches[0].first, PackageVersionProperty::ManifestSHA256Hash); + + // Regardless of what hash, it should still be a SHA256 hash + REQUIRE(hashResult); + auto hashResultBytes = SHA256::ConvertToBytes(hashResult.value()); + REQUIRE(hash.size() == hashResultBytes.size()); if (AreManifestHashesSupported(index, testVersion)) { - REQUIRE(hashResult); - auto hashResultBytes = SHA256::ConvertToBytes(hashResult.value()); - REQUIRE(hash.size() == hashResultBytes.size()); REQUIRE(std::equal(hash.begin(), hash.end(), hashResultBytes.begin())); } - else - { - REQUIRE(!hashResult); - } } TEST_CASE("SQLiteIndex_ManifestHash_Missing", "[sqliteindex]") @@ -2993,9 +3065,12 @@ TEST_CASE("SQLiteIndex_ManifestHash_Missing", "[sqliteindex]") auto results = index.Search({}); REQUIRE(results.Matches.size() == 1); - auto hashResult = index.GetPropertyByManifestId(results.Matches[0].first, PackageVersionProperty::ManifestSHA256Hash); + auto hashResult = index.GetPropertyByPrimaryId(results.Matches[0].first, PackageVersionProperty::ManifestSHA256Hash); - REQUIRE(!hashResult); + if (AreManifestHashesSupported(index, testVersion)) + { + REQUIRE(!hashResult); + } } TEST_CASE("SQLiteIndex_ManifestArpVersion_Present_Add", "[sqliteindex]") @@ -3014,7 +3089,7 @@ TEST_CASE("SQLiteIndex_ManifestArpVersion_Present_Add", "[sqliteindex]") manifest.Installers[0].AppsAndFeaturesEntries[0].DisplayVersion = "1.0"; manifest.Installers[0].AppsAndFeaturesEntries.push_back({}); manifest.Installers[0].AppsAndFeaturesEntries[1].DisplayVersion = "1.1"; - + index.AddManifest(manifest, "path"); SQLiteVersion testVersion = TestPrepareForRead(index); @@ -3022,15 +3097,15 @@ TEST_CASE("SQLiteIndex_ManifestArpVersion_Present_Add", "[sqliteindex]") auto results = index.Search({}); REQUIRE(results.Matches.size() == 1); - auto arpMin = index.GetPropertyByManifestId(results.Matches[0].first, PackageVersionProperty::ArpMinVersion); - auto arpMax = index.GetPropertyByManifestId(results.Matches[0].first, PackageVersionProperty::ArpMaxVersion); + auto arpMin = index.GetPropertyByPrimaryId(results.Matches[0].first, PackageVersionProperty::ArpMinVersion); + auto arpMax = index.GetPropertyByPrimaryId(results.Matches[0].first, PackageVersionProperty::ArpMaxVersion); if (AreArpVersionsSupported(index, testVersion)) { REQUIRE(arpMin); - REQUIRE(arpMin.value() == "1.0"); + REQUIRE(UtilityVersion(arpMin.value()) == UtilityVersion(manifest.Installers[0].AppsAndFeaturesEntries[0].DisplayVersion)); REQUIRE(arpMax); - REQUIRE(arpMax.value() == "1.1"); + REQUIRE(UtilityVersion(arpMax.value()) == UtilityVersion(manifest.Installers[0].AppsAndFeaturesEntries[1].DisplayVersion)); } else { @@ -3067,8 +3142,8 @@ TEST_CASE("SQLiteIndex_ManifestArpVersion_Present_AddThenUpdate", "[sqliteindex] auto results = index.Search({}); REQUIRE(results.Matches.size() == 1); - auto arpMin = index.GetPropertyByManifestId(results.Matches[0].first, PackageVersionProperty::ArpMinVersion); - auto arpMax = index.GetPropertyByManifestId(results.Matches[0].first, PackageVersionProperty::ArpMaxVersion); + auto arpMin = index.GetPropertyByPrimaryId(results.Matches[0].first, PackageVersionProperty::ArpMinVersion); + auto arpMax = index.GetPropertyByPrimaryId(results.Matches[0].first, PackageVersionProperty::ArpMaxVersion); if (AreArpVersionsSupported(index, testVersion)) { @@ -3101,10 +3176,10 @@ TEST_CASE("SQLiteIndex_ManifestArpVersion_Empty", "[sqliteindex]") auto results = index.Search({}); REQUIRE(results.Matches.size() == 1); - auto arpMin = index.GetPropertyByManifestId(results.Matches[0].first, PackageVersionProperty::ArpMinVersion); - auto arpMax = index.GetPropertyByManifestId(results.Matches[0].first, PackageVersionProperty::ArpMaxVersion); + auto arpMin = index.GetPropertyByPrimaryId(results.Matches[0].first, PackageVersionProperty::ArpMinVersion); + auto arpMax = index.GetPropertyByPrimaryId(results.Matches[0].first, PackageVersionProperty::ArpMaxVersion); - if (AreArpVersionsSupported(index, testVersion)) + if (AreArpVersionsSupported(index, testVersion) && !AreArpVersionsNullable(index)) { REQUIRE(arpMin); REQUIRE(arpMin.value() == ""); @@ -3251,7 +3326,7 @@ TEST_CASE("SQLiteIndex_CheckConsistency_FindEmbeddedNull", "[sqliteindex]") REQUIRE(!index.CheckConsistency(true)); } -TEST_CASE("SQLiteIndex_MapDataFolding_Tags", "[sqliteindex][mapdatafolding][V1_7]") +TEST_CASE("SQLiteIndex_MapDataFolding_Tags", "[sqliteindex][mapdatafolding]") { TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; INFO("Using temporary file named: " << tempFile.GetPath()); @@ -3262,10 +3337,7 @@ TEST_CASE("SQLiteIndex_MapDataFolding_Tags", "[sqliteindex][mapdatafolding][V1_7 SQLiteIndex index = SearchTestSetup(tempFile, { { "Id", "Name", "Publisher", "Moniker", "Version1", "", { tag1 }, { "Command" }, "Path1", {}, { "PC1" } }, { "Id", "Name", "Publisher", "Moniker", "Version2", "", { tag2 }, { "Command" }, "Path2", {}, { "PC2" } }, - }, SQLiteVersion{ 1, 7 }); - - // Apply the map data folding if it is present in the created test index. - index.PrepareForPackaging(); + }); SQLiteVersion testVersion = TestPrepareForRead(index); @@ -3275,14 +3347,14 @@ TEST_CASE("SQLiteIndex_MapDataFolding_Tags", "[sqliteindex][mapdatafolding][V1_7 SearchRequest request2; request2.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::Tag, MatchType::Exact, tag2)); - auto results2 = index.Search(request1); + auto results2 = index.Search(request2); REQUIRE(results1.Matches.size() == 1); REQUIRE(results2.Matches.size() == 1); REQUIRE(results1.Matches[0].first == results2.Matches[0].first); } -TEST_CASE("SQLiteIndex_MapDataFolding_PFNs", "[sqliteindex][mapdatafolding][V1_7]") +TEST_CASE("SQLiteIndex_MapDataFolding_PFNs", "[sqliteindex][mapdatafolding]") { TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; INFO("Using temporary file named: " << tempFile.GetPath()); @@ -3293,20 +3365,22 @@ TEST_CASE("SQLiteIndex_MapDataFolding_PFNs", "[sqliteindex][mapdatafolding][V1_7 SQLiteIndex index = SearchTestSetup(tempFile, { { "Id", "Name", "Publisher", "Moniker", "Version1", "", { }, { "Command" }, "Path1", { pfn1 }, { } }, { "Id", "Name", "Publisher", "Moniker", "Version2", "", { }, { "Command" }, "Path2", { pfn2 }, { } }, - }, SQLiteVersion{ 1, 7 }); - - // Apply the map data folding if it is present in the created test index. - index.PrepareForPackaging(); + }); SQLiteVersion testVersion = TestPrepareForRead(index); + if (!AreChannelsSupported(index)) + { + return; + } + SearchRequest request1; request1.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::PackageFamilyName, MatchType::Exact, pfn1)); auto results1 = index.Search(request1); SearchRequest request2; request2.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::PackageFamilyName, MatchType::Exact, pfn2)); - auto results2 = index.Search(request1); + auto results2 = index.Search(request2); REQUIRE(results1.Matches.size() == 1); REQUIRE(results2.Matches.size() == 1); @@ -3318,8 +3392,8 @@ TEST_CASE("SQLiteIndex_MapDataFolding_PFNs", "[sqliteindex][mapdatafolding][V1_7 auto manifestId1 = versionKeys[0].ManifestId; auto manifestId2 = versionKeys[1].ManifestId; - auto pfnValues1 = index.GetMultiPropertyByManifestId(manifestId1, PackageVersionMultiProperty::PackageFamilyName); - auto pfnValues2 = index.GetMultiPropertyByManifestId(manifestId2, PackageVersionMultiProperty::PackageFamilyName); + auto pfnValues1 = index.GetMultiPropertyByPrimaryId(manifestId1, PackageVersionMultiProperty::PackageFamilyName); + auto pfnValues2 = index.GetMultiPropertyByPrimaryId(manifestId2, PackageVersionMultiProperty::PackageFamilyName); if (IsMapDataFoldingSupported(index, testVersion)) { @@ -3350,7 +3424,7 @@ TEST_CASE("SQLiteIndex_MapDataFolding_PFNs", "[sqliteindex][mapdatafolding][V1_7 } } -TEST_CASE("SQLiteIndex_MapDataFolding_ProductCodes", "[sqliteindex][mapdatafolding][V1_7]") +TEST_CASE("SQLiteIndex_MapDataFolding_ProductCodes", "[sqliteindex][mapdatafolding]") { TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; INFO("Using temporary file named: " << tempFile.GetPath()); @@ -3361,20 +3435,22 @@ TEST_CASE("SQLiteIndex_MapDataFolding_ProductCodes", "[sqliteindex][mapdatafoldi SQLiteIndex index = SearchTestSetup(tempFile, { { "Id", "Name", "Publisher", "Moniker", "Version1", "", { }, { "Command" }, "Path1", { }, { pc1 } }, { "Id", "Name", "Publisher", "Moniker", "Version2", "", { }, { "Command" }, "Path2", { }, { pc2 } }, - }, SQLiteVersion{ 1, 7 }); - - // Apply the map data folding if it is present in the created test index. - index.PrepareForPackaging(); + }); SQLiteVersion testVersion = TestPrepareForRead(index); + if (!AreChannelsSupported(index)) + { + return; + } + SearchRequest request1; request1.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::ProductCode, MatchType::Exact, pc1)); auto results1 = index.Search(request1); SearchRequest request2; request2.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::ProductCode, MatchType::Exact, pc2)); - auto results2 = index.Search(request1); + auto results2 = index.Search(request2); REQUIRE(results1.Matches.size() == 1); REQUIRE(results2.Matches.size() == 1); @@ -3386,8 +3462,8 @@ TEST_CASE("SQLiteIndex_MapDataFolding_ProductCodes", "[sqliteindex][mapdatafoldi auto manifestId1 = versionKeys[0].ManifestId; auto manifestId2 = versionKeys[1].ManifestId; - auto pcValues1 = index.GetMultiPropertyByManifestId(manifestId1, PackageVersionMultiProperty::ProductCode); - auto pcValues2 = index.GetMultiPropertyByManifestId(manifestId2, PackageVersionMultiProperty::ProductCode); + auto pcValues1 = index.GetMultiPropertyByPrimaryId(manifestId1, PackageVersionMultiProperty::ProductCode); + auto pcValues2 = index.GetMultiPropertyByPrimaryId(manifestId2, PackageVersionMultiProperty::ProductCode); REQUIRE(pcValues1.size() == 1); REQUIRE(pcValues2.size() == 1); @@ -3536,8 +3612,10 @@ std::filesystem::path GetOnlyChild(const std::filesystem::path& parent) return result; } -void CheckIntermediates(const std::filesystem::path& intermediatesDirectory, const std::vector<std::vector<ManifestAndPath>>& expectedIntermediatesData, std::chrono::seconds sleep = 1s) +void CheckIntermediates(const std::filesystem::path& baseDirectory, const std::vector<std::vector<ManifestAndPath>>& expectedIntermediatesData, std::chrono::seconds sleep = 1s) { + std::filesystem::path intermediatesDirectory = baseDirectory / "packages"; + size_t intermediatePackageCount = std::count_if(std::filesystem::directory_iterator{ intermediatesDirectory }, std::filesystem::directory_iterator{}, [](const auto&){ return true; }); REQUIRE(intermediatePackageCount == expectedIntermediatesData.size()); @@ -3550,12 +3628,9 @@ void CheckIntermediates(const std::filesystem::path& intermediatesDirectory, con REQUIRE(std::filesystem::exists(packageDirectory)); std::filesystem::path hashDirectory = GetOnlyChild(packageDirectory); - SHA256::HashBuffer hashBytes = SHA256::ConvertToBytes(hashDirectory.filename().u8string()); std::filesystem::path versionDataFile = GetOnlyChild(hashDirectory); std::ifstream versionDataStream{ versionDataFile, std::ios_base::in | std::ios_base::binary }; auto versionDataBytes = ReadEntireStreamAsByteArray(versionDataStream); - SHA256::HashBuffer versionDataHash = SHA256::ComputeHash(versionDataBytes); - REQUIRE(SHA256::AreEqual(hashBytes, versionDataHash)); PackageVersionDataManifest versionDataManifest; versionDataManifest.Deserialize(PackageVersionDataManifest::CreateDecompressor().Decompress(versionDataBytes)); diff --git a/src/AppInstallerCLITests/SQLiteIndexSource.cpp b/src/AppInstallerCLITests/SQLiteIndexSource.cpp @@ -12,26 +12,52 @@ using namespace AppInstaller::Repository; using namespace AppInstaller::Repository::Microsoft; using namespace AppInstaller::SQLite; +using SQLiteVersion = AppInstaller::SQLite::Version; + static std::shared_ptr<SQLiteIndexSource> SimpleTestSetup(const std::string& filePath, SourceDetails& details, Manifest& manifest, std::string& relativePath) { - SQLiteIndex index = SQLiteIndex::CreateNew(filePath, Version::Latest()); + SQLiteVersion latest1 = Version::LatestForMajor(1); + SQLiteVersion latest2 = Version::LatestForMajor(2); + + const SQLiteVersion versionToUse = GENERATE_COPY(SQLiteVersion{ latest1 }, SQLiteVersion{ latest2 }); + + SQLiteIndex index = SQLiteIndex::CreateNew(filePath, versionToUse); TestDataFile testManifest("Manifest-Good.yaml"); manifest = YamlParser::CreateFromPath(testManifest); - relativePath = testManifest.GetPath().filename().u8string(); + std::filesystem::path testManifestPath = testManifest.GetPath(); + relativePath = testManifestPath.filename().u8string(); + + TempDirectory sourceFilesDirectory{ "SQLiteIndexSource" }; + std::filesystem::path sourceFilesDirectoryPath = sourceFilesDirectory.GetPath(); + std::filesystem::create_directories(sourceFilesDirectoryPath); + std::filesystem::copy_file(testManifestPath, sourceFilesDirectoryPath / relativePath); + sourceFilesDirectory.Release(); index.AddManifest(manifest, relativePath); details.Name = "TestName"; details.Type = "TestType"; - details.Arg = testManifest.GetPath().parent_path().u8string(); + details.Arg = sourceFilesDirectoryPath.u8string(); details.Data = ""; - details.Identifier = "*SimpleTestSetup"; + details.Identifier = "SimpleTestSetup"; + + if (versionToUse.MajorVersion == 2) + { + index.SetProperty(SQLiteIndex::Property::IntermediateFileOutputPath, sourceFilesDirectoryPath.u8string()); + } + + index.PrepareForPackaging(); return std::make_shared<SQLiteIndexSource>(details, std::move(index)); } +static bool SupportsChannel(const std::shared_ptr<SQLiteIndexSource>& source) +{ + return source->GetIndex().GetVersion().MajorVersion == 1; +} + TEST_CASE("SQLiteIndexSource_Search_IdExactMatch", "[sqliteindexsource]") { TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; @@ -133,7 +159,10 @@ TEST_CASE("SQLiteIndexSource_Versions", "[sqliteindexsource]") auto result = results.Matches[0].Package->GetAvailable()[0]->GetVersionKeys(); REQUIRE(result.size() == 1); REQUIRE(result[0].Version == manifest.Version); - REQUIRE(result[0].Channel == manifest.Channel); + if (SupportsChannel(source)) + { + REQUIRE(result[0].Channel == manifest.Channel); + } } TEST_CASE("SQLiteIndexSource_GetManifest", "[sqliteindexsource]") @@ -155,21 +184,27 @@ TEST_CASE("SQLiteIndexSource_GetManifest", "[sqliteindexsource]") REQUIRE(results.Matches[0].Package->GetAvailable().size() == 1); auto package = results.Matches[0].Package->GetAvailable()[0]; - auto specificResultVersion = package->GetVersion(PackageVersionKey("", manifest.Version, manifest.Channel)); + auto specificResultVersion = package->GetVersion(PackageVersionKey("", manifest.Version, SupportsChannel(source) ? manifest.Channel : "")); REQUIRE(specificResultVersion); auto specificResult = specificResultVersion->GetManifest(); REQUIRE(specificResult.Id == manifest.Id); REQUIRE(specificResult.DefaultLocalization.Get<Localization::PackageName>() == manifest.DefaultLocalization.Get<Localization::PackageName>()); REQUIRE(specificResult.Version == manifest.Version); - REQUIRE(specificResult.Channel == manifest.Channel); - - auto latestResultVersion = package->GetVersion(PackageVersionKey("", "", manifest.Channel)); - REQUIRE(latestResultVersion); - auto latestResult = latestResultVersion->GetManifest(); - REQUIRE(latestResult.Id == manifest.Id); - REQUIRE(latestResult.DefaultLocalization.Get<Localization::PackageName>() == manifest.DefaultLocalization.Get<Localization::PackageName>()); - REQUIRE(latestResult.Version == manifest.Version); - REQUIRE(latestResult.Channel == manifest.Channel); + if (SupportsChannel(source)) + { + REQUIRE(specificResult.Channel == manifest.Channel); + } + + if (SupportsChannel(source)) + { + auto latestResultVersion = package->GetVersion(PackageVersionKey("", "", manifest.Channel)); + REQUIRE(latestResultVersion); + auto latestResult = latestResultVersion->GetManifest(); + REQUIRE(latestResult.Id == manifest.Id); + REQUIRE(latestResult.DefaultLocalization.Get<Localization::PackageName>() == manifest.DefaultLocalization.Get<Localization::PackageName>()); + REQUIRE(latestResult.Version == manifest.Version); + REQUIRE(latestResult.Channel == manifest.Channel); + } auto noResultVersion = package->GetVersion(PackageVersionKey("", "blargle", "flargle")); REQUIRE(!noResultVersion); diff --git a/src/AppInstallerCLITests/TestCommon.cpp b/src/AppInstallerCLITests/TestCommon.cpp @@ -87,16 +87,19 @@ namespace TestCommon TempFile::~TempFile() try { - switch (s_TempFileDestructorBehavior) + if (m_destructionToken) { - case TempFileDestructionBehavior::Delete: - std::filesystem::remove_all(_filepath); - break; - case TempFileDestructionBehavior::Keep: - break; - case TempFileDestructionBehavior::ShellExecuteOnFailure: - s_TempFilesOnFile.emplace_back(std::move(_filepath)); - break; + switch (s_TempFileDestructorBehavior) + { + case TempFileDestructionBehavior::Delete: + std::filesystem::remove_all(_filepath); + break; + case TempFileDestructionBehavior::Keep: + break; + case TempFileDestructionBehavior::ShellExecuteOnFailure: + s_TempFilesOnFile.emplace_back(std::move(_filepath)); + break; + } } } CATCH_LOG_RETURN() @@ -107,6 +110,11 @@ namespace TestCommon _filepath = newFilePath; } + void TempFile::Release() + { + m_destructionToken = false; + } + void TempFile::SetDestructorBehavior(TempFileDestructionBehavior behavior) { s_TempFileDestructorBehavior = behavior; diff --git a/src/AppInstallerCLITests/TestCommon.h b/src/AppInstallerCLITests/TestCommon.h @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. #pragma once +#include <AppInstallerLanguageUtilities.h> #include <AppInstallerLogging.h> #include <AppInstallerProgress.h> #include <AppxPackaging.h> @@ -36,8 +37,8 @@ namespace TestCommon TempFile(const TempFile&) = delete; TempFile& operator=(const TempFile&) = delete; - TempFile(TempFile&&) = delete; - TempFile& operator=(TempFile&&) = delete; + TempFile(TempFile&&) = default; + TempFile& operator=(TempFile&&) = default; ~TempFile(); @@ -47,6 +48,8 @@ namespace TestCommon void Rename(const std::filesystem::path& newFilePath); + void Release(); + static void SetDestructorBehavior(TempFileDestructionBehavior behavior); static void SetTestFailed(bool failed); @@ -54,6 +57,7 @@ namespace TestCommon protected: TempFile() = default; std::filesystem::path _filepath; + AppInstaller::DestructionToken m_destructionToken{ true }; }; // Use to create a temporary directory for testing. diff --git a/src/AppInstallerCLITests/UserSettings.cpp b/src/AppInstallerCLITests/UserSettings.cpp @@ -326,16 +326,17 @@ TEST_CASE("SettingAutoUpdateIntervalInMinutes", "[settings]") constexpr static auto cinq = 5min; constexpr static auto cero = 0min; - constexpr static auto threehundred = 300min; - - SECTION("Default value") + constexpr static auto threehundred = 300min; + + std::chrono::minutes defaultAutoUpdateTime{}; + { + SetSetting(Stream::PrimaryUserSettings, ""); UserSettingsTest userSettingTest; - - REQUIRE(userSettingTest.Get<Setting::AutoUpdateTimeInMinutes>() == cinq); - REQUIRE(userSettingTest.GetWarnings().size() == 0); + defaultAutoUpdateTime = userSettingTest.Get<Setting::AutoUpdateTimeInMinutes>(); } - SECTION("Valid value") + + SECTION("Valid value 0") { std::string_view json = R"({ "source": { "autoUpdateIntervalInMinutes": 0 } })"; SetSetting(Stream::PrimaryUserSettings, json); @@ -344,7 +345,7 @@ TEST_CASE("SettingAutoUpdateIntervalInMinutes", "[settings]") REQUIRE(userSettingTest.Get<Setting::AutoUpdateTimeInMinutes>() == cero); REQUIRE(userSettingTest.GetWarnings().size() == 0); } - SECTION("Valid value 0") + SECTION("Valid value 300") { std::string_view json = R"({ "source": { "autoUpdateIntervalInMinutes": 300 } })"; SetSetting(Stream::PrimaryUserSettings, json); @@ -359,7 +360,7 @@ TEST_CASE("SettingAutoUpdateIntervalInMinutes", "[settings]") SetSetting(Stream::PrimaryUserSettings, json); UserSettingsTest userSettingTest; - REQUIRE(userSettingTest.Get<Setting::AutoUpdateTimeInMinutes>() == cinq); + REQUIRE(userSettingTest.Get<Setting::AutoUpdateTimeInMinutes>() == defaultAutoUpdateTime); REQUIRE(userSettingTest.GetWarnings().size() == 1); } SECTION("Invalid type string") @@ -368,7 +369,7 @@ TEST_CASE("SettingAutoUpdateIntervalInMinutes", "[settings]") SetSetting(Stream::PrimaryUserSettings, json); UserSettingsTest userSettingTest; - REQUIRE(userSettingTest.Get<Setting::AutoUpdateTimeInMinutes>() == cinq); + REQUIRE(userSettingTest.Get<Setting::AutoUpdateTimeInMinutes>() == defaultAutoUpdateTime); REQUIRE(userSettingTest.GetWarnings().size() == 1); } SECTION("Overridden by Group Policy") diff --git a/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj b/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj @@ -400,6 +400,7 @@ <ClInclude Include="Authentication\WebAccountManagerAuthenticator.h" /> <ClInclude Include="DODownloader.h" /> <ClInclude Include="Public\winget\Authentication.h" /> + <ClInclude Include="Public\winget\FileCache.h" /> <ClInclude Include="Public\winget\FolderFileWatcher.h" /> <ClInclude Include="Public\winget\MsixManifest.h" /> <ClInclude Include="Public\winget\AdminSettings.h" /> @@ -437,7 +438,7 @@ <ClInclude Include="Public\winget\MsixManifestValidation.h" /> <ClInclude Include="Public\winget\MSStore.h" /> <ClInclude Include="Public\winget\MSStoreDownload.h"> - <ExcludedFromBuild Condition="'$(Configuration)'=='Fuzzing'">true</ExcludedFromBuild> + <ExcludedFromBuild Condition="'$(Configuration)'=='Fuzzing'">true</ExcludedFromBuild> </ClInclude> <ClInclude Include="Public\winget\NameNormalization.h" /> <ClInclude Include="Public\winget\Filesystem.h" /> @@ -466,6 +467,7 @@ <ClCompile Include="Debugging.cpp" /> <ClCompile Include="DependenciesGraph.cpp" /> <ClCompile Include="DODownloader.cpp" /> + <ClCompile Include="FileCache.cpp" /> <ClCompile Include="Filesystem.cpp" /> <ClCompile Include="FolderFileWatcher.cpp" /> <ClCompile Include="Deployment.cpp" /> @@ -493,7 +495,7 @@ <ClCompile Include="MsixManifest.cpp" /> <ClCompile Include="MSStore.cpp" /> <ClCompile Include="MSStoreDownload.cpp"> - <ExcludedFromBuild Condition="'$(Configuration)'=='Fuzzing'">true</ExcludedFromBuild> + <ExcludedFromBuild Condition="'$(Configuration)'=='Fuzzing'">true</ExcludedFromBuild> </ClCompile> <ClCompile Include="NameNormalization.cpp" /> <ClCompile Include="NetworkSettings.cpp" /> @@ -504,7 +506,7 @@ <ClCompile Include="Reboot.cpp" /> <ClCompile Include="Regex.cpp" /> <ClCompile Include="Rest.cpp"> - <ExcludedFromBuild Condition="'$(Configuration)'=='Fuzzing'">true</ExcludedFromBuild> + <ExcludedFromBuild Condition="'$(Configuration)'=='Fuzzing'">true</ExcludedFromBuild> </ClCompile> <ClCompile Include="Runtime.cpp" /> <ClCompile Include="pch.cpp"> @@ -545,4 +547,4 @@ <Error Condition="!Exists('$(SolutionDir)\packages\Microsoft.Windows.CppWinRT.2.0.230706.1\build\native\Microsoft.Windows.CppWinRT.props')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\packages\Microsoft.Windows.CppWinRT.2.0.230706.1\build\native\Microsoft.Windows.CppWinRT.props'))" /> <Error Condition="!Exists('$(SolutionDir)\packages\Microsoft.Windows.CppWinRT.2.0.230706.1\build\native\Microsoft.Windows.CppWinRT.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\packages\Microsoft.Windows.CppWinRT.2.0.230706.1\build\native\Microsoft.Windows.CppWinRT.targets'))" /> </Target> -</Project> +</Project>+ \ No newline at end of file diff --git a/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj.filters b/src/AppInstallerCommonCore/AppInstallerCommonCore.vcxproj.filters @@ -204,6 +204,9 @@ <ClInclude Include="Public\winget\MSStoreDownload.h"> <Filter>Public\winget</Filter> </ClInclude> + <ClInclude Include="Public\winget\FileCache.h"> + <Filter>Public\winget</Filter> + </ClInclude> </ItemGroup> <ItemGroup> <ClCompile Include="pch.cpp"> @@ -368,6 +371,9 @@ <ClCompile Include="MSStoreDownload.cpp"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="FileCache.cpp"> + <Filter>Source Files</Filter> + </ClCompile> </ItemGroup> <ItemGroup> <None Include="PropertySheet.props" /> diff --git a/src/AppInstallerCommonCore/ExperimentalFeature.cpp b/src/AppInstallerCommonCore/ExperimentalFeature.cpp @@ -50,6 +50,8 @@ namespace AppInstaller::Settings return userSettings.Get<Setting::EFConfigureSelfElevation>(); case ExperimentalFeature::Feature::StoreDownload: return userSettings.Get<Setting::EFStoreDownload>(); + case ExperimentalFeature::Feature::IndexV2: + return userSettings.Get<Setting::EFIndexV2>(); case ExperimentalFeature::Feature::ConfigureExport: return userSettings.Get<Setting::EFConfigureExport>(); default: @@ -91,6 +93,8 @@ namespace AppInstaller::Settings return ExperimentalFeature{ "Configure Self Elevation", "configureSelfElevate", "https://aka.ms/winget-settings", Feature::ConfigureSelfElevation }; case Feature::StoreDownload: return ExperimentalFeature{ "Store Download", "storeDownload", "https://aka.ms/winget-settings", Feature::StoreDownload }; + case Feature::IndexV2: + return ExperimentalFeature{ "Index V2", "indexV2", "https://aka.ms/winget-settings", Feature::IndexV2 }; case Feature::ConfigureExport: return ExperimentalFeature{ "Configure Export", "configureExport", "https://aka.ms/winget-settings", Feature::ConfigureExport }; default: diff --git a/src/AppInstallerCommonCore/FileCache.cpp b/src/AppInstallerCommonCore/FileCache.cpp @@ -0,0 +1,244 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "Public/winget/FileCache.h" +#include <AppInstallerDownloader.h> +#include <AppInstallerLogging.h> +#include <AppInstallerStrings.h> + +namespace AppInstaller::Caching +{ + namespace anon + { + std::string_view GetNameForType(FileCache::Type type) + { + switch (type) + { + case FileCache::Type::IndexV1_Manifest: return "V1_M"; + case FileCache::Type::IndexV2_PackageVersionData: return "V2_PVD"; + case FileCache::Type::IndexV2_Manifest: return "V2_M"; +#ifndef AICLI_DISABLE_TEST_HOOKS + case FileCache::Type::Tests: return "Tests"; +#endif + } + + THROW_HR(E_UNEXPECTED); + } + + std::unique_ptr<std::stringstream> GetUpstreamFile(const std::string& basePath, const std::string& relativePath, const Utility::SHA256::HashBuffer& expectedHash) + { + // Until signed files are implemented, fail on an empty hash + THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE, expectedHash.empty()); + + std::string fullPath = basePath; + if (fullPath.back() != '/') + { + fullPath += '/'; + } + fullPath += relativePath; + + if (Utility::IsUrlRemote(fullPath)) + { + auto result = std::make_unique<std::stringstream>(); + + AICLI_LOG(Core, Verbose, << "Getting upstream file from remote: " << fullPath); + ProgressCallback emptyCallback; + + constexpr int MaxRetryCount = 2; + constexpr std::chrono::seconds maximumWaitTimeAllowed = 10s; + for (int retryCount = 0; retryCount < MaxRetryCount; ++retryCount) + { + try + { + auto downloadHash = Utility::DownloadToStream(fullPath, *result, Utility::DownloadType::Manifest, emptyCallback, !expectedHash.empty()); + + if (!expectedHash.empty() && + (!downloadHash || !Utility::SHA256::AreEqual(expectedHash, downloadHash.value()))) + { + THROW_HR(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE); + } + + break; + } + catch (const Utility::ServiceUnavailableException& sue) + { + if (retryCount < MaxRetryCount - 1) + { + auto waitSecondsForRetry = sue.RetryAfter(); + if (waitSecondsForRetry > maximumWaitTimeAllowed) + { + throw; + } + + // TODO: Get real progress callback to allow cancelation. + auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(waitSecondsForRetry); + Sleep(static_cast<DWORD>(ms.count())); + } + else + { + throw; + } + } + catch (...) + { + if (retryCount < MaxRetryCount - 1) + { + AICLI_LOG(Core, Verbose, << "Getting upstream file failed, waiting a bit and retrying: " << fullPath); + Sleep(500); + } + else + { + throw; + } + } + } + + return result; + } + else + { + AICLI_LOG(Core, Verbose, << "Getting upstream file from local: " << fullPath); + std::ifstream fileStream{ fullPath, std::ios_base::in | std::ios_base::binary }; + std::string fileContents = Utility::ReadEntireStream(fileStream); + + auto fileContentsHash = Utility::SHA256::ComputeHash(fileContents); + + if (expectedHash.empty() || Utility::SHA256::AreEqual(expectedHash, fileContentsHash)) + { + return std::make_unique<std::stringstream>(std::move(fileContents)); + } + else + { + THROW_HR(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE); + } + } + } + } + + FileCache::Details::Details(FileCache::Type type, std::string identifier) : + Type(type), Identifier(std::move(identifier)) + { + switch (type) + { + case Type::IndexV1_Manifest: + case Type::IndexV2_PackageVersionData: + case Type::IndexV2_Manifest: +#ifndef AICLI_DISABLE_TEST_HOOKS + case Type::Tests: +#endif + BasePath = Runtime::PathName::Temp; + break; + default: + THROW_HR(E_UNEXPECTED); + } + } + + std::filesystem::path FileCache::Details::GetCachePath() const + { + std::filesystem::path result = Runtime::GetPathTo(BasePath); + result /= "cache"; + result /= anon::GetNameForType(Type); + result /= Utility::ConvertToUTF16(Identifier); + return result; + } + + FileCache::FileCache(Type type, std::string identifier, std::vector<std::string> sources) : + m_details(type, std::move(identifier)), m_sources(std::move(sources)) + { + m_cacheBase = m_details.GetCachePath(); + } + + const FileCache::Details& FileCache::GetDetails() const + { + return m_details; + } + + std::unique_ptr<std::istream> FileCache::GetFile(const std::filesystem::path& relativePath, const Utility::SHA256::HashBuffer& expectedHash) const + { + std::filesystem::path cachedFilePath = m_cacheBase / relativePath; + + // Check cache for matching file + try + { + if (std::filesystem::is_regular_file(cachedFilePath)) + { + AICLI_LOG(Core, Verbose, << "Reading cached file [" << cachedFilePath << "]"); + + std::ifstream fileStream{ cachedFilePath, std::ios_base::in | std::ios_base::binary }; + std::string fileContents = Utility::ReadEntireStream(fileStream); + + auto fileContentsHash = Utility::SHA256::ComputeHash(fileContents); + + if (Utility::SHA256::AreEqual(expectedHash, fileContentsHash)) + { + return std::make_unique<std::istringstream>(std::move(fileContents)); + } + else + { + AICLI_LOG(Core, Verbose, << "Removing cached file [" << cachedFilePath << "] due to hash mismatch; expected [" << + Utility::SHA256::ConvertToString(expectedHash) << "] but was [" << Utility::SHA256::ConvertToString(fileContentsHash) << "]"); + } + } + + std::filesystem::remove_all(cachedFilePath); + } + catch (...) + { + LOG_CAUGHT_EXCEPTION_MSG("Error while attempting to read cached file"); + } + + // Making it here means that we do not have a cached file or it needed to be updated and was removed. + auto result = GetUpstreamFile(relativePath.u8string(), expectedHash); + + // GetUpstreamFile only returns with a successfully verified hash, we just need to write the file out. + // Only log failures as caching is an optimization. + try + { + std::filesystem::create_directories(cachedFilePath.parent_path()); + + AICLI_LOG(Core, Verbose, << "Writing cached file [" << cachedFilePath << "]"); + std::ofstream fileStream{ cachedFilePath, std::ios_base::out | std::ios_base::binary | std::ios_base::trunc }; + LOG_LAST_ERROR_IF(fileStream.fail()); + fileStream << result->str() << std::flush; + LOG_LAST_ERROR_IF(fileStream.fail()); + } + catch (...) + { + LOG_CAUGHT_EXCEPTION_MSG("Error while attempting to write cached file"); + } + + return result; + } + + std::unique_ptr<std::stringstream> FileCache::GetUpstreamFile(std::string relativePath, const Utility::SHA256::HashBuffer& expectedHash) const + { + // Replace backslashes with forward slashes for HTTP requests (since local can handle them). + Utility::FindAndReplace(relativePath, "\\", "/"); + + std::exception_ptr firstException; + + for (const auto& upstream : m_sources) + { + try + { + return anon::GetUpstreamFile(upstream, relativePath, expectedHash); + } + catch(...) + { + LOG_CAUGHT_EXCEPTION_MSG("GetUpstreamFile failed on source: %hs", upstream.c_str()); + if (!firstException) + { + firstException = std::current_exception(); + } + } + } + + if (firstException) + { + std::rethrow_exception(firstException); + } + + // Somewhat arbitrary error that should only happen if no upstream sources provided. + THROW_HR(E_NOT_SET); + } +} diff --git a/src/AppInstallerCommonCore/PackageVersionDataManifest.cpp b/src/AppInstallerCommonCore/PackageVersionDataManifest.cpp @@ -4,6 +4,8 @@ #include "Public/winget/PackageVersionDataManifest.h" #include "Public/winget/Yaml.h" #include <AppInstallerErrors.h> +#include <AppInstallerLogging.h> +#include <AppInstallerStrings.h> using namespace std::string_view_literals; @@ -70,6 +72,14 @@ namespace AppInstaller::Manifest return "versionData.mszyml"sv; } + std::filesystem::path PackageVersionDataManifest::GetRelativeDirectoryPath(std::string_view packageIdentifier, std::string_view manifestHash) + { + std::filesystem::path result = "packages"; + result /= Utility::ConvertToUTF16(packageIdentifier); + result /= manifestHash.substr(0, 8); + return result; + } + Compression::Compressor PackageVersionDataManifest::CreateCompressor() { Compression::Compressor result(CompressionAlgorithm); @@ -152,6 +162,8 @@ namespace AppInstaller::Manifest void PackageVersionDataManifest::Deserialize(std::string_view input) { + AICLI_LOG_LARGE_STRING(Core, Verbose, << "PackageVersionDataManifest deserializing:", input); + YAML::Node document = YAML::Load(input); THROW_HR_IF(APPINSTALLER_CLI_ERROR_INVALID_MANIFEST, !document.IsMap()); diff --git a/src/AppInstallerCommonCore/Public/winget/ExperimentalFeature.h b/src/AppInstallerCommonCore/Public/winget/ExperimentalFeature.h @@ -29,6 +29,7 @@ namespace AppInstaller::Settings ConfigureSelfElevation = 0x10, StoreDownload = 0x20, ConfigureExport = 0x40, + IndexV2 = 0x80, Max, // This MUST always be after all experimental features // Features listed after Max will not be shown with the features command diff --git a/src/AppInstallerCommonCore/Public/winget/FileCache.h b/src/AppInstallerCommonCore/Public/winget/FileCache.h @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once +#include <AppInstallerRuntime.h> +#include <AppInstallerSHA256.h> +#include <filesystem> +#include <istream> +#include <sstream> + +namespace AppInstaller::Caching +{ + // A file cache for relatively small files (they are always full loaded into memory due to the hash enforcement). + struct FileCache + { + // The supported file cache types. + enum class Type + { + // Manifests for index V1. + IndexV1_Manifest, + // Package version data files for index V2. + IndexV2_PackageVersionData, + // Manifests for index V2. + IndexV2_Manifest, +#ifndef AICLI_DISABLE_TEST_HOOKS + // The test type. + Tests, +#endif + }; + + // Contains information about a specific file cache instance. + struct Details + { + Details(Type type, std::string identifier); + + Runtime::PathName BasePath; + Type Type; + std::string Identifier; + + // Gets the full path to the cache directory. + std::filesystem::path GetCachePath() const; + }; + + // Construct a file cache for the given instance. + FileCache(Type type, std::string identifier, std::vector<std::string> sources); + + // Gets the details for this file cache. + const Details& GetDetails() const; + + // Gets a stream containing the contents of the requested file. + // The hash must match for this function to return successfully. + std::unique_ptr<std::istream> GetFile(const std::filesystem::path& relativePath, const Utility::SHA256::HashBuffer& expectedHash) const; + + private: + // Gets a stream containing the contents of the requested file from an upstream source. + // The hash must match for this function to return successfully. + std::unique_ptr<std::stringstream> GetUpstreamFile(std::string relativePath, const Utility::SHA256::HashBuffer& expectedHash) const; + + Details m_details; + std::vector<std::string> m_sources; + std::filesystem::path m_cacheBase; + }; +} diff --git a/src/AppInstallerCommonCore/Public/winget/PackageVersionDataManifest.h b/src/AppInstallerCommonCore/Public/winget/PackageVersionDataManifest.h @@ -3,6 +3,7 @@ #pragma once #include <AppInstallerVersions.h> #include <winget/Compression.h> +#include <filesystem> namespace AppInstaller::Manifest @@ -16,6 +17,9 @@ namespace AppInstaller::Manifest // The file name to use for the compressed package version data manifest. static std::string_view VersionManifestCompressedFileName(); + // Gets the relative path to the package version manifest from the inputs. + static std::filesystem::path GetRelativeDirectoryPath(std::string_view packageIdentifier, std::string_view manifestHash); + // Creates the compressor used by the PackageVersionDataManifest. static Compression::Compressor CreateCompressor(); diff --git a/src/AppInstallerCommonCore/Public/winget/UserSettings.h b/src/AppInstallerCommonCore/Public/winget/UserSettings.h @@ -75,6 +75,7 @@ namespace AppInstaller::Settings EFSideBySide, EFConfigureSelfElevation, EFStoreDownload, + EFIndexV2, EFConfigureExport, // Telemetry TelemetryDisable, @@ -148,7 +149,7 @@ namespace AppInstaller::Settings SETTINGMAPPING_SPECIALIZATION(Setting::ProgressBarVisualStyle, std::string, VisualStyle, VisualStyle::Accent, ".visual.progressBar"sv); SETTINGMAPPING_SPECIALIZATION(Setting::AnonymizePathForDisplay, bool, bool, true, ".visual.anonymizeDisplayedPaths"sv); // Source - SETTINGMAPPING_SPECIALIZATION_POLICY(Setting::AutoUpdateTimeInMinutes, uint32_t, std::chrono::minutes, 5min, ".source.autoUpdateIntervalInMinutes"sv, ValuePolicy::SourceAutoUpdateIntervalInMinutes); + SETTINGMAPPING_SPECIALIZATION_POLICY(Setting::AutoUpdateTimeInMinutes, uint32_t, std::chrono::minutes, 15min, ".source.autoUpdateIntervalInMinutes"sv, ValuePolicy::SourceAutoUpdateIntervalInMinutes); // Experimental SETTINGMAPPING_SPECIALIZATION(Setting::EFExperimentalCmd, bool, bool, false, ".experimentalFeatures.experimentalCmd"sv); SETTINGMAPPING_SPECIALIZATION(Setting::EFExperimentalArg, bool, bool, false, ".experimentalFeatures.experimentalArg"sv); @@ -158,6 +159,7 @@ namespace AppInstaller::Settings SETTINGMAPPING_SPECIALIZATION(Setting::EFSideBySide, bool, bool, false, ".experimentalFeatures.sideBySide"sv); SETTINGMAPPING_SPECIALIZATION(Setting::EFConfigureSelfElevation, bool, bool, false, ".experimentalFeatures.configureSelfElevate"sv); SETTINGMAPPING_SPECIALIZATION(Setting::EFStoreDownload, bool, bool, false, ".experimentalFeatures.storeDownload"sv); + SETTINGMAPPING_SPECIALIZATION(Setting::EFIndexV2, bool, bool, false, ".experimentalFeatures.indexV2"sv); SETTINGMAPPING_SPECIALIZATION(Setting::EFConfigureExport, bool, bool, false, ".experimentalFeatures.configureExport"sv); // Telemetry SETTINGMAPPING_SPECIALIZATION(Setting::TelemetryDisable, bool, bool, false, ".telemetry.disable"sv); diff --git a/src/AppInstallerCommonCore/UserSettings.cpp b/src/AppInstallerCommonCore/UserSettings.cpp @@ -264,6 +264,7 @@ namespace AppInstaller::Settings WINGET_VALIDATE_PASS_THROUGH(EFSideBySide) WINGET_VALIDATE_PASS_THROUGH(EFConfigureSelfElevation) WINGET_VALIDATE_PASS_THROUGH(EFStoreDownload) + WINGET_VALIDATE_PASS_THROUGH(EFIndexV2) WINGET_VALIDATE_PASS_THROUGH(EFConfigureExport) WINGET_VALIDATE_PASS_THROUGH(AnonymizePathForDisplay) WINGET_VALIDATE_PASS_THROUGH(TelemetryDisable) diff --git a/src/AppInstallerCommonCore/pch.h b/src/AppInstallerCommonCore/pch.h @@ -1,112 +1,112 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. -#pragma once - -#define NOMINMAX -#include <Windows.h> -#include <AclAPI.h> +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once + +#define NOMINMAX +#include <Windows.h> +#include <AclAPI.h> #include <appmodel.h> -#include <sddl.h> -#include <Shlobj.h> +#include <sddl.h> +#include <Shlobj.h> #include <Shlwapi.h> -#include <wow64apiset.h> -#include <icu.h> -#include <msi.h> -#include <DbgHelp.h> -#include <SoftPub.h> -#include <WinTrust.h> -#include <wincrypt.h> - -#include "TraceLogging.h" - -#define YAML_DECLARE_STATIC -#include <yaml.h> - -// TODO: See if we can get down to having just one JSON parser... -#include <json/json.h> - -#pragma warning( push ) -#pragma warning ( disable : 4458 4100 4702 6031 ) -#include <valijson/schema.hpp> -#include <valijson/schema_parser.hpp> -#include <valijson/validator.hpp> -#include <valijson/adapters/jsoncpp_adapter.hpp> -#pragma warning( pop ) - -#include <algorithm> -#include <chrono> -#include <condition_variable> -#include <cwctype> -#include <filesystem> -#include <fstream> -#include <functional> -#include <future> -#include <iomanip> -#include <iterator> -#include <limits> -#include <memory> -#include <mutex> -#include <optional> -#include <ostream> -#include <regex> -#include <set> -#include <string> -#include <sstream> -#include <stack> -#include <string_view> -#include <type_traits> -#include <unordered_set> -#include <vector> +#include <wow64apiset.h> +#include <icu.h> +#include <msi.h> +#include <DbgHelp.h> +#include <SoftPub.h> +#include <WinTrust.h> +#include <wincrypt.h> + +#include "TraceLogging.h" + +#define YAML_DECLARE_STATIC +#include <yaml.h> + +// TODO: See if we can get down to having just one JSON parser... +#include <json/json.h> + +#pragma warning( push ) +#pragma warning ( disable : 4458 4100 4702 6031 ) +#include <valijson/schema.hpp> +#include <valijson/schema_parser.hpp> +#include <valijson/validator.hpp> +#include <valijson/adapters/jsoncpp_adapter.hpp> +#pragma warning( pop ) + +#include <algorithm> +#include <chrono> +#include <condition_variable> +#include <cwctype> +#include <filesystem> +#include <fstream> +#include <functional> +#include <future> +#include <iomanip> +#include <iterator> +#include <limits> +#include <memory> +#include <mutex> +#include <optional> +#include <ostream> +#include <regex> +#include <set> +#include <string> +#include <sstream> +#include <stack> +#include <string_view> +#include <type_traits> +#include <unordered_set> +#include <vector> #include <variant> -#ifndef WINGET_DISABLE_FOR_FUZZING -#pragma warning( push ) -#pragma warning ( disable : 26495 26439 ) -#include <cpprest/http_client.h> -#include <cpprest/json.h> -#include <cpprest/uri_builder.h> -#pragma warning( pop ) -#endif - -#pragma warning( push ) -#pragma warning ( disable : 6001 6285 6287 6340 6387 6388 26451 26495 28196 ) -#include <wil/resource.h> -#include <wil/result.h> -#include <wil/result_macros.h> -#include <wil/safecast.h> -#include <wil/token_helpers.h> -#include <wil/com.h> -#include <wil/filesystem.h> -#include <wil/win32_helpers.h> -#pragma warning( pop ) - -#include <wil/cppwinrt.h> - -#include <winrt/Windows.ApplicationModel.h> -#include <winrt/Windows.ApplicationModel.AppExtensions.h> -#include <winrt/Windows.Foundation.h> -#include <winrt/Windows.Foundation.Collections.h> -#include <winrt/Windows.Management.Deployment.h> -#include <winrt/Windows.Security.Cryptography.h> -#include <winrt/Windows.Services.Store.h> -#include <winrt/Windows.Storage.h> -#include <winrt/Windows.Storage.Streams.h> -#include <winrt/Windows.System.Profile.h> -#include <winrt/Windows.System.UserProfile.h> -#include <winrt/Windows.Web.Http.h> -#include <winrt/Windows.Web.Http.Headers.h> -#include <winrt/Windows.Web.Http.Filters.h> -#include <winrt/Windows.Globalization.h> -#include <winrt/Windows.ApplicationModel.Store.Preview.InstallControl.h> -#include <winrt/Windows.Security.Authentication.Web.Core.h> -#include <winrt/Windows.Security.Credentials.h> - -#include <wrl/client.h> -#include <wrl/implements.h> - -// Stream/buffer helper APIs -#include <robuffer.h> -#include <shcore.h> - -#include <AppxPackaging.h> -#include <WebAuthenticationCoreManagerInterop.h> +#ifndef WINGET_DISABLE_FOR_FUZZING +#pragma warning( push ) +#pragma warning ( disable : 26495 26439 ) +#include <cpprest/http_client.h> +#include <cpprest/json.h> +#include <cpprest/uri_builder.h> +#pragma warning( pop ) +#endif + +#pragma warning( push ) +#pragma warning ( disable : 6001 6285 6287 6340 6387 6388 26451 26495 28196 ) +#include <wil/resource.h> +#include <wil/result.h> +#include <wil/result_macros.h> +#include <wil/safecast.h> +#include <wil/token_helpers.h> +#include <wil/com.h> +#include <wil/filesystem.h> +#include <wil/win32_helpers.h> +#pragma warning( pop ) + +#include <wil/cppwinrt.h> + +#include <winrt/Windows.ApplicationModel.h> +#include <winrt/Windows.ApplicationModel.AppExtensions.h> +#include <winrt/Windows.Foundation.h> +#include <winrt/Windows.Foundation.Collections.h> +#include <winrt/Windows.Management.Deployment.h> +#include <winrt/Windows.Security.Cryptography.h> +#include <winrt/Windows.Services.Store.h> +#include <winrt/Windows.Storage.h> +#include <winrt/Windows.Storage.Streams.h> +#include <winrt/Windows.System.Profile.h> +#include <winrt/Windows.System.UserProfile.h> +#include <winrt/Windows.Web.Http.h> +#include <winrt/Windows.Web.Http.Headers.h> +#include <winrt/Windows.Web.Http.Filters.h> +#include <winrt/Windows.Globalization.h> +#include <winrt/Windows.ApplicationModel.Store.Preview.InstallControl.h> +#include <winrt/Windows.Security.Authentication.Web.Core.h> +#include <winrt/Windows.Security.Credentials.h> + +#include <wrl/client.h> +#include <wrl/implements.h> + +// Stream/buffer helper APIs +#include <robuffer.h> +#include <shcore.h> + +#include <AppxPackaging.h> +#include <WebAuthenticationCoreManagerInterop.h> diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj @@ -412,6 +412,8 @@ <ClInclude Include="Microsoft\SQLiteIndex.h" /> <ClInclude Include="Microsoft\SQLiteIndexSource.h" /> <ClInclude Include="Microsoft\ConfigurableTestSourceFactory.h" /> + <ClInclude Include="Microsoft\SQLiteIndexSourceV1.h" /> + <ClInclude Include="Microsoft\SQLiteIndexSourceV2.h" /> <ClInclude Include="PackageDependenciesValidation.h" /> <ClInclude Include="PackageTrackingCatalogSourceFactory.h" /> <ClInclude Include="pch.h" /> @@ -511,6 +513,8 @@ <ClCompile Include="Microsoft\Schema\Checkpoint_1_0\CheckpointTable.cpp" /> <ClCompile Include="Microsoft\SQLiteIndex.cpp" /> <ClCompile Include="Microsoft\SQLiteIndexSource.cpp" /> + <ClCompile Include="Microsoft\SQLiteIndexSourceV1.cpp" /> + <ClCompile Include="Microsoft\SQLiteIndexSourceV2.cpp" /> <ClCompile Include="PackageDependenciesValidation.cpp" /> <ClCompile Include="PackageInstalledStatus.cpp" /> <ClCompile Include="PackageTrackingCatalog.cpp" /> @@ -566,4 +570,4 @@ <Error Condition="!Exists('$(SolutionDir)\packages\Microsoft.Windows.CppWinRT.2.0.230706.1\build\native\Microsoft.Windows.CppWinRT.props')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\packages\Microsoft.Windows.CppWinRT.2.0.230706.1\build\native\Microsoft.Windows.CppWinRT.props'))" /> <Error Condition="!Exists('$(SolutionDir)\packages\Microsoft.Windows.CppWinRT.2.0.230706.1\build\native\Microsoft.Windows.CppWinRT.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\packages\Microsoft.Windows.CppWinRT.2.0.230706.1\build\native\Microsoft.Windows.CppWinRT.targets'))" /> </Target> -</Project> +</Project>+ \ No newline at end of file diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters @@ -456,6 +456,12 @@ <ClInclude Include="Microsoft\Schema\SQLiteIndexContextData.h"> <Filter>Microsoft\Schema</Filter> </ClInclude> + <ClInclude Include="Microsoft\SQLiteIndexSourceV1.h"> + <Filter>Microsoft</Filter> + </ClInclude> + <ClInclude Include="Microsoft\SQLiteIndexSourceV2.h"> + <Filter>Microsoft</Filter> + </ClInclude> </ItemGroup> <ItemGroup> <ClCompile Include="pch.cpp"> @@ -710,6 +716,12 @@ <ClCompile Include="Microsoft\Schema\2_0\PackageUpdateTrackingTable.cpp"> <Filter>Microsoft\Schema\2_0</Filter> </ClCompile> + <ClCompile Include="Microsoft\SQLiteIndexSourceV1.cpp"> + <Filter>Microsoft</Filter> + </ClCompile> + <ClCompile Include="Microsoft\SQLiteIndexSourceV2.cpp"> + <Filter>Microsoft</Filter> + </ClCompile> </ItemGroup> <ItemGroup> <None Include="PropertySheet.props" /> diff --git a/src/AppInstallerRepositoryCore/ArpVersionValidation.cpp b/src/AppInstallerRepositoryCore/ArpVersionValidation.cpp @@ -24,8 +24,8 @@ namespace AppInstaller::Repository continue; } - auto arpMinVersion = index->GetPropertyByManifestId(versionKey.ManifestId, PackageVersionProperty::ArpMinVersion).value_or(""); - auto arpMaxVersion = index->GetPropertyByManifestId(versionKey.ManifestId, PackageVersionProperty::ArpMaxVersion).value_or(""); + auto arpMinVersion = index->GetPropertyByPrimaryId(versionKey.ManifestId, PackageVersionProperty::ArpMinVersion).value_or(""); + auto arpMaxVersion = index->GetPropertyByPrimaryId(versionKey.ManifestId, PackageVersionProperty::ArpMaxVersion).value_or(""); // Either both empty or both not empty THROW_HR_IF(E_UNEXPECTED, arpMinVersion.empty() != arpMaxVersion.empty()); @@ -86,4 +86,4 @@ namespace AppInstaller::Repository APPINSTALLER_CLI_ERROR_DEPENDENCIES_VALIDATION_FAILED)); } } -}- \ No newline at end of file +} diff --git a/src/AppInstallerRepositoryCore/CompositeSource.cpp b/src/AppInstallerRepositoryCore/CompositeSource.cpp @@ -144,6 +144,20 @@ namespace AppInstaller::Repository // for the default (first) available package. If we ever need to consider other sources, this function needs to be revisited. std::string GetMappedInstalledVersion(const std::string& installedVersion, const std::shared_ptr<IPackage>& availablePackage) { + // Perform an initial check to see if the latest version has a mapping; if it does not, do not attempt any more. + auto latestVersion = availablePackage->GetLatestVersion(); + if (latestVersion) + { + auto version = latestVersion->GetProperty(PackageVersionProperty::Version); + auto arpMinVersion = latestVersion->GetProperty(PackageVersionProperty::ArpMinVersion); + auto arpMaxVersion = latestVersion->GetProperty(PackageVersionProperty::ArpMaxVersion); + + if ((arpMinVersion.empty() || arpMinVersion == version) && (arpMaxVersion.empty() || arpMaxVersion == version)) + { + return installedVersion; + } + } + // Stores raw versions value strings to run a preliminary check whether version mapping is needed. std::vector<std::tuple<std::string, std::string, std::string>> rawVersionValues; auto versionKeys = availablePackage->GetVersionKeys(); diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp @@ -11,6 +11,7 @@ #include <AppInstallerDownloader.h> #include <AppInstallerMsixInfo.h> #include <winget/ManagedFile.h> +#include <winget/ExperimentalFeature.h> using namespace std::string_literals; using namespace std::string_view_literals; @@ -20,6 +21,7 @@ namespace AppInstaller::Repository::Microsoft namespace { static constexpr std::string_view s_PreIndexedPackageSourceFactory_PackageFileName = "source.msix"sv; + static constexpr std::string_view s_PreIndexedPackageSourceFactory_V2_PackageFileName = "source2.msix"sv; static constexpr std::string_view s_PreIndexedPackageSourceFactory_PackageVersionHeader = "x-ms-meta-sourceversion"sv; static constexpr std::string_view s_PreIndexedPackageSourceFactory_IndexFileName = "index.db"sv; // TODO: This being hard coded to force using the Public directory name is not ideal. @@ -38,17 +40,31 @@ namespace AppInstaller::Repository::Microsoft return result; } - std::string GetPrimaryPackageLocation(const SourceDetails& details, std::string_view fileName) + // Gets the set of package locations that should be tried, in order. + std::vector<std::string> GetPackageLocations(const SourceDetails& details) { THROW_HR_IF(E_INVALIDARG, details.Arg.empty()); - return GetPackageLocation(details.Arg, fileName); - } - std::string GetAlternatePackageLocation(const SourceDetails& details, std::string_view fileName) - { - return (details.AlternateArg.empty() ? - std::string{} : - GetPackageLocation(details.AlternateArg, fileName)); + std::vector<std::string> result; + + if (Settings::ExperimentalFeature::IsEnabled(Settings::ExperimentalFeature::Feature::IndexV2)) + { + result.emplace_back(GetPackageLocation(details.Arg, s_PreIndexedPackageSourceFactory_V2_PackageFileName)); + } + + result.emplace_back(GetPackageLocation(details.Arg, s_PreIndexedPackageSourceFactory_PackageFileName)); + + if (!details.AlternateArg.empty()) + { + if (Settings::ExperimentalFeature::IsEnabled(Settings::ExperimentalFeature::Feature::IndexV2)) + { + result.emplace_back(GetPackageLocation(details.AlternateArg, s_PreIndexedPackageSourceFactory_V2_PackageFileName)); + } + + result.emplace_back(GetPackageLocation(details.AlternateArg, s_PreIndexedPackageSourceFactory_PackageFileName)); + } + + return result; } // Abstracts the fallback for package location when the MsixInfo is needed. @@ -57,44 +73,34 @@ namespace AppInstaller::Repository::Microsoft template <typename LocationCheck> PreIndexedPackageInfo(const SourceDetails& details, LocationCheck&& locationCheck) { - // Get both locations to force the alternate location check - m_packageLocation = GetPrimaryPackageLocation(details, s_PreIndexedPackageSourceFactory_PackageFileName); - locationCheck(m_packageLocation); + std::vector<std::string> potentialLocations = GetPackageLocations(details); - std::string alternateLocation = GetAlternatePackageLocation(details, s_PreIndexedPackageSourceFactory_PackageFileName); - if (!alternateLocation.empty()) + for (const auto& location : potentialLocations) { - locationCheck(alternateLocation); + locationCheck(location); } - // Try getting the primary location's info - HRESULT primaryHR = S_OK; + std::exception_ptr primaryException; - try + for (const auto& location : potentialLocations) { - m_msixInfo = std::make_unique<Msix::MsixInfo>(m_packageLocation); - return; - } - catch (...) - { - if (alternateLocation.empty()) + try { - throw; + m_msixInfo = std::make_unique<Msix::MsixInfo>(location); + m_packageLocation = location; + return; + } + catch (...) + { + LOG_CAUGHT_EXCEPTION_MSG("PreIndexedPackageInfo failed on location: %hs", location.c_str()); + if (!primaryException) + { + primaryException = std::current_exception(); + } } - primaryHR = LOG_CAUGHT_EXCEPTION_MSG("PreIndexedPackageInfo failed on primary location"); - } - - // Try alternate location - m_packageLocation = std::move(alternateLocation); - - try - { - m_msixInfo = std::make_unique<Msix::MsixInfo>(m_packageLocation); - return; } - CATCH_LOG_MSG("PreIndexedPackageInfo failed on alternate location"); - THROW_HR(primaryHR); + std::rethrow_exception(primaryException); } const std::string& PackageLocation() const { return m_packageLocation; } @@ -110,36 +116,27 @@ namespace AppInstaller::Repository::Microsoft { PreIndexedPackageUpdateCheck(const SourceDetails& details) { - m_packageLocation = GetPrimaryPackageLocation(details, s_PreIndexedPackageSourceFactory_PackageFileName); - std::string alternateLocation = GetAlternatePackageLocation(details, s_PreIndexedPackageSourceFactory_PackageFileName); + std::vector<std::string> potentialLocations = GetPackageLocations(details); - // Try getting the primary location's info std::exception_ptr primaryException; - try - { - m_availableVersion = GetAvailableVersionFrom(m_packageLocation); - return; - } - catch (...) + for (const auto& location : potentialLocations) { - if (alternateLocation.empty()) + try { - throw; + m_availableVersion = GetAvailableVersionFrom(location); + m_packageLocation = location; + return; + } + catch (...) + { + LOG_CAUGHT_EXCEPTION_MSG("PreIndexedPackageUpdateCheck failed on location: %hs", location.c_str()); + if (!primaryException) + { + primaryException = std::current_exception(); + } } - LOG_CAUGHT_EXCEPTION_MSG("PreIndexedPackageUpdateCheck failed on primary location"); - primaryException = std::current_exception(); - } - - // Try alternate location - m_packageLocation = std::move(alternateLocation); - - try - { - m_availableVersion = GetAvailableVersionFrom(m_packageLocation); - return; } - CATCH_LOG_MSG("PreIndexedPackageUpdateCheck failed on alternate location"); std::rethrow_exception(primaryException); } diff --git a/src/AppInstallerRepositoryCore/Microsoft/PredefinedInstalledSourceFactory.cpp b/src/AppInstallerRepositoryCore/Microsoft/PredefinedInstalledSourceFactory.cpp @@ -54,7 +54,7 @@ namespace AppInstaller::Repository::Microsoft return std::nullopt; } - return cacheData.GetPropertyByManifestId(versionKey->ManifestId, PackageVersionProperty::Name); + return cacheData.GetPropertyByPrimaryId(versionKey->ManifestId, PackageVersionProperty::Name); } // Populates the index with the entries from MSIX. diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp @@ -224,16 +224,16 @@ namespace AppInstaller::Repository::Microsoft return m_interface->Search(m_dbconn, request); } - std::optional<std::string> SQLiteIndex::GetPropertyByManifestId(IdType manifestId, PackageVersionProperty property) const + std::optional<std::string> SQLiteIndex::GetPropertyByPrimaryId(IdType primaryId, PackageVersionProperty property) const { std::lock_guard<std::mutex> lockInterface{ *m_interfaceLock }; - return m_interface->GetPropertyByManifestId(m_dbconn, manifestId, property); + return m_interface->GetPropertyByPrimaryId(m_dbconn, primaryId, property); } - std::vector<std::string> SQLiteIndex::GetMultiPropertyByManifestId(IdType manifestId, PackageVersionMultiProperty property) const + std::vector<std::string> SQLiteIndex::GetMultiPropertyByPrimaryId(IdType primaryId, PackageVersionMultiProperty property) const { std::lock_guard<std::mutex> lockInterface{ *m_interfaceLock }; - return m_interface->GetMultiPropertyByManifestId(m_dbconn, manifestId, property); + return m_interface->GetMultiPropertyByPrimaryId(m_dbconn, primaryId, property); } std::optional<SQLiteIndex::IdType> SQLiteIndex::GetManifestIdByKey(IdType id, std::string_view version, std::string_view channel) const diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h @@ -116,11 +116,11 @@ namespace AppInstaller::Repository::Microsoft // Performs a search based on the given criteria. SearchResult Search(const SearchRequest& request) const; - // Gets the string for the given property and manifest id, if present. - std::optional<std::string> GetPropertyByManifestId(IdType manifestId, PackageVersionProperty property) const; + // Gets the string for the given property and primary id, if present. + std::optional<std::string> GetPropertyByPrimaryId(IdType primaryId, PackageVersionProperty property) const; - // Gets the string values for the given property and manifest id, if present. - std::vector<std::string> GetMultiPropertyByManifestId(IdType manifestId, PackageVersionMultiProperty property) const; + // Gets the string values for the given property and primary id, if present. + std::vector<std::string> GetMultiPropertyByPrimaryId(IdType primaryId, PackageVersionMultiProperty property) const; // Gets the manifest id for the given { id, version, channel }, if present. // If version is empty, gets the value for the 'latest' version. diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSource.cpp b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSource.cpp @@ -2,404 +2,28 @@ // Licensed under the MIT License. #include "pch.h" #include "Microsoft/SQLiteIndexSource.h" +#include "Microsoft/SQLiteIndexSourceV1.h" +#include "Microsoft/SQLiteIndexSourceV2.h" #include "Microsoft/PreIndexedPackageSourceFactory.h" #include <winget/ManifestYamlParser.h> - +#include <winget/PackageVersionDataManifest.h> using namespace AppInstaller::Utility; namespace AppInstaller::Repository::Microsoft { - namespace + namespace details { - // The base for the package objects. - struct SourceReference - { - SourceReference(const std::shared_ptr<SQLiteIndexSource>& source) : - m_source(source) {} - - protected: - std::shared_ptr<SQLiteIndexSource> GetReferenceSource() const - { - std::shared_ptr<SQLiteIndexSource> source = m_source.lock(); - THROW_HR_IF(E_NOT_VALID_STATE, !source); - return source; - } - - private: - std::weak_ptr<SQLiteIndexSource> m_source; - }; + SourceReference::SourceReference(const std::shared_ptr<SQLiteIndexSource>& source) : + m_source(source) {} - // The IPackageVersion impl for SQLiteIndexSource. - struct PackageVersion : public SourceReference, public IPackageVersion + std::shared_ptr<SQLiteIndexSource> SourceReference::GetReferenceSource() const { - PackageVersion(const std::shared_ptr<SQLiteIndexSource>& source, SQLiteIndex::IdType manifestId) : - SourceReference(source), m_manifestId(manifestId) {} - - // Inherited via IPackageVersion - Utility::LocIndString GetProperty(PackageVersionProperty property) const override - { - switch (property) - { - case PackageVersionProperty::SourceIdentifier: - return LocIndString{ GetReferenceSource()->GetIdentifier() }; - case PackageVersionProperty::SourceName: - return LocIndString{ GetReferenceSource()->GetDetails().Name }; - default: - // Values coming from the index will always be localized/independent. - std::optional<std::string> optValue = GetReferenceSource()->GetIndex().GetPropertyByManifestId(m_manifestId, property); - return LocIndString{ optValue ? optValue.value() : std::string{} }; - } - } - - std::vector<Utility::LocIndString> GetMultiProperty(PackageVersionMultiProperty property) const override - { - std::vector<Utility::LocIndString> result; - - for (auto&& value : GetReferenceSource()->GetIndex().GetMultiPropertyByManifestId(m_manifestId, property)) - { - // Values coming from the index will always be localized/independent. - result.emplace_back(std::move(value)); - } - - return result; - } - - Manifest::Manifest GetManifest() override - { - std::shared_ptr<SQLiteIndexSource> source = GetReferenceSource(); - - std::optional<std::string> relativePathOpt = source->GetIndex().GetPropertyByManifestId(m_manifestId, PackageVersionProperty::RelativePath); - THROW_HR_IF(E_NOT_SET, !relativePathOpt); - - std::optional<std::string> manifestHashString = source->GetIndex().GetPropertyByManifestId(m_manifestId, PackageVersionProperty::ManifestSHA256Hash); - THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE, source->RequireManifestHash() && !manifestHashString); - - SHA256::HashBuffer manifestSHA256; - if (manifestHashString) - { - manifestSHA256 = SHA256::ConvertToBytes(manifestHashString.value()); - } - - // Try the primary location - HRESULT primaryHR = S_OK; - try - { - return GetManifestFromArgAndRelativePath(source->GetDetails().Arg, relativePathOpt.value(), manifestSHA256); - } - catch (...) - { - if (source->GetDetails().AlternateArg.empty()) - { - throw; - } - primaryHR = LOG_CAUGHT_EXCEPTION_MSG("GetManifest failed on primary location"); - } - - // Try alternate location - try - { - return GetManifestFromArgAndRelativePath(source->GetDetails().AlternateArg, relativePathOpt.value(), manifestSHA256); - } - CATCH_LOG_MSG("GetManifest failed on alternate location"); - - THROW_HR(primaryHR); - } - - Source GetSource() const override - { - return Source{ GetReferenceSource() }; - } - - IPackageVersion::Metadata GetMetadata() const override - { - auto metadata = GetReferenceSource()->GetIndex().GetMetadataByManifestId(m_manifestId); - - IPackageVersion::Metadata result; - for (auto&& data : metadata) - { - result.emplace(std::move(data)); - } - - return result; - } - - private: - static Manifest::Manifest GetManifestFromArgAndRelativePath(const std::string& arg, const std::string& relativePath, const SHA256::HashBuffer& expectedHash) - { - std::string fullPath = arg; - if (fullPath.back() != '/') - { - fullPath += '/'; - } - fullPath += relativePath; - - if (Utility::IsUrlRemote(fullPath)) - { - std::ostringstream manifestStream; - - AICLI_LOG(Repo, Info, << "Downloading manifest"); - ProgressCallback emptyCallback; - - constexpr int MaxRetryCount = 2; - constexpr std::chrono::seconds maximumWaitTimeAllowed = 10s; - for (int retryCount = 0; retryCount < MaxRetryCount; ++retryCount) - { - try - { - auto downloadHash = Utility::DownloadToStream(fullPath, manifestStream, Utility::DownloadType::Manifest, emptyCallback, !expectedHash.empty()); - - if (!expectedHash.empty() && - (!downloadHash || downloadHash->size() != expectedHash.size() || !std::equal(expectedHash.begin(), expectedHash.end(), downloadHash->begin()))) - { - THROW_HR(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE); - } - - break; - } - catch (const ServiceUnavailableException& sue) - { - if (retryCount < MaxRetryCount - 1) - { - auto waitSecondsForRetry = sue.RetryAfter(); - if (waitSecondsForRetry > maximumWaitTimeAllowed) - { - throw; - } - - // TODO: Get real progress callback to allow cancelation. - auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(waitSecondsForRetry); - Sleep(static_cast<DWORD>(ms.count())); - } - else - { - throw; - } - } - catch (...) - { - if (retryCount < MaxRetryCount - 1) - { - AICLI_LOG(Repo, Info, << "Downloading manifest failed, waiting a bit and retrying: " << fullPath); - Sleep(500); - } - else - { - throw; - } - } - } - - std::string manifestContents = manifestStream.str(); - AICLI_LOG(Repo, Verbose, << "Manifest contents: " << manifestContents); - - return Manifest::YamlParser::Create(manifestContents); - } - else - { - AICLI_LOG(Repo, Info, << "Opening manifest from local file: " << fullPath); - Manifest::Manifest result = Manifest::YamlParser::CreateFromPath(fullPath); - - if (!expectedHash.empty() && - (result.StreamSha256.size() != expectedHash.size() || !std::equal(expectedHash.begin(), expectedHash.end(), result.StreamSha256.begin()))) - { - THROW_HR(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE); - } - - return result; - } - } - - SQLiteIndex::IdType m_manifestId; - }; - - // The IPackage implementation here. - struct SQLitePackage : public std::enable_shared_from_this<SQLitePackage>, public SourceReference, public IPackage, public ICompositePackage - { - static constexpr IPackageType PackageType = IPackageType::SQLitePackage; - - SQLitePackage(const std::shared_ptr<SQLiteIndexSource>& source, SQLiteIndex::IdType idId, bool isInstalled) : - SourceReference(source), m_idId(idId), m_isInstalled(isInstalled) {} - - // Inherited via IPackage - Utility::LocIndString GetProperty(PackageProperty property) const - { - Utility::LocIndString result; - - std::shared_ptr<IPackageVersion> truth = GetLatestVersion(); - if (truth) - { - switch (property) - { - case PackageProperty::Id: - return truth->GetProperty(PackageVersionProperty::Id); - case PackageProperty::Name: - return truth->GetProperty(PackageVersionProperty::Name); - default: - THROW_HR(E_UNEXPECTED); - } - } - else - { - AICLI_LOG(Repo, Verbose, << "SQLitePackage: No manifest was found for the package with id# '" << m_idId << "'"); - } - - return result; - } - - std::vector<PackageVersionKey> GetVersionKeys() const override - { - std::shared_ptr<SQLiteIndexSource> source = GetReferenceSource(); - - { - auto sharedLock = m_versionKeysLock.lock_shared(); - - if (!m_versionKeys.empty()) - { - return m_versionKeys; - } - } - - auto exclusiveLock = m_versionKeysLock.lock_exclusive(); - - if (!m_versionKeys.empty()) - { - return m_versionKeys; - } - - std::vector<SQLiteIndex::VersionKey> versions = source->GetIndex().GetVersionKeysById(m_idId); - - for (const auto& vk : versions) - { - std::string version = vk.VersionAndChannel.GetVersion().ToString(); - std::string channel = vk.VersionAndChannel.GetChannel().ToString(); - m_versionKeys.emplace_back(source->GetIdentifier(), version, channel); - m_versionKeysMap.emplace(MapKey{ std::move(version), std::move(channel) }, vk.ManifestId); - } - - return m_versionKeys; - } - - std::shared_ptr<IPackageVersion> GetLatestVersion() const override - { - std::shared_ptr<SQLiteIndexSource> source = GetReferenceSource(); - std::optional<SQLiteIndex::IdType> manifestId = source->GetIndex().GetManifestIdByKey(m_idId, {}, {}); - - if (manifestId) - { - return std::make_shared<PackageVersion>(source, manifestId.value()); - } - - return {}; - } - - std::shared_ptr<IPackageVersion> GetVersion(const PackageVersionKey& versionKey) const override - { - std::shared_ptr<SQLiteIndexSource> source = GetReferenceSource(); - - // Ensure that this key targets this (or any) source - if (!versionKey.SourceId.empty() && versionKey.SourceId != source->GetIdentifier()) - { - return {}; - } - - std::optional<SQLiteIndex::IdType> manifestId; - - { - MapKey requested{ versionKey.Version, versionKey.Channel }; - auto sharedLock = m_versionKeysLock.lock_shared(); - - auto itr = m_versionKeysMap.find(requested); - if (itr != m_versionKeysMap.end()) - { - manifestId = itr->second; - } - } - - if (!manifestId) - { - manifestId = source->GetIndex().GetManifestIdByKey(m_idId, versionKey.Version, versionKey.Channel); - } - - if (manifestId) - { - return std::make_shared<PackageVersion>(source, manifestId.value()); - } - - return {}; - } - - Source GetSource() const override - { - return Source{ GetReferenceSource() }; - } - - bool IsSame(const IPackage* other) const override - { - const SQLitePackage* otherSQLite = PackageCast<const SQLitePackage*>(other); - - if (otherSQLite) - { - return GetReferenceSource()->IsSame(otherSQLite->GetReferenceSource().get()) && m_idId == otherSQLite->m_idId; - } - - return false; - } - - const void* CastTo(IPackageType type) const override - { - if (type == PackageType) - { - return this; - } - - return nullptr; - } - - // Inherited via ICompositePackage - std::shared_ptr<IPackage> GetInstalled() override - { - return m_isInstalled ? shared_from_this() : std::shared_ptr<IPackage>{}; - } - - std::vector<std::shared_ptr<IPackage>> GetAvailable() override - { - return m_isInstalled ? std::vector<std::shared_ptr<IPackage>>{} : std::vector<std::shared_ptr<IPackage>>{ shared_from_this() }; - } - - private: - // Contains the information needed to map a version key to it's rows. - struct MapKey - { - Utility::NormalizedString Version; - Utility::NormalizedString Channel; - - bool operator<(const MapKey& other) const - { - if (Version < other.Version) - { - return true; - } - else if (Version == other.Version) - { - return Channel < other.Channel; - } - else - { - return false; - } - } - }; - - SQLiteIndex::IdType m_idId; - bool m_isInstalled; - - // To avoid removing const from the interface - mutable wil::srwlock m_versionKeysLock; - mutable std::vector<PackageVersionKey> m_versionKeys; - mutable std::map<MapKey, SQLiteIndex::IdType> m_versionKeysMap; - }; + std::shared_ptr<SQLiteIndexSource> source = m_source.lock(); + THROW_HR_IF(E_NOT_VALID_STATE, !source); + return source; + } } SQLiteIndexSource::SQLiteIndexSource( @@ -409,6 +33,25 @@ namespace AppInstaller::Repository::Microsoft bool requireManifestHash) : m_details(details), m_isInstalled(isInstalledSource), m_index(std::move(index)), m_requireManifestHash(requireManifestHash) { + std::vector<std::string> cacheSources; + cacheSources.push_back(m_details.Arg); + if (!m_details.AlternateArg.empty()) + { + cacheSources.push_back(m_details.AlternateArg); + } + + switch (m_index.GetVersion().MajorVersion) + { + case 1: + m_manifestCache = std::make_shared<Caching::FileCache>(Caching::FileCache::Type::IndexV1_Manifest, m_details.Identifier, std::move(cacheSources)); + break; + case 2: + m_manifestCache = std::make_shared<Caching::FileCache>(Caching::FileCache::Type::IndexV2_Manifest, m_details.Identifier, cacheSources); + m_packageVersionDataCache = std::make_shared<Caching::FileCache>(Caching::FileCache::Type::IndexV2_PackageVersionData, m_details.Identifier, std::move(cacheSources)); + break; + default: + THROW_WIN32(ERROR_NOT_SUPPORTED); + } } const SourceDetails& SQLiteIndexSource::GetDetails() const @@ -427,12 +70,29 @@ namespace AppInstaller::Repository::Microsoft SearchResult result; std::shared_ptr<SQLiteIndexSource> sharedThis = NonConstSharedFromThis(); + uint32_t majorVersion = m_index.GetVersion().MajorVersion; + for (auto& indexResult : indexResults.Matches) { + std::shared_ptr<ICompositePackage> package; + + switch (majorVersion) + { + case 1: + package = std::make_shared<details::V1::SQLitePackage>(sharedThis, indexResult.first, m_manifestCache, m_isInstalled); + break; + case 2: + package = std::make_shared<details::V2::SQLitePackage>(sharedThis, indexResult.first, m_manifestCache, m_packageVersionDataCache, m_isInstalled); + break; + default: + THROW_WIN32(ERROR_NOT_SUPPORTED); + } + result.Matches.emplace_back( - std::make_shared<SQLitePackage>(sharedThis, indexResult.first, m_isInstalled), + std::move(package), std::move(indexResult.second)); } + result.Truncated = indexResults.Truncated; return result; } diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSource.h b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSource.h @@ -3,7 +3,7 @@ #pragma once #include "Microsoft/SQLiteIndex.h" #include "ISource.h" - +#include <winget/FileCache.h> #include <memory> @@ -57,6 +57,8 @@ namespace AppInstaller::Repository::Microsoft SourceDetails m_details; bool m_requireManifestHash; bool m_isInstalled; + std::shared_ptr<Caching::FileCache> m_manifestCache; + std::shared_ptr<Caching::FileCache> m_packageVersionDataCache; protected: SQLiteIndex m_index; @@ -79,4 +81,19 @@ namespace AppInstaller::Repository::Microsoft // Removes a package version from the source. void RemovePackageVersion(const Manifest::Manifest& manifest, const std::filesystem::path& relativePath); }; + + namespace details + { + // For the IPackage(Version) implementations that need to hold a weak reference to a SQLiteIndexSource. + struct SourceReference + { + SourceReference(const std::shared_ptr<SQLiteIndexSource>& source); + + protected: + std::shared_ptr<SQLiteIndexSource> GetReferenceSource() const; + + private: + std::weak_ptr<SQLiteIndexSource> m_source; + }; + } } diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSourceV1.cpp b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSourceV1.cpp @@ -0,0 +1,252 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "Microsoft/SQLiteIndexSourceV1.h" +#include <winget/ManifestYamlParser.h> + +using namespace AppInstaller::Utility; + + +namespace AppInstaller::Repository::Microsoft::details::V1 +{ + // The IPackageVersion implementation for V1 index. + struct PackageVersion : public SourceReference, public IPackageVersion + { + PackageVersion(const std::shared_ptr<SQLiteIndexSource>& source, SQLiteIndex::IdType manifestId, const std::shared_ptr<Caching::FileCache>& manifestCache) : + SourceReference(source), m_manifestId(manifestId), m_manifestCache(manifestCache) {} + + // Inherited via IPackageVersion + LocIndString GetProperty(PackageVersionProperty property) const override + { + switch (property) + { + case PackageVersionProperty::SourceIdentifier: + return LocIndString{ GetReferenceSource()->GetIdentifier() }; + case PackageVersionProperty::SourceName: + return LocIndString{ GetReferenceSource()->GetDetails().Name }; + default: + // Values coming from the index will always be localized/independent. + std::optional<std::string> optValue = GetReferenceSource()->GetIndex().GetPropertyByPrimaryId(m_manifestId, property); + return LocIndString{ optValue ? optValue.value() : std::string{} }; + } + } + + std::vector<LocIndString> GetMultiProperty(PackageVersionMultiProperty property) const override + { + std::vector<LocIndString> result; + + for (auto&& value : GetReferenceSource()->GetIndex().GetMultiPropertyByPrimaryId(m_manifestId, property)) + { + // Values coming from the index will always be localized/independent. + result.emplace_back(std::move(value)); + } + + return result; + } + + Manifest::Manifest GetManifest() override + { + std::shared_ptr<SQLiteIndexSource> source = GetReferenceSource(); + + std::optional<std::string> relativePathOpt = source->GetIndex().GetPropertyByPrimaryId(m_manifestId, PackageVersionProperty::RelativePath); + THROW_HR_IF(E_NOT_SET, !relativePathOpt); + + std::optional<std::string> manifestHashString = source->GetIndex().GetPropertyByPrimaryId(m_manifestId, PackageVersionProperty::ManifestSHA256Hash); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE, source->RequireManifestHash() && !manifestHashString); + + SHA256::HashBuffer manifestSHA256; + if (manifestHashString) + { + manifestSHA256 = SHA256::ConvertToBytes(manifestHashString.value()); + } + + std::unique_ptr<std::istream> manifestStream = m_manifestCache->GetFile(ConvertToUTF16(relativePathOpt.value()), manifestSHA256); + return Manifest::YamlParser::Create(ReadEntireStream(*manifestStream)); + } + + Source GetSource() const override + { + return Source{ GetReferenceSource() }; + } + + IPackageVersion::Metadata GetMetadata() const override + { + auto metadata = GetReferenceSource()->GetIndex().GetMetadataByManifestId(m_manifestId); + + IPackageVersion::Metadata result; + for (auto&& data : metadata) + { + result.emplace(std::move(data)); + } + + return result; + } + + private: + SQLiteIndex::IdType m_manifestId; + std::shared_ptr<Caching::FileCache> m_manifestCache; + }; + + SQLitePackage::SQLitePackage(const std::shared_ptr<SQLiteIndexSource>& source, SQLiteIndex::IdType idId, const std::shared_ptr<Caching::FileCache>& manifestCache, bool isInstalled) : + SourceReference(source), m_idId(idId), m_manifestCache(manifestCache), m_isInstalled(isInstalled) {} + + LocIndString SQLitePackage::GetProperty(PackageProperty property) const + { + LocIndString result; + + std::shared_ptr<IPackageVersion> truth = GetLatestVersion(); + if (truth) + { + switch (property) + { + case PackageProperty::Id: + return truth->GetProperty(PackageVersionProperty::Id); + case PackageProperty::Name: + return truth->GetProperty(PackageVersionProperty::Name); + default: + THROW_HR(E_UNEXPECTED); + } + } + else + { + AICLI_LOG(Repo, Verbose, << "SQLitePackage: No manifest was found for the package with id# '" << m_idId << "'"); + } + + return result; + } + + std::vector<PackageVersionKey> SQLitePackage::GetVersionKeys() const + { + std::shared_ptr<SQLiteIndexSource> source = GetReferenceSource(); + + { + auto sharedLock = m_versionKeysLock.lock_shared(); + + if (!m_versionKeys.empty()) + { + return m_versionKeys; + } + } + + auto exclusiveLock = m_versionKeysLock.lock_exclusive(); + + if (!m_versionKeys.empty()) + { + return m_versionKeys; + } + + std::vector<SQLiteIndex::VersionKey> versions = source->GetIndex().GetVersionKeysById(m_idId); + + for (const auto& vk : versions) + { + std::string version = vk.VersionAndChannel.GetVersion().ToString(); + std::string channel = vk.VersionAndChannel.GetChannel().ToString(); + m_versionKeys.emplace_back(source->GetIdentifier(), version, channel); + m_versionKeysMap.emplace(MapKey{ std::move(version), std::move(channel) }, vk.ManifestId); + } + + return m_versionKeys; + } + + std::shared_ptr<IPackageVersion> SQLitePackage::GetLatestVersion() const + { + std::shared_ptr<SQLiteIndexSource> source = GetReferenceSource(); + std::optional<SQLiteIndex::IdType> manifestId = source->GetIndex().GetManifestIdByKey(m_idId, {}, {}); + + if (manifestId) + { + return std::make_shared<PackageVersion>(source, manifestId.value(), m_manifestCache); + } + + return {}; + } + + std::shared_ptr<IPackageVersion> SQLitePackage::GetVersion(const PackageVersionKey& versionKey) const + { + std::shared_ptr<SQLiteIndexSource> source = GetReferenceSource(); + + // Ensure that this key targets this (or any) source + if (!versionKey.SourceId.empty() && versionKey.SourceId != source->GetIdentifier()) + { + return {}; + } + + std::optional<SQLiteIndex::IdType> manifestId; + + { + MapKey requested{ versionKey.Version, versionKey.Channel }; + auto sharedLock = m_versionKeysLock.lock_shared(); + + auto itr = m_versionKeysMap.find(requested); + if (itr != m_versionKeysMap.end()) + { + manifestId = itr->second; + } + } + + if (!manifestId) + { + manifestId = source->GetIndex().GetManifestIdByKey(m_idId, versionKey.Version, versionKey.Channel); + } + + if (manifestId) + { + return std::make_shared<PackageVersion>(source, manifestId.value(), m_manifestCache); + } + + return {}; + } + + Source SQLitePackage::GetSource() const + { + return Source{ GetReferenceSource() }; + } + + bool SQLitePackage::IsSame(const IPackage* other) const + { + const SQLitePackage* otherSQLite = PackageCast<const SQLitePackage*>(other); + + if (otherSQLite) + { + return GetReferenceSource()->IsSame(otherSQLite->GetReferenceSource().get()) && m_idId == otherSQLite->m_idId; + } + + return false; + } + + const void* SQLitePackage::CastTo(IPackageType type) const + { + if (type == PackageType) + { + return this; + } + + return nullptr; + } + + std::shared_ptr<IPackage> SQLitePackage::GetInstalled() + { + return m_isInstalled ? shared_from_this() : std::shared_ptr<IPackage>{}; + } + + std::vector<std::shared_ptr<IPackage>> SQLitePackage::GetAvailable() + { + return m_isInstalled ? std::vector<std::shared_ptr<IPackage>>{} : std::vector<std::shared_ptr<IPackage>>{ shared_from_this() }; + } + + bool SQLitePackage::MapKey::operator<(const MapKey& other) const + { + if (Version < other.Version) + { + return true; + } + else if (Version == other.Version) + { + return Channel < other.Channel; + } + else + { + return false; + } + } +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSourceV1.h b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSourceV1.h @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once +#include "Microsoft/SQLiteIndexSource.h" + + +namespace AppInstaller::Repository::Microsoft::details::V1 +{ + // The IPackage implementation for V1 index. + struct SQLitePackage : public std::enable_shared_from_this<SQLitePackage>, public SourceReference, public IPackage, public ICompositePackage + { + static constexpr IPackageType PackageType = IPackageType::SQLitePackage1; + + SQLitePackage(const std::shared_ptr<SQLiteIndexSource>& source, SQLiteIndex::IdType idId, const std::shared_ptr<Caching::FileCache>& manifestCache, bool isInstalled); + + // Inherited via IPackage + Utility::LocIndString GetProperty(PackageProperty property) const; + + std::vector<PackageVersionKey> GetVersionKeys() const override; + + std::shared_ptr<IPackageVersion> GetLatestVersion() const override; + + std::shared_ptr<IPackageVersion> GetVersion(const PackageVersionKey& versionKey) const override; + + Source GetSource() const override; + + bool IsSame(const IPackage* other) const override; + + const void* CastTo(IPackageType type) const override; + + // Inherited via ICompositePackage + std::shared_ptr<IPackage> GetInstalled() override; + + std::vector<std::shared_ptr<IPackage>> GetAvailable() override; + + private: + // Contains the information needed to map a version key to it's rows. + struct MapKey + { + Utility::NormalizedString Version; + Utility::NormalizedString Channel; + + bool operator<(const MapKey& other) const; + }; + + SQLiteIndex::IdType m_idId; + std::shared_ptr<Caching::FileCache> m_manifestCache; + bool m_isInstalled; + + // To avoid removing const from the interface + mutable wil::srwlock m_versionKeysLock; + mutable std::vector<PackageVersionKey> m_versionKeys; + mutable std::map<MapKey, SQLiteIndex::IdType> m_versionKeysMap; + }; +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSourceV2.cpp b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSourceV2.cpp @@ -0,0 +1,601 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "Microsoft/SQLiteIndexSourceV2.h" +#include <winget/ManifestYamlParser.h> + +using namespace AppInstaller::Utility; + + +namespace AppInstaller::Repository::Microsoft::details::V2 +{ + // Get the relative path and hash for the package version data manifest. + std::pair<std::filesystem::path, std::string> CreatePackageVersionDataRelativePath(const std::shared_ptr<SQLiteIndexSource>& source, SQLiteIndex::IdType packageRowId) + { + const SQLiteIndex& index = source->GetIndex(); + + std::string identifier = index.GetPropertyByPrimaryId(packageRowId, PackageVersionProperty::Id).value(); + std::string hash = index.GetPropertyByPrimaryId(packageRowId, PackageVersionProperty::ManifestSHA256Hash).value(); + std::filesystem::path relativePath = Manifest::PackageVersionDataManifest::GetRelativeDirectoryPath(identifier, hash) / Manifest::PackageVersionDataManifest::VersionManifestCompressedFileName(); + + return std::make_pair(std::move(relativePath), std::move(hash)); + } + + // Gets package version data for the given package in the index. + Manifest::PackageVersionDataManifest GetPackageVersionData(const std::shared_ptr<SQLiteIndexSource>& source, SQLiteIndex::IdType packageRowId, const Caching::FileCache& fileCache) + { + auto pathAndHash = CreatePackageVersionDataRelativePath(source, packageRowId); + auto fileStream = fileCache.GetFile(pathAndHash.first, SHA256::ConvertToBytes(pathAndHash.second)); + auto fileBytes = ReadEntireStreamAsByteArray(*fileStream); + + Manifest::PackageVersionDataManifest result; + result.Deserialize(Manifest::PackageVersionDataManifest::CreateDecompressor().Decompress(fileBytes)); + + return result; + } + + // The IPackageVersion implementation for V2 index. + struct PackageVersion : public SourceReference, public IPackageVersion + { + PackageVersion( + const std::shared_ptr<SQLiteIndexSource>& source, + SQLiteIndex::IdType packageRowId, + std::optional<Manifest::PackageVersionDataManifest::VersionData> packageVersionData, + const std::shared_ptr<Caching::FileCache>& manifestCache, + const std::shared_ptr<Caching::FileCache>& packageVersionDataCache) : + SourceReference(source), + m_packageRowId(packageRowId), + m_packageVersionData(std::move(packageVersionData)), + m_manifestCache(manifestCache), + m_packageVersionDataCache(packageVersionDataCache) + {} + + // Inherited via IPackageVersion + LocIndString GetProperty(PackageVersionProperty property) const override + { + switch (property) + { + case PackageVersionProperty::SourceIdentifier: + return LocIndString{ GetReferenceSource()->GetIdentifier() }; + case PackageVersionProperty::SourceName: + return LocIndString{ GetReferenceSource()->GetDetails().Name }; + case PackageVersionProperty::RelativePath: + case PackageVersionProperty::ManifestSHA256Hash: + { + // These values can only come from the version data. + EnsurePackageVersionData(); + return GetPropertyFromVersionData(property); + } + break; + case PackageVersionProperty::Publisher: + { + // These values can only come from the manifest. + EnsureManifest(); + return GetPropertyFromManifest(property); + } + break; + case PackageVersionProperty::Id: + case PackageVersionProperty::Name: + case PackageVersionProperty::Moniker: + { + // These properties can come from the manifest or the index. + // The index values will be for the latest version rather than this specific one though. + auto sharedLock = m_versionAndManifestLock.lock_shared(); + + if (m_manifest) + { + return GetPropertyFromManifestWithLock(property); + } + else + { + return GetPropertyFromIndex(property); + } + } + break; + case PackageVersionProperty::Version: + case PackageVersionProperty::Channel: + case PackageVersionProperty::ArpMinVersion: + case PackageVersionProperty::ArpMaxVersion: + { + // These properties can come from the manifest, version data, or the index. + // The index values are only for the latest version, but we should always already have the version data + // for any version that is not the latest. + auto sharedLock = m_versionAndManifestLock.lock_shared(); + + if (m_manifest) + { + return GetPropertyFromManifestWithLock(property); + } + else if (m_packageVersionData) + { + return GetPropertyFromVersionDataWithLock(property); + } + else + { + return GetPropertyFromIndex(property); + } + } + break; + default: + THROW_HR(E_UNEXPECTED); + } + } + + std::vector<LocIndString> GetMultiProperty(PackageVersionMultiProperty property) const override + { + switch (property) + { + case PackageVersionMultiProperty::Locale: + { + // These values can only come from the manifest. + EnsureManifest(); + return GetMultiPropertyFromManifest(property); + } + break; + case PackageVersionMultiProperty::PackageFamilyName: + case PackageVersionMultiProperty::ProductCode: + case PackageVersionMultiProperty::UpgradeCode: + case PackageVersionMultiProperty::Name: + case PackageVersionMultiProperty::Publisher: + case PackageVersionMultiProperty::Tag: + case PackageVersionMultiProperty::Command: + { + // These properties can come from the manifest or the index. + // The index values will be for all versions rather than this specific one though. + auto sharedLock = m_versionAndManifestLock.lock_shared(); + + if (m_manifest) + { + return GetMultiPropertyFromManifestWithLock(property); + } + else + { + return GetMultiPropertyFromIndex(property); + } + } + break; + default: + THROW_HR(E_UNEXPECTED); + } + } + + Manifest::Manifest GetManifest() override + { + EnsureManifest(); + auto sharedLock = m_versionAndManifestLock.lock_shared(); + return m_manifest.value(); + } + + Source GetSource() const override + { + return Source{ GetReferenceSource() }; + } + + IPackageVersion::Metadata GetMetadata() const override + { + return {}; + } + + private: + // Ensures that the package version data is present. + void EnsurePackageVersionData() const + { + { + auto sharedLock = m_versionAndManifestLock.lock_shared(); + if (m_packageVersionData) + { + return; + } + } + + auto exclusiveLock = m_versionAndManifestLock.lock_exclusive(); + if (m_packageVersionData) + { + return; + } + + Manifest::PackageVersionDataManifest packageVersionDataManifest = GetPackageVersionData(GetReferenceSource(), m_packageRowId, *m_packageVersionDataCache); + + for (const auto& versionData : packageVersionDataManifest.Versions()) + { + // We should only ever be looking for the latest version here. + if (!m_packageVersionData || m_packageVersionData->Version < versionData.Version) + { + m_packageVersionData = versionData; + } + } + } + + // Ensures that the manifest is present. + void EnsureManifest() const + { + { + auto sharedLock = m_versionAndManifestLock.lock_shared(); + if (m_manifest) + { + return; + } + } + + // We will need the package version data to get the manifest. + EnsurePackageVersionData(); + + auto exclusiveLock = m_versionAndManifestLock.lock_exclusive(); + if (m_manifest) + { + return; + } + + std::unique_ptr<std::istream> manifestStream = + m_manifestCache->GetFile(ConvertToUTF16(m_packageVersionData->ManifestRelativePath), SHA256::ConvertToBytes(m_packageVersionData->ManifestHash)); + m_manifest = Manifest::YamlParser::Create(ReadEntireStream(*manifestStream)); + m_manifest->ApplyLocale(); + } + + LocIndString GetPropertyFromIndex(PackageVersionProperty property) const + { + switch (property) + { + case PackageVersionProperty::Id: + case PackageVersionProperty::Name: + case PackageVersionProperty::Moniker: + case PackageVersionProperty::Version: + case PackageVersionProperty::ArpMinVersion: + case PackageVersionProperty::ArpMaxVersion: + { + // Values coming from the index will always be localized/independent. + std::optional<std::string> optValue = GetReferenceSource()->GetIndex().GetPropertyByPrimaryId(m_packageRowId, property); + return LocIndString{ optValue ? optValue.value() : std::string{} }; + } + default: + return {}; + } + } + + LocIndString GetPropertyFromVersionData(PackageVersionProperty property) const + { + auto sharedLock = m_versionAndManifestLock.lock_shared(); + return GetPropertyFromVersionDataWithLock(property); + } + + LocIndString GetPropertyFromVersionDataWithLock(PackageVersionProperty property) const + { + std::string result; + + switch (property) + { + case PackageVersionProperty::RelativePath: + result = m_packageVersionData->ManifestRelativePath; + break; + case PackageVersionProperty::ManifestSHA256Hash: + result = m_packageVersionData->ManifestHash; + break; + case PackageVersionProperty::Version: + result = m_packageVersionData->Version.ToString(); + break; + case PackageVersionProperty::ArpMinVersion: + result = m_packageVersionData->ArpMinVersion.value_or(""); + break; + case PackageVersionProperty::ArpMaxVersion: + result = m_packageVersionData->ArpMaxVersion.value_or(""); + break; + } + + return LocIndString{ std::move(result) }; + } + + LocIndString GetPropertyFromManifest(PackageVersionProperty property) const + { + auto sharedLock = m_versionAndManifestLock.lock_shared(); + return GetPropertyFromManifestWithLock(property); + } + + LocIndString GetPropertyFromManifestWithLock(PackageVersionProperty property) const + { + std::string result; + + switch (property) + { + case PackageVersionProperty::Publisher: + result = m_manifest->CurrentLocalization.Get<Manifest::Localization::Publisher>(); + break; + case PackageVersionProperty::Id: + result = m_manifest->Id; + break; + case PackageVersionProperty::Name: + result = m_manifest->CurrentLocalization.Get<Manifest::Localization::PackageName>(); + break; + case PackageVersionProperty::Moniker: + result = m_manifest->Moniker; + break; + case PackageVersionProperty::Version: + result = m_manifest->Version; + break; + case PackageVersionProperty::Channel: + result = m_manifest->Channel; + break; + case PackageVersionProperty::ArpMinVersion: + { + auto versionRange = m_manifest->GetArpVersionRange(); + if (!versionRange.IsEmpty()) + { + result = versionRange.GetMinVersion().ToString(); + } + } + break; + case PackageVersionProperty::ArpMaxVersion: + { + auto versionRange = m_manifest->GetArpVersionRange(); + if (!versionRange.IsEmpty()) + { + result = versionRange.GetMaxVersion().ToString(); + } + } + break; + } + + return LocIndString{ std::move(result) }; + } + + std::vector<LocIndString> GetMultiPropertyFromIndex(PackageVersionMultiProperty property) const + { + std::vector<LocIndString> result; + + for (auto&& value : GetReferenceSource()->GetIndex().GetMultiPropertyByPrimaryId(m_packageRowId, property)) + { + // Values coming from the index will always be localized/independent. + result.emplace_back(std::move(value)); + } + + return result; + } + + std::vector<LocIndString> GetMultiPropertyFromManifest(PackageVersionMultiProperty property) const + { + auto sharedLock = m_versionAndManifestLock.lock_shared(); + return GetMultiPropertyFromManifestWithLock(property); + } + + std::vector<LocIndString> GetMultiPropertyFromManifestWithLock(PackageVersionMultiProperty property) const + { + std::vector<Manifest::string_t> intermediate; + + switch (property) + { + case PackageVersionMultiProperty::PackageFamilyName: + intermediate = m_manifest->GetPackageFamilyNames(); + break; + case PackageVersionMultiProperty::ProductCode: + intermediate = m_manifest->GetProductCodes(); + break; + case PackageVersionMultiProperty::UpgradeCode: + intermediate = m_manifest->GetUpgradeCodes(); + break; + case PackageVersionMultiProperty::Name: + intermediate = m_manifest->GetPackageNames(); + break; + case PackageVersionMultiProperty::Publisher: + intermediate = m_manifest->GetPublishers(); + break; + case PackageVersionMultiProperty::Locale: + for (const auto& localization : m_manifest->Localizations) + { + intermediate.emplace_back(localization.Locale); + } + break; + case PackageVersionMultiProperty::Tag: + intermediate = m_manifest->GetAggregatedTags(); + break; + case PackageVersionMultiProperty::Command: + intermediate = m_manifest->GetAggregatedCommands(); + break; + } + + std::vector<LocIndString> result; + + for (auto&& value : intermediate) + { + // Values coming from the manifest will always be localized/independent. + result.emplace_back(std::move(value)); + } + + return result; + } + + SQLiteIndex::IdType m_packageRowId; + + mutable wil::srwlock m_versionAndManifestLock; + mutable std::optional<Manifest::PackageVersionDataManifest::VersionData> m_packageVersionData; + mutable std::optional<Manifest::Manifest> m_manifest; + + std::shared_ptr<Caching::FileCache> m_manifestCache; + std::shared_ptr<Caching::FileCache> m_packageVersionDataCache; + }; + + SQLitePackage::SQLitePackage( + const std::shared_ptr<SQLiteIndexSource>& source, + SQLiteIndex::IdType packageRowId, + const std::shared_ptr<Caching::FileCache>& manifestCache, + const std::shared_ptr<Caching::FileCache>& packageVersionDataCache, + bool isInstalled) : + SourceReference(source), + m_packageRowId(packageRowId), + m_manifestCache(manifestCache), + m_packageVersionDataCache(packageVersionDataCache), + m_isInstalled(isInstalled) + {} + + LocIndString SQLitePackage::GetProperty(PackageProperty property) const + { + std::optional<std::string> result; + std::shared_ptr<SQLiteIndexSource> source = GetReferenceSource(); + + switch (property) + { + case PackageProperty::Id: + result = source->GetIndex().GetPropertyByPrimaryId(m_packageRowId, PackageVersionProperty::Id); + break; + case PackageProperty::Name: + result = source->GetIndex().GetPropertyByPrimaryId(m_packageRowId, PackageVersionProperty::Name); + break; + default: + THROW_HR(E_UNEXPECTED); + } + + return LocIndString{ result ? std::move(result).value() : std::string{} }; + } + + std::vector<PackageVersionKey> SQLitePackage::GetVersionKeys() const + { + std::shared_ptr<SQLiteIndexSource> source = GetReferenceSource(); + + { + auto sharedLock = m_versionKeysLock.lock_shared(); + + if (!m_versionKeys.empty()) + { + return m_versionKeys; + } + } + + EnsurePackageVersionData(source); + + auto sharedLock = m_versionKeysLock.lock_shared(); + return m_versionKeys; + } + + std::shared_ptr<IPackageVersion> SQLitePackage::GetLatestVersion() const + { + std::shared_ptr<SQLiteIndexSource> source = GetReferenceSource(); + auto sharedLock = m_versionKeysLock.lock_shared(); + return std::make_shared<PackageVersion>(source, m_packageRowId, m_latestVersionData, m_manifestCache, m_packageVersionDataCache); + } + + std::shared_ptr<IPackageVersion> SQLitePackage::GetVersion(const PackageVersionKey& versionKey) const + { + std::shared_ptr<SQLiteIndexSource> source = GetReferenceSource(); + + // Ensure that this key targets this (or any) source + if (!versionKey.SourceId.empty() && versionKey.SourceId != source->GetIdentifier()) + { + return {}; + } + + std::optional<Manifest::PackageVersionDataManifest::VersionData> versionData; + + // Check for a latest version request. + if (versionKey.IsDefaultLatest()) + { + auto sharedLock = m_versionKeysLock.lock_shared(); + return std::make_shared<PackageVersion>(source, m_packageRowId, m_latestVersionData, m_manifestCache, m_packageVersionDataCache); + } + + EnsurePackageVersionData(source); + + { + MapKey requested{ versionKey.Version, versionKey.Channel }; + auto sharedLock = m_versionKeysLock.lock_shared(); + + auto itr = m_versionKeysMap.find(requested); + if (itr != m_versionKeysMap.end()) + { + versionData = itr->second; + } + } + + if (versionData) + { + return std::make_shared<PackageVersion>(source, m_packageRowId, std::move(versionData), m_manifestCache, m_packageVersionDataCache); + } + + return {}; + } + + Source SQLitePackage::GetSource() const + { + return Source{ GetReferenceSource() }; + } + + bool SQLitePackage::IsSame(const IPackage* other) const + { + const SQLitePackage* otherSQLite = PackageCast<const SQLitePackage*>(other); + + if (otherSQLite) + { + return GetReferenceSource()->IsSame(otherSQLite->GetReferenceSource().get()) && m_packageRowId == otherSQLite->m_packageRowId; + } + + return false; + } + + const void* SQLitePackage::CastTo(IPackageType type) const + { + if (type == PackageType) + { + return this; + } + + return nullptr; + } + + std::shared_ptr<IPackage> SQLitePackage::GetInstalled() + { + return m_isInstalled ? shared_from_this() : std::shared_ptr<IPackage>{}; + } + + std::vector<std::shared_ptr<IPackage>> SQLitePackage::GetAvailable() + { + return m_isInstalled ? std::vector<std::shared_ptr<IPackage>>{} : std::vector<std::shared_ptr<IPackage>>{ shared_from_this() }; + } + + bool SQLitePackage::MapKey::operator<(const MapKey& other) const + { + if (Version < other.Version) + { + return true; + } + else if (Version == other.Version) + { + return Channel < other.Channel; + } + else + { + return false; + } + } + + // Ensures that we have the package version data present. + void SQLitePackage::EnsurePackageVersionData(const std::shared_ptr<SQLiteIndexSource>& source) const + { + { + auto sharedLock = m_versionKeysLock.lock_shared(); + + if (!m_versionKeys.empty()) + { + return; + } + } + + auto exclusiveLock = m_versionKeysLock.lock_exclusive(); + + if (!m_versionKeys.empty()) + { + return; + } + + Manifest::PackageVersionDataManifest packageVersionDataManifest = GetPackageVersionData(source, m_packageRowId, *m_packageVersionDataCache); + + for (const auto& versionData : packageVersionDataManifest.Versions()) + { + std::string version = versionData.Version.ToString(); + std::string channel; + m_versionKeys.emplace_back(source->GetIdentifier(), version, channel); + m_versionKeysMap.emplace(MapKey{ std::move(version), std::move(channel) }, versionData); + + if (!m_latestVersionData || m_latestVersionData->Version < versionData.Version) + { + m_latestVersionData = versionData; + } + } + } +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSourceV2.h b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSourceV2.h @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once +#include "Microsoft/SQLiteIndexSource.h" +#include <winget/PackageVersionDataManifest.h> + + +namespace AppInstaller::Repository::Microsoft::details::V2 +{ + // The IPackage implementation for V2 index. + struct SQLitePackage : public std::enable_shared_from_this<SQLitePackage>, public SourceReference, public IPackage, public ICompositePackage + { + static constexpr IPackageType PackageType = IPackageType::SQLitePackage2; + + SQLitePackage( + const std::shared_ptr<SQLiteIndexSource>& source, + SQLiteIndex::IdType packageRowId, + const std::shared_ptr<Caching::FileCache>& manifestCache, + const std::shared_ptr<Caching::FileCache>& packageVersionDataCache, + bool isInstalled); + + // Inherited via IPackage + Utility::LocIndString GetProperty(PackageProperty property) const; + + std::vector<PackageVersionKey> GetVersionKeys() const override; + + std::shared_ptr<IPackageVersion> GetLatestVersion() const override; + + std::shared_ptr<IPackageVersion> GetVersion(const PackageVersionKey& versionKey) const override; + + Source GetSource() const override; + + bool IsSame(const IPackage* other) const override; + + const void* CastTo(IPackageType type) const override; + + // Inherited via ICompositePackage + std::shared_ptr<IPackage> GetInstalled() override; + + std::vector<std::shared_ptr<IPackage>> GetAvailable() override; + + private: + // Contains the information needed to map a version key to it's rows. + struct MapKey + { + Utility::NormalizedString Version; + Utility::NormalizedString Channel; + + bool operator<(const MapKey& other) const; + }; + + // Ensures that we have the package version data present. + void EnsurePackageVersionData(const std::shared_ptr<SQLiteIndexSource>& source) const; + + SQLiteIndex::IdType m_packageRowId; + std::shared_ptr<Caching::FileCache> m_manifestCache; + std::shared_ptr<Caching::FileCache> m_packageVersionDataCache; + bool m_isInstalled; + + // To avoid removing const from the interface + mutable wil::srwlock m_versionKeysLock; + mutable std::vector<PackageVersionKey> m_versionKeys; + mutable std::map<MapKey, Manifest::PackageVersionDataManifest::VersionData> m_versionKeysMap; + mutable std::optional<Manifest::PackageVersionDataManifest::VersionData> m_latestVersionData; + }; +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface.h @@ -24,8 +24,8 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 void PrepareForPackaging(SQLite::Connection& connection) override; bool CheckConsistency(const SQLite::Connection& connection, bool log) const override; SearchResult Search(const SQLite::Connection& connection, const SearchRequest& request) const override; - std::optional<std::string> GetPropertyByManifestId(const SQLite::Connection& connection, SQLite::rowid_t manifestId, PackageVersionProperty property) const override; - std::vector<std::string> GetMultiPropertyByManifestId(const SQLite::Connection& connection, SQLite::rowid_t manifestId, PackageVersionMultiProperty property) const override; + std::optional<std::string> GetPropertyByPrimaryId(const SQLite::Connection& connection, SQLite::rowid_t primaryId, PackageVersionProperty property) const override; + std::vector<std::string> GetMultiPropertyByPrimaryId(const SQLite::Connection& connection, SQLite::rowid_t primaryId, PackageVersionMultiProperty property) const override; std::optional<SQLite::rowid_t> GetManifestIdByKey(const SQLite::Connection& connection, SQLite::rowid_t id, std::string_view version, std::string_view channel) const override; std::optional<SQLite::rowid_t> GetManifestIdByManifest(const SQLite::Connection& connection, const Manifest::Manifest& manifest) const override; std::vector<VersionKey> GetVersionKeysById(const SQLite::Connection& connection, SQLite::rowid_t id) const override; diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface_1_0.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface_1_0.cpp @@ -515,19 +515,19 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 return resultsTable->GetSearchResults(request.MaximumResults); } - std::optional<std::string> Interface::GetPropertyByManifestId(const SQLite::Connection& connection, SQLite::rowid_t manifestId, PackageVersionProperty property) const + std::optional<std::string> Interface::GetPropertyByPrimaryId(const SQLite::Connection& connection, SQLite::rowid_t primaryId, PackageVersionProperty property) const { - return GetPropertyByManifestIdInternal(connection, manifestId, property); + return GetPropertyByManifestIdInternal(connection, primaryId, property); } - std::vector<std::string> Interface::GetMultiPropertyByManifestId(const SQLite::Connection& connection, SQLite::rowid_t rowid, PackageVersionMultiProperty property) const + std::vector<std::string> Interface::GetMultiPropertyByPrimaryId(const SQLite::Connection& connection, SQLite::rowid_t primaryId, PackageVersionMultiProperty property) const { switch (property) { case PackageVersionMultiProperty::Tag: - return TagsTable::GetValuesByManifestId(connection, rowid); + return TagsTable::GetValuesByManifestId(connection, primaryId); case PackageVersionMultiProperty::Command: - return CommandsTable::GetValuesByManifestId(connection, rowid); + return CommandsTable::GetValuesByManifestId(connection, primaryId); default: return {}; } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_1/Interface.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_1/Interface.h @@ -19,7 +19,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_1 void PrepareForPackaging(SQLite::Connection& connection) override; bool CheckConsistency(const SQLite::Connection& connection, bool log) const override; SearchResult Search(const SQLite::Connection& connection, const SearchRequest& request) const override; - std::vector<std::string> GetMultiPropertyByManifestId(const SQLite::Connection& connection, SQLite::rowid_t manifestId, PackageVersionMultiProperty property) const override; + std::vector<std::string> GetMultiPropertyByPrimaryId(const SQLite::Connection& connection, SQLite::rowid_t primaryId, PackageVersionMultiProperty property) const override; // Version 1.1 MetadataResult GetMetadataByManifestId(const SQLite::Connection& connection, SQLite::rowid_t manifestId) const override; diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_1/Interface_1_1.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_1/Interface_1_1.cpp @@ -138,16 +138,16 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_1 return SearchInternal(connection, updatedRequest); } - std::vector<std::string> Interface::GetMultiPropertyByManifestId(const SQLite::Connection& connection, SQLite::rowid_t manifestId, PackageVersionMultiProperty property) const + std::vector<std::string> Interface::GetMultiPropertyByPrimaryId(const SQLite::Connection& connection, SQLite::rowid_t primaryId, PackageVersionMultiProperty property) const { switch (property) { case PackageVersionMultiProperty::PackageFamilyName: - return PackageFamilyNameTable::GetValuesByManifestId(connection, manifestId); + return PackageFamilyNameTable::GetValuesByManifestId(connection, primaryId); case PackageVersionMultiProperty::ProductCode: - return ProductCodeTable::GetValuesByManifestId(connection, manifestId); + return ProductCodeTable::GetValuesByManifestId(connection, primaryId); default: - return V1_0::Interface::GetMultiPropertyByManifestId(connection, manifestId, property); + return V1_0::Interface::GetMultiPropertyByPrimaryId(connection, primaryId, property); } } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_2/Interface.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_2/Interface.h @@ -19,7 +19,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_2 std::pair<bool, SQLite::rowid_t> UpdateManifest(SQLite::Connection& connection, const Manifest::Manifest& manifest, const std::optional<std::filesystem::path>& relativePath) override; void RemoveManifestById(SQLite::Connection& connection, SQLite::rowid_t manifestId) override; bool CheckConsistency(const SQLite::Connection& connection, bool log) const override; - std::vector<std::string> GetMultiPropertyByManifestId(const SQLite::Connection& connection, SQLite::rowid_t manifestId, PackageVersionMultiProperty property) const override; + std::vector<std::string> GetMultiPropertyByPrimaryId(const SQLite::Connection& connection, SQLite::rowid_t primaryId, PackageVersionMultiProperty property) const override; // Version 1.2 Utility::NormalizedName NormalizeName(std::string_view name, std::string_view publisher) const override; diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_2/Interface_1_2.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_2/Interface_1_2.cpp @@ -234,17 +234,17 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_2 return result; } - std::vector<std::string> Interface::GetMultiPropertyByManifestId(const SQLite::Connection& connection, SQLite::rowid_t manifestId, PackageVersionMultiProperty property) const + std::vector<std::string> Interface::GetMultiPropertyByPrimaryId(const SQLite::Connection& connection, SQLite::rowid_t primaryId, PackageVersionMultiProperty property) const { switch (property) { // These values are not right, as they are normalized. But they are good enough for now and all we have. case PackageVersionMultiProperty::Name: - return NormalizedPackageNameTable::GetValuesByManifestId(connection, manifestId); + return NormalizedPackageNameTable::GetValuesByManifestId(connection, primaryId); case PackageVersionMultiProperty::Publisher: - return NormalizedPackagePublisherTable::GetValuesByManifestId(connection, manifestId); + return NormalizedPackagePublisherTable::GetValuesByManifestId(connection, primaryId); default: - return V1_1::Interface::GetMultiPropertyByManifestId(connection, manifestId, property); + return V1_1::Interface::GetMultiPropertyByPrimaryId(connection, primaryId, property); } } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_5/Interface_1_5.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_5/Interface_1_5.cpp @@ -197,8 +197,8 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_5 auto versionKeys = GetVersionKeysById(connection, match.first); for (auto const& versionKey : versionKeys) { - auto arpMinVersion = GetPropertyByManifestId(connection, versionKey.ManifestId, PackageVersionProperty::ArpMinVersion).value_or(""); - auto arpMaxVersion = GetPropertyByManifestId(connection, versionKey.ManifestId, PackageVersionProperty::ArpMaxVersion).value_or(""); + auto arpMinVersion = GetPropertyByPrimaryId(connection, versionKey.ManifestId, PackageVersionProperty::ArpMinVersion).value_or(""); + auto arpMaxVersion = GetPropertyByPrimaryId(connection, versionKey.ManifestId, PackageVersionProperty::ArpMaxVersion).value_or(""); // Either both empty or both not empty THROW_HR_IF(E_UNEXPECTED, arpMinVersion.empty() != arpMaxVersion.empty()); @@ -230,4 +230,4 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_5 return false; } } -}- \ No newline at end of file +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_6/Interface.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_6/Interface.h @@ -18,7 +18,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_6 std::pair<bool, SQLite::rowid_t> UpdateManifest(SQLite::Connection& connection, const Manifest::Manifest& manifest, const std::optional<std::filesystem::path>& relativePath) override; void RemoveManifestById(SQLite::Connection& connection, SQLite::rowid_t manifestId) override; bool CheckConsistency(const SQLite::Connection& connection, bool log) const override; - std::vector<std::string> GetMultiPropertyByManifestId(const SQLite::Connection& connection, SQLite::rowid_t manifestId, PackageVersionMultiProperty property) const override; + std::vector<std::string> GetMultiPropertyByPrimaryId(const SQLite::Connection& connection, SQLite::rowid_t primaryId, PackageVersionMultiProperty property) const override; // Version 1.7 void DropTables(SQLite::Connection& connection) override; diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_6/Interface_1_6.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_6/Interface_1_6.cpp @@ -85,14 +85,14 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_6 return result; } - std::vector<std::string> Interface::GetMultiPropertyByManifestId(const SQLite::Connection& connection, SQLite::rowid_t manifestId, PackageVersionMultiProperty property) const + std::vector<std::string> Interface::GetMultiPropertyByPrimaryId(const SQLite::Connection& connection, SQLite::rowid_t primaryId, PackageVersionMultiProperty property) const { switch (property) { case PackageVersionMultiProperty::UpgradeCode: - return UpgradeCodeTable::GetValuesByManifestId(connection, manifestId); + return UpgradeCodeTable::GetValuesByManifestId(connection, primaryId); default: - return V1_5::Interface::GetMultiPropertyByManifestId(connection, manifestId, property); + return V1_5::Interface::GetMultiPropertyByPrimaryId(connection, primaryId, property); } } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_7/Interface.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_7/Interface.h @@ -21,7 +21,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_7 // Version 1.0 SQLite::Version GetVersion() const override; - std::vector<std::string> GetMultiPropertyByManifestId(const SQLite::Connection& connection, SQLite::rowid_t manifestId, PackageVersionMultiProperty property) const override; + std::vector<std::string> GetMultiPropertyByPrimaryId(const SQLite::Connection& connection, SQLite::rowid_t primaryId, PackageVersionMultiProperty property) const override; protected: void PrepareForPackaging(SQLite::Connection& connection, bool vacuum) override; diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_7/Interface_1_7.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_7/Interface_1_7.cpp @@ -48,7 +48,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_7 return { 1, 7 }; } - std::vector<std::string> Interface::GetMultiPropertyByManifestId(const SQLite::Connection& connection, SQLite::rowid_t manifestId, PackageVersionMultiProperty property) const + std::vector<std::string> Interface::GetMultiPropertyByPrimaryId(const SQLite::Connection& connection, SQLite::rowid_t primaryId, PackageVersionMultiProperty property) const { if (property == PackageVersionMultiProperty::PackageFamilyName || property == PackageVersionMultiProperty::Name || @@ -57,15 +57,15 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_7 { if (ShouldFoldPropertyLookup(connection)) { - std::optional<SQLite::rowid_t> maximumManifestId = V1_0::OneToManyTableGetMapDataFoldingManifestTargetId(connection, manifestId); + std::optional<SQLite::rowid_t> maximumManifestId = V1_0::OneToManyTableGetMapDataFoldingManifestTargetId(connection, primaryId); if (maximumManifestId) { - manifestId = maximumManifestId.value(); + primaryId = maximumManifestId.value(); } } } - return V1_6::Interface::GetMultiPropertyByManifestId(connection, manifestId, property); + return V1_6::Interface::GetMultiPropertyByPrimaryId(connection, primaryId, property); } void Interface::PrepareForPackaging(SQLite::Connection& connection, bool vacuum) diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/Interface.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/Interface.h @@ -31,8 +31,8 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 void PrepareForPackaging(const SQLiteIndexContext& context) override; bool CheckConsistency(const SQLite::Connection& connection, bool log) const override; SearchResult Search(const SQLite::Connection& connection, const SearchRequest& request) const override; - std::optional<std::string> GetPropertyByManifestId(const SQLite::Connection& connection, SQLite::rowid_t manifestId, PackageVersionProperty property) const override; - std::vector<std::string> GetMultiPropertyByManifestId(const SQLite::Connection& connection, SQLite::rowid_t manifestId, PackageVersionMultiProperty property) const override; + std::optional<std::string> GetPropertyByPrimaryId(const SQLite::Connection& connection, SQLite::rowid_t primaryId, PackageVersionProperty property) const override; + std::vector<std::string> GetMultiPropertyByPrimaryId(const SQLite::Connection& connection, SQLite::rowid_t primaryId, PackageVersionMultiProperty property) const override; std::optional<SQLite::rowid_t> GetManifestIdByKey(const SQLite::Connection& connection, SQLite::rowid_t id, std::string_view version, std::string_view channel) const override; std::optional<SQLite::rowid_t> GetManifestIdByManifest(const SQLite::Connection& connection, const Manifest::Manifest& manifest) const override; std::vector<VersionKey> GetVersionKeysById(const SQLite::Connection& connection, SQLite::rowid_t id) const override; diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/Interface_2_0.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/Interface_2_0.cpp @@ -105,7 +105,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 { EnsureInternalInterface(connection, true); SQLite::rowid_t manifestId = m_internalInterface->AddManifest(connection, manifest, relativePath); - PackageUpdateTrackingTable::Update(connection, m_internalInterface.get(), m_internalInterface->GetPropertyByManifestId(connection, manifestId, PackageVersionProperty::Id).value()); + PackageUpdateTrackingTable::Update(connection, m_internalInterface.get(), m_internalInterface->GetPropertyByPrimaryId(connection, manifestId, PackageVersionProperty::Id).value()); return manifestId; } @@ -115,7 +115,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 std::pair<bool, SQLite::rowid_t> result = m_internalInterface->UpdateManifest(connection, manifest, relativePath); if (result.first) { - PackageUpdateTrackingTable::Update(connection, m_internalInterface.get(), m_internalInterface->GetPropertyByManifestId(connection, result.second, PackageVersionProperty::Id).value()); + PackageUpdateTrackingTable::Update(connection, m_internalInterface.get(), m_internalInterface->GetPropertyByPrimaryId(connection, result.second, PackageVersionProperty::Id).value()); } return result; } @@ -137,7 +137,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 void Interface::RemoveManifestById(SQLite::Connection& connection, SQLite::rowid_t manifestId) { EnsureInternalInterface(connection, true); - std::optional<std::string> identifier = m_internalInterface->GetPropertyByManifestId(connection, manifestId, PackageVersionProperty::Id); + std::optional<std::string> identifier = m_internalInterface->GetPropertyByPrimaryId(connection, manifestId, PackageVersionProperty::Id); m_internalInterface->RemoveManifestById(connection, manifestId); if (identifier) { @@ -213,67 +213,67 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 return SearchInternal(connection, requestCopy); } - std::optional<std::string> Interface::GetPropertyByManifestId(const SQLite::Connection& connection, SQLite::rowid_t manifestId, PackageVersionProperty property) const + std::optional<std::string> Interface::GetPropertyByPrimaryId(const SQLite::Connection& connection, SQLite::rowid_t primaryId, PackageVersionProperty property) const { EnsureInternalInterface(connection); if (m_internalInterface) { - return m_internalInterface->GetPropertyByManifestId(connection, manifestId, property); + return m_internalInterface->GetPropertyByPrimaryId(connection, primaryId, property); } switch (property) { case PackageVersionProperty::Id: - return PackagesTable::GetValueById<PackagesTable::IdColumn>(connection, manifestId); + return PackagesTable::GetValueById<PackagesTable::IdColumn>(connection, primaryId); case PackageVersionProperty::Name: - return PackagesTable::GetValueById<PackagesTable::NameColumn>(connection, manifestId); + return PackagesTable::GetValueById<PackagesTable::NameColumn>(connection, primaryId); case PackageVersionProperty::Version: - return PackagesTable::GetValueById<PackagesTable::LatestVersionColumn>(connection, manifestId); + return PackagesTable::GetValueById<PackagesTable::LatestVersionColumn>(connection, primaryId); case PackageVersionProperty::Channel: return ""; case PackageVersionProperty::ManifestSHA256Hash: { - std::optional<SQLite::blob_t> hash = PackagesTable::GetValueById<PackagesTable::HashColumn>(connection, manifestId); + std::optional<SQLite::blob_t> hash = PackagesTable::GetValueById<PackagesTable::HashColumn>(connection, primaryId); return (!hash || hash->empty()) ? std::optional<std::string>{} : Utility::SHA256::ConvertToString(hash.value()); } case PackageVersionProperty::ArpMinVersion: - return PackagesTable::GetValueById<PackagesTable::ARPMinVersionColumn>(connection, manifestId); + return PackagesTable::GetValueById<PackagesTable::ARPMinVersionColumn>(connection, primaryId); case PackageVersionProperty::ArpMaxVersion: - return PackagesTable::GetValueById<PackagesTable::ARPMaxVersionColumn>(connection, manifestId); + return PackagesTable::GetValueById<PackagesTable::ARPMaxVersionColumn>(connection, primaryId); case PackageVersionProperty::Moniker: - return PackagesTable::GetValueById<PackagesTable::MonikerColumn>(connection, manifestId); + return PackagesTable::GetValueById<PackagesTable::MonikerColumn>(connection, primaryId); default: return {}; } } - std::vector<std::string> Interface::GetMultiPropertyByManifestId(const SQLite::Connection& connection, SQLite::rowid_t manifestId, PackageVersionMultiProperty property) const + std::vector<std::string> Interface::GetMultiPropertyByPrimaryId(const SQLite::Connection& connection, SQLite::rowid_t primaryId, PackageVersionMultiProperty property) const { EnsureInternalInterface(connection); if (m_internalInterface) { - return m_internalInterface->GetMultiPropertyByManifestId(connection, manifestId, property); + return m_internalInterface->GetMultiPropertyByPrimaryId(connection, primaryId, property); } switch (property) { case PackageVersionMultiProperty::PackageFamilyName: - return PackageFamilyNameTable::GetValuesByPrimaryId(connection, manifestId); + return PackageFamilyNameTable::GetValuesByPrimaryId(connection, primaryId); case PackageVersionMultiProperty::ProductCode: - return ProductCodeTable::GetValuesByPrimaryId(connection, manifestId); + return ProductCodeTable::GetValuesByPrimaryId(connection, primaryId); // These values are not right, as they are normalized. But they are good enough for now and all we have. case PackageVersionMultiProperty::Name: - return NormalizedPackageNameTable::GetValuesByPrimaryId(connection, manifestId); + return NormalizedPackageNameTable::GetValuesByPrimaryId(connection, primaryId); case PackageVersionMultiProperty::Publisher: - return NormalizedPackagePublisherTable::GetValuesByPrimaryId(connection, manifestId); + return NormalizedPackagePublisherTable::GetValuesByPrimaryId(connection, primaryId); case PackageVersionMultiProperty::UpgradeCode: - return UpgradeCodeTable::GetValuesByPrimaryId(connection, manifestId); + return UpgradeCodeTable::GetValuesByPrimaryId(connection, primaryId); case PackageVersionMultiProperty::Tag: - return TagsTable::GetValuesByPrimaryId(connection, manifestId); + return TagsTable::GetValuesByPrimaryId(connection, primaryId); case PackageVersionMultiProperty::Command: - return CommandsTable::GetValuesByPrimaryId(connection, manifestId); + return CommandsTable::GetValuesByPrimaryId(connection, primaryId); default: return {}; } @@ -403,7 +403,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 { std::vector<ISQLiteIndex::VersionKey> versionKeys = current->GetVersionKeysById(connection, packageMatch.first); ISQLiteIndex::VersionKey& latestVersionKey = versionKeys[0]; - PackageUpdateTrackingTable::Update(connection, current, current->GetPropertyByManifestId(connection, latestVersionKey.ManifestId, PackageVersionProperty::Id).value(), false); + PackageUpdateTrackingTable::Update(connection, current, current->GetPropertyByPrimaryId(connection, latestVersionKey.ManifestId, PackageVersionProperty::Id).value(), false); } savepoint.Commit(); @@ -595,7 +595,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 } // Remove any duplicate manifest entries - resultsTable->RemoveDuplicateManifestRows(); + resultsTable->RemoveDuplicatePackageRows(); // Second phase, for remaining filters, flag matching search results, then remove unflagged values. for (size_t i = filterIndex; i < request.Filters.size(); ++i) @@ -646,12 +646,12 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 // Output all of the changed package version manifests since the base time to the target location for (const auto& packageData : PackageUpdateTrackingTable::GetUpdatesSince(connection, updateBaseTime)) { - std::filesystem::path packageDirectory = baseOutputDirectory / Utility::ConvertToUTF16(packageData.PackageIdentifier); - std::filesystem::path hashDirectory = packageDirectory / Utility::SHA256::ConvertToWideString(packageData.Hash); + std::filesystem::path packageDirectory = baseOutputDirectory / + Manifest::PackageVersionDataManifest::GetRelativeDirectoryPath(packageData.PackageIdentifier, Utility::SHA256::ConvertToString(packageData.Hash)); - std::filesystem::create_directories(hashDirectory); + std::filesystem::create_directories(packageDirectory); - std::filesystem::path manifestPath = hashDirectory / Manifest::PackageVersionDataManifest::VersionManifestCompressedFileName(); + std::filesystem::path manifestPath = packageDirectory / Manifest::PackageVersionDataManifest::VersionManifestCompressedFileName(); AICLI_LOG(Repo, Info, << "Writing PackageVersionDataManifest for [" << packageData.PackageIdentifier << "] to [" << manifestPath << "]"); @@ -692,11 +692,11 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 std::vector<ISQLiteIndex::VersionKey> versionKeys = m_internalInterface->GetVersionKeysById(connection, packageMatch.first); ISQLiteIndex::VersionKey& latestVersionKey = versionKeys[0]; - std::string packageIdentifier = m_internalInterface->GetPropertyByManifestId(connection, latestVersionKey.ManifestId, PackageVersionProperty::Id).value(); + std::string packageIdentifier = m_internalInterface->GetPropertyByPrimaryId(connection, latestVersionKey.ManifestId, PackageVersionProperty::Id).value(); std::vector<PackagesTable::NameValuePair> packageData{ { PackagesTable::IdColumn::Name, packageIdentifier }, - { PackagesTable::NameColumn::Name, m_internalInterface->GetPropertyByManifestId(connection, latestVersionKey.ManifestId, PackageVersionProperty::Name).value() }, + { PackagesTable::NameColumn::Name, m_internalInterface->GetPropertyByPrimaryId(connection, latestVersionKey.ManifestId, PackageVersionProperty::Name).value() }, { PackagesTable::LatestVersionColumn::Name, latestVersionKey.VersionAndChannel.GetVersion().ToString() }, }; @@ -708,9 +708,9 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 } }; - addIfPresent(PackagesTable::MonikerColumn::Name, m_internalInterface->GetPropertyByManifestId(connection, latestVersionKey.ManifestId, PackageVersionProperty::Moniker).value()); - addIfPresent(PackagesTable::ARPMinVersionColumn::Name, m_internalInterface->GetPropertyByManifestId(connection, latestVersionKey.ManifestId, PackageVersionProperty::ArpMinVersion).value()); - addIfPresent(PackagesTable::ARPMaxVersionColumn::Name, m_internalInterface->GetPropertyByManifestId(connection, latestVersionKey.ManifestId, PackageVersionProperty::ArpMaxVersion).value()); + addIfPresent(PackagesTable::MonikerColumn::Name, m_internalInterface->GetPropertyByPrimaryId(connection, latestVersionKey.ManifestId, PackageVersionProperty::Moniker).value()); + addIfPresent(PackagesTable::ARPMinVersionColumn::Name, m_internalInterface->GetPropertyByPrimaryId(connection, latestVersionKey.ManifestId, PackageVersionProperty::ArpMinVersion).value()); + addIfPresent(PackagesTable::ARPMaxVersionColumn::Name, m_internalInterface->GetPropertyByPrimaryId(connection, latestVersionKey.ManifestId, PackageVersionProperty::ArpMaxVersion).value()); SQLite::rowid_t packageId = PackagesTable::Insert(connection, packageData); @@ -718,14 +718,14 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 for (const auto& versionKey : versionKeys) { - TagsTable::EnsureExistsAndInsert(connection, m_internalInterface->GetMultiPropertyByManifestId(connection, versionKey.ManifestId, PackageVersionMultiProperty::Tag), packageId); - CommandsTable::EnsureExistsAndInsert(connection, m_internalInterface->GetMultiPropertyByManifestId(connection, versionKey.ManifestId, PackageVersionMultiProperty::Command), packageId); - - PackageFamilyNameTable::EnsureExists(connection, m_internalInterface->GetMultiPropertyByManifestId(connection, versionKey.ManifestId, PackageVersionMultiProperty::PackageFamilyName), packageId); - ProductCodeTable::EnsureExists(connection, m_internalInterface->GetMultiPropertyByManifestId(connection, versionKey.ManifestId, PackageVersionMultiProperty::ProductCode), packageId); - NormalizedPackageNameTable::EnsureExists(connection, m_internalInterface->GetMultiPropertyByManifestId(connection, versionKey.ManifestId, PackageVersionMultiProperty::Name), packageId); - NormalizedPackagePublisherTable::EnsureExists(connection, m_internalInterface->GetMultiPropertyByManifestId(connection, versionKey.ManifestId, PackageVersionMultiProperty::Publisher), packageId); - UpgradeCodeTable::EnsureExists(connection, m_internalInterface->GetMultiPropertyByManifestId(connection, versionKey.ManifestId, PackageVersionMultiProperty::UpgradeCode), packageId); + TagsTable::EnsureExistsAndInsert(connection, m_internalInterface->GetMultiPropertyByPrimaryId(connection, versionKey.ManifestId, PackageVersionMultiProperty::Tag), packageId); + CommandsTable::EnsureExistsAndInsert(connection, m_internalInterface->GetMultiPropertyByPrimaryId(connection, versionKey.ManifestId, PackageVersionMultiProperty::Command), packageId); + + PackageFamilyNameTable::EnsureExists(connection, m_internalInterface->GetMultiPropertyByPrimaryId(connection, versionKey.ManifestId, PackageVersionMultiProperty::PackageFamilyName), packageId); + ProductCodeTable::EnsureExists(connection, m_internalInterface->GetMultiPropertyByPrimaryId(connection, versionKey.ManifestId, PackageVersionMultiProperty::ProductCode), packageId); + NormalizedPackageNameTable::EnsureExists(connection, m_internalInterface->GetMultiPropertyByPrimaryId(connection, versionKey.ManifestId, PackageVersionMultiProperty::Name), packageId); + NormalizedPackagePublisherTable::EnsureExists(connection, m_internalInterface->GetMultiPropertyByPrimaryId(connection, versionKey.ManifestId, PackageVersionMultiProperty::Publisher), packageId); + UpgradeCodeTable::EnsureExists(connection, m_internalInterface->GetMultiPropertyByPrimaryId(connection, versionKey.ManifestId, PackageVersionMultiProperty::UpgradeCode), packageId); } } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/OneToManyTableWithMap.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/OneToManyTableWithMap.cpp @@ -407,5 +407,44 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 return ((countStatement.GetColumn<int>(0) == 0) && (countMapStatement.GetColumn<int>(0) == 0)); } + + int OneToManyTableWithMapBuildSearchStatement( + SQLite::Builder::StatementBuilder& builder, + std::string_view tableName, + std::string_view valueName, + std::string_view primaryAlias, + std::string_view valueAlias, + bool useLike) + { + using QCol = SQLite::Builder::QualifiedColumn; + constexpr std::string_view s_map = "map"sv; + + // Build a statement like: + // SELECT map.package as p, table.value as v from table + // join map on table.rowid = map.value + // where table.value = <value> + builder.Select(). + Column(QCol(s_map, s_OneToManyTableWithMap_MapTable_PrimaryName)).As(primaryAlias). + Column(QCol(tableName, valueName)).As(valueAlias). + From(tableName). + Join({ tableName, s_OneToManyTableWithMap_MapTable_Suffix }).As(s_map).On(QCol(tableName, SQLite::RowIDName), QCol(s_map, valueName)). + Where(QCol(tableName, valueName)); + + int result = -1; + + if (useLike) + { + builder.Like(SQLite::Builder::Unbound); + result = builder.GetLastBindIndex(); + builder.Escape(SQLite::EscapeCharForLike); + } + else + { + builder.Equals(SQLite::Builder::Unbound); + result = builder.GetLastBindIndex(); + } + + return result; + } } } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/OneToManyTableWithMap.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/OneToManyTableWithMap.h @@ -2,6 +2,7 @@ // Licensed under the MIT License. #pragma once #include <winget/SQLiteWrapper.h> +#include <winget/SQLiteStatementBuilder.h> #include <string> #include <string_view> #include <vector> @@ -53,6 +54,16 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 // Determines if the table is empty. bool OneToManyTableWithMapIsEmpty(SQLite::Connection& connection, std::string_view tableName); + + // Builds the search select statement base on the given value. + // The return value is the bind index of the value to match against. + int OneToManyTableWithMapBuildSearchStatement( + SQLite::Builder::StatementBuilder& builder, + std::string_view tableName, + std::string_view valueName, + std::string_view primaryAlias, + std::string_view valueAlias, + bool useLike); } // A table that represents a value that is 1:N with a primary entry. @@ -115,5 +126,12 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 { return details::OneToManyTableWithMapIsEmpty(connection, TableInfo::TableName()); } + + // Builds the search select statement base on the given value. + // The return value is the bind index of the value to match against. + static int BuildSearchStatement(SQLite::Builder::StatementBuilder& builder, std::string_view primaryAlias, std::string_view valueAlias, bool useLike) + { + return details::OneToManyTableWithMapBuildSearchStatement(builder, TableInfo::TableName(), TableInfo::ValueName(), primaryAlias, valueAlias, useLike); + } }; } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/PackageUpdateTrackingTable.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/PackageUpdateTrackingTable.cpp @@ -103,10 +103,10 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 { Manifest::PackageVersionDataManifest::VersionData versionData{ key.VersionAndChannel, - internalIndex->GetPropertyByManifestId(connection, key.ManifestId, PackageVersionProperty::ArpMinVersion), - internalIndex->GetPropertyByManifestId(connection, key.ManifestId, PackageVersionProperty::ArpMaxVersion), - internalIndex->GetPropertyByManifestId(connection, key.ManifestId, PackageVersionProperty::RelativePath), - internalIndex->GetPropertyByManifestId(connection, key.ManifestId, PackageVersionProperty::ManifestSHA256Hash) + internalIndex->GetPropertyByPrimaryId(connection, key.ManifestId, PackageVersionProperty::ArpMinVersion), + internalIndex->GetPropertyByPrimaryId(connection, key.ManifestId, PackageVersionProperty::ArpMaxVersion), + internalIndex->GetPropertyByPrimaryId(connection, key.ManifestId, PackageVersionProperty::RelativePath), + internalIndex->GetPropertyByPrimaryId(connection, key.ManifestId, PackageVersionProperty::ManifestSHA256Hash) }; manifest.AddVersion(std::move(versionData)); @@ -189,7 +189,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 std::vector<ISQLiteIndex::VersionKey> versionKeys = internalIndex->GetVersionKeysById(connection, packageMatch.first); ISQLiteIndex::VersionKey& latestVersionKey = versionKeys[0]; - std::string packageIdentifier = internalIndex->GetPropertyByManifestId(connection, latestVersionKey.ManifestId, PackageVersionProperty::Id).value(); + std::string packageIdentifier = internalIndex->GetPropertyByPrimaryId(connection, latestVersionKey.ManifestId, PackageVersionProperty::Id).value(); select.Reset(); select.Bind(1, packageIdentifier); diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/PackagesTable.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/PackagesTable.cpp @@ -98,70 +98,6 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 return result; } - std::vector<int> PackagesTableBuildSearchStatement( - SQLite::Builder::StatementBuilder& builder, - std::initializer_list<std::string_view> columns, - std::string_view primaryAlias, - std::string_view valueAlias, - bool useLike) - { - using QCol = SQLite::Builder::QualifiedColumn; - - // Build a statement like: - // SELECT manifest.rowid as m, ids.id as v from manifest - // join ids on manifest.id = ids.rowid - // where ids.id = <value> - // OR - // SELECT manifest.rowid as m, tags.tag as v from manifest - // join tags_map on manifest.rowid = tags_map.manifest - // join tags on tags_map.tag = tags.rowid - // where tags.tag = <value> - // Where the joins and where portions are repeated for each table in question. - builder.Select(). - Column(QCol(s_PackagesTable_Table_Name, SQLite::RowIDName)).As(primaryAlias); - - // Value will be captured for single tables references, and left empty for multi-tables - if (columns.size() == 1) - { - builder.Column(*columns.begin()); - } - else - { - builder.LiteralColumn(""); - } - - builder.As(valueAlias).From(s_PackagesTable_Table_Name); - - std::vector<int> result; - - // Create where clause - for (const auto& column : columns) - { - if (result.empty()) - { - builder.Where(column); - } - else - { - builder.And(column); - } - - if (useLike) - { - builder.Like(SQLite::Builder::Unbound); - result.push_back(builder.GetLastBindIndex()); - builder.Escape(SQLite::EscapeCharForLike); - } - else - { - builder.Equals(SQLite::Builder::Unbound); - result.push_back(builder.GetLastBindIndex()); - } - } - - return result; - } - SQLite::Statement PackagesTableUpdateValueIdById_Statement(SQLite::Connection& connection, std::string_view valueName) { SQLite::Builder::StatementBuilder builder; @@ -345,6 +281,40 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 return static_cast<uint64_t>(countStatement.GetColumn<SQLite::rowid_t>(0)); } + int PackagesTable::BuildSearchStatement( + SQLite::Builder::StatementBuilder& builder, + std::string_view valueName, + std::string_view primaryAlias, + std::string_view valueAlias, + bool useLike) + { + using QCol = SQLite::Builder::QualifiedColumn; + + // Build a statement like: + // SELECT packages.rowid as p, packages.id as v from packages + // where packages.id = <value> + builder.Select(). + Column(QCol(s_PackagesTable_Table_Name, SQLite::RowIDName)).As(primaryAlias). + Column(QCol(s_PackagesTable_Table_Name, valueName)).As(valueAlias). + From(s_PackagesTable_Table_Name).Where(QCol(s_PackagesTable_Table_Name, valueName)); + + int result = -1; + + if (useLike) + { + builder.Like(SQLite::Builder::Unbound); + result = builder.GetLastBindIndex(); + builder.Escape(SQLite::EscapeCharForLike); + } + else + { + builder.Equals(SQLite::Builder::Unbound); + result = builder.GetLastBindIndex(); + } + + return result; + } + bool PackagesTable::IsEmpty(SQLite::Connection& connection) { SQLite::Builder::StatementBuilder builder; diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/PackagesTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/PackagesTable.h @@ -44,14 +44,6 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 std::initializer_list<std::string_view> columns, bool stepAndVerify = true); - // Builds the search select statement base on the given values. - std::vector<int> PackagesTableBuildSearchStatement( - SQLite::Builder::StatementBuilder& builder, - std::initializer_list<std::string_view> columns, - std::string_view manifestAlias, - std::string_view valueAlias, - bool useLike); - // Prepares a statement to update the value of a single column for the manifest with the given rowid. // The first bind value will be the value to set. // The second bind value will be the manifest rowid to modify. @@ -167,30 +159,25 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 template <typename... Columns> static auto GetValuesById(const SQLite::Connection& connection, SQLite::rowid_t rowid) { - return details::PackagesTableGetValuesById_Statement(connection, rowid, { Columns::Name... }).GetRow<typename SQLite::Builder::TypeInfo<Columns::Type>::value_t...>(); + return details::PackagesTableGetValuesById_Statement(connection, rowid, { Columns::Name... }).GetRow<typename SQLite::Builder::TypeInfo<Columns::Type, Columns::AllowNull>::value_t...>(); } // Gets the value requested for the package with the given rowid, if it exists. template <typename Column> - static std::optional<typename SQLite::Builder::TypeInfo<Column::Type>::value_t> GetValueById(const SQLite::Connection& connection, SQLite::rowid_t rowid) + static std::optional<typename SQLite::Builder::TypeInfo<Column::Type, false>::value_t> GetValueById(const SQLite::Connection& connection, SQLite::rowid_t rowid) { auto statement = details::PackagesTableGetValuesById_Statement(connection, rowid, { Column::Name }, false); - if (statement.Step()) { return statement.GetColumn<typename SQLite::Builder::TypeInfo<Column::Type>::value_t>(0); } + if (statement.Step()) { return statement.GetColumn<typename SQLite::Builder::TypeInfo<Column::Type, Column::AllowNull>::value_t>(0); } else { return std::nullopt; } } - // Builds the search select statement base on the given values. - // If more than one table is provided, no value will be captured. - // The return value is the bind indices of the values to match against. - template <typename... Columns> - static std::vector<int> BuildSearchStatement(SQLite::Builder::StatementBuilder& builder, std::string_view primaryAlias, std::string_view valueAlias, bool useLike) - { - return details::PackagesTableBuildSearchStatement(builder, { Columns::Name... }, primaryAlias, valueAlias, useLike); - } + // Builds the search select statement base on the given value. + // The return value is the bind index of the value to match against. + static int BuildSearchStatement(SQLite::Builder::StatementBuilder& builder, std::string_view valueName, std::string_view primaryAlias, std::string_view valueAlias, bool useLike); // Update the value of a single column for the package with the given rowid. template <typename Column> - static void UpdateValueIdById(SQLite::Connection& connection, SQLite::rowid_t id, const typename SQLite::Builder::TypeInfo<Column::Type>::value_t& value) + static void UpdateValueIdById(SQLite::Connection& connection, SQLite::rowid_t id, const typename SQLite::Builder::TypeInfo<Column::Type, Column::AllowNull>::value_t& value) { auto stmt = details::PackagesTableUpdateValueIdById_Statement(connection, Column::Name); stmt.Bind(1, value); diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/SearchResultsTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/SearchResultsTable.h @@ -27,8 +27,8 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 // Performs the requested search type on the requested field. void SearchOnField(const PackageMatchFilter& filter); - // Removes rows with manifest ids whose sort order is below the highest one. - void RemoveDuplicateManifestRows(); + // Removes rows with package ids whose sort order is below the highest one. + void RemoveDuplicatePackageRows(); // Prepares the table for a filtering pass. void PrepareToFilter(); diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/SearchResultsTable_2_0.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/SearchResultsTable_2_0.cpp @@ -7,10 +7,13 @@ #include "Microsoft/Schema/2_0/PackagesTable.h" #include "Microsoft/Schema/2_0/TagsTable.h" #include "Microsoft/Schema/2_0/CommandsTable.h" +#include "Microsoft/Schema/2_0/PackageFamilyNameTable.h" +#include "Microsoft/Schema/2_0/ProductCodeTable.h" +#include "Microsoft/Schema/2_0/UpgradeCodeTable.h" +#include "Microsoft/Schema/2_0/NormalizedPackageNameTable.h" +#include "Microsoft/Schema/2_0/NormalizedPackagePublisherTable.h" -// TODO :: The code here was copied from a previous schema version and is currently a placeholder !!! - namespace AppInstaller::Repository::Microsoft::Schema::V2_0 { namespace @@ -18,7 +21,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 using namespace std::string_literals; using namespace std::string_view_literals; - constexpr std::string_view s_SearchResultsTable_Manifest = "manifest"sv; + constexpr std::string_view s_SearchResultsTable_Package = "package"sv; constexpr std::string_view s_SearchResultsTable_MatchField = "field"sv; constexpr std::string_view s_SearchResultsTable_MatchType = "match"sv; constexpr std::string_view s_SearchResultsTable_MatchValue = "value"sv; @@ -28,7 +31,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 constexpr std::string_view s_SearchResultsTable_Index_Suffix = "_i_m"sv; constexpr std::string_view s_SearchResultsTable_SubSelect_TableAlias = "valueTable"sv; - constexpr std::string_view s_SearchResultsTable_SubSelect_ManifestAlias = "m"sv; + constexpr std::string_view s_SearchResultsTable_SubSelect_PackageAlias = "p"sv; constexpr std::string_view s_SearchResultsTable_SubSelect_ValueAlias = "v"sv; } @@ -41,7 +44,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 StatementBuilder builder; builder.CreateTable(GetQualifiedName()).BeginColumns(); - builder.Column(ColumnBuilder(s_SearchResultsTable_Manifest, Type::RowId).NotNull()); + builder.Column(ColumnBuilder(s_SearchResultsTable_Package, Type::RowId).NotNull()); builder.Column(ColumnBuilder(s_SearchResultsTable_MatchField, Type::Int).NotNull()); builder.Column(ColumnBuilder(s_SearchResultsTable_MatchType, Type::Int).NotNull()); builder.Column(ColumnBuilder(s_SearchResultsTable_MatchValue, Type::Text).NotNull()); @@ -62,7 +65,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 index.Table = indexName; StatementBuilder builder; - builder.CreateIndex(indexName).On(GetQualifiedName().Table).Columns(s_SearchResultsTable_Manifest); + builder.CreateIndex(indexName).On(GetQualifiedName().Table).Columns(s_SearchResultsTable_Package); builder.Execute(m_connection); } @@ -77,12 +80,12 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 // Create an insert statement to select values into the table as requested. // The goal is a statement like this: // INSERT INTO <tempTable> - // SELECT valueTable.m, <field>, <match>, valueTable.v, <sort>, <filter> FROM - // (SELECT manifest.rowid as m, manifest.id as v from manifest join ids on manifest.id = ids.rowid where ids.id = <value>) AS valueTable + // SELECT valueTable.p, <field>, <match>, valueTable.v, <sort>, <filter> FROM + // (SELECT packages.rowid as p, packages.id as v from packages where packages.id = <value>) AS valueTable // Where the subselect is built by the owning table. StatementBuilder builder; builder.InsertInto(GetQualifiedName()).Select(). - Column(QualifiedColumn(s_SearchResultsTable_SubSelect_TableAlias, s_SearchResultsTable_SubSelect_ManifestAlias)). + Column(QualifiedColumn(s_SearchResultsTable_SubSelect_TableAlias, s_SearchResultsTable_SubSelect_PackageAlias)). Value(filter.Field). Value(filter.Type). Column(QualifiedColumn(s_SearchResultsTable_SubSelect_TableAlias, s_SearchResultsTable_SubSelect_ValueAlias)). @@ -107,22 +110,22 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 AICLI_LOG(SQL, Verbose, << "Search found " << m_connection.GetChanges() << " rows"); } - void SearchResultsTable::RemoveDuplicateManifestRows() + void SearchResultsTable::RemoveDuplicatePackageRows() { using namespace SQLite::Builder; - // Create a delete statement to leave only one row with a given manifest. + // Create a delete statement to leave only one row with a given package. // This will arbitrarily choose one of the rows if multiple have the same lowest sort order. // The goal is a statement like this: // DELETE from <temp> where rowid not in ( // SELECT rowid from ( - // SELECT rowid, min(sort) from <temp> group by manifest + // SELECT rowid, min(sort) from <temp> group by package // ) // ) StatementBuilder builder; builder.DeleteFrom(GetQualifiedName()).Where(SQLite::RowIDName).Not().In().BeginParenthetical(). Select(SQLite::RowIDName).From().BeginParenthetical(). - Select().Column(SQLite::RowIDName).Column(Aggregate::Min, s_SearchResultsTable_SortValue).From(GetQualifiedName()).GroupBy(s_SearchResultsTable_Manifest). + Select().Column(SQLite::RowIDName).Column(Aggregate::Min, s_SearchResultsTable_SortValue).From(GetQualifiedName()).GroupBy(s_SearchResultsTable_Package). EndParenthetical(). EndParenthetical(); @@ -146,14 +149,14 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 // Create an update statement to mark rows that are found by the search. // This will arbitrarily choose one of the rows if multiple have the same lowest sort order. // The goal is a statement like this: - // UPDATE <temp> set filter = 1 where manifest in ( - // SELECT m from ( - // SELECT manifest.rowid as m, manifest.id as v from manifest join ids on manifest.id = ids.rowid where ids.id = <value> + // UPDATE <temp> set filter = 1 where package in ( + // SELECT p from ( + // SELECT packages.rowid as p, packages.id as v from packages where packages.id = <value> // ) // ) StatementBuilder builder; - builder.Update(GetQualifiedName()).Set().Column(s_SearchResultsTable_Filter).Equals(true).Where(s_SearchResultsTable_Manifest).In().BeginParenthetical(). - Select(s_SearchResultsTable_SubSelect_ManifestAlias).From().BeginParenthetical(); + builder.Update(GetQualifiedName()).Set().Column(s_SearchResultsTable_Filter).Equals(true).Where(s_SearchResultsTable_Package).In().BeginParenthetical(). + Select(s_SearchResultsTable_SubSelect_PackageAlias).From().BeginParenthetical(); // Add the field specific portion std::vector<int> bindIndex = BuildSearchStatement(builder, filter.Field, filter.Type); @@ -184,26 +187,18 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 ISQLiteIndex::SearchResult SearchResultsTable::GetSearchResults(size_t limit) { - constexpr std::string_view tempTableAlias = "t"sv; - using namespace SQLite::Builder; using QCol = QualifiedColumn; - // Select all unique ids from the results table, and their highest ordered match. - // The goal is a statement like this: - // SELECT m.id, field, match, value, min(sort) from <temp> join manifest on rowid = manifest group by m.id order by t.sort - // Through the "group by m.id", we will only ever have one row per id, and the "min(sort)" returns us one of the rows that matched - // through the earliest search. We also order by the sort value to have the earliest search matches first in the list + // Select all of the results from the table; it is expected that RemoveDuplicatePackageRows has been called. StatementBuilder builder; - //builder.Select(). - // Column(QCol(ManifestTable::TableName(), IdTable::ValueName())). - // Column(QCol(tempTableAlias, s_SearchResultsTable_MatchField)). - // Column(QCol(tempTableAlias, s_SearchResultsTable_MatchType)). - // Column(QCol(tempTableAlias, s_SearchResultsTable_MatchValue)). - // Column(Aggregate::Min, QCol(tempTableAlias, s_SearchResultsTable_SortValue)). - //From(GetQualifiedName()).As(tempTableAlias). - // Join(ManifestTable::TableName()).On(QCol(tempTableAlias, s_SearchResultsTable_Manifest), QCol(ManifestTable::TableName(), SQLite::RowIDName)). - // GroupBy(QCol(ManifestTable::TableName(), IdTable::ValueName())).OrderBy(QCol(tempTableAlias, s_SearchResultsTable_SortValue)); + builder.Select({ + s_SearchResultsTable_Package, + s_SearchResultsTable_MatchField, + s_SearchResultsTable_MatchType, + s_SearchResultsTable_MatchValue, + }). + From(GetQualifiedName()).OrderBy(s_SearchResultsTable_SortValue); SQLite::Statement select = builder.Prepare(m_connection); @@ -226,7 +221,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 std::vector<int> SearchResultsTable::BuildSearchStatement(SQLite::Builder::StatementBuilder& builder, PackageMatchField field, MatchType match) const { - return BuildSearchStatement(builder, field, s_SearchResultsTable_SubSelect_ManifestAlias, s_SearchResultsTable_SubSelect_ValueAlias, MatchUsesLike(match)); + return BuildSearchStatement(builder, field, s_SearchResultsTable_SubSelect_PackageAlias, s_SearchResultsTable_SubSelect_ValueAlias, MatchUsesLike(match)); } std::vector<int> SearchResultsTable::BuildSearchStatement( @@ -236,21 +231,40 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 std::string_view valueAlias, bool useLike) const { + std::vector<int> result; + switch (field) { case PackageMatchField::Id: - return PackagesTable::BuildSearchStatement<PackagesTable::IdColumn>(builder, manifestAlias, valueAlias, useLike); + result.push_back(PackagesTable::BuildSearchStatement(builder, PackagesTable::IdColumn::Name, manifestAlias, valueAlias, useLike)); + break; case PackageMatchField::Name: - return PackagesTable::BuildSearchStatement<PackagesTable::NameColumn>(builder, manifestAlias, valueAlias, useLike); + result.push_back(PackagesTable::BuildSearchStatement(builder, PackagesTable::NameColumn::Name, manifestAlias, valueAlias, useLike)); + break; case PackageMatchField::Moniker: - return PackagesTable::BuildSearchStatement<PackagesTable::MonikerColumn>(builder, manifestAlias, valueAlias, useLike); - //case PackageMatchField::Tag: - // return PackagesTable::BuildSearchStatement<TagsTable>(builder, manifestAlias, valueAlias, useLike); - //case PackageMatchField::Command: - // return PackagesTable::BuildSearchStatement<CommandsTable>(builder, manifestAlias, valueAlias, useLike); - default: - return {}; + result.push_back(PackagesTable::BuildSearchStatement(builder, PackagesTable::MonikerColumn::Name, manifestAlias, valueAlias, useLike)); + break; + case PackageMatchField::Tag: + result.push_back(TagsTable::BuildSearchStatement(builder, manifestAlias, valueAlias, useLike)); + break; + case PackageMatchField::Command: + result.push_back(CommandsTable::BuildSearchStatement(builder, manifestAlias, valueAlias, useLike)); + break; + case PackageMatchField::PackageFamilyName: + result.push_back(PackageFamilyNameTable::BuildSearchStatement(builder, manifestAlias, valueAlias, useLike)); + break; + case PackageMatchField::ProductCode: + result.push_back(ProductCodeTable::BuildSearchStatement(builder, manifestAlias, valueAlias, useLike)); + break; + case PackageMatchField::UpgradeCode: + result.push_back(UpgradeCodeTable::BuildSearchStatement(builder, manifestAlias, valueAlias, useLike)); + break; + case PackageMatchField::NormalizedNameAndPublisher: + result = NormalizedPackageNameTable::BuildPairedSearchStatement<NormalizedPackagePublisherTable>(builder, manifestAlias, valueAlias, useLike); + break; } + + return result; } bool SearchResultsTable::MatchUsesLike(MatchType match) @@ -297,5 +311,10 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 } BindStatementForMatchType(statement, filter.Type, bindIndex[0], filter.Value); + + if (filter.Field == PackageMatchField::NormalizedNameAndPublisher) + { + BindStatementForMatchType(statement, filter.Type, bindIndex[1], filter.Additional.value()); + } } } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/SystemReferenceStringTable.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/SystemReferenceStringTable.cpp @@ -172,5 +172,95 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 return countStatement.GetColumn<int>(0) == 0; } + + int SystemReferenceStringTableBuildSearchStatement( + SQLite::Builder::StatementBuilder& builder, + std::string_view tableName, + std::string_view valueName, + std::string_view primaryAlias, + std::string_view valueAlias, + bool useLike) + { + using QCol = SQLite::Builder::QualifiedColumn; + + // Build a statement like: + // SELECT table.package as p, table.value as v from table + // where table.value = <value> + builder.Select(). + Column(s_SystemReferenceStringTable_PrimaryName).As(primaryAlias). + Column(valueName).As(valueAlias). + From(tableName). + Where(valueName); + + int result = -1; + + if (useLike) + { + builder.Like(SQLite::Builder::Unbound); + result = builder.GetLastBindIndex(); + builder.Escape(SQLite::EscapeCharForLike); + } + else + { + builder.Equals(SQLite::Builder::Unbound); + result = builder.GetLastBindIndex(); + } + + return result; + } + + std::vector<int> SystemReferenceStringTableBuildPairedSearchStatement( + SQLite::Builder::StatementBuilder& builder, + std::string_view tableName, + std::string_view valueName, + std::string_view pairedTableName, + std::string_view pairedValueName, + std::string_view primaryAlias, + std::string_view valueAlias, + bool useLike) + { + using QCol = SQLite::Builder::QualifiedColumn; + + // Build a statement like: + // SELECT table.package as p, '' as v from table + // join paired on table.package = paired.package + // where table.value = <value1> and paired.pairedValue = <value2> + builder.Select(). + Column(QCol(tableName, s_SystemReferenceStringTable_PrimaryName)).As(primaryAlias). + LiteralColumn("").As(valueAlias). + From(tableName). + Join(pairedTableName).On(QCol(tableName, s_SystemReferenceStringTable_PrimaryName), QCol(pairedTableName, s_SystemReferenceStringTable_PrimaryName)). + Where(QCol(tableName, valueName)); + + std::vector<int> result; + + if (useLike) + { + builder.Like(SQLite::Builder::Unbound); + result.push_back(builder.GetLastBindIndex()); + builder.Escape(SQLite::EscapeCharForLike); + } + else + { + builder.Equals(SQLite::Builder::Unbound); + result.push_back(builder.GetLastBindIndex()); + } + + builder.And(QCol(pairedTableName, pairedValueName)); + + if (useLike) + { + builder.Like(SQLite::Builder::Unbound); + result.push_back(builder.GetLastBindIndex()); + builder.Escape(SQLite::EscapeCharForLike); + } + else + { + builder.Equals(SQLite::Builder::Unbound); + result.push_back(builder.GetLastBindIndex()); + } + + return result; + } } } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/SystemReferenceStringTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/2_0/SystemReferenceStringTable.h @@ -2,6 +2,7 @@ // Licensed under the MIT License. #pragma once #include <winget/SQLiteWrapper.h> +#include <winget/SQLiteStatementBuilder.h> #include <AppInstallerStrings.h> #include <string> #include <string_view> @@ -42,6 +43,28 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 // Determines if the table is empty. bool SystemReferenceStringTableIsEmpty(SQLite::Connection& connection, std::string_view tableName); + + // Builds the search select statement base on the given value. + // The return value is the bind index of the value to match against. + int SystemReferenceStringTableBuildSearchStatement( + SQLite::Builder::StatementBuilder& builder, + std::string_view tableName, + std::string_view valueName, + std::string_view primaryAlias, + std::string_view valueAlias, + bool useLike); + + // Builds the search select statement base on the given value. + // The return value is the bind index of the value to match against. + std::vector<int> SystemReferenceStringTableBuildPairedSearchStatement( + SQLite::Builder::StatementBuilder& builder, + std::string_view tableName, + std::string_view valueName, + std::string_view pairedTableName, + std::string_view pairedValueName, + std::string_view primaryAlias, + std::string_view valueAlias, + bool useLike); } // A table that represents a value that is 1:N with a primary entry. @@ -96,5 +119,20 @@ namespace AppInstaller::Repository::Microsoft::Schema::V2_0 { return details::SystemReferenceStringTableIsEmpty(connection, TableInfo::TableName()); } + + // Builds the search select statement base on the given value. + // The return value is the bind index of the value to match against. + static int BuildSearchStatement(SQLite::Builder::StatementBuilder& builder, std::string_view primaryAlias, std::string_view valueAlias, bool useLike) + { + return details::SystemReferenceStringTableBuildSearchStatement(builder, TableInfo::TableName(), TableInfo::ValueName(), primaryAlias, valueAlias, useLike); + } + + // Builds the search select statement base on the given value. + // The return value is the bind index of the value to match against. + template<typename PairedTable> + static std::vector<int> BuildPairedSearchStatement(SQLite::Builder::StatementBuilder& builder, std::string_view primaryAlias, std::string_view valueAlias, bool useLike) + { + return details::SystemReferenceStringTableBuildPairedSearchStatement(builder, TableInfo::TableName(), TableInfo::ValueName(), PairedTable::TableName(), PairedTable::ValueName(), primaryAlias, valueAlias, useLike); + } }; } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.h @@ -95,11 +95,11 @@ namespace AppInstaller::Repository::Microsoft::Schema // Performs a search based on the given criteria. virtual SearchResult Search(const SQLite::Connection& connection, const SearchRequest& request) const = 0; - // Gets the string for the given property and manifest id, if present. - virtual std::optional<std::string> GetPropertyByManifestId(const SQLite::Connection& connection, SQLite::rowid_t manifestId, PackageVersionProperty property) const = 0; + // Gets the string for the given property and primary id, if present. + virtual std::optional<std::string> GetPropertyByPrimaryId(const SQLite::Connection& connection, SQLite::rowid_t primaryId, PackageVersionProperty property) const = 0; - // Gets the string values for the given property and manifest id, if present. - virtual std::vector<std::string> GetMultiPropertyByManifestId(const SQLite::Connection& connection, SQLite::rowid_t manifestId, PackageVersionMultiProperty property) const = 0; + // Gets the string values for the given property and primary id, if present. + virtual std::vector<std::string> GetMultiPropertyByPrimaryId(const SQLite::Connection& connection, SQLite::rowid_t primaryId, PackageVersionMultiProperty property) const = 0; // Gets the manifest id for the given { id, version, channel }, if present. // If version is empty, gets the value for the 'latest' version. diff --git a/src/AppInstallerRepositoryCore/PackageDependenciesValidation.cpp b/src/AppInstallerRepositoryCore/PackageDependenciesValidation.cpp @@ -141,7 +141,7 @@ namespace AppInstaller::Repository [&](std::pair<SQLite::rowid_t, Utility::NormalizedString> row) { auto manifestRowId = index->GetManifestIdByKey(row.first, "", ""); - auto packageId = index->GetPropertyByManifestId(manifestRowId.value(), PackageVersionProperty::Id); + auto packageId = index->GetPropertyByPrimaryId(manifestRowId.value(), PackageVersionProperty::Id); Dependency dep(DependencyType::Package, packageId.value(), row.second); depList.Add(dep); }); @@ -183,8 +183,8 @@ namespace AppInstaller::Repository [&](std::pair<SQLite::rowid_t, Utility::Version> current) { DependentManifestInfo dependentManifestInfo; - dependentManifestInfo.Id = index->GetPropertyByManifestId(current.first, PackageVersionProperty::Id).value(); - dependentManifestInfo.Version = index->GetPropertyByManifestId(current.first, PackageVersionProperty::Version).value(); + dependentManifestInfo.Id = index->GetPropertyByPrimaryId(current.first, PackageVersionProperty::Id).value(); + dependentManifestInfo.Version = index->GetPropertyByPrimaryId(current.first, PackageVersionProperty::Version).value(); dependentManifestInfoToVersionPair.emplace_back(std::make_pair(dependentManifestInfo, current.second)); }); @@ -233,4 +233,4 @@ namespace AppInstaller::Repository return true; } -}- \ No newline at end of file +} diff --git a/src/AppInstallerRepositoryCore/PackageVersionSelection.cpp b/src/AppInstallerRepositoryCore/PackageVersionSelection.cpp @@ -75,6 +75,29 @@ namespace AppInstaller::Repository { package = GetAvailablePackageFromSource(m_packages, versionKey.SourceId); } + else if (m_packages.size() == 1) + { + package = m_packages[0]; + } + else if (versionKey.IsDefaultLatest()) + { + std::shared_ptr<IPackageVersion> result; + Utility::Version resultVersion; + + for (const auto& p : m_packages) + { + std::shared_ptr<IPackageVersion> latest = p->GetLatestVersion(); + Utility::Version version { latest->GetProperty(PackageVersionProperty::Version) }; + + if (!result || resultVersion < version) + { + result = std::move(latest); + resultVersion = std::move(version); + } + } + + return result; + } else { // Otherwise, find the first version that matches diff --git a/src/AppInstallerRepositoryCore/Public/winget/RepositorySearch.h b/src/AppInstallerRepositoryCore/Public/winget/RepositorySearch.h @@ -269,6 +269,9 @@ namespace AppInstaller::Repository // Determines if a well defined key (this one) is matched by the provided key. // The provided key may use empty values to indicate no specific matching requirements. bool IsMatch(const PackageVersionKey& other) const; + + // Determines if this version is the simple "latest" targeting version (version and channel are both empty). + bool IsDefaultLatest() const; }; // A property of a package. @@ -283,7 +286,8 @@ namespace AppInstaller::Repository { TestPackage, RestPackage, - SQLitePackage, + SQLitePackage1, + SQLitePackage2, PinnablePackage, CompositeInstalledPackage, }; diff --git a/src/AppInstallerRepositoryCore/RepositorySearch.cpp b/src/AppInstallerRepositoryCore/RepositorySearch.cpp @@ -100,6 +100,11 @@ namespace AppInstaller::Repository (other.Channel.empty() || Utility::ICUCaseInsensitiveEquals(other.Channel, Channel))); } + bool PackageVersionKey::IsDefaultLatest() const + { + return Version.empty() && Channel.empty(); + } + const char* UnsupportedRequestException::what() const noexcept { if (m_whatMessage.empty()) diff --git a/src/AppInstallerSharedLib/Public/winget/SQLiteStatementBuilder.h b/src/AppInstallerSharedLib/Public/winget/SQLiteStatementBuilder.h @@ -114,23 +114,35 @@ namespace AppInstaller::SQLite::Builder Integer, // Type for specifying a primary key column as a row id alias. }; - template <Type type> + template <Type type, bool allowNull> struct TypeInfo { }; template <> - struct TypeInfo<Type::Text> + struct TypeInfo<Type::Text, false> { using value_t = std::string; }; template <> - struct TypeInfo<Type::Blob> + struct TypeInfo<Type::Text, true> + { + using value_t = std::optional<std::string>; + }; + + template <> + struct TypeInfo<Type::Blob, false> { using value_t = SQLite::blob_t; }; + template <> + struct TypeInfo<Type::Blob, true> + { + using value_t = std::optional<SQLite::blob_t>; + }; + // Aggregate functions. enum class Aggregate { diff --git a/src/AppInstallerSharedLib/Public/winget/SQLiteStorageBase.h b/src/AppInstallerSharedLib/Public/winget/SQLiteStorageBase.h @@ -23,13 +23,13 @@ namespace AppInstaller::SQLite }; // Gets the last write time for the database. - std::chrono::system_clock::time_point GetLastWriteTime(); + std::chrono::system_clock::time_point GetLastWriteTime() const; // Gets the identifier written to the database when it was created. - std::string GetDatabaseIdentifier(); + std::string GetDatabaseIdentifier() const; // Gets the schema version of the database. - Version GetVersion() const { return m_version; } + const Version& GetVersion() const { return m_version; } protected: SQLiteStorageBase(const std::string& target, const Version& version); @@ -46,4 +46,4 @@ namespace AppInstaller::SQLite Version m_version; std::unique_ptr<std::mutex> m_interfaceLock = std::make_unique<std::mutex>(); }; -}- \ No newline at end of file +} diff --git a/src/AppInstallerSharedLib/Public/winget/SQLiteVersion.h b/src/AppInstallerSharedLib/Public/winget/SQLiteVersion.h @@ -33,6 +33,13 @@ namespace AppInstaller::SQLite return MinorVersion >= other.MinorVersion; } + bool operator<(const Version& other) const + { + if (MajorVersion < other.MajorVersion) return true; + if (MajorVersion > other.MajorVersion) return false; + return MinorVersion < other.MinorVersion; + } + // Gets a version that represents the latest schema known to the implementation. static Version Latest(); diff --git a/src/AppInstallerSharedLib/Public/winget/SQLiteWrapper.h b/src/AppInstallerSharedLib/Public/winget/SQLiteWrapper.h @@ -9,6 +9,8 @@ #include <AppInstallerLanguageUtilities.h> #include <memory> +#include <optional> +#include <sstream> #include <string> #include <string_view> #include <tuple> @@ -124,6 +126,50 @@ namespace AppInstaller::SQLite } }; + template <typename Opt> + struct ParameterSpecificsImpl<std::optional<Opt>> + { + using Optional = std::optional<Opt>; + + static auto ToLog(const Optional& v) + { + std::ostringstream result; + if (v) + { + result << ParameterSpecificsImpl<Opt>::ToLog(v.value()); + } + else + { + result << "{null}"; + } + return std::move(result).str(); + } + + static void Bind(sqlite3_stmt* stmt, int index, const Optional& v) + { + if (v) + { + ParameterSpecificsImpl<Opt>::Bind(stmt, index, v.value()); + } + else + { + ParameterSpecificsImpl<nullptr_t>::Bind(stmt, index, nullptr); + } + } + + static Optional GetColumn(sqlite3_stmt* stmt, int column) + { + if (sqlite3_column_type(stmt, column) == SQLITE_NULL) + { + return std::nullopt; + } + else + { + return ParameterSpecificsImpl<Opt>::GetColumn(stmt, column); + } + } + }; + template <typename T> using ParameterSpecifics = ParameterSpecificsImpl<std::decay_t<T>>; diff --git a/src/AppInstallerSharedLib/SQLiteStorageBase.cpp b/src/AppInstallerSharedLib/SQLiteStorageBase.cpp @@ -34,13 +34,13 @@ namespace AppInstaller::SQLite // Recording last write time based on MSDN documentation stating that time returns a POSIX epoch time and thus // should be consistent across systems. - std::chrono::system_clock::time_point SQLiteStorageBase::GetLastWriteTime() + std::chrono::system_clock::time_point SQLiteStorageBase::GetLastWriteTime() const { int64_t lastWriteTime = MetadataTable::GetNamedValue<int64_t>(m_dbconn, s_MetadataValueName_LastWriteTime); return Utility::ConvertUnixEpochToSystemClock(lastWriteTime); } - std::string SQLiteStorageBase::GetDatabaseIdentifier() + std::string SQLiteStorageBase::GetDatabaseIdentifier() const { return MetadataTable::TryGetNamedValue<std::string>(m_dbconn, s_MetadataValueName_DatabaseIdentifier).value_or(std::string{}); } diff --git a/src/LocalhostWebServer/Startup.cs b/src/LocalhostWebServer/Startup.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT License. namespace LocalhostWebServer @@ -59,6 +59,7 @@ namespace LocalhostWebServer provider.Mappings[".msi"] = "application/msi"; provider.Mappings[".appx"] = "application/vns.ms-appx"; provider.Mappings[".appxbundle"] = "application/vns.ms-appx"; + provider.Mappings[".mszyml"] = "application/x-ms-zip-yaml"; //Enable static file serving @@ -85,4 +86,4 @@ namespace LocalhostWebServer }); } } -}- \ No newline at end of file +} diff --git a/src/WinGetSourceCreator/Model/LocalSource.cs b/src/WinGetSourceCreator/Model/LocalSource.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT License. namespace WinGetSourceCreator.Model @@ -64,12 +64,20 @@ namespace WinGetSourceCreator.Model public string GetIndexName() { - return $"index.db"; + return "index.db"; } - public string GetSourceName() + public string GetSourceName(int version) { - return "source.msix"; + switch (version) + { + case 1: + return "source.msix"; + case 2: + return "source2.msix"; + } + + throw new ArgumentOutOfRangeException(nameof(version), version, "Unknown source major version"); } } } diff --git a/src/WinGetSourceCreator/WinGetLocalSource.cs b/src/WinGetSourceCreator/WinGetLocalSource.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT License. namespace Microsoft.WinGetSourceCreator @@ -7,6 +7,7 @@ namespace Microsoft.WinGetSourceCreator using System.Text.Json.Serialization; using System.Text.Json; using Microsoft.WinGetUtil.Api; + using Microsoft.WinGetUtil.Interfaces; public class WinGetLocalSource { @@ -66,9 +67,11 @@ namespace Microsoft.WinGetSourceCreator wingetSource.PrepareManifest(localManifest); } - var indexFile = wingetSource.CreateIndex(localSource.GetIndexName()); + var indexV2File = wingetSource.CreateIndex(localSource.GetIndexName(), 2, 0); + _ = wingetSource.CreatePackage(localSource.GetSourceName(2), localSource.AppxManifest, indexV2File, localSource.Signature); - _ = wingetSource.CreatePackage(localSource.GetSourceName(), localSource.AppxManifest, indexFile, localSource.Signature); + var indexV1File = wingetSource.CreateIndex(localSource.GetIndexName()); + _ = wingetSource.CreatePackage(localSource.GetSourceName(1), localSource.AppxManifest, indexV1File, localSource.Signature); } public WinGetLocalSource(string workingDirectory, Signature? signature) @@ -110,10 +113,17 @@ namespace Microsoft.WinGetSourceCreator } } - public string CreateIndex(string indexName) + public string CreateIndex(string indexName, uint? majorVersion = null, uint? minorVersion = null) { string fullPath = Path.Combine(this.workingDirectory, indexName); - using var indexHelper = new WinGetFactory().SQLiteIndexCreateLatestVersion(fullPath); + + if (File.Exists(fullPath)) + { + File.Delete(fullPath); + } + + WinGetFactory factory = new (); + using IWinGetSQLiteIndex indexHelper = majorVersion == null ? factory.SQLiteIndexCreateLatestVersion(fullPath) : factory.SQLiteIndexCreate(fullPath, majorVersion.Value, minorVersion.GetValueOrDefault()); Queue<string> filesQueue = new(Directory.EnumerateFiles(this.workingDirectory, "*.yaml", SearchOption.AllDirectories)); while (filesQueue.Count > 0) @@ -160,7 +170,7 @@ namespace Microsoft.WinGetSourceCreator } string appxManifestFile = Path.Combine(this.workingDirectory, "AppxManifest.xml"); - File.Copy(inputAppxManifestFile, appxManifestFile); + File.Copy(inputAppxManifestFile, appxManifestFile, true); if (signature != null && signature.Publisher != null) { @@ -170,7 +180,7 @@ namespace Microsoft.WinGetSourceCreator string mappingFile = Path.Combine(this.workingDirectory, "MappingFile.txt"); { - using StreamWriter outputFile = new(mappingFile); + using StreamWriter outputFile = new(mappingFile, false); outputFile.WriteLine("[Files]"); outputFile.WriteLine($"\"{indexPath}\" \"Public\\{Path.GetFileName(indexPath)}\""); outputFile.WriteLine($"\"{appxManifestFile}\" \"AppxManifest.xml\""); diff --git a/src/WinGetUtil/Source.def b/src/WinGetUtil/Source.def @@ -21,3 +21,5 @@ EXPORTS WinGetBeginInstallerMetadataCollection WinGetCompleteInstallerMetadataCollection WinGetMergeInstallerMetadata + WinGetSQLiteIndexMigrate + WinGetSQLiteIndexSetProperty diff --git a/templates/e2e-test.template.yml b/templates/e2e-test.template.yml @@ -5,6 +5,9 @@ parameters: type: boolean - name: filter type: string +- name: experimentalFeatures + type: string + default: 'none' steps: - task: CmdLine@2 @@ -28,14 +31,16 @@ steps: -StaticFileRootPath $(Agent.TempDirectory)\TestLocalIndex -PowerShellModulePath $(buildOutDir)\PowerShell\Microsoft.WinGet.Client\Microsoft.WinGet.Client.psd1 -LocalServerCertPath $(Agent.TempDirectory)\servercert.cer - -SkipTestSource true' + -SkipTestSource true + -ForcedExperimentalFeatures ${{ parameters.experimentalFeatures }}' ${{ else }}: overrideTestrunParameters: '-PackagedContext false -AICLIPath $(packageLayoutDir)\AppInstallerCLI\winget.exe -StaticFileRootPath $(Agent.TempDirectory)\TestLocalIndex -PowerShellModulePath $(buildOutDir)\PowerShell\Microsoft.WinGet.Client\Microsoft.WinGet.Client.psd1 -LocalServerCertPath $(Agent.TempDirectory)\servercert.cer - -SkipTestSource true' + -SkipTestSource true + -ForcedExperimentalFeatures ${{ parameters.experimentalFeatures }}' - task: CmdLine@2 displayName: Complete COM trace for ${{ parameters.title }}