commit a864c8242a0418a31ef363ba8c6e12a8f9b309bf parent f95cdb9a596b1cf0d34aacf93721d45c114c8396 Author: Flor Chacón <lechacon@users.noreply.github.com> Date: Mon, 15 Aug 2022 12:45:39 -0600 Use `UpgradeCode` for matching MSI apps (#2418) Diffstat:
25 files changed, 559 insertions(+), 61 deletions(-)
diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt @@ -267,6 +267,7 @@ ISource isspace istream istringstream +isxdigit ITest IUnknown json diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt @@ -11,7 +11,7 @@ aicli AICLIC ajor alreadyinstalled -Amd +amd amrutha anonymized APARTMENTTHREADED @@ -40,6 +40,8 @@ azurewebsites badbit Baz bcp +BEFACEF +BEFACEFE Beigi bfd BFirst @@ -67,10 +69,8 @@ cend centralus certmgr certs -Cfg cfr cgi -cgmanifest chcp ci cinq @@ -91,7 +91,6 @@ count'th countof countryregion createmanifestmetadata -CSharp cstdint ctc Ctx @@ -127,6 +126,7 @@ execustom EXEHASH experimentalfeatures fcb +FECAFEB fd fdw fedorapeople @@ -220,7 +220,7 @@ kayone Keivan KF KNOWNFOLDERID -kp +Kp ktf langs LATN @@ -276,11 +276,9 @@ myinstalldir mylog mysilent mysilentwithprogress -mytool nameof nativehandle NESTEDINSTALLER -NETFX netlify Newtonsoft NOEXPAND @@ -302,16 +300,15 @@ ofile openmode Outptr packageinuse +PACL PARAMETERMAP paramref -PACL pathparts Patil pb PCCERT PCs pcwsz -pdp PEGI pfn pfxpath @@ -444,6 +441,7 @@ unparsable UNSCOPED unvirtualized UParse +upgradecode UPSERT uris URLs @@ -460,7 +458,6 @@ VERSIE virtualization vns vscode -vstest vy wcslen webpages diff --git a/src/AppInstallerCLICore/Workflows/InstallFlow.cpp b/src/AppInstallerCLICore/Workflows/InstallFlow.cpp @@ -605,6 +605,14 @@ namespace AppInstaller::CLI::Workflow entries.push_back(std::move(entry)); } + auto upgradeCodes = correlationResult.Package->GetMultiProperty(PackageVersionMultiProperty::UpgradeCode); + for (auto&& upgradeCode : upgradeCodes) + { + AppsAndFeaturesEntry entry = baseEntry; + entry.UpgradeCode= std::move(upgradeCode).get(); + entries.push_back(std::move(entry)); + } + context.Add<Data::CorrelatedAppsAndFeaturesEntries>(std::move(entries)); } diff --git a/src/AppInstallerCLIE2ETests/ListCommand.cs b/src/AppInstallerCLIE2ETests/ListCommand.cs @@ -59,6 +59,25 @@ namespace AppInstallerCLIE2ETests ArpVersionMappingTest("AppInstallerTest.TestArpVersionSameOrder", "TestArpVersionSameOrder", "12.0", "> 2.0", "12.0"); } + [Test] + public void ListWithUpgradeCode() + { + // Installs the MSI installer using the TestMsiInstaller package. + // Then tries listing the TestMsiInstallerUpgradeCode package, which should + // be correlated to it by the UpgradeCode. + if (string.IsNullOrEmpty(TestCommon.MsiInstallerPath)) + { + Assert.Ignore("MSI installer not available"); + } + + var installDir = TestCommon.GetRandomTestDir(); + Assert.AreEqual(Constants.ErrorCode.S_OK, TestCommon.RunAICLICommand("install", $"TestMsiInstaller --silent -l {installDir}").ExitCode); + + var result = TestCommon.RunAICLICommand("list", "TestMsiInstallerUpgradeCode"); + Assert.AreEqual(Constants.ErrorCode.S_OK, result.ExitCode); + Assert.True(result.StdOut.Contains("AppInstallerTest.TestMsiInstallerUpgradeCode")); + } + private void ArpVersionMappingTest(string packageIdentifier, string displayNameOverride, string displayVersionOverride, string expectedListVersion, string notExpectedListVersion = "") { System.Guid guid = System.Guid.NewGuid(); diff --git a/src/AppInstallerCLIE2ETests/TestData/Manifests/TestMsiInstaller_UpgradeCode.yaml b/src/AppInstallerCLIE2ETests/TestData/Manifests/TestMsiInstaller_UpgradeCode.yaml @@ -0,0 +1,15 @@ +# Uses the MSI installer; doesn't list the ProductCode, only the UpgradeCode +PackageIdentifier: AppInstallerTest.TestMsiInstallerUpgradeCode +PackageVersion: 1.0.0.0 +PackageLocale: en-US +PackageName: TestMsiInstallerUpgradeCode +Publisher: AppInstallerTest +Installers: + - Architecture: x86 + InstallerUrl: https://localhost:5001/TestKit/AppInstallerTestMsiInstaller/AppInstallerTestMsiInstaller.msi + InstallerType: msi + InstallerSha256: <MSIHASH> + AppsAndFeaturesEntries: + - UpgradeCode: '{B9CF9DD5-D46F-4CE0-BFC9-633BF9D3A6F4}' +ManifestType: singleton +ManifestVersion: 1.1.0 diff --git a/src/AppInstallerRepositoryCore/ARPCorrelation.cpp b/src/AppInstallerRepositoryCore/ARPCorrelation.cpp @@ -188,15 +188,16 @@ namespace AppInstaller::Repository::Correlation } } - std::vector<std::string> productCodes; + std::set<std::string> productCodes; + std::set<std::string> upgradeCodes; for (const auto& installer : manifest.Installers) { if (!installer.ProductCode.empty()) { - if (std::find(productCodes.begin(), productCodes.end(), installer.ProductCode) == productCodes.end()) + // Add each ProductCode only once + if (productCodes.insert(installer.ProductCode).second) { manifestSearchRequest.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::ProductCode, MatchType::Exact, installer.ProductCode)); - productCodes.emplace_back(installer.ProductCode); } } @@ -208,6 +209,16 @@ namespace AppInstaller::Repository::Correlation appsAndFeaturesEntry.DisplayName, appsAndFeaturesEntry.Publisher.empty() ? defaultPublisher : appsAndFeaturesEntry.Publisher)); } + + // Add each ProductCode and UpgradeCode only once; + if (!appsAndFeaturesEntry.ProductCode.empty() && upgradeCodes.insert(appsAndFeaturesEntry.ProductCode).second) + { + manifestSearchRequest.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::ProductCode, MatchType::Exact, appsAndFeaturesEntry.ProductCode)); + } + if (!appsAndFeaturesEntry.UpgradeCode.empty() && upgradeCodes.insert(appsAndFeaturesEntry.UpgradeCode).second) + { + manifestSearchRequest.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::UpgradeCode, MatchType::Exact, appsAndFeaturesEntry.UpgradeCode)); + } } } diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj @@ -269,6 +269,9 @@ <ClInclude Include="Microsoft\Schema\1_4\Interface.h" /> <ClInclude Include="Microsoft\Schema\1_5\ArpVersionVirtualTable.h" /> <ClInclude Include="Microsoft\Schema\1_5\Interface.h" /> + <ClInclude Include="Microsoft\Schema\1_6\Interface.h" /> + <ClInclude Include="Microsoft\Schema\1_6\SearchResultsTable.h" /> + <ClInclude Include="Microsoft\Schema\1_6\UpgradeCodeTable.h" /> <ClInclude Include="Microsoft\Schema\ISQLiteIndex.h" /> <ClInclude Include="Microsoft\Schema\MetadataTable.h" /> <ClInclude Include="Microsoft\Schema\Version.h" /> @@ -344,6 +347,8 @@ <ClCompile Include="Microsoft\Schema\1_4\DependenciesTable.cpp" /> <ClCompile Include="Microsoft\Schema\1_4\Interface_1_4.cpp" /> <ClCompile Include="Microsoft\Schema\1_5\Interface_1_5.cpp" /> + <ClCompile Include="Microsoft\Schema\1_6\Interface_1_6.cpp" /> + <ClCompile Include="Microsoft\Schema\1_6\SearchResultsTable_1_6.cpp" /> <ClCompile Include="Microsoft\Schema\MetadataTable.cpp" /> <ClCompile Include="Microsoft\Schema\Version.cpp" /> <ClCompile Include="Microsoft\SQLiteIndex.cpp" /> diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters @@ -64,6 +64,9 @@ <Filter Include="Microsoft\Schema\1_5"> <UniqueIdentifier>{e31c8e5b-ed2c-43c8-b91b-db8ec4c52f71}</UniqueIdentifier> </Filter> + <Filter Include="Microsoft\Schema\1_6"> + <UniqueIdentifier>{84a55def-9fb8-4c90-8d5a-2cedc171940b}</UniqueIdentifier> + </Filter> </ItemGroup> <ItemGroup> <ClInclude Include="pch.h"> @@ -288,6 +291,15 @@ <ClInclude Include="Microsoft\Schema\1_0\VirtualTableBase.h"> <Filter>Microsoft\Schema\1_0</Filter> </ClInclude> + <ClInclude Include="Microsoft\Schema\1_6\Interface.h"> + <Filter>Microsoft\Schema\1_6</Filter> + </ClInclude> + <ClInclude Include="Microsoft\Schema\1_6\UpgradeCodeTable.h"> + <Filter>Microsoft\Schema\1_6</Filter> + </ClInclude> + <ClInclude Include="Microsoft\Schema\1_6\SearchResultsTable.h"> + <Filter>Microsoft\Schema\1_6</Filter> + </ClInclude> </ItemGroup> <ItemGroup> <ClCompile Include="pch.cpp"> @@ -452,6 +464,12 @@ <ClCompile Include="ArpVersionValidation.cpp"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="Microsoft\Schema\1_6\Interface_1_6.cpp"> + <Filter>Microsoft\Schema\1_6</Filter> + </ClCompile> + <ClCompile Include="Microsoft\Schema\1_6\SearchResultsTable_1_6.cpp"> + <Filter>Microsoft\Schema\1_6</Filter> + </ClCompile> </ItemGroup> <ItemGroup> <None Include="PropertySheet.props" /> diff --git a/src/AppInstallerRepositoryCore/CompositeSource.cpp b/src/AppInstallerRepositoryCore/CompositeSource.cpp @@ -24,6 +24,7 @@ namespace AppInstaller::Repository { case AppInstaller::Repository::PackageMatchField::PackageFamilyName: case AppInstaller::Repository::PackageMatchField::ProductCode: + case AppInstaller::Repository::PackageMatchField::UpgradeCode: return true; } @@ -738,6 +739,12 @@ namespace AppInstaller::Repository PackageMatchField::ProductCode, data); + GetSystemReferenceStrings( + version, + PackageVersionMultiProperty::UpgradeCode, + PackageMatchField::UpgradeCode, + data); + GetNameAndPublisher( version, data); diff --git a/src/AppInstallerRepositoryCore/Microsoft/ARPHelper.cpp b/src/AppInstallerRepositoryCore/Microsoft/ARPHelper.cpp @@ -8,6 +8,102 @@ namespace AppInstaller::Repository::Microsoft { using namespace AppInstaller::Registry::Portable; + namespace + { + // "Unpacks" a GUID in the format used by the UpgradesCode registry key into the usual format. + // Returns empty if it is not a valid GUID + std::optional<std::string> TryUnpackUpgradeCodeGuid(std::string_view packed) + { + // A GUID is made up of 4 parts: + // - Part 1 is made up of one 4 byte block + // - Parts 2 and 3 are made up of one 2 byte block + // - Part 4 is made up of eight 1 byte blocks + // + // The GUID strings we have in the manifests represent all of this in hex in order, + // with dashes between each part, and after the second byte of Part 4. + // The "packed" GUIDs in the registry place the blocks in the same order, + // without dashes and with opposite endian-ness. + // + // For example + // ARP: {FECAFEB5-8D0E-4AE4-8FA0-745BAA835C35} + // FECAFEB5 8D0E 4AE4 8F A0 74 5B AA 83 5C 35 + // Part 1 P2 P3 <------ Part 4 -------> + // 5BEFACEF E0D8 4EA4 F8 0A 47 B5 AA 38 C5 53 + // UpgradeCode: 5BEFACEFE0D84EA4F80A47B5AA38C553 + // + // The conversion can be done by mapping each location in the packed string + // to the appropriate location in the unpacked string. + constexpr size_t PackedLength = 32; + if (packed.length() != PackedLength || !std::all_of(packed.begin(), packed.end(), isxdigit)) + { + return {}; + } + + // PositionMapping[i] is the position to which the i-th char is mapped + // I.e., unpacked[ PositionMapping[i] ] = packed[i] + constexpr size_t PositionMapping[PackedLength] = + { + 8,7,6,5,4,3,2,1, + 13,12,11,10, + 18,17,16,15, + 21,20, 23,22, + 26,25, 28,27, 30,29, 32,31, 34,33, 36,35, + }; + + std::string unpacked("{00000000-0000-0000-0000-000000000000}"); + for (size_t i = 0; i < PackedLength; ++i) + { + unpacked[PositionMapping[i]] = packed[i]; + } + + return unpacked; + } + + // Gets a mapping from ProductCode to UpgradeCode for MSI packages. + std::map<std::string, std::string> GetUpgradeCodes() + { + // The UpgradeCode is not stored in the ARP registry keys, so we have to get it separately. + // We could use MsiGetProductProperty or MsiGetProperty from the MSI API to query it, + // but it is very slow. + // + // The UpgradeCode is also stored in the registry under + // HKLM\SOFTWARE\Microsoft\Windows\CurrentVersion\Installer\UpgradeCodes + // (Note that this key is not documented, so it is possible that it will change but very unlikely...) + // + // Under 'UpgradeCodes' there is one key for each upgrade code, and each upgrade code key + // contains the product code as a value. All the upgrade codes and product codes are GUIDs, + // but represented in an unusual way - see TryUnpackUpgradeCodeGuid() + + AICLI_LOG(Repo, Info, << "Reading MSI UpgradeCodes"); + std::map<std::string, std::string> upgradeCodes; + + // There is no UpgradeCodes key on the x86 view of the registry + Registry::Key upgradeCodesKey = Registry::Key::OpenIfExists(HKEY_LOCAL_MACHINE, "SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Installer\\UpgradeCodes", 0, KEY_READ | KEY_WOW64_64KEY); + + if (upgradeCodesKey) + { + for (const auto& upgradeCodeKeyRef : upgradeCodesKey) + { + auto upgradeCode = TryUnpackUpgradeCodeGuid(upgradeCodeKeyRef.Name()); + if (upgradeCode) + { + auto upgradeCodeKey = upgradeCodeKeyRef.Open(); + for (const auto& productCodeValue : upgradeCodeKey.Values()) + { + auto productCode = TryUnpackUpgradeCodeGuid(productCodeValue.Name()); + if (productCode) + { + upgradeCodes[*productCode] = *upgradeCode; + } + } + } + } + } + + return upgradeCodes; + } + } + Registry::Key ARPHelper::GetARPKey(Manifest::ScopeEnum scope, Utility::Architecture architecture) const { HKEY rootKey = NULL; @@ -208,18 +304,20 @@ namespace AppInstaller::Repository::Microsoft void ARPHelper::PopulateIndexFromARP(SQLiteIndex& index, Manifest::ScopeEnum scope) const { + auto upgradeCodes = GetUpgradeCodes(); + for (auto architecture : Utility::GetApplicableArchitectures()) { Registry::Key arpRootKey = GetARPKey(scope, architecture); if (arpRootKey) { - PopulateIndexFromKey(index, arpRootKey, Manifest::ScopeToString(scope), Utility::ToString(architecture)); + PopulateIndexFromKey(index, arpRootKey, Manifest::ScopeToString(scope), Utility::ToString(architecture), upgradeCodes); } } } - void ARPHelper::PopulateIndexFromKey(SQLiteIndex& index, const Registry::Key& key, std::string_view scope, std::string_view architecture) const + void ARPHelper::PopulateIndexFromKey(SQLiteIndex& index, const Registry::Key& key, std::string_view scope, std::string_view architecture, const std::map<std::string, std::string>& upgradeCodes) const { AICLI_LOG(Repo, Info, << "Examining ARP entries for " << scope << " | " << architecture); @@ -294,6 +392,23 @@ namespace AppInstaller::Repository::Microsoft //manifest.Id = normalizedName.Publisher() + '.' + normalizedName.Name(); } + // Pick up WindowsInstaller to determine if this is an MSI install. + // TODO: Could also determine Inno (and maybe other types) through detecting other keys here. + auto installedType = Manifest::InstallerTypeEnum::Exe; + + if (GetBoolValue(arpKey, WindowsInstaller)) + { + installedType = Manifest::InstallerTypeEnum::Msi; + + // If this is an MSI, look up the UpgradeCode + auto upgradeCodeItr = upgradeCodes.find(productCode); + if (upgradeCodeItr != upgradeCodes.end()) + { + manifest.Installers[0].AppsAndFeaturesEntries.emplace_back(); + manifest.Installers[0].AppsAndFeaturesEntries[0].UpgradeCode = upgradeCodeItr->second; + } + } + // TODO: If we want to keep the constructed manifest around to allow for `show` type commands // against installed packages, we should use URLInfoAbout/HelpLink for the Homepage. @@ -347,15 +462,6 @@ namespace AppInstaller::Repository::Microsoft // Pick up Language to enable proper selection of language for upgrade. AddMetadataIfPresent(arpKey, Language, index, manifestId, PackageVersionMetadata::InstalledLocale); - // Pick up WindowsInstaller to determine if this is an MSI install. - // TODO: Could also determine Inno (and maybe other types) through detecting other keys here. - auto installedType = Manifest::InstallerTypeEnum::Exe; - - if (GetBoolValue(arpKey, WindowsInstaller)) - { - installedType = Manifest::InstallerTypeEnum::Msi; - } - if (Manifest::ConvertToInstallerTypeEnum(GetStringValue(arpKey, std::wstring{ ToString(PortableValueName::WinGetInstallerType) })) == Manifest::InstallerTypeEnum::Portable) { // Portable uninstall requires the installed architecture for locating the entry in the registry. diff --git a/src/AppInstallerRepositoryCore/Microsoft/ARPHelper.h b/src/AppInstallerRepositoryCore/Microsoft/ARPHelper.h @@ -79,6 +79,6 @@ namespace AppInstaller::Repository::Microsoft // Populates the index with the ARP entries from the given key. // This entry point is primarily to allow unit tests to operate of arbitrary keys; // product code should use PopulateIndexFromARP. - void PopulateIndexFromKey(SQLiteIndex& index, const Registry::Key& key, std::string_view scope, std::string_view architecture) const; + void PopulateIndexFromKey(SQLiteIndex& index, const Registry::Key& key, std::string_view scope, std::string_view architecture, const std::map<std::string, std::string>& upgradeCodes = {}) const; }; } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface.h @@ -39,7 +39,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 // Version 1.4 Get all the dependencies for a specific manifest. std::set<std::pair<SQLite::rowid_t, Utility::NormalizedString>> GetDependenciesByManifestRowId(const SQLite::Connection& connection, SQLite::rowid_t manifestRowId) const override; std::vector<std::pair<SQLite::rowid_t, Utility::NormalizedString>> GetDependentsById(const SQLite::Connection& connection, AppInstaller::Manifest::string_t packageId) const override; - + protected: virtual bool NotNeeded(const SQLite::Connection& connection, std::string_view tableName, std::string_view valueName, SQLite::rowid_t id) const; @@ -54,5 +54,9 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 // Gets a property already knowing that the manifest id is valid. virtual std::optional<std::string> GetPropertyByManifestIdInternal(const SQLite::Connection& connection, SQLite::rowid_t manifestId, PackageVersionProperty property) const; + + // Force the database to shrink the file size. + // This *must* be done outside of an active transaction. + void Vacuum(const SQLite::Connection& connection); }; } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface_1_0.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface_1_0.cpp @@ -382,11 +382,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 savepoint.Commit(); - // Force the database to actually shrink the file size. - // This *must* be done outside of an active transaction. - SQLite::Builder::StatementBuilder builder; - builder.Vacuum(); - builder.Execute(connection); + Vacuum(connection); } bool Interface::CheckConsistency(const SQLite::Connection& connection, bool log) const @@ -644,4 +640,11 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 return {}; } } + + void Interface::Vacuum(const SQLite::Connection& connection) + { + SQLite::Builder::StatementBuilder builder; + builder.Vacuum(); + builder.Execute(connection); + } } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_1/Interface_1_1.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_1/Interface_1_1.cpp @@ -29,25 +29,36 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_1 { std::vector<Utility::NormalizedString> GetSystemReferenceStrings( const Manifest::Manifest& manifest, - std::function<const Utility::NormalizedString&(const Manifest::ManifestInstaller&)> func) + std::function<const Utility::NormalizedString&(const Manifest::ManifestInstaller&)> extractStringFromInstaller, + std::function<const Utility::NormalizedString& (const Manifest::AppsAndFeaturesEntry&)> extractStringFromAppsAndFeaturesEntry = {}) { std::set<Utility::NormalizedString> set; for (const auto& installer : manifest.Installers) { - const Utility::NormalizedString& string = func(installer); - if (!string.empty()) + const auto& installerString = extractStringFromInstaller(installer); + if (!installerString.empty()) { - set.emplace(Utility::FoldCase(string)); + set.emplace(Utility::FoldCase(installerString)); } - } - std::vector<Utility::NormalizedString> result; - for (auto&& string : set) - { - result.emplace_back(string); + if (extractStringFromAppsAndFeaturesEntry) + { + for (const auto& entry : installer.AppsAndFeaturesEntries) + { + const auto& entryString = extractStringFromAppsAndFeaturesEntry(entry); + if (!entryString.empty()) + { + set.emplace(Utility::FoldCase(entryString)); + } + } + } } + std::vector<Utility::NormalizedString> result( + std::make_move_iterator(set.begin()), + std::make_move_iterator(set.end())); + return result; } @@ -58,7 +69,10 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_1 std::vector<Utility::NormalizedString> GetProductCodes(const Manifest::Manifest& manifest) { - return GetSystemReferenceStrings(manifest, [](const Manifest::ManifestInstaller& i) -> const Utility::NormalizedString& { return i.ProductCode; }); + return GetSystemReferenceStrings( + manifest, + [](const Manifest::ManifestInstaller& i) -> const Utility::NormalizedString& { return i.ProductCode; }, + [](const Manifest::AppsAndFeaturesEntry& e) -> const Utility::NormalizedString& { return e.ProductCode; }); } } @@ -216,7 +230,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_1 std::unique_ptr<V1_0::SearchResultsTable> Interface::CreateSearchResultsTable(const SQLite::Connection& connection) const { - return std::make_unique<SearchResultsTable>(connection); + return std::make_unique<V1_1::SearchResultsTable>(connection); } void Interface::PerformQuerySearch(V1_0::SearchResultsTable& resultsTable, const RequestMatch& query) const @@ -285,11 +299,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_1 if (vacuum) { - // Force the database to actually shrink the file size. - // This *must* be done outside of an active transaction. - SQLite::Builder::StatementBuilder builder; - builder.Vacuum(); - builder.Execute(connection); + Vacuum(connection); } } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_2/Interface_1_2.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_2/Interface_1_2.cpp @@ -207,7 +207,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_2 std::unique_ptr<V1_0::SearchResultsTable> Interface::CreateSearchResultsTable(const SQLite::Connection& connection) const { - return std::make_unique<SearchResultsTable>(connection); + return std::make_unique<V1_2::SearchResultsTable>(connection); } ISQLiteIndex::SearchResult Interface::SearchInternal(const SQLite::Connection& connection, SearchRequest& request) const @@ -249,11 +249,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_2 if (vacuum) { - // Force the database to actually shrink the file size. - // This *must* be done outside of an active transaction. - SQLite::Builder::StatementBuilder builder; - builder.Vacuum(); - builder.Execute(connection); + Vacuum(connection); } } } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_3/Interface.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_3/Interface.h @@ -17,7 +17,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_3 void CreateTables(SQLite::Connection& connection, CreateOptions options) override; SQLite::rowid_t AddManifest(SQLite::Connection& connection, const Manifest::Manifest& manifest, const std::optional<std::filesystem::path>& relativePath) override; std::pair<bool, SQLite::rowid_t> UpdateManifest(SQLite::Connection& connection, const Manifest::Manifest& manifest, const std::optional<std::filesystem::path>& relativePath) override; - + protected: // Gets a property already knowing that the manifest id is valid. std::optional<std::string> GetPropertyByManifestIdInternal(const SQLite::Connection& connection, SQLite::rowid_t manifestId, PackageVersionProperty property) const override; diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_4/Interface_1_4.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_4/Interface_1_4.cpp @@ -102,11 +102,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_4 if (vacuum) { - // Force the database to actually shrink the file size. - // This *must* be done outside of an active transaction. - SQLite::Builder::StatementBuilder builder; - builder.Vacuum(); - builder.Execute(connection); + Vacuum(connection); } } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_6/Interface.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_6/Interface.h @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once +#include "Microsoft/Schema/ISQLiteIndex.h" +#include "Microsoft/Schema/1_5/Interface.h" + +namespace AppInstaller::Repository::Microsoft::Schema::V1_6 +{ + // Interface to this schema version exposed through ISQLiteIndex. + struct Interface : public V1_5::Interface + { + Interface(Utility::NormalizationVersion normVersion = Utility::NormalizationVersion::Initial); + + // Version 1.0 + Schema::Version GetVersion() const override; + void CreateTables(SQLite::Connection& connection, CreateOptions options) override; + SQLite::rowid_t AddManifest(SQLite::Connection& connection, const Manifest::Manifest& manifest, const std::optional<std::filesystem::path>& relativePath) override; + std::pair<bool, SQLite::rowid_t> UpdateManifest(SQLite::Connection& connection, const Manifest::Manifest& manifest, const std::optional<std::filesystem::path>& relativePath) override; + void RemoveManifestById(SQLite::Connection& connection, SQLite::rowid_t manifestId) override; + bool CheckConsistency(const SQLite::Connection& connection, bool log) const override; + std::vector<std::string> GetMultiPropertyByManifestId(const SQLite::Connection& connection, SQLite::rowid_t manifestId, PackageVersionMultiProperty property) const override; + + protected: + std::unique_ptr<V1_0::SearchResultsTable> CreateSearchResultsTable(const SQLite::Connection& connection) const override; + void PerformQuerySearch(V1_0::SearchResultsTable& resultsTable, const RequestMatch& query) const override; + ISQLiteIndex::SearchResult SearchInternal(const SQLite::Connection& connection, SearchRequest& request) const; + void PrepareForPackaging(SQLite::Connection& connection, bool vacuum) override; + }; +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_6/Interface_1_6.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_6/Interface_1_6.cpp @@ -0,0 +1,180 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "Microsoft/Schema/1_6/Interface.h" +#include "Microsoft/Schema/1_6/UpgradeCodeTable.h" +#include "Microsoft/Schema/1_6/SearchResultsTable.h" +#include "Microsoft/Schema/1_0/ManifestTable.h" +#include "Microsoft/Schema/1_0/VersionTable.h" + +namespace AppInstaller::Repository::Microsoft::Schema::V1_6 +{ + namespace + { + std::vector<Utility::NormalizedString> GetUpgradeCodes(const Manifest::Manifest& manifest) + { + std::set<Utility::NormalizedString> set; + + for (const auto& installer : manifest.Installers) + { + for (const auto& appsAndFeaturesEntry : installer.AppsAndFeaturesEntries) + { + const Utility::NormalizedString& string = appsAndFeaturesEntry.UpgradeCode; + if (!string.empty()) + { + set.emplace(Utility::FoldCase(string)); + } + } + } + + std::vector<Utility::NormalizedString> result( + std::make_move_iterator(set.begin()), + std::make_move_iterator(set.end())); + + return result; + } + } + + Interface::Interface(Utility::NormalizationVersion normVersion) : V1_5::Interface(normVersion) + { + } + + Schema::Version Interface::GetVersion() const + { + return { 1, 6 }; + } + + void Interface::CreateTables(SQLite::Connection& connection, CreateOptions options) + { + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, "createtables_v1_6"); + + V1_5::Interface::CreateTables(connection, options); + + UpgradeCodeTable::Create(connection); + + savepoint.Commit(); + } + + SQLite::rowid_t Interface::AddManifest(SQLite::Connection& connection, const Manifest::Manifest& manifest, const std::optional<std::filesystem::path>& relativePath) + { + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, "addmanifest_v1_6"); + + SQLite::rowid_t manifestId = V1_5::Interface::AddManifest(connection, manifest, relativePath); + + // Add the new 1.6 data + // These system reference strings are all stored with their cases folded so that they can be + // looked up ordinally; enabling the index to provide efficient searches. + UpgradeCodeTable::EnsureExistsAndInsert(connection, GetUpgradeCodes(manifest), manifestId); + + savepoint.Commit(); + + return manifestId; + } + + std::pair<bool, SQLite::rowid_t> Interface::UpdateManifest(SQLite::Connection& connection, const Manifest::Manifest& manifest, const std::optional<std::filesystem::path>& relativePath) + { + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, "updatemanifest_v1_6"); + + auto [indexModified, manifestId] = V1_5::Interface::UpdateManifest(connection, manifest, relativePath); + + // Update new 1:N tables as necessary + indexModified = UpgradeCodeTable::UpdateIfNeededByManifestId(connection, GetUpgradeCodes(manifest), manifestId) || indexModified; + + savepoint.Commit(); + + return { indexModified, manifestId }; + } + + void Interface::RemoveManifestById(SQLite::Connection& connection, SQLite::rowid_t manifestId) + { + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, "RemoveManifestById_v1_6"); + + // Removes the manifest. + V1_5::Interface::RemoveManifestById(connection, manifestId); + + // Remove all of the new 1:N data that is no longer referenced. + UpgradeCodeTable::DeleteIfNotNeededByManifestId(connection, manifestId); + + savepoint.Commit(); + } + + bool Interface::CheckConsistency(const SQLite::Connection& connection, bool log) const + { + bool result = V1_5::Interface::CheckConsistency(connection, log); + + // If the v1.5 index was consistent, or if full logging of inconsistency was requested, check the v1.6 data. + if (result || log) + { + result = UpgradeCodeTable::CheckConsistency(connection, log) && result; + } + + return result; + } + + std::vector<std::string> Interface::GetMultiPropertyByManifestId(const SQLite::Connection& connection, SQLite::rowid_t manifestId, PackageVersionMultiProperty property) const + { + switch (property) + { + case PackageVersionMultiProperty::UpgradeCode: + return UpgradeCodeTable::GetValuesByManifestId(connection, manifestId); + default: + return V1_5::Interface::GetMultiPropertyByManifestId(connection, manifestId, property); + } + } + + std::unique_ptr<V1_0::SearchResultsTable> Interface::CreateSearchResultsTable(const SQLite::Connection& connection) const + { + return std::make_unique<V1_6::SearchResultsTable>(connection); + } + + void Interface::PerformQuerySearch(V1_0::SearchResultsTable& resultsTable, const RequestMatch& query) const + { + // First, do an exact match search for the folded system reference strings + // We do this first because it is exact, and likely won't match anything else if it matches this. + PackageMatchFilter filter(PackageMatchField::UpgradeCode, MatchType::Exact, Utility::FoldCase(query.Value)); + resultsTable.SearchOnField(filter); + + // Then do the 1.5 search + V1_5::Interface::PerformQuerySearch(resultsTable, query); + } + + ISQLiteIndex::SearchResult Interface::SearchInternal(const SQLite::Connection& connection, SearchRequest& request) const + { + // Update any system reference strings to be folded + auto foldIfNeeded = [](PackageMatchFilter& filter) + { + if (filter.Field == PackageMatchField::UpgradeCode && filter.Type == MatchType::Exact) + { + filter.Value = Utility::FoldCase(filter.Value); + } + }; + + for (auto& inclusion : request.Inclusions) + { + foldIfNeeded(inclusion); + } + + for (auto& filter : request.Filters) + { + foldIfNeeded(filter); + } + + return V1_5::Interface::SearchInternal(connection, request); + } + + void Interface::PrepareForPackaging(SQLite::Connection& connection, bool vacuum) + { + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, "prepareforpackaging_v1_6"); + + V1_5::Interface::PrepareForPackaging(connection, false); + + UpgradeCodeTable::PrepareForPackaging(connection, true, true); + + savepoint.Commit(); + + if (vacuum) + { + Vacuum(connection); + } + } +}+ \ No newline at end of file diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_6/SearchResultsTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_6/SearchResultsTable.h @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once +#include "Microsoft/Schema/1_2/SearchResultsTable.h" + + +namespace AppInstaller::Repository::Microsoft::Schema::V1_6 +{ + // Table for holding temporary search results. + struct SearchResultsTable : public V1_2::SearchResultsTable + { + SearchResultsTable(const SQLite::Connection& connection) : V1_2::SearchResultsTable(connection) {} + + SearchResultsTable(const SearchResultsTable&) = delete; + SearchResultsTable& operator=(const SearchResultsTable&) = delete; + + SearchResultsTable(SearchResultsTable&&) = default; + SearchResultsTable& operator=(SearchResultsTable&&) = default; + + protected: + std::vector<int> BuildSearchStatement( + SQLite::Builder::StatementBuilder& builder, + PackageMatchField field, + std::string_view manifestAlias, + std::string_view valueAlias, + bool useLike) const override; + }; +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_6/SearchResultsTable_1_6.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_6/SearchResultsTable_1_6.cpp @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "SearchResultsTable.h" + +#include "Microsoft/Schema/1_0/ManifestTable.h" +#include "Microsoft/Schema/1_6/UpgradeCodeTable.h" + + +namespace AppInstaller::Repository::Microsoft::Schema::V1_6 +{ + std::vector<int> SearchResultsTable::BuildSearchStatement( + SQLite::Builder::StatementBuilder& builder, + PackageMatchField field, + std::string_view manifestAlias, + std::string_view valueAlias, + bool useLike) const + { + switch (field) + { + case PackageMatchField::UpgradeCode: + return V1_0::ManifestTable::BuildSearchStatement<UpgradeCodeTable>(builder, manifestAlias, valueAlias, useLike); + default: + return V1_2::SearchResultsTable::BuildSearchStatement(builder, field, manifestAlias, valueAlias, useLike); + } + } +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_6/UpgradeCodeTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_6/UpgradeCodeTable.h @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once +#include "Microsoft/Schema/1_0/OneToManyTable.h" + + +namespace AppInstaller::Repository::Microsoft::Schema::V1_6 +{ + namespace details + { + using namespace std::string_view_literals; + + struct UpgradeCodeTableInfo + { + inline static constexpr std::string_view TableName() { return "upgradecodes"sv; } + inline static constexpr std::string_view ValueName() { return "upgradecode"sv; } + }; + } + + // The table for UpgradeCode. + using UpgradeCodeTable = V1_0::OneToManyTable<details::UpgradeCodeTableInfo>; +} + diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/Version.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/Version.cpp @@ -10,6 +10,7 @@ #include "1_3/Interface.h" #include "1_4/Interface.h" #include "1_5/Interface.h" +#include "1_6/Interface.h" namespace AppInstaller::Repository::Microsoft::Schema { @@ -54,11 +55,15 @@ namespace AppInstaller::Repository::Microsoft::Schema { return std::make_unique<V1_4::Interface>(); } - else if (*this == Version{ 1, 5 } || + else if (*this == Version{ 1, 5 }) + { + return std::make_unique<V1_5::Interface>(); + } + else if (*this == Version{ 1, 6 } || this->MajorVersion == 1 || this->IsLatest()) { - return std::make_unique<V1_5::Interface>(); + return std::make_unique<V1_6::Interface>(); } // We do not have the capacity to operate on this schema version diff --git a/src/AppInstallerRepositoryCore/Public/winget/RepositorySearch.h b/src/AppInstallerRepositoryCore/Public/winget/RepositorySearch.h @@ -46,6 +46,7 @@ namespace AppInstaller::Repository Tag, PackageFamilyName, ProductCode, + UpgradeCode, NormalizedNameAndPublisher, Market, Unknown = 9999 @@ -146,6 +147,8 @@ namespace AppInstaller::Repository PackageFamilyName, // The product codes associated with the package version. ProductCode, + // The upgrade codes associated with the package version. + UpgradeCode, // TODO: Fully implement these 3; the data is not yet in the index source (name and publisher are hacks and locale is not present) // For future usage of these, be aware of the limitations. // The package names for the version; ideally these would match in number and order with both Publisher and Locale. diff --git a/src/AppInstallerRepositoryCore/RepositorySearch.cpp b/src/AppInstallerRepositoryCore/RepositorySearch.cpp @@ -183,6 +183,8 @@ namespace AppInstaller::Repository return "PackageFamilyName"sv; case PackageMatchField::ProductCode: return "ProductCode"sv; + case PackageMatchField::UpgradeCode: + return "UpgradeCode"sv; case PackageMatchField::NormalizedNameAndPublisher: return "NormalizedNameAndPublisher"sv; case PackageMatchField::Market: @@ -224,6 +226,10 @@ namespace AppInstaller::Repository { return PackageMatchField::ProductCode; } + else if (toLower == "upgradecode") + { + return PackageMatchField::UpgradeCode; + } else if (toLower == "normalizednameandpublisher") { return PackageMatchField::NormalizedNameAndPublisher;