commit 657d33ce1702fc11f1fa64ca46c7dd6c07bd12ac parent 9c5c5db1f42355b0d08b657f6cf1f0c7d681dfa6 Author: JohnMcPMS <johnmcp@microsoft.com> Date: Mon, 25 Sep 2023 11:46:04 -0700 Reduce the size of the index (#3666) This change seeks to reduce the size of the index in two ways: 1. Better schema design for the 1:N mapping tables 2. Dropping some of the mapping data that is not particularly interesting per manifest (aka per version) ### Better schema design This is achieved by the map table having no `rowid` and using a primary key with the value first. This makes the table already sorted by the value, thus the reverse lookups are fast. It also drops a fair amount of the data in the table itself to remove the `rowid`, given that it was ~1/3 of the rows. ### Dropping map data We don't actually use the fact that we know that different versions have different tags (or any other data). Thus, we can simply have one manifest entry per package identifier have all of the values and maintain the same functionality. There is a slight loss of fidelity if one is reading through the values via API, but this is deemed acceptable given the large data savings. I explicitly left the product codes alone, as this does have value to keep per version (even if we are not using it currently). Diffstat:
24 files changed, 636 insertions(+), 133 deletions(-)
diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt @@ -103,6 +103,7 @@ DACL datetimeoffset Dbg debian +dedupe deigh deleteifnotneeded DENYWR @@ -264,6 +265,7 @@ luffy Luffytaro maclachlan malware +mapdatafolding mapview Maxed maxvalue diff --git a/src/AppInstallerCLITests/SQLiteIndex.cpp b/src/AppInstallerCLITests/SQLiteIndex.cpp @@ -35,7 +35,17 @@ SQLiteIndex CreateTestIndex(const std::string& filePath, std::optional<Schema::V // If no specific version requested, then use generator to run against the last 3 versions. if (!version) { - version = GENERATE(Schema::Version{ 1, 2 }, Schema::Version{ 1, 3 }, Schema::Version{ 1, 4 }, Schema::Version::Latest()); + Schema::Version latestVersion = SQLiteIndex::GetLatestVersion(); + if (latestVersion.MajorVersion != 1) + { + throw std::exception("You added major version 2, figure out how to deal with these tests that do back compat coverage!"); + } + + // Relies on the fact that min version is already >= 2 + Schema::Version versionMinus1 = Schema::Version{ 1, latestVersion.MinorVersion - 1 }; + Schema::Version versionMinus2 = Schema::Version{ 1, latestVersion.MinorVersion - 2 }; + + version = GENERATE_COPY(Schema::Version{ versionMinus2 }, Schema::Version{ versionMinus1 }, Schema::Version{ latestVersion }); } return SQLiteIndex::CreateNew(filePath, version.value()); @@ -43,41 +53,35 @@ SQLiteIndex CreateTestIndex(const std::string& filePath, std::optional<Schema::V Schema::Version TestPrepareForRead(SQLiteIndex& index) { - if (index.GetVersion() == Schema::Version{ 1, 2 }) + Schema::Version latestVersion = SQLiteIndex::GetLatestVersion(); + if (latestVersion.MajorVersion != 1) { - Schema::Version version = GENERATE(Schema::Version{ 1, 2 }); - - if (version != Schema::Version{ 1, 2 }) - { - index.ForceVersion(version); - return version; - } + throw std::exception("You added major version 2, figure out how to deal with these tests that do back compat coverage!"); } - else if (index.GetVersion() == Schema::Version{ 1, 3 }) - { - Schema::Version version = GENERATE(Schema::Version{ 1, 2 }, Schema::Version{ 1, 3 }); - if (version != Schema::Version{ 1, 3 }) - { - index.ForceVersion(version); - return version; - } + // Relies on the fact that min version is already >= 2 + Schema::Version versionMinus1 = Schema::Version{ 1, latestVersion.MinorVersion - 1 }; + Schema::Version versionMinus2 = Schema::Version{ 1, latestVersion.MinorVersion - 2 }; + + if (index.GetVersion() == versionMinus2) + { + // Degenerate case where we don't need to do anything } - else if (index.GetVersion() == Schema::Version{ 1, 4 }) + else if (index.GetVersion() == versionMinus1) { - Schema::Version version = GENERATE(Schema::Version{ 1, 2 }, Schema::Version{ 1, 3 }, Schema::Version{ 1, 4 }); + Schema::Version version = GENERATE_COPY(Schema::Version{ versionMinus2 }, Schema::Version{ versionMinus1 }); - if (version != Schema::Version{ 1, 4 }) + if (version != versionMinus1) { index.ForceVersion(version); return version; } } - else if (index.GetVersion() == Schema::Version{ 1, 5 }) + else if (index.GetVersion() == latestVersion) { - Schema::Version version = GENERATE(Schema::Version{ 1, 2 }, Schema::Version{ 1, 3 }, Schema::Version{ 1, 4 }, Schema::Version{ 1, 5 }); + Schema::Version version = GENERATE_COPY(Schema::Version{ versionMinus2 }, Schema::Version{ versionMinus1 }, Schema::Version{ latestVersion }); - if (version != Schema::Version{ 1, 5 }) + if (version != latestVersion) { index.ForceVersion(version); return version; @@ -321,6 +325,17 @@ bool AreArpVersionsSupported(const SQLiteIndex& index, const Schema::Version& te return (index.GetVersion() >= Schema::Version{ 1, 5 } && testVersion >= Schema::Version{ 1, 5 }); } +bool IsMapDataFoldingSupported(const SQLiteIndex& index, const Schema::Version& testVersion) +{ + UNSCOPED_INFO("Index " << index.GetVersion() << " | Test " << testVersion); + return (index.GetVersion() >= Schema::Version{ 1, 7 } && testVersion >= Schema::Version{ 1, 7 }); +} + +bool IsMapDataFolded(const SQLiteIndex& index) +{ + return (index.GetVersion() >= Schema::Version{ 1, 7 }); +} + std::string GetPropertyStringByKey(const SQLiteIndex& index, SQLite::rowid_t id, PackageVersionProperty property, std::string_view version, std::string_view channel) { auto manifestId = index.GetManifestIdByKey(id, version, channel); @@ -3187,3 +3202,152 @@ TEST_CASE("SQLiteIndex_CheckConsistency_FindEmbeddedNull", "[sqliteindex]") REQUIRE(!index.CheckConsistency(true)); } + +TEST_CASE("SQLiteIndex_MapDataFolding_Tags", "[sqliteindex][mapdatafolding]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + std::string tag1 = "Tag1"; + std::string tag2 = "Tag2"; + + SQLiteIndex index = SearchTestSetup(tempFile, { + { "Id", "Name", "Publisher", "Moniker", "Version1", "", { tag1 }, { "Command" }, "Path1", {}, { "PC1" } }, + { "Id", "Name", "Publisher", "Moniker", "Version2", "", { tag2 }, { "Command" }, "Path2", {}, { "PC2" } }, + }); + + // Apply the map data folding if it is present in the created test index. + index.PrepareForPackaging(); + + Schema::Version testVersion = TestPrepareForRead(index); + + SearchRequest request1; + request1.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::Tag, MatchType::Exact, tag1)); + auto results1 = index.Search(request1); + + SearchRequest request2; + request2.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::Tag, MatchType::Exact, tag2)); + auto results2 = index.Search(request1); + + REQUIRE(results1.Matches.size() == 1); + REQUIRE(results2.Matches.size() == 1); + REQUIRE(results1.Matches[0].first == results2.Matches[0].first); +} + +TEST_CASE("SQLiteIndex_MapDataFolding_PFNs", "[sqliteindex][mapdatafolding]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + std::string pfn1 = "PFN1"; + std::string pfn2 = "PFN2"; + + SQLiteIndex index = SearchTestSetup(tempFile, { + { "Id", "Name", "Publisher", "Moniker", "Version1", "", { }, { "Command" }, "Path1", { pfn1 }, { } }, + { "Id", "Name", "Publisher", "Moniker", "Version2", "", { }, { "Command" }, "Path2", { pfn2 }, { } }, + }); + + // Apply the map data folding if it is present in the created test index. + index.PrepareForPackaging(); + + Schema::Version testVersion = TestPrepareForRead(index); + + SearchRequest request1; + request1.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::PackageFamilyName, MatchType::Exact, pfn1)); + auto results1 = index.Search(request1); + + SearchRequest request2; + request2.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::PackageFamilyName, MatchType::Exact, pfn2)); + auto results2 = index.Search(request1); + + REQUIRE(results1.Matches.size() == 1); + REQUIRE(results2.Matches.size() == 1); + REQUIRE(results1.Matches[0].first == results2.Matches[0].first); + + auto versionKeys = index.GetVersionKeysById(results1.Matches[0].first); + REQUIRE(versionKeys.size() == 2); + + auto manifestId1 = index.GetManifestIdByKey(results1.Matches[0].first, versionKeys[0].GetVersion().ToString(), versionKeys[0].GetChannel().ToString()); + auto manifestId2 = index.GetManifestIdByKey(results1.Matches[0].first, versionKeys[1].GetVersion().ToString(), versionKeys[1].GetChannel().ToString()); + + REQUIRE(manifestId1.has_value()); + REQUIRE(manifestId2.has_value()); + + auto pfnValues1 = index.GetMultiPropertyByManifestId(manifestId1.value(), PackageVersionMultiProperty::PackageFamilyName); + auto pfnValues2 = index.GetMultiPropertyByManifestId(manifestId2.value(), PackageVersionMultiProperty::PackageFamilyName); + + if (IsMapDataFoldingSupported(index, testVersion)) + { + REQUIRE(pfnValues1.size() == 2); + REQUIRE(pfnValues2.size() == 2); + REQUIRE(pfnValues1[0] != pfnValues1[1]); + } + else if (IsMapDataFolded(index)) + { + if (manifestId1 > manifestId2) + { + REQUIRE(pfnValues1.size() == 2); + REQUIRE(pfnValues2.size() == 0); + REQUIRE(pfnValues1[0] != pfnValues1[1]); + } + else + { + REQUIRE(pfnValues1.size() == 0); + REQUIRE(pfnValues2.size() == 2); + REQUIRE(pfnValues2[0] != pfnValues2[1]); + } + } + else + { + REQUIRE(pfnValues1.size() == 1); + REQUIRE(pfnValues2.size() == 1); + REQUIRE(pfnValues1[0] != pfnValues2[0]); + } +} + +TEST_CASE("SQLiteIndex_MapDataFolding_ProductCodes", "[sqliteindex][mapdatafolding]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + std::string pc1 = "PC1"; + std::string pc2 = "PC2"; + + SQLiteIndex index = SearchTestSetup(tempFile, { + { "Id", "Name", "Publisher", "Moniker", "Version1", "", { }, { "Command" }, "Path1", { }, { pc1 } }, + { "Id", "Name", "Publisher", "Moniker", "Version2", "", { }, { "Command" }, "Path2", { }, { pc2 } }, + }); + + // Apply the map data folding if it is present in the created test index. + index.PrepareForPackaging(); + + Schema::Version testVersion = TestPrepareForRead(index); + + SearchRequest request1; + request1.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::ProductCode, MatchType::Exact, pc1)); + auto results1 = index.Search(request1); + + SearchRequest request2; + request2.Inclusions.emplace_back(PackageMatchFilter(PackageMatchField::ProductCode, MatchType::Exact, pc2)); + auto results2 = index.Search(request1); + + REQUIRE(results1.Matches.size() == 1); + REQUIRE(results2.Matches.size() == 1); + REQUIRE(results1.Matches[0].first == results2.Matches[0].first); + + auto versionKeys = index.GetVersionKeysById(results1.Matches[0].first); + REQUIRE(versionKeys.size() == 2); + + auto manifestId1 = index.GetManifestIdByKey(results1.Matches[0].first, versionKeys[0].GetVersion().ToString(), versionKeys[0].GetChannel().ToString()); + auto manifestId2 = index.GetManifestIdByKey(results1.Matches[0].first, versionKeys[1].GetVersion().ToString(), versionKeys[1].GetChannel().ToString()); + + REQUIRE(manifestId1.has_value()); + REQUIRE(manifestId2.has_value()); + + auto pcValues1 = index.GetMultiPropertyByManifestId(manifestId1.value(), PackageVersionMultiProperty::ProductCode); + auto pcValues2 = index.GetMultiPropertyByManifestId(manifestId2.value(), PackageVersionMultiProperty::ProductCode); + + REQUIRE(pcValues1.size() == 1); + REQUIRE(pcValues2.size() == 1); + REQUIRE(pcValues1[0] != pcValues2[0]); +} diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj @@ -385,6 +385,7 @@ <ClInclude Include="Microsoft\Schema\1_6\Interface.h" /> <ClInclude Include="Microsoft\Schema\1_6\SearchResultsTable.h" /> <ClInclude Include="Microsoft\Schema\1_6\UpgradeCodeTable.h" /> + <ClInclude Include="Microsoft\Schema\1_7\Interface.h" /> <ClInclude Include="Microsoft\Schema\IPinningIndex.h" /> <ClInclude Include="Microsoft\Schema\IPortableIndex.h" /> <ClInclude Include="Microsoft\Schema\ISQLiteIndex.h" /> @@ -488,6 +489,7 @@ <ClCompile Include="Microsoft\Schema\1_5\Interface_1_5.cpp" /> <ClCompile Include="Microsoft\Schema\1_6\Interface_1_6.cpp" /> <ClCompile Include="Microsoft\Schema\1_6\SearchResultsTable_1_6.cpp" /> + <ClCompile Include="Microsoft\Schema\1_7\Interface_1_7.cpp" /> <ClCompile Include="Microsoft\Schema\MetadataTable.cpp" /> <ClCompile Include="Microsoft\Schema\Pinning_1_0\PinningIndexInterface_1_0.cpp" /> <ClCompile Include="Microsoft\Schema\Pinning_1_0\PinTable.cpp" /> diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters @@ -91,6 +91,9 @@ <Filter Include="Rest\Schema\1_6\Json"> <UniqueIdentifier>{b2e78f3d-931e-432c-8485-255b1dbc9db7}</UniqueIdentifier> </Filter> + <Filter Include="Microsoft\Schema\1_7"> + <UniqueIdentifier>{f610927a-6f1d-42c5-9ad9-b59790091944}</UniqueIdentifier> + </Filter> </ItemGroup> <ItemGroup> <ClInclude Include="pch.h"> @@ -387,6 +390,9 @@ <ClInclude Include="Rest\Schema\1_6\Interface.h"> <Filter>Rest\Schema\1_6</Filter> </ClInclude> + <ClInclude Include="Microsoft\Schema\1_7\Interface.h"> + <Filter>Microsoft\Schema\1_7</Filter> + </ClInclude> </ItemGroup> <ItemGroup> <ClCompile Include="pch.cpp"> @@ -614,6 +620,9 @@ <ClCompile Include="Rest\Schema\1_6\RestInterface_1_6.cpp"> <Filter>Rest\Schema\1_6</Filter> </ClCompile> + <ClCompile Include="Microsoft\Schema\1_7\Interface_1_7.cpp"> + <Filter>Microsoft\Schema\1_7</Filter> + </ClCompile> </ItemGroup> <ItemGroup> <None Include="PropertySheet.props" /> diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp @@ -13,6 +13,7 @@ #include "Schema/1_4/Interface.h" #include "Schema/1_5/Interface.h" #include "Schema/1_6/Interface.h" +#include "Schema/1_7/Interface.h" namespace AppInstaller::Repository::Microsoft { @@ -35,49 +36,36 @@ namespace AppInstaller::Repository::Microsoft return result; } - std::unique_ptr<Schema::ISQLiteIndex> SQLiteIndex::CreateISQLiteIndex() const + std::unique_ptr<Schema::ISQLiteIndex> SQLiteIndex::CreateISQLiteIndex(const Schema::Version& version) { using namespace Schema; - if (m_version == Version{ 1, 0 }) + if (version.MajorVersion == 1 || + version.IsLatest()) { - return std::make_unique<V1_0::Interface>(); - } - else if (m_version == Version{ 1, 1 }) - { - return std::make_unique<V1_1::Interface>(); - } - else if (m_version == Version{ 1, 2 }) - { - return std::make_unique<V1_2::Interface>(); - } - else if (m_version == Version{ 1, 3 }) - { - return std::make_unique<V1_3::Interface>(); - } - else if (m_version == Version{ 1, 4 }) - { - return std::make_unique<V1_4::Interface>(); - } - else if (m_version == Version{ 1, 5 }) - { - return std::make_unique<V1_5::Interface>(); - } - else if (m_version == Version{ 1, 6 } || - m_version.MajorVersion == 1 || - m_version.IsLatest()) - { - return std::make_unique<V1_6::Interface>(); + constexpr std::array<std::unique_ptr<Schema::ISQLiteIndex>(*)(), 8> versionCreatorMap = + { + []() { return std::unique_ptr<Schema::ISQLiteIndex>(std::make_unique<V1_0::Interface>()); }, + []() { return std::unique_ptr<Schema::ISQLiteIndex>(std::make_unique<V1_1::Interface>()); }, + []() { return std::unique_ptr<Schema::ISQLiteIndex>(std::make_unique<V1_2::Interface>()); }, + []() { return std::unique_ptr<Schema::ISQLiteIndex>(std::make_unique<V1_3::Interface>()); }, + []() { return std::unique_ptr<Schema::ISQLiteIndex>(std::make_unique<V1_4::Interface>()); }, + []() { return std::unique_ptr<Schema::ISQLiteIndex>(std::make_unique<V1_5::Interface>()); }, + []() { return std::unique_ptr<Schema::ISQLiteIndex>(std::make_unique<V1_6::Interface>()); }, + []() { return std::unique_ptr<Schema::ISQLiteIndex>(std::make_unique<V1_7::Interface>()); }, + }; + + return versionCreatorMap[std::min(static_cast<size_t>(version.MinorVersion), versionCreatorMap.size() - 1)](); } // We do not have the capacity to operate on this schema version THROW_HR(HRESULT_FROM_WIN32(ERROR_NOT_SUPPORTED)); } - SQLiteIndex::SQLiteIndex(const std::string& target, Schema::Version version) : SQLiteStorageBase(target, version) + SQLiteIndex::SQLiteIndex(const std::string& target, const Schema::Version& version) : SQLiteStorageBase(target, version) { m_dbconn.EnableICU(); - m_interface = CreateISQLiteIndex(); + m_interface = CreateISQLiteIndex(version); m_version = m_interface->GetVersion(); } @@ -86,15 +74,19 @@ namespace AppInstaller::Repository::Microsoft { m_dbconn.EnableICU(); AICLI_LOG(Repo, Info, << "Opened SQLite Index with version [" << m_version << "], last write [" << GetLastWriteTime() << "]"); - m_interface = CreateISQLiteIndex(); + m_interface = CreateISQLiteIndex(m_version); THROW_HR_IF(APPINSTALLER_CLI_ERROR_CANNOT_WRITE_TO_UPLEVEL_INDEX, disposition == SQLiteStorageBase::OpenDisposition::ReadWrite && m_version != m_interface->GetVersion()); } #ifndef AICLI_DISABLE_TEST_HOOKS void SQLiteIndex::ForceVersion(const Schema::Version& version) { - m_version = version; - m_interface = CreateISQLiteIndex(); + m_interface = CreateISQLiteIndex(version); + } + + Schema::Version SQLiteIndex::GetLatestVersion() + { + return CreateISQLiteIndex(Schema::Version::Latest())->GetVersion(); } #endif diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h @@ -56,6 +56,9 @@ namespace AppInstaller::Repository::Microsoft // Changes the version of the interface being used to operate on the database. // Should only be used for testing. void ForceVersion(const Schema::Version& version); + + // Gets the latest version of the index schema (the actual numbers, not just the latest sentinel values). + static Schema::Version GetLatestVersion(); #endif // Adds the manifest at the repository relative path to the index. @@ -138,7 +141,7 @@ namespace AppInstaller::Repository::Microsoft std::vector<std::pair<SQLite::rowid_t, Utility::NormalizedString>> GetDependentsById(AppInstaller::Manifest::string_t packageId) const; private: // Constructor used to create a new index. - SQLiteIndex(const std::string& target, Schema::Version version); + SQLiteIndex(const std::string& target, const Schema::Version& version); // Constructor used to open an existing index. SQLiteIndex(const std::string& target, SQLiteStorageBase::OpenDisposition disposition, Utility::ManagedFile&& indexFile); @@ -148,7 +151,7 @@ namespace AppInstaller::Repository::Microsoft bool UpdateManifestInternal(const Manifest::Manifest& manifest, const std::optional<std::filesystem::path>& relativePath); // Creates the ISQLiteIndex interface object for this version. - std::unique_ptr<Schema::ISQLiteIndex> CreateISQLiteIndex() const; + static std::unique_ptr<Schema::ISQLiteIndex> CreateISQLiteIndex(const Schema::Version& version); std::unique_ptr<Schema::ISQLiteIndex> m_interface; }; diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteStorageBase.cpp b/src/AppInstallerRepositoryCore/Microsoft/SQLiteStorageBase.cpp @@ -105,7 +105,7 @@ namespace AppInstaller::Repository::Microsoft m_version = Schema::Version::GetSchemaVersion(m_dbconn); } - SQLiteStorageBase::SQLiteStorageBase(const std::string& target, Schema::Version version) : + SQLiteStorageBase::SQLiteStorageBase(const std::string& target, const Schema::Version& version) : m_dbconn(SQLite::Connection::Create(target, SQLite::Connection::OpenDisposition::Create)) { m_version = version; diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteStorageBase.h b/src/AppInstallerRepositoryCore/Microsoft/SQLiteStorageBase.h @@ -30,7 +30,7 @@ namespace AppInstaller::Repository::Microsoft Schema::Version GetVersion() const { return m_version; } protected: - SQLiteStorageBase(const std::string& target, Schema::Version version); + SQLiteStorageBase(const std::string& target, const Schema::Version& version); SQLiteStorageBase(const std::string& filePath, SQLiteStorageBase::OpenDisposition disposition, Utility::ManagedFile&& indexFile); diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface.h @@ -3,6 +3,7 @@ #pragma once #include "Microsoft/Schema/ISQLiteIndex.h" #include "Microsoft/Schema/1_0/SearchResultsTable.h" +#include "Microsoft/Schema/1_0/OneToManyTable.h" #include <memory> #include <vector> @@ -55,6 +56,9 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 // Gets a property already knowing that the manifest id is valid. virtual std::optional<std::string> GetPropertyByManifestIdInternal(const SQLite::Connection& connection, SQLite::rowid_t manifestId, PackageVersionProperty property) const; + // Gets the one to many table schema to use. + virtual OneToManyTableSchema GetOneToManyTableSchema() const; + // Force the database to shrink the file size. // This *must* be done outside of an active transaction. void Vacuum(const SQLite::Connection& connection); diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface_1_0.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface_1_0.cpp @@ -176,8 +176,8 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 { PathPartTable::ValueName(), false, WI_IsFlagClear(options, CreateOptions::SupportPathless) } }); - TagsTable::Create_deprecated(connection); - CommandsTable::Create_deprecated(connection); + TagsTable::Create(connection, GetOneToManyTableSchema()); + CommandsTable::Create(connection, GetOneToManyTableSchema()); savepoint.Commit(); } @@ -377,8 +377,8 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 PathPartTable::ValueName(), }); - TagsTable::PrepareForPackaging_deprecated(connection); - CommandsTable::PrepareForPackaging_deprecated(connection); + TagsTable::PrepareForPackaging(connection, GetOneToManyTableSchema(), false, false); + CommandsTable::PrepareForPackaging(connection, GetOneToManyTableSchema(), false, false); savepoint.Commit(); @@ -641,6 +641,11 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 } } + OneToManyTableSchema Interface::GetOneToManyTableSchema() const + { + return OneToManyTableSchema::Version_1_0; + } + void Interface::Vacuum(const SQLite::Connection& connection) { SQLite::Builder::StatementBuilder builder; diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToManyTable.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToManyTable.cpp @@ -4,6 +4,7 @@ #include "Microsoft/Schema/1_0/OneToManyTable.h" #include "Microsoft/Schema/1_0/OneToOneTable.h" #include "Microsoft/Schema/1_0/ManifestTable.h" +#include "Microsoft/Schema/1_0/IdTable.h" #include "SQLiteStatementBuilder.h" @@ -84,6 +85,11 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 // Bind valid rowid to 1. SQLite::Statement DeleteValueById; }; + + bool SchemaVersionUsesNamedIndices(OneToManyTableSchema schemaVersion) + { + return schemaVersion != OneToManyTableSchema::Version_1_0; + } } std::string OneToManyTableGetMapTableName(std::string_view tableName) @@ -98,16 +104,31 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 return s_OneToManyTable_MapTable_ManifestName; } - void CreateOneToManyTable(SQLite::Connection& connection, bool useNamedIndices, std::string_view tableName, std::string_view valueName) + void CreateOneToManyTable(SQLite::Connection& connection, OneToManyTableSchema schemaVersion, std::string_view tableName, std::string_view valueName) { using namespace SQLite::Builder; SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, std::string{ tableName } + "_create_v1_0"); // Create the data table as a 1:1 - CreateOneToOneTable(connection, tableName, valueName, useNamedIndices); + CreateOneToOneTable(connection, tableName, valueName, SchemaVersionUsesNamedIndices(schemaVersion)); - if (useNamedIndices) + switch (schemaVersion) + { + case OneToManyTableSchema::Version_1_0: + { + // Create the mapping table + StatementBuilder createMapTableBuilder; + createMapTableBuilder.CreateTable({ tableName, s_OneToManyTable_MapTable_Suffix }).Columns({ + ColumnBuilder(s_OneToManyTable_MapTable_ManifestName, Type::Int64).NotNull(), + ColumnBuilder(valueName, Type::Int64).NotNull(), + PrimaryKeyBuilder({ valueName, s_OneToManyTable_MapTable_ManifestName }) + }); + + createMapTableBuilder.Execute(connection); + } + break; + case OneToManyTableSchema::Version_1_1: { // Create the mapping table StatementBuilder createMapTableBuilder; @@ -123,7 +144,8 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 On({ tableName, s_OneToManyTable_MapTable_Suffix }).Columns({ valueName, s_OneToManyTable_MapTable_ManifestName }); pkIndexBuilder.Execute(connection); } - else + break; + case OneToManyTableSchema::Version_1_7: { // Create the mapping table StatementBuilder createMapTableBuilder; @@ -131,14 +153,18 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 ColumnBuilder(s_OneToManyTable_MapTable_ManifestName, Type::Int64).NotNull(), ColumnBuilder(valueName, Type::Int64).NotNull(), PrimaryKeyBuilder({ valueName, s_OneToManyTable_MapTable_ManifestName }) - }); + }).WithoutRowID(); createMapTableBuilder.Execute(connection); } + break; + default: + THROW_HR(E_UNEXPECTED); + } StatementBuilder createMapTableIndexBuilder; createMapTableIndexBuilder.CreateIndex({ tableName, s_OneToManyTable_MapTable_Suffix, s_OneToManyTable_MapTable_IndexSuffix }). - On({ tableName, s_OneToManyTable_MapTable_Suffix }).Columns(s_OneToManyTable_MapTable_ManifestName); + On({ tableName, s_OneToManyTable_MapTable_Suffix }).Columns({ s_OneToManyTable_MapTable_ManifestName, valueName }); createMapTableIndexBuilder.Execute(connection); @@ -272,7 +298,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 savepoint.Commit(); } - void OneToManyTablePrepareForPackaging(SQLite::Connection& connection, std::string_view tableName, bool useNamedIndices, bool preserveManifestIndex, bool preserveValuesIndex) + void OneToManyTablePrepareForPackaging(SQLite::Connection& connection, std::string_view tableName, OneToManyTableSchema schemaVersion, bool preserveManifestIndex, bool preserveValuesIndex) { if (!preserveManifestIndex) { @@ -282,7 +308,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 dropMapTableIndexBuilder.Execute(connection); } - OneToOneTablePrepareForPackaging(connection, tableName, useNamedIndices, preserveValuesIndex); + OneToOneTablePrepareForPackaging(connection, tableName, SchemaVersionUsesNamedIndices(schemaVersion), preserveValuesIndex); } bool OneToManyTableCheckConsistency(const SQLite::Connection& connection, std::string_view tableName, std::string_view valueName, bool log) @@ -299,7 +325,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 SQLite::Builder::StatementBuilder builder; builder. - Select({ QCol(s_map, SQLite::RowIDName), QCol(s_map, s_OneToManyTable_MapTable_ManifestName) }). + Select({ QCol(s_map, s_OneToManyTable_MapTable_ManifestName), QCol(s_map, valueName) }). From({ tableName, s_OneToManyTable_MapTable_Suffix }).As(s_map). LeftOuterJoin(ManifestTable::TableName()).On(QCol(s_map, s_OneToManyTable_MapTable_ManifestName), QCol(ManifestTable::TableName(), SQLite::RowIDName)). Where(QCol(ManifestTable::TableName(), SQLite::RowIDName)).IsNull(); @@ -316,7 +342,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 } AICLI_LOG(Repo, Info, << " [INVALID] " << tableName << s_OneToManyTable_MapTable_Suffix << " [" << select.GetColumn<SQLite::rowid_t>(0) << - "] refers to " << ManifestTable::TableName() << " [" << select.GetColumn<SQLite::rowid_t>(1) << "]"); + ", " << select.GetColumn<SQLite::rowid_t>(1) << "] refers to invalid " << ManifestTable::TableName()); } } @@ -331,7 +357,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 // Select map.rowid, map.tag from tags_map as map left outer join tags on map.tag = tags.rowid where tags.tag is null SQLite::Builder::StatementBuilder builder; builder. - Select({ QCol(s_map, SQLite::RowIDName), QCol(s_map, valueName) }). + Select({ QCol(s_map, s_OneToManyTable_MapTable_ManifestName), QCol(s_map, valueName) }). From({ tableName, s_OneToManyTable_MapTable_Suffix }).As(s_map). LeftOuterJoin(tableName).On(QCol(s_map, valueName), QCol(tableName, SQLite::RowIDName)). Where(QCol(tableName, valueName)).IsNull(); @@ -349,7 +375,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 } AICLI_LOG(Repo, Info, << " [INVALID] " << tableName << s_OneToManyTable_MapTable_Suffix << " [" << select.GetColumn<SQLite::rowid_t>(0) << - "] refers to " << tableName << " [" << select.GetColumn<SQLite::rowid_t>(1) << "]"); + ", " << select.GetColumn<SQLite::rowid_t>(1) << "] refers to invalid " << tableName); } result = result && secondaryResult; @@ -383,5 +409,46 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 return ((countStatement.GetColumn<int>(0) == 0) && (countMapStatement.GetColumn<int>(0) == 0)); } + + SQLite::Statement OneToManyTablePrepareMapDataFoldingStatement(const SQLite::Connection& connection, std::string_view tableName) + { + using namespace SQLite::Builder; + StatementBuilder builder; + + // Create a statement that will collapse (and dedupe) all rows in the map to the latest (max) manifest for a given id, like: + // UPDATE OR REPLACE map SET manifest = (SELECT MAX(rowid) FROM manifest_table WHERE id = ?1) WHERE manifest IN (SELECT rowid FROM manifest_table WHERE id = ?1) + builder.UpdateOrReplace({ tableName, s_OneToManyTable_MapTable_Suffix }).Set().Column(s_OneToManyTable_MapTable_ManifestName).Equals() + .BeginParenthetical() + .Select().Column(Aggregate::Max, SQLite::RowIDName).From(ManifestTable::TableName()).Where(IdTable::ValueName()).Equals(Unbound, 1) + .EndParenthetical() + .Where(s_OneToManyTable_MapTable_ManifestName).In() + .BeginParenthetical() + .Select(SQLite::RowIDName).From(ManifestTable::TableName()).Where(IdTable::ValueName()).Equals(Unbound, 1) + .EndParenthetical(); + + return builder.Prepare(connection); + } + } + + std::optional<SQLite::rowid_t> OneToManyTableGetMapDataFoldingManifestTargetId(const SQLite::Connection& connection, SQLite::rowid_t manifestId) + { + using namespace SQLite::Builder; + StatementBuilder builder; + + // Select the maximum manifest rowid from the manifests whose id is the same as the row with the given manifest rowid, like: + // SELECT MAX(rowid) FROM manifest_table WHERE id = (SELECT id FROM manifest_table WHERE rowid = ?) + builder.Select().Column(Aggregate::Max, SQLite::RowIDName).From(ManifestTable::TableName()).Where(IdTable::ValueName()).Equals() + .BeginParenthetical() + .Select(IdTable::ValueName()).From(ManifestTable::TableName()).Where(SQLite::RowIDName).Equals(manifestId) + .EndParenthetical(); + + SQLite::Statement statement = builder.Prepare(connection); + + if (statement.Step() && !statement.GetColumnIsNull(0)) + { + return statement.GetColumn<SQLite::rowid_t>(0); + } + + return std::nullopt; } } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToManyTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToManyTable.h @@ -9,6 +9,20 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 { + // Allow the different schema version to indicate which they are. + enum class OneToManyTableSchema + { + // Does not used a named unique index for either table. + // Map table has primary key and rowid. + Version_1_0, + // Uses a named unique index for both tables. + // Map table has rowid. + Version_1_1, + // Uses a named unique index for data table. (No change from 1.1) + // Map table has primary key and no rowid. + Version_1_7, + }; + namespace details { // Returns the map table name for a given table. @@ -18,7 +32,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 std::string_view OneToManyTableGetManifestColumnName(); // Create the tables. - void CreateOneToManyTable(SQLite::Connection& connection, bool useNamedIndices, std::string_view tableName, std::string_view valueName); + void CreateOneToManyTable(SQLite::Connection& connection, OneToManyTableSchema schemaVersion, std::string_view tableName, std::string_view valueName); // Gets all values associated with the given manifest id. std::vector<std::string> OneToManyTableGetValuesByManifestId( @@ -41,7 +55,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 void OneToManyTableDeleteIfNotNeededByManifestId(SQLite::Connection& connection, std::string_view tableName, std::string_view valueName, SQLite::rowid_t manifestId); // Removes data that is no longer needed for an index that is to be published. - void OneToManyTablePrepareForPackaging(SQLite::Connection& connection, std::string_view tableName, bool useNamedIndices, bool preserveManifestIndex, bool preserveValuesIndex); + void OneToManyTablePrepareForPackaging(SQLite::Connection& connection, std::string_view tableName, OneToManyTableSchema schemaVersion, bool preserveManifestIndex, bool preserveValuesIndex); // Checks the consistency of the index to ensure that every referenced row exists. // Returns true if index is consistent; false if it is not. @@ -49,8 +63,14 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 // Determines if the table is empty. bool OneToManyTableIsEmpty(SQLite::Connection& connection, std::string_view tableName); + + // Prepares a statement for replacing all of the map data to point to a single manifest for a given id. + SQLite::Statement OneToManyTablePrepareMapDataFoldingStatement(const SQLite::Connection& connection, std::string_view tableName); } + // Gets the manifest id that the PrepareMapDataFoldingStatement will fold to for the given manifest id. + std::optional<SQLite::rowid_t> OneToManyTableGetMapDataFoldingManifestTargetId(const SQLite::Connection& connection, SQLite::rowid_t manifestId); + // A table that represents a value that is 1:N with a primary entry. template <typename TableInfo> struct OneToManyTable @@ -74,15 +94,9 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 } // Creates the table with named indices. - static void Create(SQLite::Connection& connection) - { - details::CreateOneToManyTable(connection, true, TableInfo::TableName(), TableInfo::ValueName()); - } - - // Creates the table with standard primary keys. - static void Create_deprecated(SQLite::Connection& connection) + static void Create(SQLite::Connection& connection, OneToManyTableSchema schemaVersion) { - details::CreateOneToManyTable(connection, false, TableInfo::TableName(), TableInfo::ValueName()); + details::CreateOneToManyTable(connection, schemaVersion, TableInfo::TableName(), TableInfo::ValueName()); } // Gets all values associated with the given manifest id. @@ -110,15 +124,11 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 } // Removes data that is no longer needed for an index that is to be published. - static void PrepareForPackaging(SQLite::Connection& connection, bool preserveManifestIndex, bool preserveValuesIndex = false) + // Preserving the manifest index will improve the efficiency of finding the values associated with a manifest. + // Preserving the values index will improve searching when it is primarily done by equality. + static void PrepareForPackaging(SQLite::Connection& connection, OneToManyTableSchema schemaVersion, bool preserveManifestIndex, bool preserveValuesIndex) { - details::OneToManyTablePrepareForPackaging(connection, TableInfo::TableName(), true, preserveManifestIndex, preserveValuesIndex); - } - - // Removes data that is no longer needed for an index that is to be published. - static void PrepareForPackaging_deprecated(SQLite::Connection& connection) - { - details::OneToManyTablePrepareForPackaging(connection, TableInfo::TableName(), false, false, false); + details::OneToManyTablePrepareForPackaging(connection, TableInfo::TableName(), schemaVersion, preserveManifestIndex, preserveValuesIndex); } // Checks the consistency of the index to ensure that every referenced row exists. @@ -133,5 +143,11 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 { return details::OneToManyTableIsEmpty(connection, TableInfo::TableName()); } + + // Prepares a statement for replacing all of the map data to point to a single manifest for a given id. + static SQLite::Statement PrepareMapDataFoldingStatement(const SQLite::Connection& connection) + { + return details::OneToManyTablePrepareMapDataFoldingStatement(connection, TableInfo::TableName()); + } }; } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToOneTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToOneTable.h @@ -113,6 +113,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 } // Removes data that is no longer needed for an index that is to be published. + // Preserving the values index will improve searching when it is primarily done by equality. static void PrepareForPackaging(SQLite::Connection& connection, bool preserveValuesIndex = false) { details::OneToOneTablePrepareForPackaging(connection, TableInfo::TableName(), true, preserveValuesIndex); diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_1/Interface.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_1/Interface.h @@ -28,6 +28,8 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_1 protected: std::unique_ptr<V1_0::SearchResultsTable> CreateSearchResultsTable(const SQLite::Connection& connection) const override; void PerformQuerySearch(V1_0::SearchResultsTable& resultsTable, const RequestMatch& query) const override; + V1_0::OneToManyTableSchema GetOneToManyTableSchema() const override; + virtual SearchResult SearchInternal(const SQLite::Connection& connection, SearchRequest& request) const; virtual void PrepareForPackaging(SQLite::Connection& connection, bool vacuum); diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_1/Interface_1_1.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_1/Interface_1_1.cpp @@ -51,10 +51,10 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_1 { V1_0::PathPartTable::ValueName(), false, WI_IsFlagClear(options, CreateOptions::SupportPathless) } }); - V1_0::TagsTable::Create(connection); - V1_0::CommandsTable::Create(connection); - PackageFamilyNameTable::Create(connection); - ProductCodeTable::Create(connection); + V1_0::TagsTable::Create(connection, GetOneToManyTableSchema()); + V1_0::CommandsTable::Create(connection, GetOneToManyTableSchema()); + PackageFamilyNameTable::Create(connection, GetOneToManyTableSchema()); + ProductCodeTable::Create(connection, GetOneToManyTableSchema()); savepoint.Commit(); } @@ -196,6 +196,11 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_1 V1_0::Interface::PerformQuerySearch(resultsTable, query); } + V1_0::OneToManyTableSchema Interface::GetOneToManyTableSchema() const + { + return V1_0::OneToManyTableSchema::Version_1_1; + } + ISQLiteIndex::SearchResult Interface::SearchInternal(const SQLite::Connection& connection, SearchRequest& request) const { // Update any system reference strings to be folded @@ -239,10 +244,10 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_1 V1_0::PathPartTable::ValueName(), }); - V1_0::TagsTable::PrepareForPackaging(connection, false); - V1_0::CommandsTable::PrepareForPackaging(connection, false); - PackageFamilyNameTable::PrepareForPackaging(connection, true, true); - ProductCodeTable::PrepareForPackaging(connection, true, true); + V1_0::TagsTable::PrepareForPackaging(connection, GetOneToManyTableSchema(), false, false); + V1_0::CommandsTable::PrepareForPackaging(connection, GetOneToManyTableSchema(), false, false); + PackageFamilyNameTable::PrepareForPackaging(connection, GetOneToManyTableSchema(), true, true); + ProductCodeTable::PrepareForPackaging(connection, GetOneToManyTableSchema(), true, true); savepoint.Commit(); diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_2/Interface_1_2.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_2/Interface_1_2.cpp @@ -165,8 +165,8 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_2 // This will mean that one can match cross locale name and publisher, but the chance that this // leads to a confusion between packages is very small. More likely would be intentional attempts // to confuse the correlation, which could be fairly easily carried out even with linked values. - NormalizedPackageNameTable::Create(connection); - NormalizedPackagePublisherTable::Create(connection); + NormalizedPackageNameTable::Create(connection, GetOneToManyTableSchema()); + NormalizedPackagePublisherTable::Create(connection, GetOneToManyTableSchema()); savepoint.Commit(); } @@ -320,8 +320,8 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_2 V1_1::Interface::PrepareForPackaging(connection, false); - NormalizedPackageNameTable::PrepareForPackaging(connection, true, true); - NormalizedPackagePublisherTable::PrepareForPackaging(connection, true, true); + NormalizedPackageNameTable::PrepareForPackaging(connection, GetOneToManyTableSchema(), true, true); + NormalizedPackagePublisherTable::PrepareForPackaging(connection, GetOneToManyTableSchema(), true, true); savepoint.Commit(); diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_6/Interface_1_6.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_6/Interface_1_6.cpp @@ -24,7 +24,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_6 V1_5::Interface::CreateTables(connection, options); - UpgradeCodeTable::Create(connection); + UpgradeCodeTable::Create(connection, GetOneToManyTableSchema()); savepoint.Commit(); } @@ -142,7 +142,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_6 V1_5::Interface::PrepareForPackaging(connection, false); - UpgradeCodeTable::PrepareForPackaging(connection, true, true); + UpgradeCodeTable::PrepareForPackaging(connection, GetOneToManyTableSchema(), true, true); savepoint.Commit(); diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_7/Interface.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_7/Interface.h @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once +#include "Microsoft/Schema/ISQLiteIndex.h" +#include "Microsoft/Schema/1_6/Interface.h" + +namespace AppInstaller::Repository::Microsoft::Schema::V1_7 +{ + using namespace std::string_view_literals; + + // Interface to this schema version exposed through ISQLiteIndex. + struct Interface : public V1_6::Interface + { + static constexpr std::string_view MapDataFolded_VersionSpecifier = "1.7"sv; + + Interface(Utility::NormalizationVersion normVersion = Utility::NormalizationVersion::Initial); + + // Version 1.0 + Schema::Version GetVersion() const override; + std::vector<std::string> GetMultiPropertyByManifestId(const SQLite::Connection& connection, SQLite::rowid_t manifestId, PackageVersionMultiProperty property) const override; + + protected: + void PrepareForPackaging(SQLite::Connection& connection, bool vacuum) override; + V1_0::OneToManyTableSchema GetOneToManyTableSchema() const override; + }; +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_7/Interface_1_7.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_7/Interface_1_7.cpp @@ -0,0 +1,114 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "Microsoft/Schema/1_7/Interface.h" +#include "Microsoft/Schema/MetadataTable.h" +#include "Microsoft/Schema/1_0/CommandsTable.h" +#include "Microsoft/Schema/1_0/IdTable.h" +#include "Microsoft/Schema/1_0/TagsTable.h" +#include "Microsoft/Schema/1_1/PackageFamilyNameTable.h" +#include "Microsoft/Schema/1_2/NormalizedPackageNameTable.h" +#include "Microsoft/Schema/1_2/NormalizedPackagePublisherTable.h" +#include "Microsoft/Schema/1_6/UpgradeCodeTable.h" + +namespace AppInstaller::Repository::Microsoft::Schema::V1_7 +{ + namespace + { + bool ShouldFoldPropertyLookup(const SQLite::Connection& connection) + { + // Get the metadata indicator that we folded these multi properties. + // If it contains the value for folding these properties in the 1.7 manner, also fold the incoming manifest + // to the same value that it would have been folded to so that all manifest entries will have all of these properties. + std::optional<std::string> mapDataFolded = MetadataTable::TryGetNamedValue<std::string>(connection, s_MetadataValueName_MapDataFolded); + + if (mapDataFolded) + { + std::vector<std::string> foldedSplit = Utility::Split(mapDataFolded.value(), s_MetadataValue_MapDataFolded_Separator); + + for (const std::string& splitValue : foldedSplit) + { + if (splitValue == Interface::MapDataFolded_VersionSpecifier) + { + return true; + } + } + } + + return false; + } + } + + Interface::Interface(Utility::NormalizationVersion normVersion) : V1_6::Interface(normVersion) + { + } + + Schema::Version Interface::GetVersion() const + { + return { 1, 7 }; + } + + std::vector<std::string> Interface::GetMultiPropertyByManifestId(const SQLite::Connection& connection, SQLite::rowid_t manifestId, PackageVersionMultiProperty property) const + { + if (property == PackageVersionMultiProperty::PackageFamilyName || + property == PackageVersionMultiProperty::Name || + property == PackageVersionMultiProperty::Publisher || + property == PackageVersionMultiProperty::UpgradeCode) + { + if (ShouldFoldPropertyLookup(connection)) + { + std::optional<SQLite::rowid_t> maximumManifestId = V1_0::OneToManyTableGetMapDataFoldingManifestTargetId(connection, manifestId); + if (maximumManifestId) + { + manifestId = maximumManifestId.value(); + } + } + } + + return V1_6::Interface::GetMultiPropertyByManifestId(connection, manifestId, property); + } + + void Interface::PrepareForPackaging(SQLite::Connection& connection, bool vacuum) + { + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, "prepareforpackaging_v1_7"); + + // Remove data that is not particularly interesting per-manifest + std::array<SQLite::Statement, 6> dataRemovalStatements{ + V1_0::CommandsTable::PrepareMapDataFoldingStatement(connection), + V1_0::TagsTable::PrepareMapDataFoldingStatement(connection), + V1_1::PackageFamilyNameTable::PrepareMapDataFoldingStatement(connection), + V1_2::NormalizedPackageNameTable::PrepareMapDataFoldingStatement(connection), + V1_2::NormalizedPackagePublisherTable::PrepareMapDataFoldingStatement(connection), + V1_6::UpgradeCodeTable::PrepareMapDataFoldingStatement(connection), + }; + + std::vector<SQLite::rowid_t> allIdentifiers = V1_0::IdTable::GetAllRowIds(connection); + + for (SQLite::rowid_t id : allIdentifiers) + { + for (SQLite::Statement& statement : dataRemovalStatements) + { + statement.Reset(); + statement.Bind(1, id); + + statement.Execute(); + } + } + + MetadataTable::SetNamedValue(connection, s_MetadataValueName_MapDataFolded, MapDataFolded_VersionSpecifier); + + V1_6::Interface::PrepareForPackaging(connection, false); + + savepoint.Commit(); + + if (vacuum) + { + Vacuum(connection); + } + } + + V1_0::OneToManyTableSchema Interface::GetOneToManyTableSchema() const + { + return V1_0::OneToManyTableSchema::Version_1_7; + } +} diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/MetadataTable.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/MetadataTable.cpp @@ -27,16 +27,31 @@ CREATE TABLE [metadata]( create.Execute(); } - SQLite::Statement MetadataTable::GetNamedValueStatement(SQLite::Connection& connection, std::string_view name) + SQLite::Statement MetadataTable::GetNamedValueStatement(const SQLite::Connection& connection, std::string_view name) + { + std::optional<SQLite::Statement> result = TryGetNamedValueStatement(connection, name); + THROW_HR_IF(E_NOT_SET, !result); + return std::move(result).value(); + } + + std::optional<SQLite::Statement> MetadataTable::TryGetNamedValueStatement(const SQLite::Connection& connection, std::string_view name) { THROW_HR_IF(E_INVALIDARG, name.empty()); + SQLite::Statement result = SQLite::Statement::Create(connection, s_MetadataTableStmt_GetNamedValue); result.Bind(1, name); - THROW_HR_IF(E_NOT_SET, !result.Step()); - return result; + + if (result.Step()) + { + return result; + } + else + { + return std::nullopt; + } } - SQLite::Statement MetadataTable::SetNamedValueStatement(SQLite::Connection& connection, std::string_view name) + SQLite::Statement MetadataTable::SetNamedValueStatement(const SQLite::Connection& connection, std::string_view name) { THROW_HR_IF(E_INVALIDARG, name.empty()); SQLite::Statement result = SQLite::Statement::Create(connection, s_MetadataTableStmt_SetNamedValue); diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/MetadataTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/MetadataTable.h @@ -16,6 +16,10 @@ namespace AppInstaller::Repository::Microsoft::Schema static constexpr std::string_view s_MetadataValueName_MinorVersion = "minorVersion"sv; static constexpr std::string_view s_MetadataValueName_LastWriteTime = "lastwritetime"sv; + // Version 1.7 + static constexpr std::string_view s_MetadataValueName_MapDataFolded = "mapDataFolded"sv; + static constexpr char s_MetadataValue_MapDataFolded_Separator = ';'; + // The metadata table for the index. // Contains a fixed-schema set of named values that can be used to determine how to read the rest of the index. struct MetadataTable @@ -24,15 +28,31 @@ namespace AppInstaller::Repository::Microsoft::Schema // Gets the named value from the metadata table, interpreting it as the given type. template <typename Value> - static Value GetNamedValue(SQLite::Connection& connection, std::string_view name) + static Value GetNamedValue(const SQLite::Connection& connection, std::string_view name) { SQLite::Statement statement = GetNamedValueStatement(connection, name); return statement.GetColumn<Value>(0); } + // Gets the named value from the metadata table, interpreting it as the given type. + // Returns nullopt if the value is not present. + template <typename Value> + static std::optional<Value> TryGetNamedValue(const SQLite::Connection& connection, std::string_view name) + { + std::optional<SQLite::Statement> statement = TryGetNamedValueStatement(connection, name); + if (statement) + { + return statement->GetColumn<Value>(0); + } + else + { + return std::nullopt; + } + } + // Sets the named value into the metadata table. template <typename Value> - static void SetNamedValue(SQLite::Connection& connection, std::string_view name, Value&& v) + static void SetNamedValue(const SQLite::Connection& connection, std::string_view name, Value&& v) { SQLite::Statement statement = SetNamedValueStatement(connection, name); statement.Bind(2, std::forward<Value>(v)); @@ -41,9 +61,12 @@ namespace AppInstaller::Repository::Microsoft::Schema private: // Internal function that gets the named value. - static SQLite::Statement GetNamedValueStatement(SQLite::Connection& connection, std::string_view name); + static SQLite::Statement GetNamedValueStatement(const SQLite::Connection& connection, std::string_view name); + + // Internal function that gets the named value, or nullopt if it is not present. + static std::optional<SQLite::Statement> TryGetNamedValueStatement(const SQLite::Connection& connection, std::string_view name); // Internal function that sets the named value. - static SQLite::Statement SetNamedValueStatement(SQLite::Connection& connection, std::string_view name); + static SQLite::Statement SetNamedValueStatement(const SQLite::Connection& connection, std::string_view name); }; } diff --git a/src/AppInstallerRepositoryCore/SQLiteStatementBuilder.cpp b/src/AppInstallerRepositoryCore/SQLiteStatementBuilder.cpp @@ -84,6 +84,9 @@ namespace AppInstaller::Repository::SQLite::Builder case Aggregate::Min: out << "MIN"; break; + case Aggregate::Max: + out << "MAX"; + break; default: THROW_HR(E_UNEXPECTED); } @@ -330,9 +333,23 @@ namespace AppInstaller::Repository::SQLite::Builder return *this; } - StatementBuilder& StatementBuilder::Equals(details::unbound_t) + StatementBuilder& StatementBuilder::Equals(details::unbound_t, std::optional<size_t> index) { - AppendOpAndBinder(Op::Equals); + AppendOpAndBinder(Op::Equals, index); + return *this; + } + + StatementBuilder& StatementBuilder::Equals(std::nullptr_t) + { + // This is almost certainly not what you want. + // In SQL, value = NULL is always false. + // Use StatementBuilder::IsNull instead. + THROW_HR(E_NOTIMPL); + } + + StatementBuilder& StatementBuilder::Equals() + { + m_stream << " ="; return *this; } @@ -366,14 +383,6 @@ namespace AppInstaller::Repository::SQLite::Builder return *this; } - StatementBuilder& StatementBuilder::Equals(std::nullptr_t) - { - // This is almost certainly not what you want. - // In SQL, value = NULL is always false. - // Use StatementBuilder::IsNull instead. - THROW_HR(E_NOTIMPL); - } - StatementBuilder& StatementBuilder::Not() { m_stream << " NOT"; @@ -794,6 +803,24 @@ namespace AppInstaller::Repository::SQLite::Builder return *this; } + StatementBuilder& StatementBuilder::UpdateOrReplace(std::string_view table) + { + OutputOperationAndTable(m_stream, "UPDATE OR REPLACE", table); + return *this; + } + + StatementBuilder& StatementBuilder::UpdateOrReplace(QualifiedTable table) + { + OutputOperationAndTable(m_stream, "UPDATE OR REPLACE", table); + return *this; + } + + StatementBuilder& StatementBuilder::UpdateOrReplace(std::initializer_list<std::string_view> table) + { + OutputOperationAndTable(m_stream, "UPDATE OR REPLACE", table); + return *this; + } + StatementBuilder& StatementBuilder::Set() { m_stream << " SET "; @@ -819,6 +846,13 @@ namespace AppInstaller::Repository::SQLite::Builder return *this; } + StatementBuilder& StatementBuilder::WithoutRowID() + { + m_stream << " WITHOUT ROWID"; + return *this; + } + + StatementBuilder& StatementBuilder::As(std::string_view alias) { OutputOperationAndTable(m_stream, " AS", alias); @@ -840,7 +874,7 @@ namespace AppInstaller::Repository::SQLite::Builder Prepare(connection).Execute(); } - int StatementBuilder::AppendOpAndBinder(Op op) + int StatementBuilder::AppendOpAndBinder(Op op, std::optional<size_t> index) { switch (op) { @@ -860,6 +894,11 @@ namespace AppInstaller::Repository::SQLite::Builder THROW_HR(E_UNEXPECTED); } + if (index) + { + m_stream << index.value(); + } + return m_bindIndex++; } diff --git a/src/AppInstallerRepositoryCore/SQLiteStatementBuilder.h b/src/AppInstallerRepositoryCore/SQLiteStatementBuilder.h @@ -116,7 +116,8 @@ namespace AppInstaller::Repository::SQLite::Builder // Aggregate functions. enum class Aggregate { - Min + Min, + Max, }; // Helper to mark create an integer primary key for rowid, making it stable across vacuum. @@ -246,8 +247,10 @@ namespace AppInstaller::Repository::SQLite::Builder return IsNull(); } } - StatementBuilder& Equals(details::unbound_t); + // The optional index value can be used to specify the parameter index. + StatementBuilder& Equals(details::unbound_t, std::optional<size_t> index = {}); StatementBuilder& Equals(std::nullptr_t); + StatementBuilder& Equals(); StatementBuilder& LikeWithEscape(std::string_view value); StatementBuilder& Like(details::unbound_t); @@ -258,8 +261,8 @@ namespace AppInstaller::Repository::SQLite::Builder StatementBuilder& Not(); StatementBuilder& In(); - - //Appends a set of value binders for the In clause. + + // Appends a set of value binders for the In clause. StatementBuilder& In(size_t count); // IsNull(true) means the value is null; IsNull(false) means the value is not null. @@ -394,6 +397,12 @@ namespace AppInstaller::Repository::SQLite::Builder StatementBuilder& Update(QualifiedTable table); StatementBuilder& Update(std::initializer_list<std::string_view> table); + // Begin an `update or replace` statement. + // The initializer_list form enables the table name to be constructed from multiple parts. + StatementBuilder& UpdateOrReplace(std::string_view table); + StatementBuilder& UpdateOrReplace(QualifiedTable table); + StatementBuilder& UpdateOrReplace(std::initializer_list<std::string_view> table); + // Output the set portion of an update statement. StatementBuilder& Set(); @@ -404,6 +413,9 @@ namespace AppInstaller::Repository::SQLite::Builder StatementBuilder& BeginParenthetical(); StatementBuilder& EndParenthetical(); + // Adds the `without rowid` clause. + StatementBuilder& WithoutRowID(); + // Assign an alias to the previous item. StatementBuilder& As(std::string_view alias); @@ -427,7 +439,8 @@ namespace AppInstaller::Repository::SQLite::Builder }; // Appends given the operation. - int AppendOpAndBinder(Op op); + // The optional index value can be used to specify the parameter index. + int AppendOpAndBinder(Op op, std::optional<size_t> index = {}); // Appends a set of binders for the values clause of an insert. int AppendValuesAndBinders(size_t count); diff --git a/src/AppInstallerRepositoryCore/pch.h b/src/AppInstallerRepositoryCore/pch.h @@ -46,6 +46,7 @@ #include <wrl/client.h> #include <algorithm> +#include <array> #include <chrono> #include <filesystem> #include <fstream>