commit 9f03555c96e147bc4cbc46619ff998025edef46a parent 54d4b67572d10888bc2d3dcf25d3bd732c1e81e8 Author: Akinwale Alagbe <24412729+hackean-msft@users.noreply.github.com> Date: Fri, 3 Dec 2021 14:54:04 -0800 Add Package Dependencies to index. (#1667) * first tests for show depenencies feature * show dependencies feature for show, install, need to change message * showflow output changed * install commands shows dependencies as in specs * tests for informing dependencies on commands: show, install, upgrade, import * refactor install flow * change import command from install flow (install multiple) * import shows all dependencies together * show dependencies for validate and uninstall * tests for validate and uninstall * create show dependencies exp feature * test for validate command * put functionality under experimental feature check * enable show dep experimental feature on unit test cases * change experimental feature name * remove info stream characters on testcase * move check for exp feature inside report function, create new DependenciesFlow * DependenciesFlow header and cpp inside Workflow * –change representation of Dependency, create DependencyType and DependencyList; add Dependency to context data * dependencies context data is of type DependencyList (not optional) * fix spelling errors * ApplyTo function, can receibe a lambda function to apply on specific DpeendencyType * localize user strings, code style changes, validate report dependencies task divided, creates ValidateFlow * change uninstall flow (gets dependencies from package version), make DependencyList.dependencies private, code style * add missing report dep on upgrade * Update src/AppInstallerCLIPackage/Shared/Strings/en-us/winget.resw Co-authored-by: JohnMcPMS <johnmcp@microsoft.com> * compare id with ICU case insensitive * move dependencies related functions to DependenciesFlow, ReportDependencies is now a WorkflowTask receiving resource string id * TODO to get dependencies on uninstall context, remove testcase for now, move dependency related function to DependenciesFlow * wfIds not temporary * DependencyList Add function checks for existence and updates min version if needed * root dependencies are used when installer are not present, otherwise installer are preferred * merge commits from master * style and details * fix merge conflicts * spellcheck exception * detail when adding dep * proto * divide identity report and installation disclaimer from actual installation * proto tests * testing * questions * test for dependency graph and fixes * check loop logic change * const & dep graph * with installation order * two more tests * change map key to Dependency (missing min version check) * Dependency implements operand< * Dependency.MinVersion type is AppInstaller::Utility::Version * choose installer using ManifestComparator * getting composite source * Drop const from the returned type of GetSource * add check for min version * spelling * add dependencies source * new parameter dependency source, open source can set up both: source and dependency source * spelling * DependencyGraph struct, manages only node/adjacent addition, loop check * OpenSource() missing * add adjacents to spellchecker * graph logic is ok, installer selection and test source settings needs to be fixed * dependency-source argument * typo * DependencyGraph receives function to search for dependencies, auto-builds the graph * dependency table, create with id * dependency graph logic moved to manifest common * refactor * break InstallMultiple workflow into two * SelectInstallerFromMetadata does not receive context * leaving workflow to work as in report dependencies, with the addition of managing package dep * move dependencies to context data * dependency graph + installation, all tests passed * information detail, delete temp folder * more testing for dependencies on install command * valid min version test * typo * temp stash * temp stash * Fixed merge errors * Merge branch 'master' of https://github.com/hackean-msft/winget-cli into user/akalagbe/floppy-dependency-graph * More cleanup * temp stash * Removed prompt for failed dependencies * Removed declarations for dependencies table * Spell check fix * In case of fire, push * WingetUtil changes done checkpoint1 * Fixed spellcheck error * Addressed pr comments. * Addressed PR comments 2 * Code cleanup * Fixed spell check error * Refactored DependencyGraph * Fixed typos * Fixed typo * Code refactor and typo fix * throw exception when package is dependency * Bug fix * Fixed typo * Error messages * Replaced file path for pointer for manifest validation. * Fixed bug * Fixed spellcheck error * MultiManifestPackageHasDependencies error message * Code refactor * Moved to 1.4 schema * Moved to schema 1.4 * Moved dependencies validation from index * Moved validation out of index * Adding error for missing package ids * throw when manifest id not found when adding dependencies * Addressed PR comments * Removed duplicate interface file * Code cleanup * Using single api for addition and deletion operation * Moved validation out of dependencies table * Fixed spell check errors * Code refactor * update flag value * Created a new enum for validation manifest dependencies option * Code refactor * create dependencies save point fix. * Refactored Insert Depedencies function * cleanup * more cleanup * return set from get dependencies * Addressed PR comments * Addressed PR comments * Tabs vs spaces * Addressed PR comments * More tests * Code refactor * Addressed PR comments * Using initializer list in place of vector * Fixed bugs * Code refactor * Implemented not needed check for min version in dependencies table * Delete versions if there are not referenced * Fixed spellcheck error Co-authored-by: Florencia Zanollo <fzanollo@dc.uba.ar> Co-authored-by: fzanollo <fzanollo@users.noreply.github.com> Co-authored-by: JohnMcPMS <johnmcp@microsoft.com> Co-authored-by: Akinwale Alagbe <akalagbe@microsoft.com> Diffstat:
32 files changed, 1743 insertions(+), 107 deletions(-)
diff --git a/src/AppInstallerCLITests/DependenciesTestSource.h b/src/AppInstallerCLITests/DependenciesTestSource.h @@ -3,11 +3,13 @@ #include "pch.h" #include "TestSource.h" #include "TestCommon.h" +#include "AppInstallerVersions.h" #include <Public/winget/RepositorySource.h> #include <winget/ManifestYamlParser.h> using namespace AppInstaller::Repository; using namespace AppInstaller::Manifest; +using namespace AppInstaller::Utility; namespace TestCommon { @@ -23,6 +25,8 @@ namespace TestCommon installer.ProductId = input; installer.Dependencies.Clear(); + string_t defaultFakeVersion("0.0.1-defaultFakeVersion"); + if (input == "withoutInstallers") { manifest.Installers.clear(); @@ -50,32 +54,32 @@ namespace TestCommon //-- predefined if (input == "C") { - installer.Dependencies.Add(Dependency(DependencyType::Package, "B")); + installer.Dependencies.Add(Dependency(DependencyType::Package, "B", defaultFakeVersion)); } if (input == "D") { - installer.Dependencies.Add(Dependency(DependencyType::Package, "E")); + installer.Dependencies.Add(Dependency(DependencyType::Package, "E", defaultFakeVersion)); } if (input == "E") { - installer.Dependencies.Add(Dependency(DependencyType::Package, "D")); + installer.Dependencies.Add(Dependency(DependencyType::Package, "D", defaultFakeVersion)); } if (input == "F") { - installer.Dependencies.Add(Dependency(DependencyType::Package, "B")); + installer.Dependencies.Add(Dependency(DependencyType::Package, "B", defaultFakeVersion)); } if (input == "G") { - installer.Dependencies.Add(Dependency(DependencyType::Package, "C")); + installer.Dependencies.Add(Dependency(DependencyType::Package, "C", defaultFakeVersion)); } if (input == "H") { - installer.Dependencies.Add(Dependency(DependencyType::Package, "G")); - installer.Dependencies.Add(Dependency(DependencyType::Package, "B")); + installer.Dependencies.Add(Dependency(DependencyType::Package, "G", defaultFakeVersion)); + installer.Dependencies.Add(Dependency(DependencyType::Package, "B", defaultFakeVersion)); } if (input == "installed1") { - installer.Dependencies.Add(Dependency(DependencyType::Package, "installed1Dep")); + installer.Dependencies.Add(Dependency(DependencyType::Package, "installed1Dep", defaultFakeVersion)); } if (input == "minVersion1.0") { @@ -95,30 +99,30 @@ namespace TestCommon // depends on test if (input == "StackOrderIsOk") { - installer.Dependencies.Add(Dependency(DependencyType::Package, "C")); + installer.Dependencies.Add(Dependency(DependencyType::Package, "C", defaultFakeVersion)); } if (input == "NeedsToInstallBFirst") { - installer.Dependencies.Add(Dependency(DependencyType::Package, "B")); - installer.Dependencies.Add(Dependency(DependencyType::Package, "C")); + installer.Dependencies.Add(Dependency(DependencyType::Package, "B", defaultFakeVersion)); + installer.Dependencies.Add(Dependency(DependencyType::Package, "C", defaultFakeVersion)); } if (input == "EasyToSeeLoop") { - installer.Dependencies.Add(Dependency(DependencyType::Package, "D")); + installer.Dependencies.Add(Dependency(DependencyType::Package, "D", defaultFakeVersion)); } if (input == "DependencyAlreadyInStackButNoLoop") { - installer.Dependencies.Add(Dependency(DependencyType::Package, "C")); - installer.Dependencies.Add(Dependency(DependencyType::Package, "F")); + installer.Dependencies.Add(Dependency(DependencyType::Package, "C", defaultFakeVersion)); + installer.Dependencies.Add(Dependency(DependencyType::Package, "F", defaultFakeVersion)); } if (input == "PathBetweenBranchesButNoLoop") { - installer.Dependencies.Add(Dependency(DependencyType::Package, "C")); - installer.Dependencies.Add(Dependency(DependencyType::Package, "H")); + installer.Dependencies.Add(Dependency(DependencyType::Package, "C", defaultFakeVersion)); + installer.Dependencies.Add(Dependency(DependencyType::Package, "H", defaultFakeVersion)); } if (input == "DependenciesInstalled") { - installer.Dependencies.Add(Dependency(DependencyType::Package, "installed1")); + installer.Dependencies.Add(Dependency(DependencyType::Package, "installed1", defaultFakeVersion)); } if (input == "DependenciesValidMinVersions") { diff --git a/src/AppInstallerCLITests/SQLiteIndex.cpp b/src/AppInstallerCLITests/SQLiteIndex.cpp @@ -3,6 +3,7 @@ #include "pch.h" #include "TestCommon.h" #include <SQLiteWrapper.h> +#include <PackageDependenciesValidation.h> #include <Microsoft/SQLiteIndex.h> #include <winget/Manifest.h> #include <AppInstallerStrings.h> @@ -17,6 +18,7 @@ #include <Microsoft/Schema/1_0/TagsTable.h> #include <Microsoft/Schema/1_0/CommandsTable.h> #include <Microsoft/Schema/1_0/SearchResultsTable.h> +#include <Microsoft/Schema/1_4/DependenciesTable.h> using namespace std::string_literals; using namespace std::string_view_literals; @@ -32,7 +34,7 @@ SQLiteIndex CreateTestIndex(const std::string& filePath, std::optional<Schema::V // If no specific version requested, then use generator to run against the last 3 versions. if (!version) { - version = GENERATE(Schema::Version{ 1, 1 }, Schema::Version{ 1, 2 }, Schema::Version::Latest()); + version = GENERATE(Schema::Version{ 1, 2 }, Schema::Version{ 1, 3 }, Schema::Version::Latest()); } return SQLiteIndex::CreateNew(filePath, version.value()); @@ -40,15 +42,9 @@ SQLiteIndex CreateTestIndex(const std::string& filePath, std::optional<Schema::V Schema::Version TestPrepareForRead(SQLiteIndex& index) { - // This will only be called for tests that want to support cross version checks. - // Based on the version of the incoming, we only want to generate versions less or equal to it. - if (index.GetVersion() == Schema::Version{ 1, 1 }) + if (index.GetVersion() == Schema::Version{ 1, 2 }) { - // Nothing to do here - } - else if (index.GetVersion() == Schema::Version{ 1, 2 }) - { - Schema::Version version = GENERATE(Schema::Version{ 1, 1 }, Schema::Version{ 1, 2 }); + Schema::Version version = GENERATE(Schema::Version{ 1, 2 }); if (version != Schema::Version{ 1, 2 }) { @@ -58,7 +54,7 @@ Schema::Version TestPrepareForRead(SQLiteIndex& index) } else if (index.GetVersion() == Schema::Version{ 1, 3 }) { - Schema::Version version = GENERATE(Schema::Version{ 1, 1 }, Schema::Version{ 1, 2 }, Schema::Version{ 1, 3 }); + Schema::Version version = GENERATE(Schema::Version{ 1, 2 }, Schema::Version{ 1, 3 }); if (version != Schema::Version{ 1, 3 }) { @@ -66,44 +62,50 @@ Schema::Version TestPrepareForRead(SQLiteIndex& index) return version; } } + else if (index.GetVersion() == Schema::Version{ 1, 4 }) + { + Schema::Version version = GENERATE(Schema::Version{ 1, 2 }, Schema::Version{ 1, 3 }, Schema::Version{ 1, 4 }); + + if (version != Schema::Version{ 1, 4 }) + { + index.ForceVersion(version); + return version; + } + } return index.GetVersion(); } -SQLiteIndex SimpleTestSetup(const std::string& filePath, Manifest& manifest, std::string& relativePath, std::optional<Schema::Version> version = {}) +std::string GetPathFromManifest(Manifest& manifest) { - SQLiteIndex index = CreateTestIndex(filePath, version); + auto publisher = manifest.Id; + AppInstaller::Utility::FindAndReplace(publisher, ".", "/"); + + return AppInstaller::Utility::ToLower(publisher).append("/").append(manifest.Version); +} +void CreateFakeManifest(Manifest& manifest, string_t publisher, string_t version = "1.0.0") +{ manifest.Installers.push_back({}); - manifest.Id = "Test.Id"; - manifest.DefaultLocalization.Add<Localization::PackageName>("Test Name"); + manifest.Id = publisher.append(".").append("Id"); + manifest.DefaultLocalization.Add<Localization::PackageName>(publisher.append(" Name")); manifest.Moniker = "testmoniker"; - manifest.Version = "1.0.0"; + manifest.Version = version; manifest.Channel = "test"; manifest.DefaultLocalization.Add<Localization::Tags>({ "t1", "t2" }); manifest.Installers[0].Commands = { "test1", "test2" }; - - relativePath = "test/id/1.0.0.yaml"; - - index.AddManifest(manifest, relativePath); - - return index; } SQLiteIndex SimpleTestSetup(const std::string& filePath, Manifest& manifest, std::optional<Schema::Version> version = {}) { SQLiteIndex index = CreateTestIndex(filePath, version); - manifest.Installers.push_back({}); - manifest.Id = "Test.Id"; - manifest.DefaultLocalization.Add<Localization::PackageName>("Test Name"); - manifest.Moniker = "testmoniker"; - manifest.Version = "1.0.0"; - manifest.Channel = "test"; - manifest.DefaultLocalization.Add<Localization::Tags>({ "t1", "t2" }); - manifest.Installers[0].Commands = { "test1", "test2" }; + string_t publisher = "Test"; + CreateFakeManifest(manifest, publisher); - index.AddManifest(manifest); + auto relativePath = GetPathFromManifest(manifest); + + index.AddManifest(manifest, relativePath); return index; } @@ -338,9 +340,9 @@ TEST_CASE("SQLiteIndexCreateAndAddManifest", "[sqliteindex]") INFO("Using temporary file named: " << tempFile.GetPath()); Manifest manifest; - std::string relativePath; + std::string relativePath = "test/id/1.0.0.yaml"; - SQLiteIndex index = SimpleTestSetup(tempFile, manifest, relativePath); + SQLiteIndex index = SimpleTestSetup(tempFile, manifest, Schema::Version::Latest()); } TEST_CASE("SQLiteIndexCreateAndAddManifestFile", "[sqliteindex]") @@ -362,9 +364,9 @@ TEST_CASE("SQLiteIndexCreateAndAddManifestDuplicate", "[sqliteindex]") INFO("Using temporary file named: " << tempFile.GetPath()); Manifest manifest; - std::string relativePath; - SQLiteIndex index = SimpleTestSetup(tempFile, manifest, relativePath); + SQLiteIndex index = SimpleTestSetup(tempFile, manifest); + auto relativePath = GetPathFromManifest(manifest); // Attempting to add the same manifest at a different path should fail. REQUIRE_THROWS_HR(index.AddManifest(manifest, "differentpath.yaml"), HRESULT_FROM_WIN32(ERROR_ALREADY_EXISTS)); @@ -378,6 +380,181 @@ TEST_CASE("SQLiteIndexCreateAndAddManifestDuplicate", "[sqliteindex]") REQUIRE_THROWS_HR(index.AddManifest(manifest, relativePath), HRESULT_FROM_WIN32(ERROR_ALREADY_EXISTS)); } +TEST_CASE("SQLiteIndex_VersionReferencedByDependenciesClearsUnusedVersionAndKeepUsedVersion", "[sqliteindex][V1_4]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + Manifest dependencyManifest1, dependencyManifest2, manifest, isolatedManifest; + SQLiteIndex index = SimpleTestSetup(tempFile, dependencyManifest1, Schema::Version::Latest()); + + auto& publisher2 = "Test2"; + CreateFakeManifest(dependencyManifest2, publisher2); + index.AddManifest(dependencyManifest2, GetPathFromManifest(dependencyManifest2)); + + auto& publisher3 = "Test3"; + CreateFakeManifest(manifest, publisher3); + std::string dependencyOnlyVersion = "0.0.5"; + manifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, dependencyManifest1.Id, "1.0.0")); + manifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, dependencyManifest2.Id, dependencyOnlyVersion)); + + index.AddManifest(manifest, GetPathFromManifest(manifest)); + + // Create a new manifest that depends on v0.0.5 + auto& publisher4 = "Test4"; + CreateFakeManifest(isolatedManifest, publisher4); + isolatedManifest.Version = dependencyOnlyVersion; + index.AddManifest(isolatedManifest); + + index.RemoveManifest(isolatedManifest); + // After deletion that the version(v0.0.5) must be present because it still referenced via dependencies table. + { + Connection connection = Connection::Create(tempFile, Connection::OpenDisposition::ReadOnly); + REQUIRE(Schema::V1_0::VersionTable::SelectIdByValue(connection, dependencyOnlyVersion).has_value()); + } + + index.RemoveManifest(manifest); + // Now, that we've deleted the manifest depending on version(v0.0.5), it should be absent. + { + Connection connection = Connection::Create(tempFile, Connection::OpenDisposition::ReadOnly); + REQUIRE(!Schema::V1_0::VersionTable::SelectIdByValue(connection, dependencyOnlyVersion).has_value()); + } + index.RemoveManifest(dependencyManifest1); + index.RemoveManifest(dependencyManifest2); + + // Final sanity check, nothing should be in the version table. + { + Connection connection = Connection::Create(tempFile, Connection::OpenDisposition::ReadOnly); + REQUIRE(Schema::V1_0::VersionTable::IsEmpty(connection)); + } +} + +TEST_CASE("SQLiteIndex_AddManifestWithDependencies", "[sqliteindex][V1_4]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + Manifest dependencyManifest1, dependencyManifest2, manifest; + SQLiteIndex index = SimpleTestSetup(tempFile, dependencyManifest1, Schema::Version::Latest()); + + auto& publisher2 = "Test2"; + CreateFakeManifest(dependencyManifest2, publisher2); + index.AddManifest(dependencyManifest2, GetPathFromManifest(dependencyManifest2)); + + auto& publisher3 = "Test3"; + CreateFakeManifest(manifest, publisher3); + manifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, dependencyManifest1.Id, "1.0.0")); + manifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, dependencyManifest2.Id, "1.0.0")); + + index.AddManifest(manifest, GetPathFromManifest(manifest)); +} + +TEST_CASE("SQLiteIndex_AddManifestWithDependencies_MissingPackage", "[sqliteindex][V1_4]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + Manifest dependencyManifest1, dependencyManifest2, manifest; + SQLiteIndex index = SimpleTestSetup(tempFile, dependencyManifest1, Schema::Version::Latest()); + + // Publisher2 is not present + auto& publisher2 = "Test2"; + CreateFakeManifest(dependencyManifest2, publisher2); + + auto& publisher3 = "Test3"; + CreateFakeManifest(manifest, publisher3); + manifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, dependencyManifest1.Id, "1.0.0")); + manifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, dependencyManifest2.Id, "1.0.0")); + + REQUIRE_THROWS_HR(index.AddManifest(manifest, GetPathFromManifest(manifest)), APPINSTALLER_CLI_ERROR_MISSING_PACKAGE); +} + +TEST_CASE("SQLiteIndex_AddManifestWithDependencies_MissingVersion", "[sqliteindex][V1_4]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + Manifest dependencyManifest1, dependencyManifest2, manifest; + SQLiteIndex index = SimpleTestSetup(tempFile, dependencyManifest1, Schema::Version::Latest()); + + auto& publisher2 = "Test2"; + CreateFakeManifest(dependencyManifest2, publisher2); + index.AddManifest(dependencyManifest2, GetPathFromManifest(dependencyManifest2)); + + auto& publisher3 = "Test3"; + CreateFakeManifest(manifest, publisher3); + manifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, dependencyManifest1.Id, "0.0.1")); + manifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, dependencyManifest2.Id, "0.0.2")); + + index.AddManifest(manifest, GetPathFromManifest(manifest)); +} + +TEST_CASE("SQLiteIndex_AddManifestWithDependencies_EmptyManifestVersion", "[sqliteindex][V1_4]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + Manifest dependencyManifest1, dependencyManifest2, manifest; + SQLiteIndex index = SimpleTestSetup(tempFile, dependencyManifest1, Schema::Version::Latest()); + + auto& publisher2 = "Test2"; + CreateFakeManifest(dependencyManifest2, publisher2); + index.AddManifest(dependencyManifest2, GetPathFromManifest(dependencyManifest2)); + + auto& publisher3 = "Test3"; + CreateFakeManifest(manifest, publisher3); + manifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, dependencyManifest1.Id)); + manifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, dependencyManifest2.Id)); + + index.AddManifest(manifest, GetPathFromManifest(manifest)); +} + +TEST_CASE("SQLiteIndex_DependenciesTable_CheckConsistency", "[sqliteindex][V1_1]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + { + Manifest levelOneManifest, levelTwoManifest, levelThreeManifest, topLevelManifest; + SQLiteIndex index = SimpleTestSetup(tempFile, levelThreeManifest, Schema::Version::Latest()); + + constexpr std::string_view levelTwoManifestPublisher = "LevelTwoManifest"; + CreateFakeManifest(levelTwoManifest, levelTwoManifestPublisher); + + levelTwoManifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, levelThreeManifest.Id, "1.0.0")); + index.AddManifest(levelTwoManifest, GetPathFromManifest(levelTwoManifest)); + + constexpr std::string_view levelOneManifestPublisher = "LevelOneManifest"; + CreateFakeManifest(levelOneManifest, levelOneManifestPublisher); + levelOneManifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, levelTwoManifest.Id, "1.0.0")); + index.AddManifest(levelOneManifest, GetPathFromManifest(levelOneManifest)); + + constexpr std::string_view topLevelManifestPublisher = "TopLevelManifest"; + CreateFakeManifest(topLevelManifest, topLevelManifestPublisher); + topLevelManifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, levelOneManifest.Id, "1.0.0")); + } + + { + // Open it directly to modify the table + Connection connection = Connection::Create(tempFile, Connection::OpenDisposition::ReadWrite); + SQLite::rowid_t nonExistentRowId = 40; + SQLite::rowid_t nonExistentManifest = 41; + SQLite::rowid_t nonExistentVersion = 42; + SQLite::rowid_t nonExistentPackageId = 43; + + SQLite::Builder::StatementBuilder builder; + builder.InsertInto(Schema::V1_4::DependenciesTable::TableName()) + .Values(nonExistentRowId, nonExistentManifest, nonExistentVersion, nonExistentPackageId); + builder.Execute(connection); + } + + { + SQLiteIndex index = SQLiteIndex::Open(tempFile, SQLiteIndex::OpenDisposition::ReadWrite); + + REQUIRE(!index.CheckConsistency(true)); + } +} + TEST_CASE("SQLiteIndex_RemoveManifestFile_NotPresent", "[sqliteindex]") { SQLiteIndex index = CreateTestIndex(SQLITE_MEMORY_DB_CONNECTION_TARGET); @@ -462,6 +639,232 @@ TEST_CASE("SQLiteIndex_RemoveManifest", "[sqliteindex][V1_0]") REQUIRE(Schema::V1_0::CommandsTable::IsEmpty(connection)); } +TEST_CASE("SQLiteIndex_RemoveManifestWithDependencies", "[sqliteindex][V1_4]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + Manifest dependencyManifest1, dependencyManifest2, manifest; + SQLiteIndex index = SimpleTestSetup(tempFile, dependencyManifest1, Schema::Version::Latest()); + + auto& publisher2 = "Test2"; + CreateFakeManifest(dependencyManifest2, publisher2); + index.AddManifest(dependencyManifest2, GetPathFromManifest(dependencyManifest2)); + + auto& publisher3 = "Test3"; + CreateFakeManifest(manifest, publisher3); + manifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, dependencyManifest1.Id, "1.0.0")); + manifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, dependencyManifest2.Id, "1.0.0")); + + index.AddManifest(manifest, GetPathFromManifest(manifest)); + + index.RemoveManifest(manifest, GetPathFromManifest(manifest)); +} + +TEST_CASE("SQLiteIndex_ValidateManifestWithDependencies", "[sqliteindex][V1_4]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + Manifest levelOneManifest, levelTwoManifest, levelThreeManifest, topLevelManifest; + SQLiteIndex index = SimpleTestSetup(tempFile, levelThreeManifest, Schema::Version::Latest()); + + constexpr std::string_view levelTwoManifestPublisher = "LevelTwoManifest"; + CreateFakeManifest(levelTwoManifest, levelTwoManifestPublisher); + + levelTwoManifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, levelThreeManifest.Id, "1.0.0")); + index.AddManifest(levelTwoManifest, GetPathFromManifest(levelTwoManifest)); + + constexpr std::string_view levelOneManifestPublisher = "LevelOneManifest"; + CreateFakeManifest(levelOneManifest, levelOneManifestPublisher); + levelOneManifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, levelTwoManifest.Id, "1.0.0")); + index.AddManifest(levelOneManifest, GetPathFromManifest(levelOneManifest)); + + constexpr std::string_view topLevelManifestPublisher = "TopLevelManifest"; + CreateFakeManifest(topLevelManifest, topLevelManifestPublisher); + topLevelManifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, levelOneManifest.Id, "1.0.0")); + REQUIRE(PackageDependenciesValidation::ValidateManifestDependencies(&index, topLevelManifest)); +} + +TEST_CASE("SQLiteIndex_ValidateManifestWithDependenciesHasLoops", "[sqliteindex][V1_4]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + Manifest levelOneManifest, levelTwoManifest, levelThreeManifest, topLevelManifest; + SQLiteIndex index = SimpleTestSetup(tempFile, levelThreeManifest, Schema::Version::Latest()); + + constexpr std::string_view levelTwoManifestPublisher = "LevelTwoManifest"; + CreateFakeManifest(levelTwoManifest, levelTwoManifestPublisher); + + levelTwoManifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, levelThreeManifest.Id, "1.0.0")); + index.AddManifest(levelTwoManifest, GetPathFromManifest(levelTwoManifest)); + + constexpr std::string_view levelOneManifestPublisher = "LevelOneManifest"; + CreateFakeManifest(levelOneManifest, levelOneManifestPublisher); + levelOneManifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, levelTwoManifest.Id, "1.0.0")); + index.AddManifest(levelOneManifest, GetPathFromManifest(levelOneManifest)); + + constexpr std::string_view topLevelManifestPublisher = "TopLevelManifest"; + CreateFakeManifest(topLevelManifest, topLevelManifestPublisher); + topLevelManifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, levelOneManifest.Id, "1.0.0")); + index.AddManifest(topLevelManifest, GetPathFromManifest(topLevelManifest)); + + levelThreeManifest.Installers.push_back(ManifestInstaller{}); + levelThreeManifest.Installers[1].Dependencies.Add(Dependency(DependencyType::Package, topLevelManifest.Id, "1.0.0")); + REQUIRE_THROWS_HR( + PackageDependenciesValidation::ValidateManifestDependencies(&index, levelThreeManifest), + APPINSTALLER_CLI_ERROR_DEPENDENCIES_VALIDATION_FAILED); +} + +TEST_CASE("SQLiteIndex_ValidateManifestWithDependenciesMissingNode", "[sqliteindex][V1_4]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + Manifest levelOneManifest, levelTwoManifest, levelThreeManifest, topLevelManifest; + SQLiteIndex index = SimpleTestSetup(tempFile, levelThreeManifest, Schema::Version::Latest()); + + constexpr std::string_view levelTwoManifestPublisher = "LevelTwoManifest"; + CreateFakeManifest(levelTwoManifest, levelTwoManifestPublisher); + + levelTwoManifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, levelThreeManifest.Id, "1.0.0")); + index.AddManifest(levelTwoManifest, GetPathFromManifest(levelTwoManifest)); + + // This node is missing, because it's not in the index. + constexpr std::string_view levelOneManifestPublisher = "LevelOneManifest"; + CreateFakeManifest(levelOneManifest, levelOneManifestPublisher); + levelOneManifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, levelTwoManifest.Id, "1.0.0")); + + constexpr std::string_view topLevelManifestPublisher = "TopLevelManifest"; + CreateFakeManifest(topLevelManifest, topLevelManifestPublisher); + topLevelManifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, levelOneManifest.Id, "1.0.0")); + REQUIRE_THROWS_HR( + PackageDependenciesValidation::ValidateManifestDependencies(&index, topLevelManifest), + APPINSTALLER_CLI_ERROR_DEPENDENCIES_VALIDATION_FAILED); +} + +TEST_CASE("SQLiteIndex_ValidateManifestWithDependenciesNoSuitableMinVersion", "[sqliteindex][V1_4]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + Manifest levelOneManifest, levelTwoManifest, levelThreeManifest, topLevelManifest; + SQLiteIndex index = SimpleTestSetup(tempFile, levelThreeManifest, Schema::Version::Latest()); + + constexpr std::string_view levelTwoManifestPublisher = "LevelTwoManifest"; + CreateFakeManifest(levelTwoManifest, levelTwoManifestPublisher); + + levelTwoManifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, levelThreeManifest.Id, "1.0.0")); + index.AddManifest(levelTwoManifest, GetPathFromManifest(levelTwoManifest)); + + constexpr std::string_view levelOneManifestPublisher = "LevelOneManifest"; + CreateFakeManifest(levelOneManifest, levelOneManifestPublisher); + levelOneManifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, levelTwoManifest.Id, "1.0.0")); + index.AddManifest(levelOneManifest, GetPathFromManifest(levelOneManifest)); + + constexpr std::string_view topLevelManifestPublisher = "TopLevelManifest"; + CreateFakeManifest(topLevelManifest, topLevelManifestPublisher); + topLevelManifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, levelOneManifest.Id, "2.0.0")); + + REQUIRE_THROWS_HR( + PackageDependenciesValidation::ValidateManifestDependencies(&index, topLevelManifest), + APPINSTALLER_CLI_ERROR_DEPENDENCIES_VALIDATION_FAILED); +} + +TEST_CASE("SQLiteIndex_ValidateManifestWhenManifestIsDependency_StructureBroken", "[sqliteindex][V1_4]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + Manifest levelOneManifest, levelTwoManifest, levelThreeManifest, topLevelManifest; + SQLiteIndex index = SimpleTestSetup(tempFile, levelThreeManifest, Schema::Version::Latest()); + + constexpr std::string_view levelTwoManifestPublisher = "LevelTwoManifest"; + CreateFakeManifest(levelTwoManifest, levelTwoManifestPublisher); + + levelTwoManifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, levelThreeManifest.Id, "1.0.0")); + index.AddManifest(levelTwoManifest, GetPathFromManifest(levelTwoManifest)); + + constexpr std::string_view levelOneManifestPublisher = "LevelOneManifest"; + CreateFakeManifest(levelOneManifest, levelOneManifestPublisher); + levelOneManifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, levelTwoManifest.Id, "1.0.0")); + index.AddManifest(levelOneManifest, GetPathFromManifest(levelOneManifest)); + + constexpr std::string_view topLevelManifestPublisher = "TopLevelManifest"; + CreateFakeManifest(topLevelManifest, topLevelManifestPublisher); + topLevelManifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, levelOneManifest.Id, "1.0.0")); + index.AddManifest(topLevelManifest, GetPathFromManifest(topLevelManifest)); + + REQUIRE_THROWS_HR( + PackageDependenciesValidation::VerifyDependenciesStructureForManifestDelete(&index, levelThreeManifest), + APPINSTALLER_CLI_ERROR_DEPENDENCIES_VALIDATION_FAILED); +} + +TEST_CASE("SQLiteIndex_ValidateManifestWhenManifestIsDependency_StructureNotBroken", "[sqliteindex][V1_4]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + Manifest levelOneManifest, levelTwoManifest, levelThreeManifest, topLevelManifest, levelThreeManifestV2; + SQLiteIndex index = SimpleTestSetup(tempFile, levelThreeManifest, Schema::Version::Latest()); + + constexpr std::string_view levelTwoManifestPublisher = "LevelTwoManifest"; + CreateFakeManifest(levelTwoManifest, levelTwoManifestPublisher); + + levelTwoManifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, levelThreeManifest.Id, "1.0.0")); + index.AddManifest(levelTwoManifest, GetPathFromManifest(levelTwoManifest)); + + constexpr std::string_view levelOneManifestPublisher = "LevelOneManifest"; + CreateFakeManifest(levelOneManifest, levelOneManifestPublisher); + levelOneManifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, levelTwoManifest.Id, "1.0.0")); + index.AddManifest(levelOneManifest, GetPathFromManifest(levelOneManifest)); + + constexpr std::string_view topLevelManifestPublisher = "TopLevelManifest"; + CreateFakeManifest(topLevelManifest, topLevelManifestPublisher); + topLevelManifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, levelOneManifest.Id, "1.0.0")); + index.AddManifest(topLevelManifest, GetPathFromManifest(topLevelManifest)); + + constexpr std::string_view levelThreeManifestV2Publisher = "Test"; + CreateFakeManifest(levelThreeManifestV2, levelThreeManifestV2Publisher, "2.0.0"); + index.AddManifest(levelThreeManifestV2, GetPathFromManifest(levelThreeManifestV2)); + + REQUIRE(PackageDependenciesValidation::VerifyDependenciesStructureForManifestDelete(&index, levelThreeManifest)); +} + +TEST_CASE("SQLiteIndex_ValidateManifestWhenManifestIsDependency_StructureBroken_NoSuitableOldManifest", "[sqliteindex][V1_4]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + Manifest levelOneManifest, levelTwoManifest, levelThreeManifest, topLevelManifest, levelThreeManifestV2; + SQLiteIndex index = SimpleTestSetup(tempFile, levelThreeManifest, Schema::Version::Latest()); + + constexpr std::string_view levelThreeManifestV2Publisher = "Test"; + CreateFakeManifest(levelThreeManifestV2, levelThreeManifestV2Publisher, "2.0.0"); + index.AddManifest(levelThreeManifestV2, GetPathFromManifest(levelThreeManifestV2)); + + constexpr std::string_view levelTwoManifestPublisher = "LevelTwoManifest"; + CreateFakeManifest(levelTwoManifest, levelTwoManifestPublisher); + + levelTwoManifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, levelThreeManifest.Id, "2.0.0")); + index.AddManifest(levelTwoManifest, GetPathFromManifest(levelTwoManifest)); + + constexpr std::string_view levelOneManifestPublisher = "LevelOneManifest"; + CreateFakeManifest(levelOneManifest, levelOneManifestPublisher); + levelOneManifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, levelTwoManifest.Id, "1.0.0")); + index.AddManifest(levelOneManifest, GetPathFromManifest(levelOneManifest)); + + constexpr std::string_view topLevelManifestPublisher = "TopLevelManifest"; + CreateFakeManifest(topLevelManifest, topLevelManifestPublisher); + topLevelManifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, levelOneManifest.Id, "1.0.0")); + index.AddManifest(topLevelManifest, GetPathFromManifest(topLevelManifest)); + + REQUIRE_THROWS( + PackageDependenciesValidation::VerifyDependenciesStructureForManifestDelete(&index, levelThreeManifestV2), + APPINSTALLER_CLI_ERROR_DEPENDENCIES_VALIDATION_FAILED); +} + TEST_CASE("SQLiteIndex_RemoveManifest_EnsureConsistentRowId", "[sqliteindex]") { TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; @@ -509,9 +912,12 @@ TEST_CASE("SQLiteIndex_RemoveManifest_EnsureConsistentRowId", "[sqliteindex]") // Now remove manifest1 and prepare index.RemoveManifest(manifest1, manifest1Path); index.PrepareForPackaging(); + // Checking consistency will also uncover issues, but not potentially the same ones as below. REQUIRE(index.CheckConsistency(true)); + + // Repeat search to ensure consistent ids result = index.Search(request); REQUIRE(result.Matches.size() == 1); @@ -653,6 +1059,66 @@ TEST_CASE("SQLiteIndex_UpdateManifest", "[sqliteindex][V1_0]") REQUIRE(Schema::V1_0::CommandsTable::IsEmpty(connection)); } +TEST_CASE("SQLiteIndex_UpdateManifestWithDependencies", "[sqliteindex][V1_4]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + Manifest dependencyManifest1, dependencyManifest2, manifest, updateManifest; + SQLiteIndex index = SimpleTestSetup(tempFile, dependencyManifest1, Schema::Version::Latest()); + + auto& publisher2 = "Test2"; + CreateFakeManifest(dependencyManifest2, publisher2); + index.AddManifest(dependencyManifest2, GetPathFromManifest(dependencyManifest2)); + + auto& publisher3 = "Test3"; + CreateFakeManifest(manifest, publisher3); + const std::string dependencyPath3 = GetPathFromManifest(manifest); + + manifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, dependencyManifest1.Id, "1.0.0")); + manifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, dependencyManifest2.Id, "1.0.0")); + + index.AddManifest(manifest, dependencyPath3); + + auto& publisher4 = "Test4"; + CreateFakeManifest(updateManifest, publisher4); + index.AddManifest(updateManifest, GetPathFromManifest(updateManifest)); + manifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, updateManifest.Id, "1.0.0")); + + REQUIRE(index.UpdateManifest(manifest, dependencyPath3)); +} + +TEST_CASE("SQLiteIndex_UpdateManifestWithDependenciesDeleteAndAdd", "[sqliteindex][V1_4]") +{ + TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; + INFO("Using temporary file named: " << tempFile.GetPath()); + + Manifest dependencyManifest1, dependencyManifest2, manifest, updateManifest; + SQLiteIndex index = SimpleTestSetup(tempFile, dependencyManifest1, Schema::Version::Latest()); + + auto& publisher2 = "Test2"; + CreateFakeManifest(dependencyManifest2, publisher2); + index.AddManifest(dependencyManifest2, GetPathFromManifest(dependencyManifest2)); + + auto& publisher3 = "Test3"; + CreateFakeManifest(manifest, publisher3); + const std::string dependencyPath3 = GetPathFromManifest(manifest); + + manifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, dependencyManifest1.Id, "1.0.0")); + manifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, dependencyManifest2.Id, "1.0.0")); + + index.AddManifest(manifest, dependencyPath3); + + manifest.Installers[0].Dependencies.Clear(); + + auto& publisher4 = "Test4"; + CreateFakeManifest(updateManifest, publisher4); + index.AddManifest(updateManifest, GetPathFromManifest(updateManifest)); + manifest.Installers[0].Dependencies.Add(Dependency(DependencyType::Package, updateManifest.Id, "1.0.0")); + + REQUIRE(index.UpdateManifest(manifest, dependencyPath3)); +} + TEST_CASE("SQLiteIndex_UpdateManifestChangePath", "[sqliteindex][V1_0]") { TempFile tempFile{ "repolibtest_tempdb"s, ".db"s }; @@ -1079,8 +1545,8 @@ TEST_CASE("SQLiteIndex_Search_IdExactMatch", "[sqliteindex]") INFO("Using temporary file named: " << tempFile.GetPath()); Manifest manifest; - std::string relativePath; - SQLiteIndex index = SimpleTestSetup(tempFile, manifest, relativePath); + std::string relativePath = "test/id/1.0.0.yaml"; + SQLiteIndex index = SimpleTestSetup(tempFile, manifest); TestPrepareForRead(index); @@ -1100,8 +1566,8 @@ TEST_CASE("SQLiteIndex_Search_MultipleMatch", "[sqliteindex]") INFO("Using temporary file named: " << tempFile.GetPath()); Manifest manifest; - std::string relativePath; - SQLiteIndex index = SimpleTestSetup(tempFile, manifest, relativePath); + std::string relativePath = "test/id/1.0.0.yaml"; + SQLiteIndex index = SimpleTestSetup(tempFile, manifest); manifest.Version = "2.0.0"; index.AddManifest(manifest, relativePath + "2"); @@ -1124,8 +1590,7 @@ TEST_CASE("SQLiteIndex_Search_NoMatch", "[sqliteindex]") INFO("Using temporary file named: " << tempFile.GetPath()); Manifest manifest; - std::string relativePath; - SQLiteIndex index = SimpleTestSetup(tempFile, manifest, relativePath); + SQLiteIndex index = SimpleTestSetup(tempFile, manifest); TestPrepareForRead(index); @@ -1142,8 +1607,7 @@ TEST_CASE("SQLiteIndex_IdString", "[sqliteindex]") INFO("Using temporary file named: " << tempFile.GetPath()); Manifest manifest; - std::string relativePath; - SQLiteIndex index = SimpleTestSetup(tempFile, manifest, relativePath); + SQLiteIndex index = SimpleTestSetup(tempFile, manifest); TestPrepareForRead(index); @@ -1163,8 +1627,7 @@ TEST_CASE("SQLiteIndex_NameString", "[sqliteindex]") INFO("Using temporary file named: " << tempFile.GetPath()); Manifest manifest; - std::string relativePath; - SQLiteIndex index = SimpleTestSetup(tempFile, manifest, relativePath); + SQLiteIndex index = SimpleTestSetup(tempFile, manifest); TestPrepareForRead(index); @@ -1184,8 +1647,8 @@ TEST_CASE("SQLiteIndex_PathString", "[sqliteindex]") INFO("Using temporary file named: " << tempFile.GetPath()); Manifest manifest; - std::string relativePath; - SQLiteIndex index = SimpleTestSetup(tempFile, manifest, relativePath); + SQLiteIndex index = SimpleTestSetup(tempFile, manifest); + auto relativePath = GetPathFromManifest(manifest); TestPrepareForRead(index); @@ -1209,7 +1672,10 @@ TEST_CASE("SQLiteIndex_PathlessString", "[sqliteindex]") Manifest manifest; std::string relativePath; - SQLiteIndex index = SimpleTestSetup(tempFile, manifest); + + SQLiteIndex index = CreateTestIndex(tempFile); + CreateFakeManifest(manifest, "Test"); + index.AddManifest(manifest); TestPrepareForRead(index); @@ -1232,8 +1698,8 @@ TEST_CASE("SQLiteIndex_Versions", "[sqliteindex]") INFO("Using temporary file named: " << tempFile.GetPath()); Manifest manifest; - std::string relativePath; - SQLiteIndex index = SimpleTestSetup(tempFile, manifest, relativePath); + std::string relativePath = "test/id/1.0.0.yaml"; + SQLiteIndex index = SimpleTestSetup(tempFile, manifest); TestPrepareForRead(index); @@ -1388,9 +1854,8 @@ TEST_CASE("SQLiteIndex_SearchResultsTableSearches", "[sqliteindex][V1_0]") INFO("Using temporary file named: " << tempFile.GetPath()); Manifest manifest; - std::string relativePath; { - (void)SimpleTestSetup(tempFile, manifest, relativePath, Schema::Version{ 1, 0 }); + (void)SimpleTestSetup(tempFile, manifest, Schema::Version{ 1, 0 }); } Connection connection = Connection::Create(tempFile, Connection::OpenDisposition::ReadOnly); diff --git a/src/AppInstallerCommonCore/Public/AppInstallerErrors.h b/src/AppInstallerCommonCore/Public/AppInstallerErrors.h @@ -89,6 +89,11 @@ #define APPINSTALLER_CLI_ERROR_INVALID_MSIEXEC_ARGUMENT ((HRESULT)0x8A15004A) #define APPINSTALLER_CLI_ERROR_FAILED_TO_OPEN_ALL_SOURCES ((HRESULT)0x8A15004B) +// Error associated with dependencies operations. +#define APPINSTALLER_CLI_ERROR_DEPENDENCIES_VALIDATION_FAILED ((HRESULT)0x8A15004C) +#define APPINSTALLER_CLI_ERROR_MISSING_PACKAGE ((HRESULT)0x8A15004D) +#define APPINSTALLER_CLI_ERROR_INVALID_TABLE_COLUMN ((HRESULT)0x8A15004E) + #define APPINSTALLER_CLI_ERROR_INSTALL_PACKAGE_IN_USE ((HRESULT)0x8A150101) #define APPINSTALLER_CLI_ERROR_INSTALL_INSTALL_IN_PROGRESS ((HRESULT)0x8A150102) #define APPINSTALLER_CLI_ERROR_INSTALL_FILE_IN_USE ((HRESULT)0x8A150103) diff --git a/src/AppInstallerCommonCore/Public/winget/DependenciesGraph.h b/src/AppInstallerCommonCore/Public/winget/DependenciesGraph.h @@ -34,7 +34,7 @@ namespace AppInstaller::Manifest const Dependency& m_root; std::map<Dependency, std::set<Dependency>> m_adjacents; std::function<const DependencyList(const Dependency&)> getDependencies; - bool m_HasLoop; + bool m_HasLoop = false; bool m_rootDependencyEvaluated = false; std::vector<Dependency> m_installationOrder; std::vector<Dependency> m_toCheck; diff --git a/src/AppInstallerCommonCore/Public/winget/ManifestCommon.h b/src/AppInstallerCommonCore/Public/winget/ManifestCommon.h @@ -174,23 +174,26 @@ bool HasExtension(std::string_view extension) const; string_t Id; std::optional<Utility::Version> MinVersion; - Dependency(DependencyType type, string_t id, string_t minVersion) : Type(type), Id(std::move(id)), MinVersion(Utility::Version(minVersion)) {} - Dependency(DependencyType type, string_t id) : Type(type), Id(std::move(id)) {} + Dependency(DependencyType type, string_t id, string_t minVersion) : Type(type), Id(std::move(id)), MinVersion(Utility::Version(minVersion)), m_foldedId(FoldCase(Id)) {} + Dependency(DependencyType type, string_t id) : Type(type), Id(std::move(id)), m_foldedId(FoldCase(Id)){} Dependency(DependencyType type) : Type(type) {} bool operator==(const Dependency& rhs) const { - return Type == rhs.Type && ICUCaseInsensitiveEquals(Id, rhs.Id) && MinVersion == rhs.MinVersion; + return Type == rhs.Type && m_foldedId == rhs.m_foldedId && MinVersion == rhs.MinVersion; } bool operator <(const Dependency& rhs) const { - return Id < rhs.Id; + return m_foldedId < rhs.m_foldedId; } bool IsVersionOk(Utility::Version version) { return MinVersion <= Utility::Version(version); } + + private: + std::string m_foldedId; }; struct DependencyList diff --git a/src/AppInstallerCommonCore/Public/winget/ManifestValidation.h b/src/AppInstallerCommonCore/Public/winget/ManifestValidation.h @@ -43,6 +43,11 @@ namespace AppInstaller::Manifest const char* const BothAllowedAndExcludedMarketsDefined = "Both AllowedMarkets and ExcludedMarkets defined."; const char* const DuplicateReturnCodeEntry = "Duplicate installer return code found."; const char* const FieldRequireVerifiedPublisher = "Field usage requires verified publishers."; + const char* const SingleManifestPackageHasDependencies = "Package has a single manifest and is a dependency of other manifests."; + const char* const MultiManifestPackageHasDependencies = "Deleting the manifest will be break the following dependencies."; + const char* const MissingManifestDependenciesNode = "Dependency not found: "; + const char* const NoSuitableMinVersion = "No Suitable Minimum Version: "; + const char* const FoundLoop = "Loop found."; } struct ValidationError diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj @@ -261,12 +261,15 @@ <ClInclude Include="Microsoft\Schema\1_2\SearchResultsTable.h" /> <ClInclude Include="Microsoft\Schema\1_3\HashVirtualTable.h" /> <ClInclude Include="Microsoft\Schema\1_3\Interface.h" /> + <ClInclude Include="Microsoft\Schema\1_4\DependenciesTable.h" /> + <ClInclude Include="Microsoft\Schema\1_4\Interface.h" /> <ClInclude Include="Microsoft\Schema\ISQLiteIndex.h" /> <ClInclude Include="Microsoft\Schema\MetadataTable.h" /> <ClInclude Include="Microsoft\Schema\Version.h" /> <ClInclude Include="Microsoft\SQLiteIndex.h" /> <ClInclude Include="Microsoft\SQLiteIndexSource.h" /> <ClInclude Include="Microsoft\ConfigurableTestSourceFactory.h" /> + <ClInclude Include="PackageDependenciesValidation.h" /> <ClInclude Include="PackageTrackingCatalogSourceFactory.h" /> <ClInclude Include="pch.h" /> <ClInclude Include="Public\winget\PackageTrackingCatalog.h" /> @@ -324,10 +327,13 @@ <ClCompile Include="Microsoft\Schema\1_2\Interface_1_2.cpp" /> <ClCompile Include="Microsoft\Schema\1_2\SearchResultsTable_1_2.cpp" /> <ClCompile Include="Microsoft\Schema\1_3\Interface_1_3.cpp" /> + <ClCompile Include="Microsoft\Schema\1_4\DependenciesTable.cpp" /> + <ClCompile Include="Microsoft\Schema\1_4\Interface_1_4.cpp" /> <ClCompile Include="Microsoft\Schema\MetadataTable.cpp" /> <ClCompile Include="Microsoft\Schema\Version.cpp" /> <ClCompile Include="Microsoft\SQLiteIndex.cpp" /> <ClCompile Include="Microsoft\SQLiteIndexSource.cpp" /> + <ClCompile Include="PackageDependenciesValidation.cpp" /> <ClCompile Include="PackageTrackingCatalog.cpp" /> <ClCompile Include="pch.cpp"> <PrecompiledHeader>Create</PrecompiledHeader> diff --git a/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters b/src/AppInstallerRepositoryCore/AppInstallerRepositoryCore.vcxproj.filters @@ -58,6 +58,9 @@ <Filter Include="Public\winget"> <UniqueIdentifier>{aa7315bc-4eb0-4280-9572-f5a25f6e73ad}</UniqueIdentifier> </Filter> + <Filter Include="Microsoft\Schema\1_4"> + <UniqueIdentifier>{dcae9c55-cdd7-4381-8acd-3554896608a5}</UniqueIdentifier> + </Filter> </ItemGroup> <ItemGroup> <ClInclude Include="pch.h"> @@ -249,9 +252,18 @@ <ClInclude Include="ISource.h"> <Filter>Header Files</Filter> </ClInclude> + <ClInclude Include="Microsoft\Schema\1_4\DependenciesTable.h"> + <Filter>Microsoft\Schema\1_4</Filter> + </ClInclude> + <ClInclude Include="Microsoft\Schema\1_4\Interface.h"> + <Filter>Microsoft\Schema\1_4</Filter> + </ClInclude> <ClInclude Include="PackageTrackingCatalogSourceFactory.h"> <Filter>Header Files</Filter> </ClInclude> + <ClInclude Include="PackageDependenciesValidation.h"> + <Filter>Header Files</Filter> + </ClInclude> </ItemGroup> <ItemGroup> <ClCompile Include="pch.cpp"> @@ -392,6 +404,15 @@ <ClCompile Include="RepositorySearch.cpp"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="Microsoft\Schema\1_4\DependenciesTable.cpp"> + <Filter>Microsoft\Schema\1_4</Filter> + </ClCompile> + <ClCompile Include="Microsoft\Schema\1_4\Interface_1_4.cpp"> + <Filter>Microsoft\Schema\1_4</Filter> + </ClCompile> + <ClCompile Include="PackageDependenciesValidation.cpp"> + <Filter>Source Files</Filter> + </ClCompile> </ItemGroup> <ItemGroup> <None Include="PropertySheet.props" /> diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.cpp @@ -314,6 +314,16 @@ namespace AppInstaller::Repository::Microsoft return m_interface->NormalizeName(name, publisher); } + std::set<std::pair<SQLite::rowid_t, Utility::NormalizedString>> SQLiteIndex::GetDependenciesByManifestRowId(SQLite::rowid_t manifestRowId) const + { + return m_interface->GetDependenciesByManifestRowId(m_dbconn, manifestRowId); + } + + std::vector<std::pair<SQLite::rowid_t, Utility::NormalizedString>> SQLiteIndex::GetDependentsById(AppInstaller::Manifest::string_t packageId) const + { + return m_interface->GetDependentsById(m_dbconn, packageId); + } + // Recording last write time based on MSDN documentation stating that time returns a POSIX epoch time and thus // should be consistent across systems. void SQLiteIndex::SetLastWriteTime() diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndex.h @@ -103,7 +103,7 @@ namespace AppInstaller::Repository::Microsoft // Removes the manifest with matching { Id, Version, Channel } from the index. void RemoveManifest(const Manifest::Manifest& manifest, const std::filesystem::path& relativePath); - + // Removes the manifest with matching { Id, Version, Channel } from the index. void RemoveManifest(const Manifest::Manifest& manifest); @@ -146,6 +146,9 @@ namespace AppInstaller::Repository::Microsoft // Largely a utility function; should not be used to do work on behalf of the index by the caller. Utility::NormalizedName NormalizeName(std::string_view name, std::string_view publisher) const; + // Get all the dependencies for a specific manifest. + std::set<std::pair<SQLite::rowid_t, Utility::NormalizedString>> GetDependenciesByManifestRowId(SQLite::rowid_t manifestRowId) const; + std::vector<std::pair<SQLite::rowid_t, Utility::NormalizedString>> GetDependentsById(AppInstaller::Manifest::string_t packageId) const; private: // Constructor used to open an existing index. SQLiteIndex(const std::string& target, SQLite::Connection::OpenDisposition disposition, SQLite::Connection::OpenFlags flags); diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface.h @@ -36,7 +36,13 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 // Version 1.2 Utility::NormalizedName NormalizeName(std::string_view name, std::string_view publisher) const override; + // Version 1.4 Get all the dependencies for a specific manifest. + std::set<std::pair<SQLite::rowid_t, Utility::NormalizedString>> GetDependenciesByManifestRowId(const SQLite::Connection& connection, SQLite::rowid_t manifestRowId) const override; + std::vector<std::pair<SQLite::rowid_t, Utility::NormalizedString>> GetDependentsById(const SQLite::Connection& connection, AppInstaller::Manifest::string_t packageId) const override; + protected: + virtual bool NotNeeded(const SQLite::Connection& connection, std::string_view tableName, std::string_view valueName, SQLite::rowid_t id) const; + // Creates the search results table. virtual std::unique_ptr<SearchResultsTable> CreateSearchResultsTable(const SQLite::Connection& connection) const; diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface_1_0.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/Interface_1_0.cpp @@ -123,6 +123,11 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 } } + bool NotNeededInternal(const SQLite::Connection& connection, std::string_view, std::string_view valueName, SQLite::rowid_t id) + { + return !ManifestTable::IsValueReferenced(connection, valueName, id); + } + // Updates the manifest column and related table based on the given value. template <typename Table> void UpdateManifestValueById(SQLite::Connection& connection, const typename Table::value_t& value, SQLite::rowid_t manifestId, bool overwriteLikeMatch = false) @@ -133,7 +138,10 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 ManifestTable::UpdateValueIdById<Table>(connection, manifestId, newValueId); - Table::DeleteIfNotNeededById(connection, oldValueId); + if (NotNeededInternal(connection, Table::TableName(), Table::ValueName(), oldValueId)) + { + Table::DeleteById(connection, oldValueId); + } } } @@ -306,11 +314,30 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 ManifestTable::DeleteById(connection, manifestId); // Remove all of the 1:1 data that is no longer referenced. - IdTable::DeleteIfNotNeededById(connection, idId); - NameTable::DeleteIfNotNeededById(connection, nameId); - MonikerTable::DeleteIfNotNeededById(connection, monikerId); - VersionTable::DeleteIfNotNeededById(connection, versionId); - ChannelTable::DeleteIfNotNeededById(connection, channelId); + if (NotNeeded(connection, IdTable::TableName(), IdTable::ValueName(), idId)) + { + IdTable::DeleteById(connection, idId); + } + + if (NotNeeded(connection, NameTable::TableName(), NameTable::ValueName(), nameId)) + { + NameTable::DeleteById(connection, nameId); + } + + if (NotNeeded(connection, MonikerTable::TableName(), MonikerTable::ValueName(), monikerId)) + { + MonikerTable::DeleteById(connection, monikerId); + } + + if (NotNeeded(connection, VersionTable::TableName(), VersionTable::ValueName(), versionId)) + { + VersionTable::DeleteById(connection, versionId); + } + + if (NotNeeded(connection, ChannelTable::TableName(), ChannelTable::ValueName(), channelId)) + { + ChannelTable::DeleteById(connection, channelId); + } // Remove the path PathPartTable::RemovePathById(connection, pathLeafId); @@ -322,6 +349,11 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 savepoint.Commit(); } + bool Interface::NotNeeded(const SQLite::Connection& connection, std::string_view tableName, std::string_view valueName, SQLite::rowid_t id) const + { + return NotNeededInternal(connection, tableName, valueName, id); + } + void Interface::PrepareForPackaging(SQLite::Connection& connection) { SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, "prepareforpackaging_v1_0"); @@ -519,6 +551,16 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 return GetExistingManifestId(connection, manifest); } + std::set<std::pair<SQLite::rowid_t, Utility::NormalizedString>> Interface::GetDependenciesByManifestRowId(const SQLite::Connection&, SQLite::rowid_t) const + { + return {}; + } + + std::vector<std::pair<SQLite::rowid_t, Utility::NormalizedString>> Interface::GetDependentsById(const SQLite::Connection&, AppInstaller::Manifest::string_t) const + { + return {}; + } + std::vector<Utility::VersionAndChannel> Interface::GetVersionKeysById(const SQLite::Connection& connection, SQLite::rowid_t id) const { auto versionsAndChannels = ManifestTable::GetAllValuesById<IdTable, VersionTable, ChannelTable>(connection, id); diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/ManifestTable.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/ManifestTable.cpp @@ -488,6 +488,11 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 savepoint.Commit(); } + bool ManifestTable::IsValueReferenced(const SQLite::Connection& connection, std::string_view valueName, SQLite::rowid_t valueRowId) + { + return details::ManifestTableSelectByValueIds(connection, { valueName }, { valueRowId }).has_value(); + } + bool ManifestTable::IsEmpty(SQLite::Connection& connection) { SQLite::Builder::StatementBuilder builder; diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/ManifestTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/ManifestTable.h @@ -177,6 +177,9 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 // Removes data that is no longer needed for an index that is to be published. static void PrepareForPackaging_deprecated(SQLite::Connection& connection, std::initializer_list<std::string_view> values); + // Checks if the row id is present in the column denoted by the value supplied. + static bool IsValueReferenced(const SQLite::Connection& connection, std::string_view valueName, SQLite::rowid_t valueRowId); + // Checks the consistency of the index to ensure that every referenced row exists. // Returns true if index is consistent; false if it is not. template <typename Table> diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToOneTable.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToOneTable.cpp @@ -142,20 +142,6 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 return connection.GetLastInsertRowID(); } - void OneToOneTableDeleteIfNotNeededById(SQLite::Connection& connection, std::string_view tableName, std::string_view valueName, SQLite::rowid_t id) - { - // If a manifest is found that references this id, then we are done. - if (ManifestTableSelectByValueIds(connection, { valueName }, { id })) - { - return; - } - - SQLite::Builder::StatementBuilder builder; - builder.DeleteFrom(tableName).Where(SQLite::RowIDName).Equals(id); - - builder.Execute(connection); - } - void OneToOneTablePrepareForPackaging(SQLite::Connection& connection, std::string_view tableName, bool useNamedIndices, bool preserveValuesIndex) { if (useNamedIndices && !preserveValuesIndex) @@ -182,5 +168,13 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 { return (OneToOneTableGetCount(connection, tableName) == 0); } + + void OneToOneTableDeleteById(SQLite::Connection& connection, std::string_view tableName, SQLite::rowid_t id) + { + SQLite::Builder::StatementBuilder builder; + builder.DeleteFrom(tableName).Where(SQLite::RowIDName).Equals(id); + + builder.Execute(connection); + } } } diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToOneTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_0/OneToOneTable.h @@ -26,10 +26,6 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 // Ensures that the values exists in the table. SQLite::rowid_t OneToOneTableEnsureExists(SQLite::Connection& connection, std::string_view tableName, std::string_view valueName, std::string_view value, bool overwriteLikeMatch = false); - - // Removes the given row by its rowid if it is no longer referenced. - void OneToOneTableDeleteIfNotNeededById(SQLite::Connection& connection, std::string_view tableName, std::string_view valueName, SQLite::rowid_t id); - // Removes data that is no longer needed for an index that is to be published. void OneToOneTablePrepareForPackaging(SQLite::Connection& connection, std::string_view tableName, bool useNamedIndices, bool preserveValuesIndex); @@ -38,6 +34,9 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 // Determines if the table is empty. bool OneToOneTableIsEmpty(SQLite::Connection& connection, std::string_view tableName); + + // Removes the given row by its rowid if it is no longer referenced. + void OneToOneTableDeleteById(SQLite::Connection& connection, std::string_view tableName, SQLite::rowid_t id); } // A table that represents a value that is 1:1 with a primary entry. @@ -105,9 +104,9 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_0 } // Removes the given row by its rowid if it is no longer referenced. - static void DeleteIfNotNeededById(SQLite::Connection& connection, SQLite::rowid_t id) + static void DeleteById(SQLite::Connection& connection, SQLite::rowid_t id) { - return details::OneToOneTableDeleteIfNotNeededById(connection, TableInfo::TableName(), TableInfo::ValueName(), id); + return details::OneToOneTableDeleteById(connection, TableInfo::TableName(), id); } // Removes data that is no longer needed for an index that is to be published. diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_3/Interface.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_3/Interface.h @@ -17,7 +17,7 @@ namespace AppInstaller::Repository::Microsoft::Schema::V1_3 void CreateTables(SQLite::Connection& connection, CreateOptions options) override; SQLite::rowid_t AddManifest(SQLite::Connection& connection, const Manifest::Manifest& manifest, const std::optional<std::filesystem::path>& relativePath) override; std::pair<bool, SQLite::rowid_t> UpdateManifest(SQLite::Connection& connection, const Manifest::Manifest& manifest, const std::optional<std::filesystem::path>& relativePath) override; - + protected: // Gets a property already knowing that the manifest id is valid. std::optional<std::string> GetPropertyByManifestIdInternal(const SQLite::Connection& connection, SQLite::rowid_t manifestId, PackageVersionProperty property) const override; diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_3/Interface_1_3.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_3/Interface_1_3.cpp @@ -5,9 +5,8 @@ #include <AppInstallerSHA256.h> #include "Microsoft/Schema/1_0/ManifestTable.h" - #include "Microsoft/Schema/1_3/HashVirtualTable.h" - +#include <winget/ManifestValidation.h> namespace AppInstaller::Repository::Microsoft::Schema::V1_3 { diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_4/DependenciesTable.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_4/DependenciesTable.cpp @@ -0,0 +1,483 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "DependenciesTable.h" +#include "SQLiteStatementBuilder.h" +#include "winget\DependenciesGraph.h" +#include "Microsoft/Schema/1_0/OneToOneTable.h" +#include "Microsoft/Schema/1_0/IdTable.h" +#include "Microsoft/Schema/1_0/ManifestTable.h" +#include "Microsoft/Schema/1_0/VersionTable.h" +#include "Microsoft/Schema/1_0/Interface.h" +#include "Microsoft/Schema/1_0/ChannelTable.h" + +namespace AppInstaller::Repository::Microsoft::Schema::V1_4 +{ + using namespace AppInstaller; + using namespace std::string_view_literals; + using namespace SQLite::Builder; + using namespace Schema::V1_0; + using QCol = SQLite::Builder::QualifiedColumn; + + static constexpr std::string_view s_DependenciesTable_Table_Name = "dependencies"sv; + static constexpr std::string_view s_DependenciesTable_Index_Name = "dependencies_pkindex"sv; + static constexpr std::string_view s_DependenciesTable_Manifest_Column_Name = "manifest"sv; + static constexpr std::string_view s_DependenciesTable_MinVersion_Column_Name = "min_version"sv; + static constexpr std::string_view s_DependenciesTable_PackageId_Column_Name = "package_id"; + + namespace + { + struct DependencyTableRow + { + SQLite::rowid_t m_packageRowId; + SQLite::rowid_t m_manifestRowId; + + // Ideally this should be version row id, the version string is more needed than row id, + // this prevents converting back and forth between version row id and version string. + std::optional<Utility::NormalizedString> m_version; + + bool operator <(const DependencyTableRow& rhs) const + { + auto lhsVersion = m_version.has_value() ? m_version.value() : ""; + auto rhsVersion = rhs.m_version.has_value() ? rhs.m_version.value() : ""; + return std::tie(m_packageRowId, m_manifestRowId, lhsVersion) < std::tie(rhs.m_packageRowId, rhs.m_manifestRowId, rhsVersion); + } + }; + + void ThrowOnMissingPackageNodes(std::vector<Manifest::Dependency>& missingPackageNodes) + { + if (!missingPackageNodes.empty()) + { + std::string missingPackages{ missingPackageNodes.begin()->Id }; + std::for_each( + missingPackageNodes.begin() + 1, + missingPackageNodes.end(), + [&](auto& dep) { missingPackages.append(", " + dep.Id); }); + THROW_HR_MSG(APPINSTALLER_CLI_ERROR_MISSING_PACKAGE, "Missing packages: %hs", missingPackages.c_str()); + } + } + + std::set<DependencyTableRow> GetAndLinkDependencies( + SQLite::Connection& connection, + const Manifest::Manifest& manifest, + SQLite::rowid_t manifestRowId, + Manifest::DependencyType dependencyType) + { + std::set<DependencyTableRow> dependencies; + std::vector<Manifest::Dependency> missingPackageNodes; + + for (const auto& installer : manifest.Installers) + { + installer.Dependencies.ApplyToType(dependencyType, [&](Manifest::Dependency dependency) + { + auto packageRowId = IdTable::SelectIdByValue(connection, dependency.Id); + std::optional<Utility::NormalizedString> version; + + if (!packageRowId.has_value()) + { + missingPackageNodes.emplace_back(dependency); + return; + } + + if (dependency.MinVersion.has_value()) + { + version = dependency.MinVersion.value().ToString(); + } + + dependencies.emplace(DependencyTableRow{ packageRowId.value(), manifestRowId, version }); + }); + } + + ThrowOnMissingPackageNodes(missingPackageNodes); + + return dependencies; + } + + void RemoveDependenciesByRowIds(SQLite::Connection& connection, std::vector<DependencyTableRow> dependencyTableRows) + { + using namespace SQLite::Builder; + if (dependencyTableRows.empty()) + { + return; + } + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, std::string{ s_DependenciesTable_Table_Name } + "remove_dependencies_by_rowid"); + + SQLite::Builder::StatementBuilder builder; + builder + .DeleteFrom(s_DependenciesTable_Table_Name) + .Where(s_DependenciesTable_PackageId_Column_Name).Equals(Unbound) + .And(s_DependenciesTable_Manifest_Column_Name).Equals(Unbound); + + SQLite::Statement deleteStmt = builder.Prepare(connection); + for (auto row : dependencyTableRows) + { + deleteStmt.Reset(); + deleteStmt.Bind(1, row.m_packageRowId); + deleteStmt.Bind(2, row.m_manifestRowId); + deleteStmt.Execute(true); + } + + savepoint.Commit(); + } + + void InsertManifestDependencies( + SQLite::Connection& connection, + std::set<DependencyTableRow>& dependenciesTableRows) + { + using namespace SQLite::Builder; + using namespace Schema::V1_0; + + StatementBuilder insertBuilder; + insertBuilder.InsertInto(s_DependenciesTable_Table_Name) + .Columns({ s_DependenciesTable_Manifest_Column_Name, s_DependenciesTable_MinVersion_Column_Name, s_DependenciesTable_PackageId_Column_Name }) + .Values(Unbound, Unbound, Unbound); + SQLite::Statement insert = insertBuilder.Prepare(connection); + + for (const auto& dep : dependenciesTableRows) + { + insert.Reset(); + insert.Bind(1, dep.m_manifestRowId); + + if (dep.m_version.has_value()) + { + insert.Bind(2, VersionTable::EnsureExists(connection, dep.m_version.value())); + } + else + { + insert.Bind(2, nullptr); + } + + insert.Bind(3, dep.m_packageRowId); + + insert.Execute(true); + } + } + } + + bool DependenciesTable::Exists(const SQLite::Connection& connection) + { + using namespace SQLite; + + Builder::StatementBuilder builder; + builder.Select(Builder::RowCount).From(Builder::Schema::MainTable). + Where(Builder::Schema::TypeColumn).Equals(Builder::Schema::Type_Table).And(Builder::Schema::NameColumn).Equals(s_DependenciesTable_Table_Name); + + Statement statement = builder.Prepare(connection); + THROW_HR_IF(E_UNEXPECTED, !statement.Step()); + return statement.GetColumn<int64_t>(0) != 0; + } + + std::string_view DependenciesTable::TableName() + { + return s_DependenciesTable_Table_Name; + } + + void DependenciesTable::Create(SQLite::Connection& connection) + { + using namespace SQLite::Builder; + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, "createDependencyTable_v1_4"); + constexpr std::string_view dependencyIndexByVersionId = "dependencies_version_id_index"; + constexpr std::string_view dependencyIndexByPackageId = "dependencies_package_id_index"; + + StatementBuilder createTableBuilder; + createTableBuilder.CreateTable(TableName()).BeginColumns(); + createTableBuilder.Column(IntegerPrimaryKey()); + + std::array<DependenciesTableColumnInfo, 2> notNullableDependenciesColumns + { + DependenciesTableColumnInfo{ s_DependenciesTable_Manifest_Column_Name }, + DependenciesTableColumnInfo{ s_DependenciesTable_PackageId_Column_Name } + }; + + std::array<DependenciesTableColumnInfo, 1> nullableDependenciesColumns + { + DependenciesTableColumnInfo{ s_DependenciesTable_MinVersion_Column_Name } + }; + + // Add dependencies column tables not null columns. + for (const DependenciesTableColumnInfo& value : notNullableDependenciesColumns) + { + createTableBuilder.Column(ColumnBuilder(value.Name, Type::RowId).NotNull()); + } + + // Add dependencies column tables null columns. + for (const DependenciesTableColumnInfo& value : nullableDependenciesColumns) + { + createTableBuilder.Column(ColumnBuilder(value.Name, Type::RowId)); + } + + createTableBuilder.EndColumns(); + + createTableBuilder.Execute(connection); + + // Primary key index by package rowid and manifest rowid. + StatementBuilder createPKIndexBuilder; + createPKIndexBuilder.CreateUniqueIndex(s_DependenciesTable_Index_Name).On(s_DependenciesTable_Table_Name).Columns({ s_DependenciesTable_Manifest_Column_Name, s_DependenciesTable_PackageId_Column_Name }); + createPKIndexBuilder.Execute(connection); + + // Index of dependency by Manifest id. + StatementBuilder createIndexByManifestIdBuilder; + createIndexByManifestIdBuilder.CreateIndex(dependencyIndexByVersionId).On(s_DependenciesTable_Table_Name).Columns({ s_DependenciesTable_MinVersion_Column_Name }); + createIndexByManifestIdBuilder.Execute(connection); + + // Index of dependency by package id. + StatementBuilder createIndexByPackageIdBuilder; + createIndexByPackageIdBuilder.CreateIndex(dependencyIndexByPackageId).On(s_DependenciesTable_Table_Name).Columns({ s_DependenciesTable_PackageId_Column_Name }); + createIndexByPackageIdBuilder.Execute(connection); + + savepoint.Commit(); + } + + void DependenciesTable::AddDependencies(SQLite::Connection& connection, const Manifest::Manifest& manifest, SQLite::rowid_t manifestRowId) + { + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, std::string{ s_DependenciesTable_Table_Name } + "add_dependencies_v1_4"); + + auto dependencies = GetAndLinkDependencies(connection, manifest, manifestRowId, Manifest::DependencyType::Package); + if (!dependencies.size()) + { + return; + } + + InsertManifestDependencies(connection, dependencies); + + savepoint.Commit(); + } + + bool DependenciesTable::UpdateDependencies(SQLite::Connection& connection, const Manifest::Manifest& manifest, SQLite::rowid_t manifestRowId) + { + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, std::string{ s_DependenciesTable_Table_Name } + "update_dependencies_v1_4"); + + const auto dependencies = GetAndLinkDependencies(connection, manifest, manifestRowId, Manifest::DependencyType::Package); + auto existingDependencies = GetDependenciesByManifestRowId(connection, manifestRowId); + + // Get dependencies to add. + std::set<DependencyTableRow> toAddDependencies; + std::copy_if( + dependencies.begin(), + dependencies.end(), + std::inserter(toAddDependencies, toAddDependencies.begin()), + [&](DependencyTableRow dep) + { + Utility::NormalizedString version = dep.m_version.has_value() ? dep.m_version.value() : ""; + return existingDependencies.find(std::make_pair(dep.m_packageRowId, version)) == existingDependencies.end(); + } + ); + + // Get dependencies to remove. + std::vector<DependencyTableRow> toRemoveDependencies; + std::for_each( + existingDependencies.begin(), + existingDependencies.end(), + [&](std::pair<SQLite::rowid_t, Utility::NormalizedString> row) + { + if (dependencies.find(DependencyTableRow{row.first, manifestRowId, row.second}) == dependencies.end()) + { + toRemoveDependencies.emplace_back(DependencyTableRow{ row.first, manifestRowId }); + } + } + ); + + InsertManifestDependencies(connection, toAddDependencies); + RemoveDependenciesByRowIds(connection, toRemoveDependencies); + savepoint.Commit(); + + return true; + } + + void DependenciesTable::RemoveDependencies(SQLite::Connection& connection, SQLite::rowid_t manifestRowId) + { + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, std::string{ s_DependenciesTable_Table_Name } + "remove_dependencies_by_manifest_v1_4"); + SQLite::Builder::StatementBuilder builder; + builder.DeleteFrom(s_DependenciesTable_Table_Name).Where(s_DependenciesTable_Manifest_Column_Name).Equals(manifestRowId); + + builder.Execute(connection); + savepoint.Commit(); + } + + std::vector<std::pair<SQLite::rowid_t, Utility::NormalizedString>> DependenciesTable::GetDependentsById(const SQLite::Connection& connection, Manifest::string_t packageId) + { + constexpr std::string_view depTableAlias = "dep"; + constexpr std::string_view minVersionAlias = "minV"; + constexpr std::string_view packageIdAlias = "pId"; + + + StatementBuilder builder; + // Find all manifest that depend on this package. + // SELECT [dep].[manifest], [pId].[id], [minV].[version] FROM [dependencies] AS [dep] + // JOIN [versions] AS [minV] ON [dep].[min_version] = [minV].[rowid] + // JOIN [ids] AS [pId] ON [pId].[rowid] = [dep].[package_id] + // WHERE [pId].[id] = ? + builder.Select() + .Column(QCol(depTableAlias, s_DependenciesTable_Manifest_Column_Name)) + .Column(QCol(packageIdAlias, IdTable::ValueName())) + .Column(QCol(minVersionAlias, VersionTable::ValueName())) + .From({ s_DependenciesTable_Table_Name }).As(depTableAlias) + .Join({ VersionTable::TableName() }).As(minVersionAlias) + .On(QCol(depTableAlias, s_DependenciesTable_MinVersion_Column_Name), QCol(minVersionAlias, SQLite::RowIDName)) + .Join({ IdTable::TableName() }).As(packageIdAlias) + .On(QCol(packageIdAlias, SQLite::RowIDName), QCol(depTableAlias, s_DependenciesTable_PackageId_Column_Name)) + .Where(QCol(packageIdAlias, IdTable::ValueName())).Equals(Unbound); + + SQLite::Statement stmt = builder.Prepare(connection); + stmt.Bind(1, std::string{ packageId }); + + std::vector<std::pair<SQLite::rowid_t, Utility::NormalizedString>> resultSet; + + while (stmt.Step()) + { + resultSet.emplace_back( + std::make_pair(stmt.GetColumn<SQLite::rowid_t>(0), Utility::NormalizedString(stmt.GetColumn<std::string>(2)))); + } + + return resultSet; + } + + std::set<std::pair<SQLite::rowid_t, Utility::NormalizedString>> DependenciesTable::GetDependenciesByManifestRowId(const SQLite::Connection& connection, SQLite::rowid_t manifestRowId) + { + SQLite::Builder::StatementBuilder builder; + + constexpr std::string_view depTableAlias = "dep"; + constexpr std::string_view minVersionAlias = "minV"; + + std::set<std::pair<SQLite::rowid_t, Utility::NormalizedString>> resultSet; + + // SELECT [dep].[package_id], [minV].[version] FROM [dependencies] AS [dep] + // JOIN [versions] AS [minV] ON [minV].[rowid] = [dep].[min_version] + // WHERE [dep].[manifest] = ? + builder.Select() + .Column(QCol(depTableAlias, s_DependenciesTable_PackageId_Column_Name)) + .Column(QCol(minVersionAlias, VersionTable::ValueName())) + .From({ s_DependenciesTable_Table_Name }).As(depTableAlias) + .Join({ VersionTable::TableName() }).As(minVersionAlias) + .On(QCol(minVersionAlias, SQLite::RowIDName), QCol(depTableAlias, s_DependenciesTable_MinVersion_Column_Name)) + .Where(QCol(depTableAlias, s_DependenciesTable_Manifest_Column_Name)).Equals(Unbound); + + SQLite::Statement select = builder.Prepare(connection); + + select.Bind(1, manifestRowId); + while (select.Step()) + { + Utility::NormalizedString version = ""; + if (!select.GetColumnIsNull(1)) + { + version = select.GetColumn<std::string>(1); + } + resultSet.emplace(std::make_pair(select.GetColumn<SQLite::rowid_t>(0), version)); + } + + return resultSet; + } + + void DependenciesTable::PrepareForPackaging(SQLite::Connection& connection) + { + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, "prepareForPacking_V1_4"); + + StatementBuilder dropIndexBuilder; + dropIndexBuilder.DropIndex({ s_DependenciesTable_Index_Name }); + dropIndexBuilder.Execute(connection); + + StatementBuilder dropTableBuilder; + dropTableBuilder.DropTable({ s_DependenciesTable_Table_Name }); + dropTableBuilder.Execute(connection); + + savepoint.Commit(); + } + + bool DependenciesTable::DependenciesTableCheckConsistency(const SQLite::Connection& connection, bool log) + { + StatementBuilder builder; + + if (!Exists(connection)) + { + return true; + } + + builder.Select(QCol(s_DependenciesTable_Table_Name, SQLite::RowIDName)) + .From(s_DependenciesTable_Table_Name) + .LeftOuterJoin(IdTable::TableName()) + .On(QCol(s_DependenciesTable_Table_Name, s_DependenciesTable_PackageId_Column_Name), QCol(IdTable::TableName(), SQLite::RowIDName)) + .LeftOuterJoin(ManifestTable::TableName()) + .On(QCol(s_DependenciesTable_Table_Name, s_DependenciesTable_Manifest_Column_Name), QCol(ManifestTable::TableName(), SQLite::RowIDName)) + .LeftOuterJoin(VersionTable::TableName()) + .On(QCol(s_DependenciesTable_Table_Name, s_DependenciesTable_MinVersion_Column_Name), QCol(VersionTable::TableName(), SQLite::RowIDName)) + .Where(QCol(ManifestTable::TableName(), SQLite::RowIDName)).IsNull() + .Or(QCol(VersionTable::TableName(), SQLite::RowIDName)).IsNull() + .Or(QCol(IdTable::TableName(), SQLite::RowIDName)).IsNull(); + + SQLite::Statement select = builder.Prepare(connection); + + bool result = true; + + while (select.Step()) + { + result = false; + + if (!log) + { + break; + } + + AICLI_LOG(Repo, Info, << " [INVALID] rowid [" << select.GetColumn<SQLite::rowid_t>(0) << "]"); + } + + return result; + } + + bool DependenciesTable::IsValueReferenced(const SQLite::Connection& connection, std::string_view tableName, SQLite::rowid_t valueRowId) + { + StatementBuilder builder; + + if (tableName != V1_0::VersionTable::TableName()) + { + return false; + } + + std::array<std::string_view, 1> columns = { s_DependenciesTable_MinVersion_Column_Name }; + bool referenced = false; + + for(auto column: columns) + { + builder.Select(SQLite::RowIDName).From(s_DependenciesTable_Table_Name).Where(column).Equals(Unbound).Limit(1); + + SQLite::Statement select = builder.Prepare(connection); + + select.Bind(1, valueRowId); + if (select.Step()) + { + referenced = true; + break; + } + } + + return referenced; + } + + std::vector<SQLite::rowid_t> DependenciesTable::GetDependenciesMinVersionsRowIdByManifestId(const SQLite::Connection& connection, SQLite::rowid_t manifestRowId) + { + StatementBuilder builder; + + std::vector<SQLite::rowid_t> result; + + // Find all versions for manifest row. + // SELECT [min_version] FROM [dependencies] + // WHERE [manifest] = ? + builder.Select() + .Column(s_DependenciesTable_MinVersion_Column_Name) + .From({ s_DependenciesTable_Table_Name }) + .Where(s_DependenciesTable_Manifest_Column_Name).Equals(Unbound); + + auto select = builder.Prepare(connection); + + select.Bind(1, manifestRowId); + + while (select.Step()) + { + if (!select.GetColumnIsNull(0)) + { + result.emplace_back(select.GetColumn<SQLite::rowid_t>(0)); + } + } + + return result; + } +}+ \ No newline at end of file diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_4/DependenciesTable.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_4/DependenciesTable.h @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once +#include "pch.h" +#include "SQLiteWrapper.h" +#include "SQLiteStatementBuilder.h" +#include <winget/Manifest.h> + +namespace AppInstaller::Repository::Microsoft::Schema::V1_4 +{ + using namespace AppInstaller; + + struct DependenciesTableColumnInfo + { + std::string_view Name; + }; + + struct DependenciesTable + { + // Get the table name. + static std::string_view TableName(); + + // Creates the table with named indices. + static void Create(SQLite::Connection& connection); + + static bool Exists(const SQLite::Connection& connection); + + // Add the dependencies for the specific manifest. + static void AddDependencies(SQLite::Connection& connection, const Manifest::Manifest& manifest, SQLite::rowid_t manifestRowId); + + // update the dependencies for the specific manifest. + static bool UpdateDependencies(SQLite::Connection& connection, const Manifest::Manifest& manifest, SQLite::rowid_t manifestRowId); + + // Remove the dependencies by manifest id + static void RemoveDependencies(SQLite::Connection& connection, SQLite::rowid_t manifestRowId); + + // Get dependencies the dependencies + static std::set<std::pair<SQLite::rowid_t, Utility::NormalizedString>> GetDependenciesByManifestRowId(const SQLite::Connection& connection, SQLite::rowid_t manifestRowId); + + // Get dependencies by package id. + static std::vector<std::pair<SQLite::rowid_t, Utility::NormalizedString>> GetDependentsById(const SQLite::Connection& connection, AppInstaller::Manifest::string_t packageId); + + // Check dependencies table consistency. + static bool DependenciesTableCheckConsistency(const SQLite::Connection& connection, bool log); + + // Checks if the row id is present in the column denoted by the value supplied. + static bool IsValueReferenced(const SQLite::Connection& connection, std::string_view valueName, SQLite::rowid_t valueRowId); + + static void PrepareForPackaging(SQLite::Connection& connection); + + static std::vector<SQLite::rowid_t> GetDependenciesMinVersionsRowIdByManifestId(const SQLite::Connection& connection, SQLite::rowid_t manifestRowId); + }; +}+ \ No newline at end of file diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_4/Interface.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_4/Interface.h @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once +#include "Microsoft/Schema/ISQLiteIndex.h" +#include "Microsoft/Schema/1_3/Interface.h" +#include "Microsoft/Schema/1_2/Interface.h" + +namespace AppInstaller::Repository::Microsoft::Schema::V1_4 +{ + // Interface to this schema version exposed through ISQLiteIndex. + struct Interface : public V1_3::Interface + { + Interface(Utility::NormalizationVersion normVersion = Utility::NormalizationVersion::Initial); + + // Version 1.0 + Schema::Version GetVersion() const override; + void CreateTables(SQLite::Connection& connection, CreateOptions options) override; + SQLite::rowid_t AddManifest(SQLite::Connection& connection, const Manifest::Manifest& manifest, const std::optional<std::filesystem::path>& relativePath) override; + std::pair<bool, SQLite::rowid_t> UpdateManifest(SQLite::Connection& connection, const Manifest::Manifest& manifest, const std::optional<std::filesystem::path>& relativePath) override; + void RemoveManifestById(SQLite::Connection& connection, SQLite::rowid_t manifestId) override; + bool CheckConsistency(const SQLite::Connection& connection, bool log) const override; + void PrepareForPackaging(SQLite::Connection& connection, bool vacuum) override; + + std::set<std::pair<SQLite::rowid_t, Utility::NormalizedString>> GetDependenciesByManifestRowId(const SQLite::Connection& connection, SQLite::rowid_t manifestRowId) const override; + std::vector<std::pair<SQLite::rowid_t, Utility::NormalizedString>> GetDependentsById(const SQLite::Connection& connection, AppInstaller::Manifest::string_t packageId) const override; + + bool NotNeeded(const SQLite::Connection& connection, std::string_view tableName, std::string_view valueName, SQLite::rowid_t id) const override; + }; +}+ \ No newline at end of file diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/1_4/Interface_1_4.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/1_4/Interface_1_4.cpp @@ -0,0 +1,133 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#include "pch.h" +#include "Microsoft/Schema/1_4/Interface.h" +#include "Microsoft/Schema/1_0/VersionTable.h" +#include <AppInstallerSHA256.h> + +#include "Microsoft/Schema/1_4/DependenciesTable.h" + +namespace AppInstaller::Repository::Microsoft::Schema::V1_4 +{ + Interface::Interface(Utility::NormalizationVersion normVersion) : V1_3::Interface(normVersion) + { + } + + Schema::Version Interface::GetVersion() const + { + return { 1, 4 }; + } + + void Interface::CreateTables(SQLite::Connection& connection, CreateOptions options) + { + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, "createtables_v1_4"); + + V1_3::Interface::CreateTables(connection, options); + + DependenciesTable::Create(connection); + + savepoint.Commit(); + } + + SQLite::rowid_t Interface::AddManifest(SQLite::Connection& connection, const Manifest::Manifest& manifest, const std::optional<std::filesystem::path>& relativePath) + { + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, "addmanifest_v1_4"); + + SQLite::rowid_t manifestId = V1_3::Interface::AddManifest(connection, manifest, relativePath); + + DependenciesTable::AddDependencies(connection, manifest, manifestId); + + savepoint.Commit(); + + return manifestId; + } + + std::pair<bool, SQLite::rowid_t> Interface::UpdateManifest(SQLite::Connection& connection, const Manifest::Manifest& manifest, const std::optional<std::filesystem::path>& relativePath) + { + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, "updatemanifest_v1_4"); + + auto [indexModified, manifestId] = V1_3::Interface::UpdateManifest(connection, manifest, relativePath); + + bool dependenciesModified = DependenciesTable::UpdateDependencies(connection, manifest, manifestId); + indexModified = indexModified || dependenciesModified; + + savepoint.Commit(); + + return { indexModified, manifestId }; + } + + void Interface::RemoveManifestById(SQLite::Connection& connection, SQLite::rowid_t manifestId) + { + // Get all versions that need cleaning from the version table. + auto minVersions = DependenciesTable::GetDependenciesMinVersionsRowIdByManifestId(connection, manifestId); + + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, "removemanifest_v1_4"); + + // Removes dependences for the manifest id. + DependenciesTable::RemoveDependencies(connection, manifestId); + + // Removes the manifest. + V1_2::Interface::RemoveManifestById(connection, manifestId); + + // Remove the versions that are not needed. + for (auto minVersion : minVersions) + { + if (NotNeeded(connection, Schema::V1_0::VersionTable::TableName(), Schema::V1_0::VersionTable::ValueName(), minVersion)) + { + Schema::V1_0::VersionTable::DeleteById(connection, minVersion); + } + } + + savepoint.Commit(); + } + + bool Interface::NotNeeded(const SQLite::Connection& connection, std::string_view tableName, std::string_view valueName, SQLite::rowid_t id) const + { + bool result = V1_0::Interface::NotNeeded(connection, tableName, valueName, id); + + return !DependenciesTable::IsValueReferenced(connection, tableName, id) && result; + } + + void Interface::PrepareForPackaging(SQLite::Connection& connection, bool vacuum) + { + SQLite::Savepoint savepoint = SQLite::Savepoint::Create(connection, "prepareforpackaging_v1_4"); + + V1_2::Interface::PrepareForPackaging(connection, false); + + DependenciesTable::PrepareForPackaging(connection); + + savepoint.Commit(); + + if (vacuum) + { + // Force the database to actually shrink the file size. + // This *must* be done outside of an active transaction. + SQLite::Builder::StatementBuilder builder; + builder.Vacuum(); + builder.Execute(connection); + } + } + + bool Interface::CheckConsistency(const SQLite::Connection& connection, bool log) const + { + bool result = V1_2::Interface::CheckConsistency(connection, log); + + // If the v1.3 index was consistent, or if full logging of inconsistency was requested, check the v1.4 data. + if (result || log) + { + result = DependenciesTable::DependenciesTableCheckConsistency(connection, log) && result; + } + + return result; + } + + std::set<std::pair<SQLite::rowid_t, Utility::NormalizedString>> Interface::GetDependenciesByManifestRowId(const SQLite::Connection& connection, SQLite::rowid_t manifestRowId) const + { + return DependenciesTable::GetDependenciesByManifestRowId(connection, manifestRowId); + } + + std::vector<std::pair<SQLite::rowid_t, Utility::NormalizedString>> Interface::GetDependentsById(const SQLite::Connection& connection, AppInstaller::Manifest::string_t packageId) const + { + return DependenciesTable::GetDependentsById(connection, packageId); + } +}+ \ No newline at end of file diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.h b/src/AppInstallerRepositoryCore/Microsoft/Schema/ISQLiteIndex.h @@ -103,6 +103,11 @@ namespace AppInstaller::Repository::Microsoft::Schema // Normalizes a name using the internal rules used by the index. // Largely a utility function; should not be used to do work on behalf of the index by the caller. virtual Utility::NormalizedName NormalizeName(std::string_view name, std::string_view publisher) const = 0; + + // Get all the dependencies for a specific manifest. + virtual std::set<std::pair<SQLite::rowid_t, Utility::NormalizedString>> GetDependenciesByManifestRowId(const SQLite::Connection& connection, SQLite::rowid_t manifestRowId) const = 0; + + virtual std::vector<std::pair<SQLite::rowid_t, Utility::NormalizedString>> GetDependentsById(const SQLite::Connection& connection, AppInstaller::Manifest::string_t packageId) const = 0; }; DEFINE_ENUM_FLAG_OPERATORS(ISQLiteIndex::CreateOptions); diff --git a/src/AppInstallerRepositoryCore/Microsoft/Schema/Version.cpp b/src/AppInstallerRepositoryCore/Microsoft/Schema/Version.cpp @@ -8,6 +8,7 @@ #include "1_1/Interface.h" #include "1_2/Interface.h" #include "1_3/Interface.h" +#include "1_4/Interface.h" namespace AppInstaller::Repository::Microsoft::Schema { @@ -44,11 +45,15 @@ namespace AppInstaller::Repository::Microsoft::Schema { return std::make_unique<V1_2::Interface>(); } - else if (*this == Version{ 1, 3 } || + else if (*this == Version{ 1, 3 }) + { + return std::make_unique<V1_3::Interface>(); + } + else if (*this == Version{ 1, 4 } || this->MajorVersion == 1 || this->IsLatest()) { - return std::make_unique<V1_3::Interface>(); + return std::make_unique<V1_4::Interface>(); } // We do not have the capacity to operate on this schema version diff --git a/src/AppInstallerRepositoryCore/PackageDependenciesValidation.cpp b/src/AppInstallerRepositoryCore/PackageDependenciesValidation.cpp @@ -0,0 +1,247 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once +#include "pch.h" +#include <AppInstallerVersions.h> +#include <winget/Manifest.h> +#include <winget/ManifestValidation.h> +#include <SQLiteWrapper.h> +#include <PackageDependenciesValidation.h> +#include <Microsoft/Schema/1_4/DependenciesTable.h> +#include "Microsoft/Schema/1_0/ManifestTable.h" +#include <winget/DependenciesGraph.h> + +namespace AppInstaller::Repository +{ + using namespace Microsoft::Schema::V1_4; + using namespace Microsoft::Schema::V1_0; + + namespace + { + struct DependentManifestInfo + { + Utility::NormalizedString Id; + Utility::NormalizedString Version; + }; + + Manifest::DependencyList GetDependencies( + const Manifest::Manifest& manifest, AppInstaller::Manifest::DependencyType dependencyType) + { + Manifest::DependencyList depList; + std::vector<AppInstaller::Manifest::Dependency> dependencies; + + for (const auto& installer : manifest.Installers) + { + installer.Dependencies.ApplyToType(dependencyType, [&](AppInstaller::Manifest::Dependency dependency) + { + depList.Add(dependency); + }); + } + + return depList; + } + + std::optional<std::pair<SQLite::rowid_t, Utility::Version>> GetPackageLatestVersion( + SQLiteIndex* index, Manifest::string_t packageId, std::set<Utility::Version> exclusions = {}) + { + SearchRequest request; + request.Filters.emplace_back(PackageMatchField::Id, MatchType::CaseInsensitive, packageId); + + auto results = index->Search(request); + + if (results.Matches.empty()) + { + return {}; + } + + auto packageRowId = results.Matches[0].first; + auto vac = index->GetVersionKeysById(packageRowId); + + if (vac.empty()) + { + return {}; + } + + Utility::VersionAndChannel maxVersion(Utility::Version::CreateUnknown(), Utility::Channel("")); + + for (auto& v : vac) + { + auto currentVersion = v.GetVersion(); + if (exclusions.find(currentVersion) != exclusions.end()) + { + continue; + } + + if (currentVersion > maxVersion.GetVersion()) + { + maxVersion = v; + } + } + + if (maxVersion.GetVersion().IsUnknown()) + { + return {}; + } + + auto manifestRowId = index->GetManifestIdByKey( + packageRowId, maxVersion.GetVersion().ToString(), maxVersion.GetChannel().ToString()); + + return std::make_pair(manifestRowId.value(), maxVersion.GetVersion()); + } + + void ThrowOnManifestValidationFailed( + std::vector<std::pair<DependentManifestInfo, Utility::Version>> failedManifests, std::string error) + { + auto itrStart = failedManifests.begin(); + std::string dependentPackages{ itrStart->first.Id + "." + itrStart->first.Version }; + + std::for_each( + itrStart + 1, + failedManifests.end(), + [&](std::pair<DependentManifestInfo, Utility::Version> current) + { + dependentPackages.append(", " + current.first.Id + "." + current.first.Version); + }); + + error.append("\n" + dependentPackages); + THROW_EXCEPTION( + Manifest::ManifestException({ Manifest::ValidationError(error) }, + APPINSTALLER_CLI_ERROR_DEPENDENCIES_VALIDATION_FAILED)); + } + }; + + bool PackageDependenciesValidation::ValidateManifestDependencies(SQLiteIndex* index, const Manifest::Manifest manifest) + { + using namespace Manifest; + + Dependency rootId(DependencyType::Package, manifest.Id, manifest.Version); + std::vector<ValidationError> dependenciesError; + bool foundErrors = false; + + DependencyGraph graph(rootId, [&](const Dependency& node) { + + DependencyList depList; + if (node.Id == rootId.Id) + { + return GetDependencies(manifest, DependencyType::Package); + } + + auto packageLatest = GetPackageLatestVersion(index, node.Id); + if (!packageLatest.has_value()) + { + std::string error = ManifestError::MissingManifestDependenciesNode; + error.append(" ").append(node.Id); + dependenciesError.emplace_back(ValidationError(error)); + foundErrors = true; + return depList; + } + + if (node.MinVersion > packageLatest.value().second) + { + std::string error = ManifestError::NoSuitableMinVersion; + error.append(" ").append(node.Id); + dependenciesError.emplace_back(ValidationError(error)); + foundErrors = true; + return depList; + } + + auto packageLatestDependencies = index->GetDependenciesByManifestRowId(packageLatest.value().first); + std::for_each( + packageLatestDependencies.begin(), + packageLatestDependencies.end(), + [&](std::pair<SQLite::rowid_t, Utility::NormalizedString> row) + { + auto manifestRowId = index->GetManifestIdByKey(row.first, "", ""); + auto packageId = index->GetPropertyByManifestId(manifestRowId.value(), PackageVersionProperty::Id); + Dependency dep(DependencyType::Package, packageId.value(), row.second); + depList.Add(dep); + }); + + return depList; + }); + + graph.BuildGraph(); + + if (foundErrors) + { + THROW_EXCEPTION(ManifestException(std::move(dependenciesError), APPINSTALLER_CLI_ERROR_DEPENDENCIES_VALIDATION_FAILED)); + } + + if (graph.HasLoop()) + { + std::string error = ManifestError::FoundLoop; + dependenciesError.emplace_back(error); + THROW_EXCEPTION(ManifestException(std::move(dependenciesError), APPINSTALLER_CLI_ERROR_DEPENDENCIES_VALIDATION_FAILED)); + } + + return true; + } + + bool PackageDependenciesValidation::VerifyDependenciesStructureForManifestDelete(SQLiteIndex* index, const Manifest::Manifest manifest) + { + auto dependentsSet = index->GetDependentsById(manifest.Id); + + if (!dependentsSet.size()) + { + // all good this manifest is not a dependency of any manifest. + return true; + } + + std::vector<std::pair<DependentManifestInfo, Utility::Version>> dependentManifestInfoToVersionPair; + std::for_each( + dependentsSet.begin(), + dependentsSet.end(), + [&](std::pair<SQLite::rowid_t, Utility::Version> current) + { + DependentManifestInfo dependentManifestInfo; + dependentManifestInfo.Id = index->GetPropertyByManifestId(current.first, PackageVersionProperty::Id).value(); + dependentManifestInfo.Version = index->GetPropertyByManifestId(current.first, PackageVersionProperty::Version).value(); + + dependentManifestInfoToVersionPair.emplace_back(std::make_pair(dependentManifestInfo, current.second)); + }); + + auto packageLatest = GetPackageLatestVersion(index, manifest.Id); + + if (!packageLatest.has_value()) + { + // this is a fatal error, a manifest should exists in the very least(including the current manifest being deleted), + // since this is a delete operation. + THROW_HR(APPINSTALLER_CLI_ERROR_MISSING_PACKAGE); + } + + if (Utility::Version(manifest.Version) < packageLatest.value().second) + { + // all good, since it's min version the criteria is still satisfied. + return true; + } + + auto nextLatestAfterDelete = GetPackageLatestVersion(index, manifest.Id, { packageLatest.value().second }); + + if (!nextLatestAfterDelete.has_value()) + { + ThrowOnManifestValidationFailed( + dependentManifestInfoToVersionPair, Manifest::ManifestError::SingleManifestPackageHasDependencies); + } + + std::vector<std::pair<DependentManifestInfo, Utility::Version>> breakingManifests; + + // Gets breaking manifests. + std::copy_if( + dependentManifestInfoToVersionPair.begin(), + dependentManifestInfoToVersionPair.end(), + std::back_inserter(breakingManifests), + [&](std::pair<DependentManifestInfo, Utility::Version> current) + { + return current.second > nextLatestAfterDelete.value().second; + } + ); + + if (breakingManifests.size()) + { + ThrowOnManifestValidationFailed( + breakingManifests, Manifest::ManifestError::MultiManifestPackageHasDependencies); + } + + return true; + } +}+ \ No newline at end of file diff --git a/src/AppInstallerRepositoryCore/PackageDependenciesValidation.h b/src/AppInstallerRepositoryCore/PackageDependenciesValidation.h @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#pragma once +#include <Microsoft/SQLiteIndex.h> +#include <AppInstallerVersions.h> +#include <winget/Manifest.h> +#include <SQLiteWrapper.h> + +namespace AppInstaller::Repository +{ + using namespace AppInstaller::Repository::Microsoft; + + struct PackageDependenciesValidation + { + // Validate the dependencies of the given manifest. + static bool ValidateManifestDependencies(SQLiteIndex* index, const Manifest::Manifest manifest); + + static bool VerifyDependenciesStructureForManifestDelete(SQLiteIndex* index, const Manifest::Manifest manifest); + }; +} diff --git a/src/AppInstallerRepositoryCore/SQLiteStatementBuilder.cpp b/src/AppInstallerRepositoryCore/SQLiteStatementBuilder.cpp @@ -363,6 +363,19 @@ namespace AppInstaller::Repository::SQLite::Builder return *this; } + StatementBuilder& StatementBuilder::In(size_t count) + { + m_stream << " IN ("; + for (size_t i = 0; i < count; ++i) + { + m_stream << (i == 0 ? "?" : ", ?"); + } + m_stream << ')'; + + m_bindIndex += static_cast<int>(count); + return *this; + } + StatementBuilder& StatementBuilder::IsNull(bool isNull) { m_stream << " IS " << (isNull ? "" : "NOT ") << "NULL"; @@ -381,6 +394,12 @@ namespace AppInstaller::Repository::SQLite::Builder return *this; } + StatementBuilder& StatementBuilder::Or(const QualifiedColumn& column) + { + OutputColumns(m_stream, " OR ", column); + return *this; + } + StatementBuilder& StatementBuilder::Join(std::string_view table) { OutputOperationAndTable(m_stream, " JOIN", table); diff --git a/src/AppInstallerRepositoryCore/SQLiteStatementBuilder.h b/src/AppInstallerRepositoryCore/SQLiteStatementBuilder.h @@ -249,6 +249,9 @@ namespace AppInstaller::Repository::SQLite::Builder StatementBuilder& Not(); StatementBuilder& In(); + + //Appends a set of value binders for the In clause. + StatementBuilder& In(size_t count); // IsNull(true) means the value is null; IsNull(false) means the value is not null. StatementBuilder& IsNull(bool isNull = true); @@ -257,6 +260,7 @@ namespace AppInstaller::Repository::SQLite::Builder // Operators for combining filter clauses. StatementBuilder& And(std::string_view column); StatementBuilder& And(const QualifiedColumn& column); + StatementBuilder& Or(const QualifiedColumn& column); // Begin a join clause. // The initializer_list form enables the table name to be constructed from multiple parts. diff --git a/src/IndexCreationTool/WinGetUtilWrapper.cs b/src/IndexCreationTool/WinGetUtilWrapper.cs @@ -126,7 +126,7 @@ namespace IndexCreationTool Console.WriteLine($"Error to update manifest {manifestPath} with relative path {relativePath}. {Environment.NewLine}{e.ToString()}"); throw; } -} + } /// <summary> /// Delete manifest from index. @@ -261,4 +261,4 @@ namespace IndexCreationTool [DllImport(DllName, CallingConvention = CallingConvention.StdCall, CharSet = CharSet.Unicode, PreserveSig = false)] private static extern IntPtr WinGetSQLiteIndexPrepareForPackaging(IntPtr index); } -} +}+ \ No newline at end of file diff --git a/src/WinGetUtil/Exports.cpp b/src/WinGetUtil/Exports.cpp @@ -10,10 +10,12 @@ #include <AppInstallerTelemetry.h> #include <Microsoft/SQLiteIndex.h> #include <winget/ManifestYamlParser.h> +#include <PackageDependenciesValidation.h> #include <winget/ThreadGlobals.h> using namespace AppInstaller::Utility; using namespace AppInstaller::Manifest; +using namespace AppInstaller::Repository; using namespace AppInstaller::Repository::Microsoft; extern "C" @@ -245,6 +247,48 @@ extern "C" } CATCH_RETURN() + WINGET_UTIL_API WinGetValidateManifestDependencies( + WINGET_STRING inputPath, + BOOL* succeeded, + WINGET_STRING_OUT* message, + WINGET_SQLITE_INDEX_HANDLE index, + WinGetValidateManifestDependenciesOption validationOption) try + { + THROW_HR_IF(E_INVALIDARG, !inputPath); + THROW_HR_IF(E_INVALIDARG, !succeeded); + + try + { + Manifest manifest = YamlParser::CreateFromPath(inputPath); + SQLiteIndex* sqliteIndex(reinterpret_cast<SQLiteIndex*>(index)); + + switch (validationOption) + { + case WinGetValidateManifestDependenciesOption::DefaultValidation: + PackageDependenciesValidation::ValidateManifestDependencies(sqliteIndex, manifest); + break; + case WinGetValidateManifestDependenciesOption::ForDelete: + PackageDependenciesValidation::VerifyDependenciesStructureForManifestDelete(sqliteIndex, manifest); + break; + default: + THROW_HR(E_INVALIDARG); + } + + *succeeded = TRUE; + } + catch (const ManifestException& e) + { + *succeeded = e.IsWarningOnly(); + if (message) + { + *message = ::SysAllocString(ConvertToUTF16(e.GetManifestErrorMessage()).c_str()); + } + } + + return S_OK; + } + CATCH_RETURN() + WINGET_UTIL_API WinGetDownload( WINGET_STRING url, WINGET_STRING filePath, diff --git a/src/WinGetUtil/Source.def b/src/WinGetUtil/Source.def @@ -14,3 +14,4 @@ EXPORTS WinGetDownload WinGetCompareVersions WinGetValidateManifestV2 + WinGetValidateManifestDependencies diff --git a/src/WinGetUtil/WinGetUtil.h b/src/WinGetUtil/WinGetUtil.h @@ -24,6 +24,12 @@ extern "C" ErrorOnVerifiedPublisherFields = 0x2, }; + enum WinGetValidateManifestDependenciesOption + { + DefaultValidation = 0, + ForDelete = 0x1, + }; + DEFINE_ENUM_FLAG_OPERATORS(WinGetValidateManifestOption); // Initializes the logging infrastructure. @@ -100,6 +106,17 @@ extern "C" WINGET_STRING mergedManifestPath, WinGetValidateManifestOption option); + // Validates a given manifest with dependencies. Returns a bool for validation result and + // a string representing validation errors if validation failed. + // If mergedManifestPath is provided, this method will write a merged manifest + // to the location specified by mergedManifestPath + WINGET_UTIL_API WinGetValidateManifestDependencies( + WINGET_STRING inputPath, + BOOL* succeeded, + WINGET_STRING_OUT* message, + WINGET_SQLITE_INDEX_HANDLE index, + WinGetValidateManifestDependenciesOption dependenciesValidationOption); + // Downloads a file to the given path, returning the SHA 256 hash of the file. WINGET_UTIL_API WinGetDownload( WINGET_STRING url,